mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-20 07:18:27 +01:00
Compare commits
22
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
91586e85f8 | ||
|
|
56e5982951 | ||
|
|
defe4062aa | ||
|
|
f74c1c0e70 | ||
|
|
b5dede14f2 | ||
|
|
8d37f966c3 | ||
|
|
95165cc39a | ||
|
|
ccb72126ea | ||
|
|
d5315c3827 | ||
|
|
8131c32c33 | ||
|
|
0013af99d4 | ||
|
|
06ff81cd7f | ||
|
|
e19b0874c0 | ||
|
|
b290eaea43 | ||
|
|
2a2b543dd6 | ||
|
|
356dfa0327 | ||
|
|
0f4e8a23c8 | ||
|
|
333edb2cbf | ||
|
|
a65523a86a | ||
|
|
fb42429107 | ||
|
|
e40e378fc9 | ||
|
|
b1aaeb94c0 |
@@ -14,11 +14,11 @@ jobs:
|
|||||||
name: run benchmarks
|
name: run benchmarks
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/setup-python@v6
|
- uses: actions/setup-python@v7
|
||||||
with:
|
with:
|
||||||
python-version: "3.14"
|
python-version: "3.14"
|
||||||
|
|
||||||
- uses: actions/checkout@v6
|
- uses: actions/checkout@v7
|
||||||
with:
|
with:
|
||||||
ref: ${{ github.event.pull_request.base.sha }}
|
ref: ${{ github.event.pull_request.base.sha }}
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
@@ -32,7 +32,7 @@ jobs:
|
|||||||
- name: run benchmarks on base branch
|
- name: run benchmarks on base branch
|
||||||
run: pytest --benchmark-autosave dvc/testing/benchmarks/ -k 'test_init or test_help'
|
run: pytest --benchmark-autosave dvc/testing/benchmarks/ -k 'test_init or test_help'
|
||||||
|
|
||||||
- uses: actions/checkout@v6
|
- uses: actions/checkout@v7
|
||||||
with:
|
with:
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
clean: false
|
clean: false
|
||||||
|
|||||||
@@ -19,11 +19,11 @@ jobs:
|
|||||||
name: Build & Verify package
|
name: Build & Verify package
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v6
|
- uses: actions/checkout@v7
|
||||||
with:
|
with:
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
|
|
||||||
- uses: actions/setup-python@v6
|
- uses: actions/setup-python@v7
|
||||||
with:
|
with:
|
||||||
python-version: "3.14"
|
python-version: "3.14"
|
||||||
|
|
||||||
@@ -41,7 +41,7 @@ jobs:
|
|||||||
- name: Check dist
|
- name: Check dist
|
||||||
run: uv tool run twine check --strict dist/*
|
run: uv tool run twine check --strict dist/*
|
||||||
|
|
||||||
- uses: actions/upload-artifact@v6
|
- uses: actions/upload-artifact@v7
|
||||||
with:
|
with:
|
||||||
name: Packages
|
name: Packages
|
||||||
path: dist/
|
path: dist/
|
||||||
@@ -61,7 +61,7 @@ jobs:
|
|||||||
id-token: write
|
id-token: write
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/download-artifact@v7
|
- uses: actions/download-artifact@v8
|
||||||
with:
|
with:
|
||||||
name: Packages
|
name: Packages
|
||||||
path: dist
|
path: dist
|
||||||
@@ -86,7 +86,7 @@ jobs:
|
|||||||
id-token: write
|
id-token: write
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/download-artifact@v7
|
- uses: actions/download-artifact@v8
|
||||||
with:
|
with:
|
||||||
name: Packages
|
name: Packages
|
||||||
path: dist
|
path: dist
|
||||||
|
|||||||
@@ -28,13 +28,13 @@ jobs:
|
|||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout repository
|
- name: Checkout repository
|
||||||
uses: actions/checkout@v6
|
uses: actions/checkout@v7
|
||||||
|
|
||||||
- name: Initialize CodeQL
|
- name: Initialize CodeQL
|
||||||
uses: github/codeql-action/init@v4
|
uses: github/codeql-action/init@v4.37.4
|
||||||
with:
|
with:
|
||||||
languages: ${{ matrix.language }}
|
languages: ${{ matrix.language }}
|
||||||
queries: security-extended
|
queries: security-extended
|
||||||
|
|
||||||
- name: Perform CodeQL Analysis
|
- name: Perform CodeQL Analysis
|
||||||
uses: github/codeql-action/analyze@v4
|
uses: github/codeql-action/analyze@v4.37.4
|
||||||
|
|||||||
@@ -23,18 +23,18 @@ jobs:
|
|||||||
plugin: ["dvc-s3"]
|
plugin: ["dvc-s3"]
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v6
|
- uses: actions/checkout@v7
|
||||||
with:
|
with:
|
||||||
path: dvc
|
path: dvc
|
||||||
|
|
||||||
- uses: actions/checkout@v6
|
- uses: actions/checkout@v7
|
||||||
with:
|
with:
|
||||||
repository: treeverse/${{ matrix.plugin }}
|
repository: treeverse/${{ matrix.plugin }}
|
||||||
ref: main
|
ref: main
|
||||||
path: ${{ matrix.plugin }}
|
path: ${{ matrix.plugin }}
|
||||||
|
|
||||||
- name: Set up Python
|
- name: Set up Python
|
||||||
uses: actions/setup-python@v6
|
uses: actions/setup-python@v7
|
||||||
with:
|
with:
|
||||||
python-version: ${{ matrix.pyv }}
|
python-version: ${{ matrix.pyv }}
|
||||||
|
|
||||||
|
|||||||
@@ -26,11 +26,11 @@ jobs:
|
|||||||
matrix:
|
matrix:
|
||||||
os: [ubuntu-latest, macos-latest, windows-latest]
|
os: [ubuntu-latest, macos-latest, windows-latest]
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v6
|
- uses: actions/checkout@v7
|
||||||
with:
|
with:
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
- name: Set up Python
|
- name: Set up Python
|
||||||
uses: actions/setup-python@v6
|
uses: actions/setup-python@v7
|
||||||
with:
|
with:
|
||||||
python-version: '3.14'
|
python-version: '3.14'
|
||||||
- uses: astral-sh/setup-uv@v7
|
- uses: astral-sh/setup-uv@v7
|
||||||
@@ -42,7 +42,7 @@ jobs:
|
|||||||
UV_LINK_MODE: ${{ runner.os == 'Windows' && 'symlink' || 'hardlink' }}
|
UV_LINK_MODE: ${{ runner.os == 'Windows' && 'symlink' || 'hardlink' }}
|
||||||
|
|
||||||
- name: Cache mypy
|
- name: Cache mypy
|
||||||
uses: actions/cache@v5
|
uses: actions/cache@v6
|
||||||
with:
|
with:
|
||||||
path: .mypy_cache
|
path: .mypy_cache
|
||||||
key: mypy-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}
|
key: mypy-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}
|
||||||
@@ -130,11 +130,11 @@ jobs:
|
|||||||
pyv: "3.14"
|
pyv: "3.14"
|
||||||
pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
|
pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v6
|
- uses: actions/checkout@v7
|
||||||
with:
|
with:
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
- name: Set up Python
|
- name: Set up Python
|
||||||
uses: actions/setup-python@v6
|
uses: actions/setup-python@v7
|
||||||
with:
|
with:
|
||||||
python-version: ${{ matrix.pyv }}
|
python-version: ${{ matrix.pyv }}
|
||||||
- uses: astral-sh/setup-uv@v7
|
- uses: astral-sh/setup-uv@v7
|
||||||
@@ -155,7 +155,7 @@ jobs:
|
|||||||
pytest ${{ matrix.pytestargs }} -n=logical --dist=worksteal --timeout=300 --durations=0
|
pytest ${{ matrix.pytestargs }} -n=logical --dist=worksteal --timeout=300 --durations=0
|
||||||
--cov --cov-report=xml --cov-report=term --durations-path=./.github/.test_durations
|
--cov --cov-report=xml --cov-report=term --durations-path=./.github/.test_durations
|
||||||
- name: upload coverage report
|
- name: upload coverage report
|
||||||
uses: codecov/codecov-action@v5
|
uses: codecov/codecov-action@v7
|
||||||
check:
|
check:
|
||||||
if: always()
|
if: always()
|
||||||
needs: [tests]
|
needs: [tests]
|
||||||
@@ -170,7 +170,7 @@ jobs:
|
|||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- name: Slack Notification
|
- name: Slack Notification
|
||||||
uses: rtCamp/action-slack-notify@v2.3.3
|
uses: rtCamp/action-slack-notify@v2.4.0
|
||||||
env:
|
env:
|
||||||
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
|
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
|
||||||
SLACK_COLOR: ${{ job.status }}
|
SLACK_COLOR: ${{ job.status }}
|
||||||
|
|||||||
@@ -22,13 +22,13 @@ repos:
|
|||||||
- id: sort-simple-yaml
|
- id: sort-simple-yaml
|
||||||
- id: trailing-whitespace
|
- id: trailing-whitespace
|
||||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||||
rev: 'v0.14.13'
|
rev: 'v0.16.10'
|
||||||
hooks:
|
hooks:
|
||||||
- id: ruff-check
|
- id: ruff-check
|
||||||
args: [--fix, --exit-non-zero-on-fix]
|
args: [--fix, --exit-non-zero-on-fix]
|
||||||
- id: ruff-format
|
- id: ruff-format
|
||||||
- repo: https://github.com/codespell-project/codespell
|
- repo: https://github.com/codespell-project/codespell
|
||||||
rev: v2.4.1
|
rev: v2.4.3
|
||||||
hooks:
|
hooks:
|
||||||
- id: codespell
|
- id: codespell
|
||||||
additional_dependencies: ["tomli"]
|
additional_dependencies: ["tomli"]
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ from PyInstaller.utils.hooks import collect_submodules, is_module_or_submodule
|
|||||||
_EXCLUDES = ("celery.bin", "celery.contrib")
|
_EXCLUDES = ("celery.bin", "celery.contrib")
|
||||||
hiddenimports = collect_submodules(
|
hiddenimports = collect_submodules(
|
||||||
"celery",
|
"celery",
|
||||||
filter=lambda name: not any(
|
filter=lambda name: (
|
||||||
is_module_or_submodule(name, module) for module in _EXCLUDES
|
not any(is_module_or_submodule(name, module) for module in _EXCLUDES)
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|||||||
+11
-4
@@ -40,9 +40,16 @@ class CmdGC(CmdBase):
|
|||||||
msg += " and all git commits"
|
msg += " and all git commits"
|
||||||
else:
|
else:
|
||||||
if self.args.all_branches and self.args.all_tags:
|
if self.args.all_branches and self.args.all_tags:
|
||||||
msg += " and all git branches and tags"
|
if self.args.num:
|
||||||
|
msg += f" and last {self.args.num} commits from all git branches"
|
||||||
|
msg += " and all git tags"
|
||||||
|
else:
|
||||||
|
msg += " and all git branches and tags"
|
||||||
elif self.args.all_branches:
|
elif self.args.all_branches:
|
||||||
msg += " and all git branches"
|
if self.args.num:
|
||||||
|
msg += f" and last {self.args.num} commits from all git branches"
|
||||||
|
else:
|
||||||
|
msg += " and all git branches"
|
||||||
elif self.args.all_tags:
|
elif self.args.all_tags:
|
||||||
msg += " and all git tags"
|
msg += " and all git tags"
|
||||||
if self.args.commit_date:
|
if self.args.commit_date:
|
||||||
@@ -126,8 +133,8 @@ def add_parser(subparsers, parent_parser):
|
|||||||
metavar="<num>",
|
metavar="<num>",
|
||||||
help=(
|
help=(
|
||||||
"Keep data files used in the last `num` commits "
|
"Keep data files used in the last `num` commits "
|
||||||
"starting from the `--rev` <commit>. "
|
"starting from each selected revision root. "
|
||||||
"Only used if `--rev` is also provided. "
|
"Can be used with `--rev` and `--all-branches`. "
|
||||||
"Defaults to `1`."
|
"Defaults to `1`."
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -109,8 +109,6 @@ def _build_tree_structure(
|
|||||||
|
|
||||||
|
|
||||||
def show_tree(entries, with_color=False, with_size=False, with_hash=False):
|
def show_tree(entries, with_color=False, with_size=False, with_hash=False):
|
||||||
import tabulate
|
|
||||||
|
|
||||||
rows = _build_tree_structure(
|
rows = _build_tree_structure(
|
||||||
entries,
|
entries,
|
||||||
with_color=with_color,
|
with_color=with_color,
|
||||||
@@ -119,13 +117,7 @@ def show_tree(entries, with_color=False, with_size=False, with_hash=False):
|
|||||||
)
|
)
|
||||||
|
|
||||||
colalign = ("right",) if with_size else None
|
colalign = ("right",) if with_size else None
|
||||||
|
ui.table(rows, colalign=colalign, preserve_whitespace=True)
|
||||||
_orig = tabulate.PRESERVE_WHITESPACE
|
|
||||||
tabulate.PRESERVE_WHITESPACE = True
|
|
||||||
try:
|
|
||||||
ui.table(rows, colalign=colalign)
|
|
||||||
finally:
|
|
||||||
tabulate.PRESERVE_WHITESPACE = _orig
|
|
||||||
|
|
||||||
|
|
||||||
class CmdList(CmdBaseNoRepo):
|
class CmdList(CmdBaseNoRepo):
|
||||||
|
|||||||
@@ -37,13 +37,13 @@ KEYCRE = re.compile(
|
|||||||
def get_parser():
|
def get_parser():
|
||||||
from pyparsing import CharsNotIn, ParserElement, Suppress, ZeroOrMore
|
from pyparsing import CharsNotIn, ParserElement, Suppress, ZeroOrMore
|
||||||
|
|
||||||
ParserElement.enablePackrat()
|
ParserElement.enable_packrat()
|
||||||
|
|
||||||
word = CharsNotIn(f"{PERIOD}{LBRACK}{RBRACK}")
|
word = CharsNotIn(f"{PERIOD}{LBRACK}{RBRACK}")
|
||||||
idx = Suppress(LBRACK) + word + Suppress(RBRACK)
|
idx = Suppress(LBRACK) + word + Suppress(RBRACK)
|
||||||
attr = Suppress(PERIOD) + word
|
attr = Suppress(PERIOD) + word
|
||||||
parser = word + ZeroOrMore(attr ^ idx)
|
parser = word + ZeroOrMore(attr ^ idx)
|
||||||
parser.setParseAction(PERIOD.join)
|
parser.set_parse_action(PERIOD.join)
|
||||||
|
|
||||||
return parser
|
return parser
|
||||||
|
|
||||||
@@ -178,7 +178,7 @@ def parse_expr(s: str):
|
|||||||
from pyparsing import ParseException
|
from pyparsing import ParseException
|
||||||
|
|
||||||
try:
|
try:
|
||||||
result = get_parser().parseString(s, parse_all=True)
|
result = get_parser().parse_string(s, parse_all=True)
|
||||||
except ParseException as exc:
|
except ParseException as exc:
|
||||||
format_and_raise_parse_error(exc)
|
format_and_raise_parse_error(exc)
|
||||||
raise AssertionError("unreachable") # noqa: B904
|
raise AssertionError("unreachable") # noqa: B904
|
||||||
|
|||||||
@@ -188,7 +188,7 @@ class Experiments:
|
|||||||
rev_names = self.get_exact_name(revs)
|
rev_names = self.get_exact_name(revs)
|
||||||
for rev in revs:
|
for rev in revs:
|
||||||
name = rev_names[rev]
|
name = rev_names[rev]
|
||||||
names.append(name if name else rev[:7])
|
names.append(name or rev[:7])
|
||||||
ui.write("\nRan experiment(s): {}".format(", ".join(names)))
|
ui.write("\nRan experiment(s): {}".format(", ".join(names)))
|
||||||
if tmp_dir:
|
if tmp_dir:
|
||||||
ui.write(
|
ui.write(
|
||||||
|
|||||||
@@ -328,7 +328,7 @@ def collect(
|
|||||||
name=baseline_names.get(baseline_rev),
|
name=baseline_names.get(baseline_rev),
|
||||||
data=baseline_data.data,
|
data=baseline_data.data,
|
||||||
error=baseline_data.error,
|
error=baseline_data.error,
|
||||||
experiments=experiments if experiments else None,
|
experiments=experiments or None,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
return result
|
return result
|
||||||
|
|||||||
@@ -757,7 +757,7 @@ class BaseExecutor(ABC):
|
|||||||
logger.debug("Commit to current experiment branch '%s'", branch)
|
logger.debug("Commit to current experiment branch '%s'", branch)
|
||||||
else:
|
else:
|
||||||
baseline_rev = scm.get_ref(EXEC_BASELINE)
|
baseline_rev = scm.get_ref(EXEC_BASELINE)
|
||||||
name = exp_name if exp_name else f"exp-{exp_hash[:5]}"
|
name = exp_name or f"exp-{exp_hash[:5]}"
|
||||||
ref_info = ExpRefInfo(baseline_rev, name)
|
ref_info = ExpRefInfo(baseline_rev, name)
|
||||||
branch = str(ref_info)
|
branch = str(ref_info)
|
||||||
old_ref = None
|
old_ref = None
|
||||||
|
|||||||
@@ -90,7 +90,7 @@ class BaseStashQueue(ABC):
|
|||||||
"""Construct a queue.
|
"""Construct a queue.
|
||||||
|
|
||||||
Arguments:
|
Arguments:
|
||||||
scm: Git SCM instance for this queue.
|
repo: DVC repo instance for this queue.
|
||||||
ref: Git stash ref for this queue.
|
ref: Git stash ref for this queue.
|
||||||
failed_ref: Failed run Git stash ref for this queue.
|
failed_ref: Failed run Git stash ref for this queue.
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -84,8 +84,8 @@ def cleanup_exp(executor: TempDirExecutor, infofile: str) -> None:
|
|||||||
"""Cleanup after an experiment.
|
"""Cleanup after an experiment.
|
||||||
|
|
||||||
Arguments:
|
Arguments:
|
||||||
tmp_dir: Temp directory to be removed.
|
executor: Executor to clean up.
|
||||||
entry_dict: Serialized QueueEntry for this experiment.
|
infofile: Path to the executor info file.
|
||||||
"""
|
"""
|
||||||
executor.cleanup(infofile)
|
executor.cleanup(infofile)
|
||||||
|
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ class ExpRefInfo:
|
|||||||
|
|
||||||
def __init__(self, baseline_sha: str, name: Optional[str] = None):
|
def __init__(self, baseline_sha: str, name: Optional[str] = None):
|
||||||
self.baseline_sha = baseline_sha
|
self.baseline_sha = baseline_sha
|
||||||
self.name: str = name if name else ""
|
self.name: str = name or ""
|
||||||
|
|
||||||
def __str__(self):
|
def __str__(self):
|
||||||
return "/".join(self.parts)
|
return "/".join(self.parts)
|
||||||
|
|||||||
@@ -67,7 +67,7 @@ class ExpStash(Stash):
|
|||||||
branch: Optional[str] = None,
|
branch: Optional[str] = None,
|
||||||
) -> str:
|
) -> str:
|
||||||
msg = cls.MESSAGE_FORMAT.format(
|
msg = cls.MESSAGE_FORMAT.format(
|
||||||
rev=rev, baseline_rev=baseline_rev, name=name if name else ""
|
rev=rev, baseline_rev=baseline_rev, name=name or ""
|
||||||
)
|
)
|
||||||
branch_msg = f":{branch}" if branch else ""
|
branch_msg = f":{branch}" if branch else ""
|
||||||
return f"{msg}{branch_msg}"
|
return f"{msg}{branch_msg}"
|
||||||
@@ -124,9 +124,7 @@ class ApplyStash(Stash):
|
|||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def format_message(cls, head_rev: str, rev: str, name: Optional[str] = None) -> str:
|
def format_message(cls, head_rev: str, rev: str, name: Optional[str] = None) -> str:
|
||||||
return cls.MESSAGE_FORMAT.format(
|
return cls.MESSAGE_FORMAT.format(head_rev=head_rev, rev=rev, name=name or "")
|
||||||
head_rev=head_rev, rev=rev, name=name if name else ""
|
|
||||||
)
|
|
||||||
|
|
||||||
@contextmanager
|
@contextmanager
|
||||||
def preserve_workspace(
|
def preserve_workspace(
|
||||||
|
|||||||
@@ -200,7 +200,7 @@ def exp_commits(
|
|||||||
) -> Iterable[str]:
|
) -> Iterable[str]:
|
||||||
"""Iterate over all experiment commits."""
|
"""Iterate over all experiment commits."""
|
||||||
shas: set[str] = set()
|
shas: set[str] = set()
|
||||||
refs = ref_infos if ref_infos else exp_refs(scm)
|
refs = ref_infos or exp_refs(scm)
|
||||||
for ref_info in refs:
|
for ref_info in refs:
|
||||||
shas.update(scm.branch_revs(str(ref_info), ref_info.baseline_sha))
|
shas.update(scm.branch_revs(str(ref_info), ref_info.baseline_sha))
|
||||||
yield from shas
|
yield from shas
|
||||||
|
|||||||
+4
-2
@@ -28,8 +28,10 @@ def _validate_args(**kwargs):
|
|||||||
"`--all-experiments`, `--all-commits`, `--date` or `--rev` "
|
"`--all-experiments`, `--all-commits`, `--date` or `--rev` "
|
||||||
"needs to be set."
|
"needs to be set."
|
||||||
)
|
)
|
||||||
if kwargs.get("num") and not kwargs.get("rev"):
|
if kwargs.get("num") and not (kwargs.get("rev") or kwargs.get("all_branches")):
|
||||||
raise InvalidArgumentError("`--num` can only be used alongside `--rev`")
|
raise InvalidArgumentError(
|
||||||
|
"`--num` can only be used alongside `--rev` or `--all-branches`"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _used_obj_ids_not_in_remote(
|
def _used_obj_ids_not_in_remote(
|
||||||
|
|||||||
@@ -10,6 +10,8 @@ from dvc.utils import relpath
|
|||||||
from dvc.utils.collections import ensure_list
|
from dvc.utils.collections import ensure_list
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
|
from typing_extensions import Self
|
||||||
|
|
||||||
from dvc.repo import Repo
|
from dvc.repo import Repo
|
||||||
from dvc.scm import Base
|
from dvc.scm import Base
|
||||||
|
|
||||||
@@ -96,7 +98,7 @@ class SCMContext:
|
|||||||
@contextmanager
|
@contextmanager
|
||||||
def __call__(
|
def __call__(
|
||||||
self, autostage: Optional[bool] = None, quiet: Optional[bool] = None
|
self, autostage: Optional[bool] = None, quiet: Optional[bool] = None
|
||||||
) -> Iterator["SCMContext"]:
|
) -> Iterator["Self"]:
|
||||||
try:
|
try:
|
||||||
yield self
|
yield self
|
||||||
except Exception:
|
except Exception:
|
||||||
@@ -131,9 +133,10 @@ class SCMContext:
|
|||||||
|
|
||||||
self.files_to_track = set()
|
self.files_to_track = set()
|
||||||
|
|
||||||
def __enter__(self) -> "SCMContext":
|
def __enter__(self) -> "Self":
|
||||||
self._cm = self()
|
self._cm = self()
|
||||||
return self._cm.__enter__()
|
self._cm.__enter__()
|
||||||
|
return self
|
||||||
|
|
||||||
def __exit__(self, *exc_args) -> None:
|
def __exit__(self, *exc_args) -> None:
|
||||||
assert self._cm
|
assert self._cm
|
||||||
|
|||||||
+5
-2
@@ -234,7 +234,7 @@ def iter_revs(
|
|||||||
results.extend(scm.list_all_commits())
|
results.extend(scm.list_all_commits())
|
||||||
else:
|
else:
|
||||||
if all_branches:
|
if all_branches:
|
||||||
results.extend(scm.list_branches())
|
results.extend(_get_n_commits(scm, scm.list_branches(), num))
|
||||||
|
|
||||||
if all_tags:
|
if all_tags:
|
||||||
results.extend(scm.list_tags())
|
results.extend(scm.list_tags())
|
||||||
@@ -258,7 +258,10 @@ def iter_revs(
|
|||||||
results.extend(exp_commits(scm))
|
results.extend(exp_commits(scm))
|
||||||
|
|
||||||
rev_resolver = partial(resolve_rev, scm)
|
rev_resolver = partial(resolve_rev, scm)
|
||||||
return group_by(rev_resolver, results)
|
grouped = group_by(rev_resolver, results)
|
||||||
|
for rev, names in grouped.items():
|
||||||
|
grouped[rev] = list(dict.fromkeys(names))
|
||||||
|
return grouped
|
||||||
|
|
||||||
|
|
||||||
def lfs_prefetch(fs: "FileSystem", paths: list[str]):
|
def lfs_prefetch(fs: "FileSystem", paths: list[str]):
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ import os
|
|||||||
import pathlib
|
import pathlib
|
||||||
import posixpath
|
import posixpath
|
||||||
import sys
|
import sys
|
||||||
from typing import Callable, ClassVar
|
from typing import Callable, ClassVar, Union
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
from dvc.utils import relpath
|
from dvc.utils import relpath
|
||||||
@@ -79,15 +79,17 @@ class PathInfo(pathlib.PurePath, _BasePath):
|
|||||||
and self._cparts[:n] == other._cparts # type: ignore[attr-defined]
|
and self._cparts[:n] == other._cparts # type: ignore[attr-defined]
|
||||||
)
|
)
|
||||||
|
|
||||||
def relative_to(self, other, *args, **kwargs):
|
def relative_to(
|
||||||
|
self, *other: Union[str, "os.PathLike[str]"], **kwargs
|
||||||
|
) -> "PathInfo":
|
||||||
# pathlib relative_to raises exception when one path is not a direct
|
# pathlib relative_to raises exception when one path is not a direct
|
||||||
# descendant of the other when os.path.relpath would return abspath.
|
# descendant of the other when os.path.relpath would return abspath.
|
||||||
# For DVC PathInfo we only need the relpath behavior.
|
# For DVC PathInfo we only need the relpath behavior.
|
||||||
# See: https://bugs.python.org/issue40358
|
# See: https://bugs.python.org/issue40358
|
||||||
try:
|
try:
|
||||||
path = super().relative_to(other, *args, **kwargs)
|
path = super().relative_to(*other, **kwargs)
|
||||||
except ValueError:
|
except ValueError:
|
||||||
path = relpath(self, other)
|
path = relpath(self, other[0])
|
||||||
return self.__class__(path)
|
return self.__class__(path)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+3
-1
@@ -289,7 +289,7 @@ class Console:
|
|||||||
|
|
||||||
return console.Console(stderr=True)
|
return console.Console(stderr=True)
|
||||||
|
|
||||||
def table(
|
def table( # noqa: PLR0913
|
||||||
self,
|
self,
|
||||||
data: "TableData",
|
data: "TableData",
|
||||||
headers: Optional["Headers"] = None,
|
headers: Optional["Headers"] = None,
|
||||||
@@ -301,6 +301,7 @@ class Console:
|
|||||||
row_styles: Optional[Sequence["Styles"]] = None,
|
row_styles: Optional[Sequence["Styles"]] = None,
|
||||||
borders: Union[bool, str] = False,
|
borders: Union[bool, str] = False,
|
||||||
colalign: Optional[tuple[str, ...]] = None,
|
colalign: Optional[tuple[str, ...]] = None,
|
||||||
|
preserve_whitespace: bool = False,
|
||||||
) -> None:
|
) -> None:
|
||||||
from dvc.ui import table as t
|
from dvc.ui import table as t
|
||||||
|
|
||||||
@@ -329,6 +330,7 @@ class Console:
|
|||||||
pager=pager,
|
pager=pager,
|
||||||
force=force,
|
force=force,
|
||||||
colalign=colalign,
|
colalign=colalign,
|
||||||
|
preserve_whitespace=preserve_whitespace,
|
||||||
)
|
)
|
||||||
|
|
||||||
def status(self, status: str, **kwargs: Any) -> "Status":
|
def status(self, status: str, **kwargs: Any) -> "Status":
|
||||||
|
|||||||
+27
-9
@@ -30,19 +30,37 @@ def plain_table(
|
|||||||
pager: bool = False,
|
pager: bool = False,
|
||||||
force: bool = True,
|
force: bool = True,
|
||||||
colalign: Optional[tuple[str, ...]] = None,
|
colalign: Optional[tuple[str, ...]] = None,
|
||||||
|
preserve_whitespace: bool = False,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
import tabulate as tabulate_mod
|
||||||
from funcy import nullcontext
|
from funcy import nullcontext
|
||||||
|
from packaging.version import Version
|
||||||
from tabulate import tabulate
|
from tabulate import tabulate
|
||||||
|
|
||||||
text: str = tabulate(
|
# NOTE: tabulate 0.10.0+ supports preserve_whitespace as a kwarg,
|
||||||
data,
|
# while 0.9.0 only respects the global PRESERVE_WHITESPACE.
|
||||||
headers if headers is not None else (),
|
kwargs: dict = {}
|
||||||
tablefmt="github" if markdown else "plain",
|
_orig = tabulate_mod.PRESERVE_WHITESPACE
|
||||||
disable_numparse=True,
|
if preserve_whitespace:
|
||||||
# None will be shown as "" by default, overriding
|
if Version(tabulate_mod.__version__) >= Version("0.10"):
|
||||||
missingval="-",
|
kwargs["preserve_whitespace"] = True
|
||||||
colalign=colalign,
|
else:
|
||||||
)
|
tabulate_mod.PRESERVE_WHITESPACE = True
|
||||||
|
|
||||||
|
try:
|
||||||
|
text: str = tabulate(
|
||||||
|
data,
|
||||||
|
headers if headers is not None else (),
|
||||||
|
tablefmt="github" if markdown else "plain",
|
||||||
|
disable_numparse=True,
|
||||||
|
# None will be shown as "" by default, overriding
|
||||||
|
missingval="-",
|
||||||
|
colalign=colalign,
|
||||||
|
**kwargs,
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
tabulate_mod.PRESERVE_WHITESPACE = _orig
|
||||||
|
|
||||||
if markdown:
|
if markdown:
|
||||||
# NOTE: md table is incomplete without the trailing newline
|
# NOTE: md table is incomplete without the trailing newline
|
||||||
text += "\n"
|
text += "\n"
|
||||||
|
|||||||
@@ -53,7 +53,7 @@ def test_new_simple(tmp_dir, scm, dvc, exp_stage, mocker, name, workspace):
|
|||||||
if workspace:
|
if workspace:
|
||||||
assert (tmp_dir / "metrics.yaml").read_text().strip() == "foo: 2"
|
assert (tmp_dir / "metrics.yaml").read_text().strip() == "foo: 2"
|
||||||
|
|
||||||
exp_name = name if name else ref_info.name
|
exp_name = name or ref_info.name
|
||||||
assert dvc.experiments.get_exact_name([exp])[exp] == exp_name
|
assert dvc.experiments.get_exact_name([exp])[exp] == exp_name
|
||||||
assert resolve_rev(scm, exp_name) == exp
|
assert resolve_rev(scm, exp_name) == exp
|
||||||
|
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ def test_exp_save(tmp_dir, dvc, scm, name):
|
|||||||
assert ref_info
|
assert ref_info
|
||||||
assert ref_info.baseline_sha == baseline
|
assert ref_info.baseline_sha == baseline
|
||||||
|
|
||||||
exp_name = name if name else ref_info.name
|
exp_name = name or ref_info.name
|
||||||
assert dvc.experiments.get_exact_name([exp])[exp] == exp_name
|
assert dvc.experiments.get_exact_name([exp])[exp] == exp_name
|
||||||
assert resolve_rev(scm, exp_name) == exp
|
assert resolve_rev(scm, exp_name) == exp
|
||||||
|
|
||||||
|
|||||||
@@ -92,6 +92,27 @@ def test_gc_multiple_dvc_repos(tmp_dir, scm, dvc, erepo_dir):
|
|||||||
assert len(list(odb.all())) == 2
|
assert len(list(odb.all())) == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_gc_multiple_dvc_repos_all_branches_num(tmp_dir, scm, dvc, erepo_dir):
|
||||||
|
tmp_dir.dvc_gen("main", "main", commit="main")
|
||||||
|
|
||||||
|
erepo_dir.dvc.cache.local.path = dvc.cache.local.path
|
||||||
|
with erepo_dir.chdir():
|
||||||
|
erepo_dir.scm_gen("base", "base", commit="base")
|
||||||
|
with erepo_dir.branch("feature", new=True):
|
||||||
|
erepo_dir.dvc_gen("shared", "feature-v1", commit="feature-v1")
|
||||||
|
erepo_dir.dvc.remove("shared.dvc")
|
||||||
|
erepo_dir.dvc_gen("shared", "feature-v2", commit="feature-v2")
|
||||||
|
|
||||||
|
odb = dvc.cache.local
|
||||||
|
assert len(list(odb.all())) == 3
|
||||||
|
|
||||||
|
dvc.gc(all_branches=True, num=2, repos=[erepo_dir])
|
||||||
|
assert len(list(odb.all())) == 3
|
||||||
|
|
||||||
|
dvc.gc(all_branches=True, repos=[erepo_dir])
|
||||||
|
assert len(list(odb.all())) == 2
|
||||||
|
|
||||||
|
|
||||||
def test_all_commits(tmp_dir, scm, dvc):
|
def test_all_commits(tmp_dir, scm, dvc):
|
||||||
tmp_dir.dvc_gen("testfile", "uncommitted")
|
tmp_dir.dvc_gen("testfile", "uncommitted")
|
||||||
tmp_dir.dvc_gen("testfile", "committed", commit="committed")
|
tmp_dir.dvc_gen("testfile", "committed", commit="committed")
|
||||||
|
|||||||
@@ -196,7 +196,7 @@ def test_diff_show_markdown_and_hash(mocker, show_hash, dvc):
|
|||||||
cmd = args.func(args)
|
cmd = args.func(args)
|
||||||
|
|
||||||
diff = {}
|
diff = {}
|
||||||
show_hash = show_hash if show_hash else False
|
show_hash = show_hash or False
|
||||||
mock_show_markdown = mocker.patch("dvc.commands.diff._show_markdown")
|
mock_show_markdown = mocker.patch("dvc.commands.diff._show_markdown")
|
||||||
mocker.patch("dvc.repo.Repo.diff", return_value=diff.copy())
|
mocker.patch("dvc.repo.Repo.diff", return_value=diff.copy())
|
||||||
|
|
||||||
|
|||||||
@@ -65,6 +65,30 @@ def test_(dvc, scm, mocker):
|
|||||||
with pytest.raises(InvalidArgumentError):
|
with pytest.raises(InvalidArgumentError):
|
||||||
cmd.run()
|
cmd.run()
|
||||||
|
|
||||||
|
cli_args = parse_args(["gc", "--all-branches", "--num", "2", "--force"])
|
||||||
|
cmd = cli_args.func(cli_args)
|
||||||
|
|
||||||
|
assert cmd.run() == 0
|
||||||
|
|
||||||
|
m.assert_called_with(
|
||||||
|
workspace=False,
|
||||||
|
all_tags=False,
|
||||||
|
all_branches=True,
|
||||||
|
all_commits=False,
|
||||||
|
all_experiments=False,
|
||||||
|
commit_date=None,
|
||||||
|
cloud=False,
|
||||||
|
remote=None,
|
||||||
|
force=True,
|
||||||
|
jobs=None,
|
||||||
|
repos=None,
|
||||||
|
rev=None,
|
||||||
|
num=2,
|
||||||
|
not_in_remote=False,
|
||||||
|
dry=False,
|
||||||
|
skip_failed=False,
|
||||||
|
)
|
||||||
|
|
||||||
cli_args = parse_args(["gc", "--cloud", "--not-in-remote"])
|
cli_args = parse_args(["gc", "--cloud", "--not-in-remote"])
|
||||||
cmd = cli_args.func(cli_args)
|
cmd = cli_args.func(cli_args)
|
||||||
with pytest.raises(InvalidArgumentError):
|
with pytest.raises(InvalidArgumentError):
|
||||||
|
|||||||
@@ -47,7 +47,7 @@ def test_help(caplog, capsys, command_tuples):
|
|||||||
out, err = capsys.readouterr()
|
out, err = capsys.readouterr()
|
||||||
|
|
||||||
# validate metavars are all in lowercase
|
# validate metavars are all in lowercase
|
||||||
usage = "\n".join(takewhile(lambda o: bool(o), out.splitlines()))
|
usage = "\n".join(takewhile(bool, out.splitlines()))
|
||||||
|
|
||||||
message = (
|
message = (
|
||||||
"metavars not lowercased, you are likely missing formatter_class=XXX "
|
"metavars not lowercased, you are likely missing formatter_class=XXX "
|
||||||
|
|||||||
@@ -47,6 +47,13 @@ def test_iter_revs(tmp_dir, scm, mocker):
|
|||||||
assert gen == {rev_old: ["tag"]}
|
assert gen == {rev_old: ["tag"]}
|
||||||
gen = iter_revs(scm, all_branches=True)
|
gen = iter_revs(scm, all_branches=True)
|
||||||
assert gen == {rev_old: [old], rev_new: ["new"], rev_other: ["other"]}
|
assert gen == {rev_old: [old], rev_new: ["new"], rev_other: ["other"]}
|
||||||
|
gen = iter_revs(scm, all_branches=True, num=2)
|
||||||
|
assert gen == {
|
||||||
|
rev_old: [old, rev_old],
|
||||||
|
rev_root: [rev_root],
|
||||||
|
rev_new: ["new"],
|
||||||
|
rev_other: ["other"],
|
||||||
|
}
|
||||||
gen = iter_revs(scm, all_tags=True)
|
gen = iter_revs(scm, all_tags=True)
|
||||||
assert gen == {rev_old: ["tag"]}
|
assert gen == {rev_old: ["tag"]}
|
||||||
gen = iter_revs(scm, all_commits=True)
|
gen = iter_revs(scm, all_commits=True)
|
||||||
|
|||||||
Reference in New Issue
Block a user