mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-04 23:09:44 +01:00
Compare commits
22
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7594d2f2ed | ||
|
|
56e5982951 | ||
|
|
defe4062aa | ||
|
|
f74c1c0e70 | ||
|
|
b5dede14f2 | ||
|
|
8d37f966c3 | ||
|
|
95165cc39a | ||
|
|
ccb72126ea | ||
|
|
d5315c3827 | ||
|
|
8131c32c33 | ||
|
|
0013af99d4 | ||
|
|
06ff81cd7f | ||
|
|
e19b0874c0 | ||
|
|
b290eaea43 | ||
|
|
2a2b543dd6 | ||
|
|
356dfa0327 | ||
|
|
0f4e8a23c8 | ||
|
|
333edb2cbf | ||
|
|
a65523a86a | ||
|
|
fb42429107 | ||
|
|
e40e378fc9 | ||
|
|
b1aaeb94c0 |
@@ -14,11 +14,11 @@ jobs:
|
||||
name: run benchmarks
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/setup-python@v6
|
||||
- uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: "3.14"
|
||||
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.base.sha }}
|
||||
fetch-depth: 0
|
||||
@@ -32,7 +32,7 @@ jobs:
|
||||
- name: run benchmarks on base branch
|
||||
run: pytest --benchmark-autosave dvc/testing/benchmarks/ -k 'test_init or test_help'
|
||||
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
clean: false
|
||||
|
||||
@@ -19,11 +19,11 @@ jobs:
|
||||
name: Build & Verify package
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: actions/setup-python@v6
|
||||
- uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: "3.14"
|
||||
|
||||
@@ -41,7 +41,7 @@ jobs:
|
||||
- name: Check dist
|
||||
run: uv tool run twine check --strict dist/*
|
||||
|
||||
- uses: actions/upload-artifact@v6
|
||||
- uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: Packages
|
||||
path: dist/
|
||||
@@ -61,7 +61,7 @@ jobs:
|
||||
id-token: write
|
||||
|
||||
steps:
|
||||
- uses: actions/download-artifact@v7
|
||||
- uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: Packages
|
||||
path: dist
|
||||
@@ -86,7 +86,7 @@ jobs:
|
||||
id-token: write
|
||||
|
||||
steps:
|
||||
- uses: actions/download-artifact@v7
|
||||
- uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: Packages
|
||||
path: dist
|
||||
|
||||
@@ -28,13 +28,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v4
|
||||
uses: github/codeql-action/init@v4.37.4
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
queries: security-extended
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v4
|
||||
uses: github/codeql-action/analyze@v4.37.4
|
||||
|
||||
@@ -23,18 +23,18 @@ jobs:
|
||||
plugin: ["dvc-s3"]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
path: dvc
|
||||
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
repository: treeverse/${{ matrix.plugin }}
|
||||
ref: main
|
||||
path: ${{ matrix.plugin }}
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v6
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: ${{ matrix.pyv }}
|
||||
|
||||
|
||||
@@ -26,11 +26,11 @@ jobs:
|
||||
matrix:
|
||||
os: [ubuntu-latest, macos-latest, windows-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v6
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: '3.14'
|
||||
- uses: astral-sh/setup-uv@v7
|
||||
@@ -42,7 +42,7 @@ jobs:
|
||||
UV_LINK_MODE: ${{ runner.os == 'Windows' && 'symlink' || 'hardlink' }}
|
||||
|
||||
- name: Cache mypy
|
||||
uses: actions/cache@v5
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: .mypy_cache
|
||||
key: mypy-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}
|
||||
@@ -130,11 +130,11 @@ jobs:
|
||||
pyv: "3.14"
|
||||
pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v6
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: ${{ matrix.pyv }}
|
||||
- uses: astral-sh/setup-uv@v7
|
||||
@@ -155,7 +155,7 @@ jobs:
|
||||
pytest ${{ matrix.pytestargs }} -n=logical --dist=worksteal --timeout=300 --durations=0
|
||||
--cov --cov-report=xml --cov-report=term --durations-path=./.github/.test_durations
|
||||
- name: upload coverage report
|
||||
uses: codecov/codecov-action@v5
|
||||
uses: codecov/codecov-action@v7
|
||||
check:
|
||||
if: always()
|
||||
needs: [tests]
|
||||
@@ -170,7 +170,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Slack Notification
|
||||
uses: rtCamp/action-slack-notify@v2.3.3
|
||||
uses: rtCamp/action-slack-notify@v2.4.0
|
||||
env:
|
||||
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
|
||||
SLACK_COLOR: ${{ job.status }}
|
||||
|
||||
@@ -22,13 +22,13 @@ repos:
|
||||
- id: sort-simple-yaml
|
||||
- id: trailing-whitespace
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
rev: 'v0.15.0'
|
||||
rev: 'v0.15.20'
|
||||
hooks:
|
||||
- id: ruff-check
|
||||
args: [--fix, --exit-non-zero-on-fix]
|
||||
- id: ruff-format
|
||||
- repo: https://github.com/codespell-project/codespell
|
||||
rev: v2.4.1
|
||||
rev: v2.4.2
|
||||
hooks:
|
||||
- id: codespell
|
||||
additional_dependencies: ["tomli"]
|
||||
|
||||
+11
-4
@@ -40,9 +40,16 @@ class CmdGC(CmdBase):
|
||||
msg += " and all git commits"
|
||||
else:
|
||||
if self.args.all_branches and self.args.all_tags:
|
||||
msg += " and all git branches and tags"
|
||||
if self.args.num:
|
||||
msg += f" and last {self.args.num} commits from all git branches"
|
||||
msg += " and all git tags"
|
||||
else:
|
||||
msg += " and all git branches and tags"
|
||||
elif self.args.all_branches:
|
||||
msg += " and all git branches"
|
||||
if self.args.num:
|
||||
msg += f" and last {self.args.num} commits from all git branches"
|
||||
else:
|
||||
msg += " and all git branches"
|
||||
elif self.args.all_tags:
|
||||
msg += " and all git tags"
|
||||
if self.args.commit_date:
|
||||
@@ -126,8 +133,8 @@ def add_parser(subparsers, parent_parser):
|
||||
metavar="<num>",
|
||||
help=(
|
||||
"Keep data files used in the last `num` commits "
|
||||
"starting from the `--rev` <commit>. "
|
||||
"Only used if `--rev` is also provided. "
|
||||
"starting from each selected revision root. "
|
||||
"Can be used with `--rev` and `--all-branches`. "
|
||||
"Defaults to `1`."
|
||||
),
|
||||
)
|
||||
|
||||
@@ -109,8 +109,6 @@ def _build_tree_structure(
|
||||
|
||||
|
||||
def show_tree(entries, with_color=False, with_size=False, with_hash=False):
|
||||
import tabulate
|
||||
|
||||
rows = _build_tree_structure(
|
||||
entries,
|
||||
with_color=with_color,
|
||||
@@ -119,13 +117,7 @@ def show_tree(entries, with_color=False, with_size=False, with_hash=False):
|
||||
)
|
||||
|
||||
colalign = ("right",) if with_size else None
|
||||
|
||||
_orig = tabulate.PRESERVE_WHITESPACE
|
||||
tabulate.PRESERVE_WHITESPACE = True
|
||||
try:
|
||||
ui.table(rows, colalign=colalign)
|
||||
finally:
|
||||
tabulate.PRESERVE_WHITESPACE = _orig
|
||||
ui.table(rows, colalign=colalign, preserve_whitespace=True)
|
||||
|
||||
|
||||
class CmdList(CmdBaseNoRepo):
|
||||
|
||||
+48
-37
@@ -60,13 +60,16 @@ class DvcIgnorePatterns(DvcIgnore):
|
||||
self.pattern_list: list[PatternInfo] = []
|
||||
self.dirname = dirname
|
||||
self.find_matching_pattern = functools.cache(self._find_matching_pattern)
|
||||
self._ignore_parents = functools.cache(self._ignore_parents)
|
||||
|
||||
regex_pattern_list: list[tuple[str, bool, bool, PatternInfo]] = []
|
||||
for count, pattern_info in enumerate(pattern_infos):
|
||||
for pattern_info in pattern_infos:
|
||||
regex, ignore = GitIgnoreSpecPattern.pattern_to_regex(pattern_info.patterns)
|
||||
if regex is not None and ignore is not None:
|
||||
self.pattern_list.append(pattern_info)
|
||||
regex = regex.replace(f"<{_DIR_MARK}>", f"<{_DIR_MARK}{count}>")
|
||||
# DVC resolves directory matches itself; only the outer rule
|
||||
# group is needed to identify a match in the combined regex.
|
||||
regex = regex.replace(f"(?P<{_DIR_MARK}>", "(?:")
|
||||
regex_pattern_list.append(
|
||||
(regex, ignore, pattern_info.patterns.endswith("/"), pattern_info)
|
||||
)
|
||||
@@ -126,8 +129,20 @@ class DvcIgnorePatterns(DvcIgnore):
|
||||
def __call__(
|
||||
self, root: str, dirs: list[str], files: list[str]
|
||||
) -> tuple[list[str], list[str]]:
|
||||
files = [f for f in files if not self.matches(root, f)]
|
||||
dirs = [d for d in dirs if not self.matches(root, d, True)]
|
||||
# Entries are basenames from one directory listing, so normalize the
|
||||
# shared prefix and check its ancestors only once.
|
||||
prefix = self._get_normalize_path(root, "")
|
||||
if prefix is None:
|
||||
return dirs.copy(), files.copy()
|
||||
if prefix and self._ignore_parents(prefix[:-1])[0]:
|
||||
return [], []
|
||||
|
||||
files = [
|
||||
f for f in files if not self.find_matching_pattern(f"{prefix}{f}", False)[0]
|
||||
]
|
||||
dirs = [
|
||||
d for d in dirs if not self.find_matching_pattern(f"{prefix}{d}", True)[0]
|
||||
]
|
||||
|
||||
return dirs, files
|
||||
|
||||
@@ -189,55 +204,51 @@ class DvcIgnorePatterns(DvcIgnore):
|
||||
_match: list[PatternInfo] = []
|
||||
if path:
|
||||
result, _match = self._ignore(path, is_dir)
|
||||
return (result, _match) if details else result
|
||||
return (result, _match.copy()) if details else result
|
||||
|
||||
def _find_matching_pattern(
|
||||
self, path: str, is_dir: bool
|
||||
) -> tuple[bool, list[PatternInfo]]:
|
||||
paths = [path]
|
||||
if is_dir and not path.endswith("/"):
|
||||
paths.append(f"{path}/")
|
||||
|
||||
for pattern, ignore, dir_only_pattern, pattern_map in reversed(
|
||||
self.ignore_spec
|
||||
):
|
||||
if dir_only_pattern and not is_dir:
|
||||
continue
|
||||
for p in paths:
|
||||
match = pattern.match(p)
|
||||
if not match:
|
||||
continue
|
||||
if ignore:
|
||||
group_name, _match = next(
|
||||
(
|
||||
(name, _match)
|
||||
for name, _match in match.groupdict().items()
|
||||
if name.startswith("rule_") and _match is not None
|
||||
)
|
||||
)
|
||||
else:
|
||||
# unignored patterns are not combined with `|`,
|
||||
# so there are no groups.
|
||||
group_name = None
|
||||
_regex, pattern_info = pattern_map[group_name]
|
||||
return ignore, [pattern_info]
|
||||
match = pattern.match(path)
|
||||
# Only directory-only patterns need the trailing slash. Probing it
|
||||
# for other patterns would make X/** exclude X itself.
|
||||
if not match and dir_only_pattern and not path.endswith("/"):
|
||||
match = pattern.match(f"{path}/")
|
||||
if not match:
|
||||
continue
|
||||
if ignore:
|
||||
group_name = match.lastgroup
|
||||
else:
|
||||
# unignored patterns are not combined with `|`,
|
||||
# so there are no groups.
|
||||
group_name = None
|
||||
_regex, pattern_info = pattern_map[group_name]
|
||||
return ignore, [pattern_info]
|
||||
return False, []
|
||||
|
||||
def _ignore(self, path: str, is_dir: bool) -> tuple[bool, list[PatternInfo]]:
|
||||
parent, sep, _ = path.rpartition("/")
|
||||
if sep:
|
||||
result = self._ignore_parents(parent)
|
||||
if result[0]:
|
||||
return result
|
||||
return self.find_matching_pattern(path, is_dir)
|
||||
|
||||
def _ignore_parents(self, path: str) -> tuple[bool, list[PatternInfo]]:
|
||||
# Files in the same directory share the ancestor verdict, not just
|
||||
# individual regex matches. Cache it to avoid rebuilding every prefix.
|
||||
parts = path.split("/")
|
||||
result = False
|
||||
matches: list[PatternInfo] = []
|
||||
for i in range(1, len(parts) + 1):
|
||||
rel_path = "/".join(parts[:i])
|
||||
result, _matches = self.find_matching_pattern(
|
||||
rel_path, is_dir or i < len(parts)
|
||||
)
|
||||
if i < len(parts) and not result:
|
||||
continue
|
||||
matches.extend(_matches)
|
||||
result, matches = self.find_matching_pattern(rel_path, True)
|
||||
if result:
|
||||
break
|
||||
return result, matches
|
||||
return result, matches
|
||||
return False, []
|
||||
|
||||
def __hash__(self) -> int:
|
||||
return hash(self.dirname + ":" + str(self.pattern_list))
|
||||
|
||||
@@ -37,13 +37,13 @@ KEYCRE = re.compile(
|
||||
def get_parser():
|
||||
from pyparsing import CharsNotIn, ParserElement, Suppress, ZeroOrMore
|
||||
|
||||
ParserElement.enablePackrat()
|
||||
ParserElement.enable_packrat()
|
||||
|
||||
word = CharsNotIn(f"{PERIOD}{LBRACK}{RBRACK}")
|
||||
idx = Suppress(LBRACK) + word + Suppress(RBRACK)
|
||||
attr = Suppress(PERIOD) + word
|
||||
parser = word + ZeroOrMore(attr ^ idx)
|
||||
parser.setParseAction(PERIOD.join)
|
||||
parser.set_parse_action(PERIOD.join)
|
||||
|
||||
return parser
|
||||
|
||||
@@ -178,7 +178,7 @@ def parse_expr(s: str):
|
||||
from pyparsing import ParseException
|
||||
|
||||
try:
|
||||
result = get_parser().parseString(s, parse_all=True)
|
||||
result = get_parser().parse_string(s, parse_all=True)
|
||||
except ParseException as exc:
|
||||
format_and_raise_parse_error(exc)
|
||||
raise AssertionError("unreachable") # noqa: B904
|
||||
|
||||
@@ -90,7 +90,7 @@ class BaseStashQueue(ABC):
|
||||
"""Construct a queue.
|
||||
|
||||
Arguments:
|
||||
scm: Git SCM instance for this queue.
|
||||
repo: DVC repo instance for this queue.
|
||||
ref: Git stash ref for this queue.
|
||||
failed_ref: Failed run Git stash ref for this queue.
|
||||
"""
|
||||
|
||||
@@ -84,8 +84,8 @@ def cleanup_exp(executor: TempDirExecutor, infofile: str) -> None:
|
||||
"""Cleanup after an experiment.
|
||||
|
||||
Arguments:
|
||||
tmp_dir: Temp directory to be removed.
|
||||
entry_dict: Serialized QueueEntry for this experiment.
|
||||
executor: Executor to clean up.
|
||||
infofile: Path to the executor info file.
|
||||
"""
|
||||
executor.cleanup(infofile)
|
||||
|
||||
|
||||
+4
-2
@@ -28,8 +28,10 @@ def _validate_args(**kwargs):
|
||||
"`--all-experiments`, `--all-commits`, `--date` or `--rev` "
|
||||
"needs to be set."
|
||||
)
|
||||
if kwargs.get("num") and not kwargs.get("rev"):
|
||||
raise InvalidArgumentError("`--num` can only be used alongside `--rev`")
|
||||
if kwargs.get("num") and not (kwargs.get("rev") or kwargs.get("all_branches")):
|
||||
raise InvalidArgumentError(
|
||||
"`--num` can only be used alongside `--rev` or `--all-branches`"
|
||||
)
|
||||
|
||||
|
||||
def _used_obj_ids_not_in_remote(
|
||||
|
||||
+5
-2
@@ -234,7 +234,7 @@ def iter_revs(
|
||||
results.extend(scm.list_all_commits())
|
||||
else:
|
||||
if all_branches:
|
||||
results.extend(scm.list_branches())
|
||||
results.extend(_get_n_commits(scm, scm.list_branches(), num))
|
||||
|
||||
if all_tags:
|
||||
results.extend(scm.list_tags())
|
||||
@@ -258,7 +258,10 @@ def iter_revs(
|
||||
results.extend(exp_commits(scm))
|
||||
|
||||
rev_resolver = partial(resolve_rev, scm)
|
||||
return group_by(rev_resolver, results)
|
||||
grouped = group_by(rev_resolver, results)
|
||||
for rev, names in grouped.items():
|
||||
grouped[rev] = list(dict.fromkeys(names))
|
||||
return grouped
|
||||
|
||||
|
||||
def lfs_prefetch(fs: "FileSystem", paths: list[str]):
|
||||
|
||||
@@ -2,7 +2,7 @@ import os
|
||||
import pathlib
|
||||
import posixpath
|
||||
import sys
|
||||
from typing import Callable, ClassVar
|
||||
from typing import Callable, ClassVar, Union
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from dvc.utils import relpath
|
||||
@@ -79,15 +79,17 @@ class PathInfo(pathlib.PurePath, _BasePath):
|
||||
and self._cparts[:n] == other._cparts # type: ignore[attr-defined]
|
||||
)
|
||||
|
||||
def relative_to(self, other, *args, **kwargs):
|
||||
def relative_to(
|
||||
self, *other: Union[str, "os.PathLike[str]"], **kwargs
|
||||
) -> "PathInfo":
|
||||
# pathlib relative_to raises exception when one path is not a direct
|
||||
# descendant of the other when os.path.relpath would return abspath.
|
||||
# For DVC PathInfo we only need the relpath behavior.
|
||||
# See: https://bugs.python.org/issue40358
|
||||
try:
|
||||
path = super().relative_to(other, *args, **kwargs)
|
||||
path = super().relative_to(*other, **kwargs)
|
||||
except ValueError:
|
||||
path = relpath(self, other)
|
||||
path = relpath(self, other[0])
|
||||
return self.__class__(path)
|
||||
|
||||
|
||||
|
||||
+3
-1
@@ -289,7 +289,7 @@ class Console:
|
||||
|
||||
return console.Console(stderr=True)
|
||||
|
||||
def table(
|
||||
def table( # noqa: PLR0913
|
||||
self,
|
||||
data: "TableData",
|
||||
headers: Optional["Headers"] = None,
|
||||
@@ -301,6 +301,7 @@ class Console:
|
||||
row_styles: Optional[Sequence["Styles"]] = None,
|
||||
borders: Union[bool, str] = False,
|
||||
colalign: Optional[tuple[str, ...]] = None,
|
||||
preserve_whitespace: bool = False,
|
||||
) -> None:
|
||||
from dvc.ui import table as t
|
||||
|
||||
@@ -329,6 +330,7 @@ class Console:
|
||||
pager=pager,
|
||||
force=force,
|
||||
colalign=colalign,
|
||||
preserve_whitespace=preserve_whitespace,
|
||||
)
|
||||
|
||||
def status(self, status: str, **kwargs: Any) -> "Status":
|
||||
|
||||
+27
-9
@@ -30,19 +30,37 @@ def plain_table(
|
||||
pager: bool = False,
|
||||
force: bool = True,
|
||||
colalign: Optional[tuple[str, ...]] = None,
|
||||
preserve_whitespace: bool = False,
|
||||
) -> None:
|
||||
import tabulate as tabulate_mod
|
||||
from funcy import nullcontext
|
||||
from packaging.version import Version
|
||||
from tabulate import tabulate
|
||||
|
||||
text: str = tabulate(
|
||||
data,
|
||||
headers if headers is not None else (),
|
||||
tablefmt="github" if markdown else "plain",
|
||||
disable_numparse=True,
|
||||
# None will be shown as "" by default, overriding
|
||||
missingval="-",
|
||||
colalign=colalign,
|
||||
)
|
||||
# NOTE: tabulate 0.10.0+ supports preserve_whitespace as a kwarg,
|
||||
# while 0.9.0 only respects the global PRESERVE_WHITESPACE.
|
||||
kwargs: dict = {}
|
||||
_orig = tabulate_mod.PRESERVE_WHITESPACE
|
||||
if preserve_whitespace:
|
||||
if Version(tabulate_mod.__version__) >= Version("0.10"):
|
||||
kwargs["preserve_whitespace"] = True
|
||||
else:
|
||||
tabulate_mod.PRESERVE_WHITESPACE = True
|
||||
|
||||
try:
|
||||
text: str = tabulate(
|
||||
data,
|
||||
headers if headers is not None else (),
|
||||
tablefmt="github" if markdown else "plain",
|
||||
disable_numparse=True,
|
||||
# None will be shown as "" by default, overriding
|
||||
missingval="-",
|
||||
colalign=colalign,
|
||||
**kwargs,
|
||||
)
|
||||
finally:
|
||||
tabulate_mod.PRESERVE_WHITESPACE = _orig
|
||||
|
||||
if markdown:
|
||||
# NOTE: md table is incomplete without the trailing newline
|
||||
text += "\n"
|
||||
|
||||
@@ -19,6 +19,36 @@ def test_check_ignore(tmp_dir, dvc, file, ret, output, caplog, capsys):
|
||||
assert "Having any troubles?" not in caplog.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"pattern, exception, ignored",
|
||||
[
|
||||
("volumes/functions/**", "volumes/functions/deno.json", False),
|
||||
("volumes/functions/**", "volumes/functions/deno.json*", False),
|
||||
("volumes/functions/", "volumes/functions/deno.json", True),
|
||||
("ignore.txt", "no-ignore.txt", False),
|
||||
("*ignore.txt", "no-ignore.txt", False),
|
||||
],
|
||||
)
|
||||
def test_check_ignore_reinclude(tmp_dir, dvc, pattern, exception, ignored):
|
||||
tmp_dir.gen(
|
||||
{
|
||||
".dvcignore": f"{pattern}\n!{exception}\n",
|
||||
"volumes": {"functions": {"deno.json": "", "other.py": ""}},
|
||||
"ignore.txt": "",
|
||||
"no-ignore.txt": "",
|
||||
}
|
||||
)
|
||||
file = exception.rstrip("*").replace("/", os.sep)
|
||||
sibling = (
|
||||
os.path.join("volumes", "functions", "other.py")
|
||||
if pattern.startswith("volumes/")
|
||||
else "ignore.txt"
|
||||
)
|
||||
|
||||
assert main(["check-ignore", "-q", file]) == (0 if ignored else 1)
|
||||
assert main(["check-ignore", "-q", sibling]) == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"file,ret,output",
|
||||
[
|
||||
|
||||
@@ -92,6 +92,27 @@ def test_gc_multiple_dvc_repos(tmp_dir, scm, dvc, erepo_dir):
|
||||
assert len(list(odb.all())) == 2
|
||||
|
||||
|
||||
def test_gc_multiple_dvc_repos_all_branches_num(tmp_dir, scm, dvc, erepo_dir):
|
||||
tmp_dir.dvc_gen("main", "main", commit="main")
|
||||
|
||||
erepo_dir.dvc.cache.local.path = dvc.cache.local.path
|
||||
with erepo_dir.chdir():
|
||||
erepo_dir.scm_gen("base", "base", commit="base")
|
||||
with erepo_dir.branch("feature", new=True):
|
||||
erepo_dir.dvc_gen("shared", "feature-v1", commit="feature-v1")
|
||||
erepo_dir.dvc.remove("shared.dvc")
|
||||
erepo_dir.dvc_gen("shared", "feature-v2", commit="feature-v2")
|
||||
|
||||
odb = dvc.cache.local
|
||||
assert len(list(odb.all())) == 3
|
||||
|
||||
dvc.gc(all_branches=True, num=2, repos=[erepo_dir])
|
||||
assert len(list(odb.all())) == 3
|
||||
|
||||
dvc.gc(all_branches=True, repos=[erepo_dir])
|
||||
assert len(list(odb.all())) == 2
|
||||
|
||||
|
||||
def test_all_commits(tmp_dir, scm, dvc):
|
||||
tmp_dir.dvc_gen("testfile", "uncommitted")
|
||||
tmp_dir.dvc_gen("testfile", "committed", commit="committed")
|
||||
|
||||
@@ -31,6 +31,50 @@ def test_ignore(tmp_dir, dvc, filename):
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("suffix, ignored", [("/**", False), ("/", True), ("", True)])
|
||||
@pytest.mark.parametrize("exception", ["deno.json", "deno.json*"])
|
||||
def test_reinclude_directory_contents(tmp_dir, dvc, suffix, ignored, exception):
|
||||
tmp_dir.gen(
|
||||
{
|
||||
".dvcignore": (
|
||||
f"volumes/functions{suffix}\n!volumes/functions/{exception}\n"
|
||||
),
|
||||
"volumes": {"functions": {"deno.json": "", "other.py": ""}},
|
||||
}
|
||||
)
|
||||
dvc._reset()
|
||||
|
||||
directory = tmp_dir / "volumes" / "functions"
|
||||
assert dvc.dvcignore.is_ignored_dir(str(directory)) is ignored
|
||||
assert dvc.dvcignore.is_ignored_file(str(directory / "deno.json")) is ignored
|
||||
assert dvc.dvcignore.is_ignored_file(str(directory / "other.py"))
|
||||
|
||||
files = {
|
||||
os.path.join(root, name)
|
||||
for root, _, names in dvc.dvcignore.walk(dvc.fs, tmp_dir)
|
||||
for name in names
|
||||
}
|
||||
expected = {str(tmp_dir / ".dvcignore")}
|
||||
if not ignored:
|
||||
expected.add(str(directory / "deno.json"))
|
||||
assert files == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pattern", ["ignore.txt", "*ignore.txt"])
|
||||
def test_reinclude_root_file(tmp_dir, dvc, pattern):
|
||||
tmp_dir.gen(
|
||||
{
|
||||
".dvcignore": f"{pattern}\n!no-ignore.txt\n",
|
||||
"ignore.txt": "",
|
||||
"no-ignore.txt": "",
|
||||
}
|
||||
)
|
||||
dvc._reset()
|
||||
|
||||
assert dvc.dvcignore.is_ignored_file(str(tmp_dir / "ignore.txt"))
|
||||
assert not dvc.dvcignore.is_ignored_file(str(tmp_dir / "no-ignore.txt"))
|
||||
|
||||
|
||||
def test_walk(tmp_dir, dvc):
|
||||
tmp_dir.gen(
|
||||
{
|
||||
|
||||
@@ -65,6 +65,30 @@ def test_(dvc, scm, mocker):
|
||||
with pytest.raises(InvalidArgumentError):
|
||||
cmd.run()
|
||||
|
||||
cli_args = parse_args(["gc", "--all-branches", "--num", "2", "--force"])
|
||||
cmd = cli_args.func(cli_args)
|
||||
|
||||
assert cmd.run() == 0
|
||||
|
||||
m.assert_called_with(
|
||||
workspace=False,
|
||||
all_tags=False,
|
||||
all_branches=True,
|
||||
all_commits=False,
|
||||
all_experiments=False,
|
||||
commit_date=None,
|
||||
cloud=False,
|
||||
remote=None,
|
||||
force=True,
|
||||
jobs=None,
|
||||
repos=None,
|
||||
rev=None,
|
||||
num=2,
|
||||
not_in_remote=False,
|
||||
dry=False,
|
||||
skip_failed=False,
|
||||
)
|
||||
|
||||
cli_args = parse_args(["gc", "--cloud", "--not-in-remote"])
|
||||
cmd = cli_args.func(cli_args)
|
||||
with pytest.raises(InvalidArgumentError):
|
||||
|
||||
@@ -47,6 +47,13 @@ def test_iter_revs(tmp_dir, scm, mocker):
|
||||
assert gen == {rev_old: ["tag"]}
|
||||
gen = iter_revs(scm, all_branches=True)
|
||||
assert gen == {rev_old: [old], rev_new: ["new"], rev_other: ["other"]}
|
||||
gen = iter_revs(scm, all_branches=True, num=2)
|
||||
assert gen == {
|
||||
rev_old: [old, rev_old],
|
||||
rev_root: [rev_root],
|
||||
rev_new: ["new"],
|
||||
rev_other: ["other"],
|
||||
}
|
||||
gen = iter_revs(scm, all_tags=True)
|
||||
assert gen == {rev_old: ["tag"]}
|
||||
gen = iter_revs(scm, all_commits=True)
|
||||
|
||||
@@ -4,6 +4,64 @@ from os.path import join
|
||||
import pytest
|
||||
|
||||
from dvc.ignore import DvcIgnorePatterns
|
||||
from dvc.pathspec_math import PatternInfo
|
||||
|
||||
|
||||
def test_match_details_combined_patterns():
|
||||
patterns = [
|
||||
PatternInfo("*.tmp", ".dvcignore:1:*.tmp"),
|
||||
PatternInfo("*.log", ".dvcignore:2:*.log"),
|
||||
PatternInfo("build/", ".dvcignore:3:build/"),
|
||||
PatternInfo("cache/", ".dvcignore:4:cache/"),
|
||||
PatternInfo("!keep.log", ".dvcignore:5:!keep.log"),
|
||||
]
|
||||
ignore = DvcIgnorePatterns(patterns, "/repo", "/")
|
||||
for _ in range(2):
|
||||
assert ignore.matches("/repo", "data/error.log", details=True) == (
|
||||
True,
|
||||
[patterns[1]],
|
||||
)
|
||||
assert ignore.matches("/repo", "cache/keep.log", details=True) == (
|
||||
True,
|
||||
[patterns[3]],
|
||||
)
|
||||
assert ignore.matches("/repo", "data/keep.log", details=True) == (
|
||||
False,
|
||||
[patterns[4]],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["data/file.tmp", "build/file.txt"])
|
||||
def test_match_details_mutation(path):
|
||||
ignore = DvcIgnorePatterns(["*.tmp", "build/"], "/repo", "/")
|
||||
result, details = ignore.matches("/repo", path, details=True)
|
||||
expected = details.copy()
|
||||
details.clear()
|
||||
|
||||
assert result
|
||||
assert expected
|
||||
assert ignore.matches("/repo", path, details=True) == (True, expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"root, expected_dirs, expected_files",
|
||||
[
|
||||
("/repo", ["child"], ["keep.txt", "logs"]),
|
||||
("/repo/dir", [], ["keep.txt"]),
|
||||
("/repo/ignored/dir", [], []),
|
||||
("/repo2", ["logs", "ignored", "child"], ["keep.txt", "other.tmp", "logs"]),
|
||||
],
|
||||
)
|
||||
def test_filter_directory_listing(root, expected_dirs, expected_files):
|
||||
ignore = DvcIgnorePatterns(
|
||||
["ignored/", "dir/**", "!dir/keep.txt", "logs/", "*.tmp"], "/repo", "/"
|
||||
)
|
||||
dirs = ["logs", "ignored", "child"]
|
||||
files = ["keep.txt", "other.tmp", "logs"]
|
||||
for _ in range(2):
|
||||
assert ignore(root, dirs, files) == (expected_dirs, expected_files)
|
||||
assert dirs == ["logs", "ignored", "child"]
|
||||
assert files == ["keep.txt", "other.tmp", "logs"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
Reference in New Issue
Block a user