diff: don't forget to use dvcignore and state (#6595)

This commit is contained in:
Ruslan Kuprieiev
2021-09-11 00:13:07 +03:00
committed by GitHub
parent 61fe8bc59c
commit 3a100382bc
+31 -30
View File
@@ -4,7 +4,6 @@ from collections import defaultdict
from typing import Dict, List
from dvc.exceptions import PathMissingError
from dvc.objects.stage import get_file_hash
from dvc.repo import locked
from dvc.repo.experiments.utils import fix_exp_head
@@ -45,7 +44,7 @@ def diff(self, a_rev="HEAD", b_rev=None, targets=None):
repo_fs, targets
)
results[rev] = _paths_checksums(self, repo_fs, targets_path_infos)
results[rev] = _paths_checksums(self, targets_path_infos)
if targets is not None:
# check for overlapping missing targets between a_rev and b_rev
@@ -98,7 +97,7 @@ def diff(self, a_rev="HEAD", b_rev=None, targets=None):
return ret if any(ret.values()) else {}
def _paths_checksums(repo, repo_fs, targets):
def _paths_checksums(repo, targets):
"""
A dictionary of checksums addressed by relpaths collected from
the current fs outputs.
@@ -110,10 +109,10 @@ def _paths_checksums(repo, repo_fs, targets):
file: "data"
"""
return dict(_output_paths(repo, repo_fs, targets))
return dict(_output_paths(repo, targets))
def _output_paths(repo, repo_fs, targets):
def _output_paths(repo, targets):
from dvc.fs.local import LocalFileSystem
from dvc.objects.stage import stage as ostage
@@ -131,45 +130,47 @@ def _output_paths(repo, repo_fs, targets):
else os.path.join(str(output), "")
)
def _to_checksum(output):
if on_working_fs:
return ostage(
repo.odb.local,
output.path_info,
repo.odb.local.fs,
"md5",
dry_run=True,
)[1].hash_info.value
return output.hash_info.value
for output in repo.index.outs:
if _exists(output):
yield_output = targets is None or any(
output.path_info.isin_or_eq(target) for target in targets
)
if on_working_fs:
_, obj = ostage(
repo.odb.local,
output.path_info,
repo.odb.local.fs,
"md5",
dry_run=True,
dvcignore=output.dvcignore,
)
hash_info = obj.hash_info
else:
hash_info = output.hash_info
obj = output.get_obj()
if yield_output:
yield _to_path(output), _to_checksum(output)
yield _to_path(output), hash_info.value
if not obj:
continue
if output.is_dir_checksum and (
yield_output
or any(target.isin(output.path_info) for target in targets)
):
yield from _dir_output_paths(repo_fs, output, targets)
yield from _dir_output_paths(output.path_info, obj, targets)
def _dir_output_paths(repo_fs, output, targets=None):
from dvc.config import NoRemoteError
try:
for fname in repo_fs.walk_files(output.path_info):
if targets is None or any(
fname.isin_or_eq(target) for target in targets
):
# pylint: disable=no-member
yield str(fname), get_file_hash(fname, repo_fs, "md5").value
except NoRemoteError:
logger.warning("dir cache entry for '%s' is missing", output)
def _dir_output_paths(path_info, obj, targets=None):
for key, entry_obj in obj:
fname = path_info.joinpath(*key)
if targets is None or any(
fname.isin_or_eq(target) for target in targets
):
# pylint: disable=no-member
yield str(fname), entry_obj.hash_info.value
def _filter_missing(repo_fs, paths):