From 8b3020aaf3a3ad5523b2f784b164f16d6deee33c Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Fri, 8 Jul 2022 20:58:51 +0300 Subject: [PATCH] deps: bump dvc-data to 0.0.20 --- dvc/cli/__init__.py | 2 +- dvc/dependency/repo.py | 24 ++++++++++++------------ dvc/output.py | 14 +++++++------- dvc/repo/diff.py | 4 ++-- setup.cfg | 2 +- tests/func/test_add.py | 2 +- tests/func/test_external_repo.py | 4 ++-- tests/func/test_ignore.py | 2 +- tests/func/test_remote.py | 12 ++++++------ tests/unit/cli/test_main.py | 2 +- tests/unit/data/db/test_local.py | 8 ++++---- tests/unit/fs/test_data.py | 10 +++++----- tests/unit/fs/test_dvc.py | 22 +++++++++++----------- 13 files changed, 54 insertions(+), 54 deletions(-) diff --git a/dvc/cli/__init__.py b/dvc/cli/__init__.py index d4c132a30..590f956ef 100644 --- a/dvc/cli/__init__.py +++ b/dvc/cli/__init__.py @@ -121,7 +121,7 @@ def _log_exceptions(exc: Exception) -> Optional[int]: ) return None - from dvc_data.stage import IgnoreInCollectedDirError + from dvc_data.build import IgnoreInCollectedDirError if isinstance(exc, IgnoreInCollectedDirError): logger.exception("") diff --git a/dvc/dependency/repo.py b/dvc/dependency/repo.py index 3fdad1ffc..404613f70 100644 --- a/dvc/dependency/repo.py +++ b/dvc/dependency/repo.py @@ -31,7 +31,7 @@ class RepoDependency(Dependency): def __init__(self, def_repo, stage, *args, **kwargs): self.def_repo = def_repo - self._staged_objs: Dict[str, "HashFile"] = {} + self._objs: Dict[str, "HashFile"] = {} super().__init__(stage, *args, **kwargs) def _parse_path(self, fs, fs_path): @@ -103,8 +103,8 @@ class RepoDependency(Dependency): from dvc.config import NoRemoteError from dvc.exceptions import NoOutputOrStageError, PathMissingError from dvc.utils import as_posix + from dvc_data.build import build from dvc_data.objects.tree import Tree, TreeError - from dvc_data.stage import stage local_odb = self.repo.odb.local locked = kwargs.pop("locked", True) @@ -133,7 +133,7 @@ class RepoDependency(Dependency): pass try: - staging, _, staged_obj = stage( + object_store, _, obj = build( local_odb, as_posix(self.def_path), repo.dvcfs, @@ -143,14 +143,14 @@ class RepoDependency(Dependency): raise PathMissingError( self.def_path, self.def_repo[self.PARAM_URL] ) from exc - staging = copy(staging) - staging.read_only = True + object_store = copy(object_store) + object_store.read_only = True - self._staged_objs[rev] = staged_obj - used_obj_ids[staging].add(staged_obj.hash_info) - if isinstance(staged_obj, Tree): - used_obj_ids[staging].update(oid for _, _, oid in staged_obj) - return used_obj_ids, staged_obj + self._objs[rev] = obj + used_obj_ids[object_store].add(obj.hash_info) + if isinstance(obj, Tree): + used_obj_ids[object_store].update(oid for _, _, oid in obj) + return used_obj_ids, obj def _check_circular_import(self, odb, obj_ids): from dvc.exceptions import CircularImportError @@ -190,8 +190,8 @@ class RepoDependency(Dependency): def get_obj(self, filter_info=None, **kwargs): locked = kwargs.get("locked", True) rev = self._get_rev(locked=locked) - if rev in self._staged_objs: - return self._staged_objs[rev] + if rev in self._objs: + return self._objs[rev] _, obj = self._get_used_and_obj( obj_only=True, filter_info=filter_info, **kwargs ) diff --git a/dvc/output.py b/dvc/output.py index d4a349997..e545f24dd 100644 --- a/dvc/output.py +++ b/dvc/output.py @@ -20,11 +20,11 @@ from dvc.exceptions import ( from dvc_data import Tree from dvc_data import check as ocheck from dvc_data import load as oload +from dvc_data.build import build from dvc_data.checkout import checkout from dvc_data.hashfile.hash_info import HashInfo from dvc_data.hashfile.istextfile import istextfile from dvc_data.hashfile.meta import Meta -from dvc_data.stage import stage as ostage from dvc_data.transfer import transfer as otransfer from dvc_objects.errors import ObjectFormatError @@ -427,7 +427,7 @@ class Output: else: odb = self.repo.odb.local name = self.fs.PARAM_CHECKSUM - _, _, obj = ostage( + _, _, obj = build( odb, self.fs_path, self.fs, @@ -549,7 +549,7 @@ class Output: self.verify_metric() if not self.use_cache: - _, self.meta, obj = ostage( + _, self.meta, obj = build( self.repo.odb.local, self.fs_path, self.fs, @@ -566,7 +566,7 @@ class Output: assert not self.IS_DEPENDENCY - _, self.meta, self.obj = ostage( + _, self.meta, self.obj = build( self.odb, self.fs_path, self.fs, @@ -608,7 +608,7 @@ class Output: if granular: obj = self._commit_granular_dir(filter_info) else: - staging, _, obj = ostage( + staging, _, obj = build( self.odb, filter_info or self.fs_path, self.fs, @@ -637,7 +637,7 @@ class Output: prefix = self.fs.path.parts( self.fs.path.relpath(filter_info, self.fs_path) ) - staging, _, save_obj = ostage( + staging, _, save_obj = build( self.odb, self.fs_path, self.fs, @@ -843,7 +843,7 @@ class Output: upload = not (update and from_fs.isdir(from_info)) jobs = jobs or min((from_fs.jobs, odb.fs.jobs)) - staging, self.meta, obj = ostage( + staging, self.meta, obj = build( odb, from_info, from_fs, diff --git a/dvc/repo/diff.py b/dvc/repo/diff.py index 163a5d9d1..540934b18 100644 --- a/dvc/repo/diff.py +++ b/dvc/repo/diff.py @@ -115,7 +115,7 @@ def _paths_checksums(repo, targets): def _output_paths(repo, targets): from dvc.fs import LocalFileSystem - from dvc_data.stage import stage as ostage + from dvc_data.build import build on_working_fs = isinstance(repo.fs, LocalFileSystem) @@ -139,7 +139,7 @@ def _output_paths(repo, targets): ) if on_working_fs: - _, _, obj = ostage( + _, _, obj = build( repo.odb.local, output.fs_path, repo.odb.local.fs, diff --git a/setup.cfg b/setup.cfg index 0b93f26a7..4c03e26a7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -68,7 +68,7 @@ install_requires = scmrepo==0.0.25 dvc-render==0.0.6 dvclive>=0.7.3 - dvc-data==0.0.19 + dvc-data==0.0.20 [options.extras_require] all = diff --git a/tests/func/test_add.py b/tests/func/test_add.py index 0911d9f38..2ebf7b54e 100644 --- a/tests/func/test_add.py +++ b/tests/func/test_add.py @@ -457,7 +457,7 @@ class TestAddCommit(TestDvc): def test_should_collect_dir_cache_only_once(mocker, tmp_dir, dvc): tmp_dir.gen({"data/data": "foo"}) - counter = mocker.spy(dvc_data.stage, "_stage_tree") + counter = mocker.spy(dvc_data.build, "_build_tree") ret = main(["add", "data"]) assert ret == 0 assert counter.mock.call_count == 3 diff --git a/tests/func/test_external_repo.py b/tests/func/test_external_repo.py index 6e31edef4..ecc3119ba 100644 --- a/tests/func/test_external_repo.py +++ b/tests/func/test_external_repo.py @@ -7,7 +7,7 @@ from dvc.external_repo import CLONES, external_repo from dvc.testing.tmp_dir import make_subrepo from dvc.utils import relpath from dvc.utils.fs import makedirs, remove -from dvc_data.stage import stage +from dvc_data.build import build from dvc_data.transfer import transfer @@ -202,7 +202,7 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir): remove(cache_dir) makedirs(cache_dir) - staging, _, obj = stage( + staging, _, obj = build( repo.odb.local, "dir", repo.dvcfs, diff --git a/tests/func/test_ignore.py b/tests/func/test_ignore.py index 752ac082d..670d1b5df 100644 --- a/tests/func/test_ignore.py +++ b/tests/func/test_ignore.py @@ -10,8 +10,8 @@ from dvc.pathspec_math import PatternInfo, merge_patterns from dvc.repo import Repo from dvc.testing.tmp_dir import TmpDir from dvc.types import List +from dvc_data.build import IgnoreInCollectedDirError from dvc_data.hashfile.utils import get_mtime_and_size -from dvc_data.stage import IgnoreInCollectedDirError def _to_pattern_info_list(str_list: List): diff --git a/tests/func/test_remote.py b/tests/func/test_remote.py index e2e2c8334..363e82dbe 100644 --- a/tests/func/test_remote.py +++ b/tests/func/test_remote.py @@ -139,8 +139,8 @@ def test_upper_case_remote(tmp_dir, dvc, local_cloud): def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc): + from dvc_data.build import build from dvc_data.objects.tree import Tree - from dvc_data.stage import stage tmp_dir.gen({"data": {"1": "1 content", "2": "2 content"}}) @@ -150,19 +150,19 @@ def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc): [{"relpath": "1", "md5": "1"}, {"relpath": "2", "md5": "2"}] ) tree.digest() - with patch("dvc_data.stage._stage_tree", return_value=(None, tree)): - _, _, obj = stage(dvc.odb.local, path, dvc.odb.local.fs, "md5") + with patch("dvc_data.build._build_tree", return_value=(None, tree)): + _, _, obj = build(dvc.odb.local, path, dvc.odb.local.fs, "md5") hash1 = obj.hash_info - # remove the raw dir obj to force building the tree on the next stage call + # remove the raw dir obj to force building the tree on the next build call dvc.odb.local.fs.remove(dvc.odb.local.oid_to_path(hash1.as_raw().value)) tree = Tree.from_list( [{"md5": "1", "relpath": "1"}, {"md5": "2", "relpath": "2"}] ) tree.digest() - with patch("dvc_data.stage._stage_tree", return_value=(None, tree)): - _, _, obj = stage(dvc.odb.local, path, dvc.odb.local.fs, "md5") + with patch("dvc_data.build._build_tree", return_value=(None, tree)): + _, _, obj = build(dvc.odb.local, path, dvc.odb.local.fs, "md5") hash2 = obj.hash_info assert hash1 == hash2 diff --git a/tests/unit/cli/test_main.py b/tests/unit/cli/test_main.py index 0378142f2..73cf3bf72 100644 --- a/tests/unit/cli/test_main.py +++ b/tests/unit/cli/test_main.py @@ -4,8 +4,8 @@ import pytest from funcy import raiser from dvc.cli import main +from dvc_data.build import IgnoreInCollectedDirError from dvc_data.hashfile.cache import DiskError -from dvc_data.stage import IgnoreInCollectedDirError from dvc_objects.fs.base import FileSystem, RemoteMissingDepsError diff --git a/tests/unit/data/db/test_local.py b/tests/unit/data/db/test_local.py index 2fe65e177..b755cf64f 100644 --- a/tests/unit/data/db/test_local.py +++ b/tests/unit/data/db/test_local.py @@ -86,14 +86,14 @@ def test_set_exec_ignore_errors(tmp_dir, dvc, mocker, err): def test_staging_file(tmp_dir, dvc): from dvc_data import check - from dvc_data.stage import stage + from dvc_data.build import build from dvc_data.transfer import transfer tmp_dir.gen("foo", "foo") fs = LocalFileSystem() local_odb = dvc.odb.local - staging_odb, _, obj = stage( + staging_odb, _, obj = build( local_odb, (tmp_dir / "foo").fs_path, fs, "md5" ) @@ -114,14 +114,14 @@ def test_staging_file(tmp_dir, dvc): def test_staging_dir(tmp_dir, dvc): from dvc_data import check - from dvc_data.stage import stage + from dvc_data.build import build from dvc_data.transfer import transfer tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}}) fs = LocalFileSystem() local_odb = dvc.odb.local - staging_odb, _, obj = stage( + staging_odb, _, obj = build( local_odb, (tmp_dir / "dir").fs_path, fs, "md5" ) diff --git a/tests/unit/fs/test_data.py b/tests/unit/fs/test_data.py index 2c7f0b3aa..77f2ad309 100644 --- a/tests/unit/fs/test_data.py +++ b/tests/unit/fs/test_data.py @@ -7,8 +7,8 @@ import dvc_data from dvc.config import NoRemoteError from dvc.fs.data import DataFileSystem from dvc.utils.fs import remove +from dvc_data.build import build from dvc_data.hashfile.hash_info import HashInfo -from dvc_data.stage import stage @pytest.mark.parametrize( @@ -243,13 +243,13 @@ def test_get_hash_granular(tmp_dir, dvc): fs = DataFileSystem(repo=dvc) subdir = "dir/subdir" assert fs.info(subdir).get("md5") is None - _, _, obj = stage(dvc.odb.local, subdir, fs, "md5", dry_run=True) + _, _, obj = build(dvc.odb.local, subdir, fs, "md5", dry_run=True) assert obj.hash_info == HashInfo( "md5", "af314506f1622d107e0ed3f14ec1a3b5.dir" ) data = posixpath.join(subdir, "data") assert fs.info(data)["md5"] == "8d777f385d3dfec8815d20f7496026dc" - _, _, obj = stage(dvc.odb.local, data, fs, "md5", dry_run=True) + _, _, obj = build(dvc.odb.local, data, fs, "md5", dry_run=True) assert obj.hash_info == HashInfo("md5", "8d777f385d3dfec8815d20f7496026dc") @@ -260,7 +260,7 @@ def test_get_hash_dirty_file(tmp_dir, dvc): fs = DataFileSystem(repo=dvc) expected = "8c7dd922ad47494fc02c388e12c00eac" assert fs.info("file").get("md5") == expected - _, _, obj = stage(dvc.odb.local, "file", fs, "md5", dry_run=True) + _, _, obj = build(dvc.odb.local, "file", fs, "md5", dry_run=True) assert obj.hash_info == HashInfo("md5", expected) @@ -271,5 +271,5 @@ def test_get_hash_dirty_dir(tmp_dir, dvc): fs = DataFileSystem(repo=dvc) expected = "5ea40360f5b4ec688df672a4db9c17d1.dir" assert fs.info("dir").get("md5") == expected - _, _, obj = stage(dvc.odb.local, "dir", fs, "md5", dry_run=True) + _, _, obj = build(dvc.odb.local, "dir", fs, "md5", dry_run=True) assert obj.hash_info == HashInfo("md5", expected) diff --git a/tests/unit/fs/test_dvc.py b/tests/unit/fs/test_dvc.py index f7c6dec7b..60fd96969 100644 --- a/tests/unit/fs/test_dvc.py +++ b/tests/unit/fs/test_dvc.py @@ -7,8 +7,8 @@ import pytest from dvc.fs.dvc import DvcFileSystem from dvc.testing.tmp_dir import make_subrepo +from dvc_data.build import build from dvc_data.hashfile.hash_info import HashInfo -from dvc_data.stage import stage def test_exists(tmp_dir, dvc): @@ -498,7 +498,7 @@ def test_get_hash_cached_file(tmp_dir, dvc, mocker): fs = DvcFileSystem(repo=dvc) expected = "acbd18db4cc2f85cedef654fccc4a4d8" assert fs.info("foo").get("md5") is None - _, _, obj = stage(dvc.odb.local, "foo", fs, "md5") + _, _, obj = build(dvc.odb.local, "foo", fs, "md5") assert obj.hash_info == HashInfo("md5", expected) (tmp_dir / "foo").unlink() assert fs.info("foo")["md5"] == expected @@ -511,14 +511,14 @@ def test_get_hash_cached_dir(tmp_dir, dvc, mocker): fs = DvcFileSystem(repo=dvc) expected = "8761c4e9acad696bee718615e23e22db.dir" assert fs.info("dir").get("md5") is None - _, _, obj = stage(dvc.odb.local, "dir", fs, "md5") + _, _, obj = build(dvc.odb.local, "dir", fs, "md5") assert obj.hash_info == HashInfo( "md5", "8761c4e9acad696bee718615e23e22db.dir" ) shutil.rmtree(tmp_dir / "dir") assert fs.info("dir")["md5"] == expected - _, _, obj = stage(dvc.odb.local, "dir", fs, "md5") + _, _, obj = build(dvc.odb.local, "dir", fs, "md5") assert obj.hash_info == HashInfo( "md5", "8761c4e9acad696bee718615e23e22db.dir" ) @@ -531,12 +531,12 @@ def test_get_hash_cached_granular(tmp_dir, dvc, mocker): fs = DvcFileSystem(repo=dvc) subdir = "dir/subdir" assert fs.info(subdir).get("md5") is None - _, _, obj = stage(dvc.odb.local, subdir, fs, "md5") + _, _, obj = build(dvc.odb.local, subdir, fs, "md5") assert obj.hash_info == HashInfo( "md5", "af314506f1622d107e0ed3f14ec1a3b5.dir" ) assert fs.info(posixpath.join(subdir, "data")).get("md5") is None - _, _, obj = stage(dvc.odb.local, posixpath.join(subdir, "data"), fs, "md5") + _, _, obj = build(dvc.odb.local, posixpath.join(subdir, "data"), fs, "md5") assert obj.hash_info == HashInfo("md5", "8d777f385d3dfec8815d20f7496026dc") (tmp_dir / "dir" / "subdir" / "data").unlink() assert ( @@ -558,7 +558,7 @@ def test_get_hash_mixed_dir(tmp_dir, scm, dvc): tmp_dir.scm.commit("add dir") fs = DvcFileSystem(repo=dvc) - _, _, obj = stage(dvc.odb.local, "dir", fs, "md5") + _, _, obj = build(dvc.odb.local, "dir", fs, "md5") assert obj.hash_info == HashInfo( "md5", "e1d9e8eae5374860ae025ec84cfd85c7.dir" ) @@ -578,7 +578,7 @@ def test_get_hash_dirty_file(tmp_dir, dvc): # hash_file(file) should return workspace hash, not DVC cached hash fs = DvcFileSystem(repo=dvc) assert fs.info("file").get("md5") is None - staging, _, obj = stage(dvc.odb.local, "file", fs, "md5") + staging, _, obj = build(dvc.odb.local, "file", fs, "md5") assert obj.hash_info == something_hash_info check(staging, obj) @@ -588,9 +588,9 @@ def test_get_hash_dirty_file(tmp_dir, dvc): _, hash_info = hash_file("file", fs, "md5", state=dvc.state) assert hash_info == file_hash_info - # tmp_dir/file can be staged even though it is missing in workspace since + # tmp_dir/file can be built even though it is missing in workspace since # repofs will use the DVC cached hash (and refer to the local cache object) - _, _, obj = stage(dvc.odb.local, "file", fs, "md5") + _, _, obj = build(dvc.odb.local, "file", fs, "md5") assert obj.hash_info == file_hash_info @@ -599,7 +599,7 @@ def test_get_hash_dirty_dir(tmp_dir, dvc): (tmp_dir / "dir" / "baz").write_text("baz") fs = DvcFileSystem(repo=dvc) - _, meta, obj = stage(dvc.odb.local, "dir", fs, "md5") + _, meta, obj = build(dvc.odb.local, "dir", fs, "md5") assert obj.hash_info == HashInfo( "md5", "ba75a2162ca9c29acecb7957105a0bc2.dir" )