diff --git a/dvc/testing/test_remote.py b/dvc/testing/test_remote.py index 88d5d634b..3c556f4a9 100644 --- a/dvc/testing/test_remote.py +++ b/dvc/testing/test_remote.py @@ -71,7 +71,7 @@ class TestRemote: _check_status(status_dir, ok=dir_hashes) # Remove and check status - remove(dvc.odb.local.path) + dvc.odb.local.clear() status = dvc.cloud.status(foo_hashes) _check_status(status, deleted={foo_hash}) diff --git a/setup.cfg b/setup.cfg index 348686eef..6be6b5b11 100644 --- a/setup.cfg +++ b/setup.cfg @@ -66,7 +66,7 @@ install_requires = dvc-render==0.0.11 dvc-task==0.1.2 dvclive>=0.10.0 - dvc-data==0.8.0 + dvc-data==0.10.0 dvc-http==2.19.1 hydra-core>=1.1.0 diff --git a/tests/func/test_add.py b/tests/func/test_add.py index 0c628fd29..b2d9f7bf5 100644 --- a/tests/func/test_add.py +++ b/tests/func/test_add.py @@ -453,7 +453,9 @@ class TestAddCommit(TestDvc): ret = main(["commit", self.FOO + ".dvc"]) self.assertEqual(ret, 0) self.assertTrue(os.path.isfile(self.FOO)) - self.assertEqual(len(os.listdir(self.dvc.odb.local.path)), 1) + self.assertTrue( + self.dvc.odb.local.exists("acbd18db4cc2f85cedef654fccc4a4d8") + ) def test_should_collect_dir_cache_only_once(mocker, tmp_dir, dvc): @@ -832,9 +834,9 @@ def test_add_symlink_file(tmp_dir, dvc): assert (tmp_dir / "dir" / "foo").read_text() == "bar" assert (tmp_dir / "dir" / "bar").read_text() == "bar" - assert (tmp_dir / ".dvc" / "cache").read_text() == { - "37": {"b51d194a7513e45b56f6524f2d51f2": "bar"} - } + assert ( + tmp_dir / ".dvc" / "cache" / "37" / "b51d194a7513e45b56f6524f2d51f2" + ).read_text() == "bar" assert not ( tmp_dir / ".dvc" / "cache" / "37" / "b51d194a7513e45b56f6524f2d51f2" ).is_symlink() @@ -891,9 +893,9 @@ def test_add_with_cache_link_error(tmp_dir, dvc, mocker, capsys): assert (tmp_dir / "foo").exists() assert (tmp_dir / "foo.dvc").exists() - assert (tmp_dir / ".dvc" / "cache").read_text() == { - "ac": {"bd18db4cc2f85cedef654fccc4a4d8": "foo"} - } + assert ( + tmp_dir / ".dvc" / "cache" / "ac" / "bd18db4cc2f85cedef654fccc4a4d8" + ).read_text() == "foo" def test_add_preserve_fields(tmp_dir, dvc): diff --git a/tests/func/test_data_cloud.py b/tests/func/test_data_cloud.py index e4727f34e..a5d56db52 100644 --- a/tests/func/test_data_cloud.py +++ b/tests/func/test_data_cloud.py @@ -49,7 +49,7 @@ def test_cloud_cli(tmp_dir, dvc, remote, mocker): for (_args, _kwargs) in oids_exist.call_args_list ) - remove(dvc.odb.local.path) + dvc.odb.local.clear() oids_exist.reset_mock() assert main(["fetch"] + args) == 0 @@ -164,7 +164,7 @@ def test_missing_cache(tmp_dir, dvc, local_remote, caplog): tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"}) # purge cache - remove(dvc.odb.local.path) + dvc.odb.local.clear() header = ( "Some of the cache files do not exist " @@ -205,7 +205,7 @@ def test_verify_hashes( # remove artifacts and cache to trigger fetching remove("file") remove("dir") - remove(dvc.odb.local.path) + dvc.odb.local.clear() hash_spy = mocker.spy(dvc_data.hashfile.hash, "file_md5") @@ -213,7 +213,7 @@ def test_verify_hashes( assert hash_spy.call_count == 0 # Removing cache will invalidate existing state entries - remove(dvc.odb.local.path) + dvc.odb.local.clear() dvc.config["remote"]["upstream"]["verify"] = True @@ -310,18 +310,19 @@ def test_pull_external_dvc_imports_mixed( def clean(outs, dvc=None): if dvc: - outs = outs + [dvc.odb.local.path] + dvc.odb.local.clear() for path in outs: print(path) remove(path) if dvc: - os.makedirs(dvc.odb.local.path, exist_ok=True) clean_repos() def recurse_list_dir(d): return [ - os.path.join(d, f) for _, _, filenames in os.walk(d) for f in filenames + os.path.join(root, f) + for root, _, filenames in os.walk(d) + for f in filenames ] @@ -488,11 +489,11 @@ def test_push_pull_fetch_pipeline_stages(tmp_dir, dvc, run_copy, local_remote): dvc.pull("copy-foo-bar") assert (tmp_dir / "bar").exists() - assert len(recurse_list_dir(dvc.odb.local.path)) == 1 + assert len(recurse_list_dir(dvc.odb.local.path)) == 2 clean(["bar"], dvc) dvc.fetch("copy-foo-bar") - assert len(recurse_list_dir(dvc.odb.local.path)) == 1 + assert len(recurse_list_dir(dvc.odb.local.path)) == 2 def test_pull_partial(tmp_dir, dvc, local_remote): diff --git a/tests/func/test_ignore.py b/tests/func/test_ignore.py index fd374ca63..adfb45077 100644 --- a/tests/func/test_ignore.py +++ b/tests/func/test_ignore.py @@ -421,8 +421,6 @@ def test_run_dvcignored_dep(tmp_dir, dvc, run_copy): def test_pull_ignore(tmp_dir, dvc, local_cloud): - from dvc.utils.fs import remove - tmp_dir.dvc_gen( { ".dvcignore": "data/processed/", @@ -437,7 +435,7 @@ def test_pull_ignore(tmp_dir, dvc, local_cloud): foo_path.unlink() assert not foo_path.exists() - remove(tmp_dir / ".dvc/cache") + dvc.odb.local.clear() dvc.pull() assert foo_path.exists() diff --git a/tests/func/test_odb.py b/tests/func/test_odb.py index 4467bd755..4d858cfe4 100644 --- a/tests/func/test_odb.py +++ b/tests/func/test_odb.py @@ -112,18 +112,15 @@ def test_shared_cache_dir(tmp_dir): assert not os.path.exists(os.path.join("dir1", ".dvc", "cache")) assert not os.path.exists(os.path.join("dir2", ".dvc", "cache")) - - subdirs = list( - filter( - lambda x: os.path.isdir(os.path.join(cache_dir, x)), - os.listdir(cache_dir), - ) + assert os.path.exists( + os.path.join(cache_dir, "dc", "f6c2fa538b445a3a095255c3641dfc") + ) + assert os.path.exists( + os.path.join(cache_dir, "b4", "333c8cfa2ebba7ef20ec6c3265902b") + ) + assert os.path.exists( + os.path.join(cache_dir, "9e", "fab2399c7c560b34de477b9aa0a465") ) - assert len(subdirs) == 3 - assert len(os.listdir(os.path.join(cache_dir, subdirs[0]))) == 1 - - assert len(os.listdir(os.path.join(cache_dir, subdirs[1]))) == 1 - assert len(os.listdir(os.path.join(cache_dir, subdirs[2]))) == 1 def test_cache_link_type(tmp_dir, scm, dvc): @@ -167,10 +164,9 @@ def test_cmd_cache_relative_path(tmp_dir, scm, dvc, make_tmp_dir): tmp_dir.dvc_gen({"foo": "foo"}) - subdirs = os.listdir(cache_dir) - assert len(subdirs) == 1 - files = os.listdir(os.path.join(cache_dir, subdirs[0])) - assert len(files) == 1 + assert os.path.exists( + os.path.join(cache_dir, "ac", "bd18db4cc2f85cedef654fccc4a4d8") + ) def test_default_cache_type(dvc): @@ -194,28 +190,13 @@ def test_shared_cache(tmp_dir, dvc, group): {"file": "file content", "dir": {"file2": "file 2 " "content"}} ) - actual = {} - for root, dnames, fnames in os.walk(cache_dir): - for name in dnames + fnames: - path = os.path.join(root, name) - actual[path] = oct(stat.S_IMODE(os.stat(path).st_mode)) - file_mode = oct(0o444) dir_mode = oct(0o2775 if group else (0o777 & ~system.umask)) + for root, dnames, fnames in os.walk(cache_dir): + for dname in dnames: + path = os.path.join(root, dname) + assert oct(stat.S_IMODE(os.stat(path).st_mode)) == dir_mode - expected = { - os.path.join(cache_dir, "17"): dir_mode, - os.path.join( - cache_dir, "17", "4eaa1dd94050255b7b98a7e1924b31.dir" - ): file_mode, - os.path.join(cache_dir, "97"): dir_mode, - os.path.join( - cache_dir, "97", "e17781c198500e2766ea56bd697c03" - ): file_mode, - os.path.join(cache_dir, "d1"): dir_mode, - os.path.join( - cache_dir, "d1", "0b4c3ff123b26dc068d43a8bef2d23" - ): file_mode, - } - - assert expected == actual + for fname in fnames: + path = os.path.join(root, fname) + assert oct(stat.S_IMODE(os.stat(path).st_mode)) == file_mode diff --git a/tests/func/test_remote.py b/tests/func/test_remote.py index ac596cc82..cd08ec9f3 100644 --- a/tests/func/test_remote.py +++ b/tests/func/test_remote.py @@ -201,7 +201,7 @@ def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory, local_remote): # Push everything and delete local cache dvc.push() - remove(dvc.odb.local.path) + dvc.odb.local.clear() baz._collect_used_dir_cache() with patch.object(generic, "transfer", side_effect=Exception): diff --git a/tests/func/test_run_single_stage.py b/tests/func/test_run_single_stage.py index 193122ad2..8d9a7c7a7 100644 --- a/tests/func/test_run_single_stage.py +++ b/tests/func/test_run_single_stage.py @@ -726,7 +726,7 @@ class TestRunCommit(TestDvc): ret = main(["commit", fname + ".dvc"]) self.assertEqual(ret, 0) self.assertTrue(os.path.isfile(fname)) - self.assertEqual(len(os.listdir(self.dvc.odb.local.path)), 1) + self.assertEqual(len(list(self.dvc.odb.local.all())), 1) class TestRunPersist(TestDvc):