diff --git a/dvc/repo/data.py b/dvc/repo/data.py index 629f64c0b..17b318d52 100644 --- a/dvc/repo/data.py +++ b/dvc/repo/data.py @@ -94,7 +94,7 @@ def _diff( with_missing and change.old and change.old.hash_info - and not old.storage_map[change.key].cache.exists(change.old.hash_info.value) + and not old.storage_map.cache_exists(change.old) ): # NOTE: emulating previous behaviour _add_change("not_in_cache", change) diff --git a/dvc/repo/imports.py b/dvc/repo/imports.py index c454772a8..3405cfa09 100644 --- a/dvc/repo/imports.py +++ b/dvc/repo/imports.py @@ -115,7 +115,7 @@ def save_imports( desc="Downloading imports from source", unit="files", ) as cb: - checkout(data_view, tmpdir, cache.fs, callback=cb) + checkout(data_view, tmpdir, cache.fs, callback=cb, storage="data") md5(data_view) save(data_view, odb=cache, hardlink=True) diff --git a/dvc/repo/index.py b/dvc/repo/index.py index a97c99367..e6996b22f 100644 --- a/dvc/repo/index.py +++ b/dvc/repo/index.py @@ -286,7 +286,7 @@ class Index: from collections import defaultdict from dvc.config import NoRemoteError - from dvc_data.index import DataIndex, DataIndexEntry, Storage + from dvc_data.index import DataIndex, DataIndexEntry, FileStorage, ObjectStorage by_workspace: dict = defaultdict(DataIndex) @@ -306,9 +306,22 @@ class Index: hash_info=out.hash_info, ) - storage = Storage(odb=out.odb, cache=out.cache) + storage_map = data_index.storage_map + if out.odb: + storage_map.add_data(ObjectStorage(key, out.odb)) + storage_map.add_cache(ObjectStorage(key, out.cache)) try: - storage.remote = self.repo.cloud.get_remote_odb(out.remote) + remote = self.repo.cloud.get_remote(out.remote) + if remote.fs.version_aware: + storage_map.add_remote( + FileStorage( + key=key, + fs=remote.fs, + path=remote.fs.path.join(remote.path, *key), + ) + ) + else: + storage_map.add_remote(ObjectStorage(key, remote.odb)) except NoRemoteError: pass @@ -318,12 +331,9 @@ class Index: entry.meta = dep.meta entry.hash_info = dep.hash_info - storage.odb = dep.odb - storage.fs = dep.fs - storage.path = dep.fs_path + storage_map.add_data(FileStorage(key, dep.fs, dep.fs_path)) data_index.add(entry) - data_index.storage_map[key] = storage return dict(by_workspace) diff --git a/dvc/repo/worktree.py b/dvc/repo/worktree.py index a60baee4b..4dd13e4ad 100644 --- a/dvc/repo/worktree.py +++ b/dvc/repo/worktree.py @@ -119,12 +119,6 @@ def fetch_worktree( ): remote_obj = _get_remote(repo, remote_name, remote, "fetch") index = view.data["repo"] - for key in index.storage_map: - storage = index.storage_map[key] - storage.fs = remote_obj.fs - storage.path = remote_obj.fs.path.join(remote_obj.path, *key) - index.storage_map[key] = storage - total = len(index) with Callback.as_tqdm_callback( unit="file", @@ -132,7 +126,7 @@ def fetch_worktree( disable=total == 0, ) as cb: cb.set_size(total) - transferred += save(index, callback=cb, jobs=jobs) + transferred += save(index, callback=cb, jobs=jobs, storage="remote") return transferred @@ -357,6 +351,7 @@ def _fetch_out_changes( update_meta=False, meta_only=True, meta_cmp_key=partial(_meta_checksum, remote.fs), + storage="data", callback=cb, ) out.save() diff --git a/pyproject.toml b/pyproject.toml index a04015b2b..400437a7d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,7 +53,7 @@ dependencies = [ "dvc-render>=0.1.2", "dvc-task==0.1.11", "dvclive>=1.2.2", - "dvc-data==0.39.3", + "dvc-data==0.40.1", "dvc-http", "hydra-core>=1.1.0", "iterative-telemetry==0.0.7",