Compare commits

...
8 Commits
Author SHA1 Message Date
Ruslan Kuprieiev f57ac609e9 deps: bump dvc-data to 0.27.1 2023-07-27 20:44:34 +03:00
Ruslan Kuprieiev cc4f505f99 api: open/read: introduce remote_config 2023-07-26 16:29:14 +03:00
Ruslan Kuprieiev 4d011718a6 out: str: don't try to relpath paths outside of the repo
This is not very useful overall, but also results in a ValueError from
relpath on windows if your repo and output are on different drives.
2023-07-25 23:49:27 +03:00
Ruslan Kuprieiev 9c89841637 fetch: support passing config
For now not exposing it in CLI since there is no need/use yet.

Requirement for https://github.com/iterative/studio/pull/6541
2023-07-25 20:03:38 +03:00
Peter Rowlands 92cae74512 exp.collect: strip metrics from queued experiments 2023-07-25 20:57:15 +09:00
Ruslan Kuprieiev 73dad4780f repo: brancher: reset parsed fields
Related #9754 and fixes `dvc fetch`.
2023-07-25 14:00:33 +03:00
Ruslan Kuprieiev 65ba9ba527 fs: get_cloud_fs: accept config instead of repo
Config is a more immutable thing and using `repo` is confusing and might be error
prone in the bigger picture.

Related to #9754
2023-07-25 00:58:47 +03:00
Peter Rowlands 7806afd4ad fs: prefer generic.copy over fsspec fs.get() 2023-07-24 17:01:34 +09:00
23 changed files with 198 additions and 110 deletions
+28 -9
View File
@@ -75,6 +75,7 @@ def open( # noqa, pylint: disable=redefined-builtin
mode: str = "r",
encoding: Optional[str] = None,
config: Optional[Dict[str, Any]] = None,
remote_config: Optional[Dict[str, Any]] = None,
):
"""
Opens a file tracked in a DVC project.
@@ -118,6 +119,9 @@ def open( # noqa, pylint: disable=redefined-builtin
Mirrors the namesake parameter in builtin `open()`_.
config(dict, optional): config to be passed to the DVC repository.
Defaults to None.
remote_config(dict, optional): remote config to be passed to the DVC
repository.
Defaults to None.
Returns:
_OpenContextManager: A context manager that generatse a corresponding
@@ -214,22 +218,27 @@ def open( # noqa, pylint: disable=redefined-builtin
"mode": mode,
"encoding": encoding,
"config": config,
"remote_config": remote_config,
}
return _OpenContextManager(_open, args, kwargs)
def _open(path, repo=None, rev=None, remote=None, mode="r", encoding=None, config=None):
if remote:
if config is not None:
raise ValueError(
"can't specify both `remote` and `config` at the same time"
)
config = {"core": {"remote": remote}}
def _open(
path,
repo=None,
rev=None,
remote=None,
mode="r",
encoding=None,
config=None,
remote_config=None,
):
repo_kwargs: Dict[str, Any] = {
"subrepos": True,
"uninitialized": True,
"remote": remote,
"config": config,
"remote_config": remote_config,
}
with Repo.open(repo, rev=rev, **repo_kwargs) as _repo:
@@ -265,7 +274,16 @@ def _open(path, repo=None, rev=None, remote=None, mode="r", encoding=None, confi
raise DvcIsADirectoryError(f"'{path}' is a directory") from exc
def read(path, repo=None, rev=None, remote=None, mode="r", encoding=None, config=None):
def read(
path,
repo=None,
rev=None,
remote=None,
mode="r",
encoding=None,
config=None,
remote_config=None,
):
"""
Returns the contents of a tracked file (by DVC or Git). For Git repos, HEAD
is used unless a rev argument is supplied. The default remote is tried
@@ -279,5 +297,6 @@ def read(path, repo=None, rev=None, remote=None, mode="r", encoding=None, config
mode=mode,
encoding=encoding,
config=config,
remote_config=remote_config,
) as fd:
return fd.read()
+1 -1
View File
@@ -24,7 +24,7 @@ def _get_odb(
if not settings:
return None
cls, config, fs_path = get_cloud_fs(repo, **settings)
cls, config, fs_path = get_cloud_fs(repo.config, **settings)
fs = fs or cls(**config)
if prefix:
fs_path = fs.path.join(fs_path, *prefix)
+23 -2
View File
@@ -93,6 +93,8 @@ class Config(dict):
validate: bool = True,
fs: Optional["FileSystem"] = None,
config: Optional["DictStrAny"] = None,
remote: Optional[str] = None,
remote_config: Optional["DictStrAny"] = None,
): # pylint: disable=super-init-not-called
from dvc.fs import LocalFileSystem
@@ -103,7 +105,9 @@ class Config(dict):
if dvc_dir:
self.dvc_dir = self.fs.path.abspath(self.fs.path.realpath(dvc_dir))
self.load(validate=validate, config=config)
self.load(
validate=validate, config=config, remote=remote, remote_config=remote_config
)
@classmethod
def from_cwd(cls, fs: Optional["FileSystem"] = None, **kwargs):
@@ -154,7 +158,13 @@ class Config(dict):
with open(config_file, "w+", encoding="utf-8"):
return Config(dvc_dir)
def load(self, validate: bool = True, config: Optional["DictStrAny"] = None):
def load(
self,
validate: bool = True,
config: Optional["DictStrAny"] = None,
remote: Optional[str] = None,
remote_config: Optional["DictStrAny"] = None,
):
"""Loads config from all the config files.
Raises:
@@ -169,6 +179,17 @@ class Config(dict):
conf = self.validate(conf)
self.clear()
if remote:
conf["core"]["remote"] = remote
if remote_config:
remote = remote or conf["core"].get("remote")
if not remote:
raise ValueError("Missing remote name")
merge(conf, {"remote": {remote: remote_config}})
self.update(conf)
def _get_fs(self, level):
+1 -1
View File
@@ -90,7 +90,7 @@ class DataCloud:
if name:
from dvc.fs import get_cloud_fs
cls, config, fs_path = get_cloud_fs(self.repo, name=name)
cls, config, fs_path = get_cloud_fs(self.repo.config, name=name)
if config.get("worktree"):
version_aware = config.get("version_aware")
+1 -1
View File
@@ -50,7 +50,7 @@ class Dependency(Output):
self.fs_path = self.fs.path.version_path(self.fs_path, self.meta.version_id)
def download(self, to, jobs=None):
fs_download(self.fs, self.fs_path, to, jobs=jobs)
fs_download(self.fs, self.fs_path, to.fs_path, jobs=jobs)
def save(self):
super().save()
+35 -11
View File
@@ -1,3 +1,4 @@
from typing import Optional
from urllib.parse import urlparse
from dvc_http import HTTPFileSystem, HTTPSFileSystem # noqa: F401
@@ -18,7 +19,7 @@ from dvc_objects.fs import ( # noqa: F401
system,
utils,
)
from dvc_objects.fs.base import AnyFSPath, FileSystem # noqa: F401
from dvc_objects.fs.base import AnyFSPath, FileSystem # noqa: F401, TCH001
from dvc_objects.fs.errors import ( # noqa: F401
AuthError,
ConfigError,
@@ -48,18 +49,45 @@ known_implementations.update(
# pylint: enable=unused-import
def download(fs, fs_path, to, jobs=None):
def download(fs: "FileSystem", fs_path: str, to: str, jobs: Optional[int] = None):
with Callback.as_tqdm_callback(
desc=f"Downloading {fs.path.name(fs_path)}",
unit="files",
) as cb:
fs.get(fs_path, to.fs_path, batch_size=jobs, callback=cb)
# NOTE: We use dvc-objects generic.copy over fs.get since it makes file
# download atomic and avoids fsspec glob/regex path expansion.
if fs.isdir(fs_path):
from_infos = [
path
for path in fs.find(fs_path)
if not path.endswith(fs.path.flavour.sep)
]
if not from_infos:
return localfs.makedirs(to, exist_ok=True)
to_infos = [
localfs.path.join(to, *fs.path.relparts(info, fs_path))
for info in from_infos
]
else:
from_infos = [fs_path]
to_infos = [to]
cb.set_size(len(from_infos))
jobs = jobs or fs.jobs
generic.copy(
fs,
from_infos,
localfs,
to_infos,
callback=cb,
batch_size=jobs,
)
def parse_external_url(url, config=None):
remote_config = dict(config) if config else {}
remote_config["url"] = url
fs_cls, fs_config, fs_path = get_cloud_fs(None, **remote_config)
fs_cls, fs_config, fs_path = get_cloud_fs({}, **remote_config)
fs = fs_cls(**fs_config)
return fs, fs_path
@@ -105,18 +133,14 @@ def _resolve_remote_refs(config, remote_conf):
return remote_conf
base = get_fs_config(config, name=parsed.netloc)
cls, _, _ = _get_cloud_fs(config, **base)
cls, _, _ = get_cloud_fs(config, **base)
relpath = parsed.path.lstrip("/").replace("/", cls.sep)
url = cls.sep.join((base["url"], relpath))
return {**base, **remote_conf, "url": url}
def get_cloud_fs(repo, **kwargs):
repo_config = repo.config if repo else {}
return _get_cloud_fs(repo_config, **kwargs)
def _get_cloud_fs(repo_config, **kwargs):
def get_cloud_fs(repo_config, **kwargs):
repo_config = repo_config or {}
core_config = repo_config.get("core", {})
remote_conf = get_fs_config(repo_config, **kwargs)
+1 -1
View File
@@ -352,7 +352,7 @@ class _DVCFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
dvc_info = dvc_fs.fs.index.info(subkey)
dvc_path = _get_dvc_path(dvc_fs, subkey)
dvc_info["name"] = dvc_path
except FileNotFoundError:
except KeyError:
pass
fs_info = None
+11 -2
View File
@@ -354,7 +354,11 @@ class Output:
if fs_config is not None:
fs_kwargs.update(**fs_config)
fs_cls, fs_config, fs_path = get_cloud_fs(self.repo, url=path, **fs_kwargs)
fs_cls, fs_config, fs_path = get_cloud_fs(
self.repo.config if self.repo else {},
url=path,
**fs_kwargs,
)
self.fs = fs_cls(**fs_config)
if (
@@ -484,6 +488,9 @@ class Output:
):
return str(self.def_path)
if not self.fs.path.isin(self.fs_path, self.repo.root_dir):
return self.fs_path
cur_dir = self.fs.path.getcwd()
if self.fs.path.isin(cur_dir, self.repo.root_dir):
return self.fs.path.relpath(self.fs_path, cur_dir)
@@ -1017,7 +1024,9 @@ class Output:
if odb is None:
odb = self.cache
cls, config, from_info = get_cloud_fs(self.repo, url=source)
cls, config, from_info = get_cloud_fs(
self.repo.config if self.repo else {}, url=source
)
from_fs = cls(**config)
# When running import-url --to-remote / add --to-remote/-o ... we
+19 -3
View File
@@ -133,7 +133,7 @@ class Repo:
assert root_dir
return root_dir, dvc_dir
def __init__( # noqa: PLR0915
def __init__( # noqa: PLR0915, PLR0913
self,
root_dir: Optional[str] = None,
fs: Optional["FileSystem"] = None,
@@ -144,9 +144,10 @@ class Repo:
url: Optional[str] = None,
repo_factory: Optional[Callable] = None,
scm: Optional[Union["Git", "NoSCM"]] = None,
remote: Optional[str] = None,
remote_config: Optional["DictStrAny"] = None,
):
from dvc.cachemgr import CacheManager
from dvc.config import Config
from dvc.data_cloud import DataCloud
from dvc.fs import GitFileSystem, LocalFileSystem, localfs
from dvc.lock import LockNoop, make_lock
@@ -163,6 +164,9 @@ class Repo:
self._fs_conf = {"repo_factory": repo_factory}
self._fs = fs or localfs
self._scm = scm
self._config = config
self._remote = remote
self._remote_config = remote_config
self._data_index = None
if rev and not fs:
@@ -182,7 +186,6 @@ class Repo:
scm=scm,
)
self.config: Config = Config(self.dvc_dir, fs=self.fs, config=config)
self._uninitialized = uninitialized
# used by DVCFileSystem to determine if it should traverse subrepos
@@ -234,6 +237,18 @@ class Repo:
def __str__(self):
return self.url or self.root_dir
@cached_property
def config(self):
from dvc.config import Config
return Config(
self.dvc_dir,
fs=self.fs,
config=self._config,
remote=self._remote,
remote_config=self._remote_config,
)
@cached_property
def local_dvc_dir(self):
from dvc.fs import GitFileSystem, LocalFileSystem
@@ -629,6 +644,7 @@ class Repo:
self.__dict__.pop("dvcignore", None)
self.__dict__.pop("dvcfs", None)
self.__dict__.pop("datafs", None)
self.__dict__.pop("config", None)
def __enter__(self):
return self
+8
View File
@@ -67,6 +67,7 @@ def brancher( # noqa: E302
saved_fs = self.fs
saved_root = self.root_dir
saved_dvc_dir = self.dvc_dir
scm = self.scm
@@ -101,6 +102,8 @@ def brancher( # noqa: E302
finally:
self.fs = saved_fs
self.root_dir = saved_root
self.dvc_dir = saved_dvc_dir
self._reset() # pylint: disable=protected-access
def _switch_fs(
@@ -125,6 +128,8 @@ def _switch_fs(
repo.fs = fs
repo.root_dir = root_dir
repo.dvc_dir = fs.path.join(root_dir, repo.DVC_DIR)
repo._reset() # pylint: disable=protected-access
if cwd_parts:
cwd = repo.fs.path.join("/", *cwd_parts)
@@ -149,9 +154,12 @@ def switch(repo: "Repo", rev: str) -> Iterator[str]:
saved_fs = repo.fs
saved_root = repo.root_dir
saved_dvc_dir = repo.dvc_dir
try:
_switch_fs(repo, rev, repo_root_parts, cwd_parts)
yield rev
finally:
repo.fs = saved_fs
repo.root_dir = saved_root
repo.dvc_dir = saved_dvc_dir
repo._reset() # pylint: disable=protected-access
+18 -1
View File
@@ -1,6 +1,7 @@
import itertools
import logging
import os
from dataclasses import fields
from datetime import datetime
from typing import (
TYPE_CHECKING,
@@ -152,7 +153,23 @@ def collect_queued(
"""
if not baseline_revs:
return {}
return repo.experiments.celery_queue.collect_queued_data(baseline_revs, **kwargs)
queued_data = {}
for rev, ranges in repo.experiments.celery_queue.collect_queued_data(
baseline_revs, **kwargs
).items():
for exp_range in ranges:
for exp_state in exp_range.revs:
if exp_state.data:
attrs = [f.name for f in fields(SerializableExp)]
exp_state.data = SerializableExp(
**{
attr: getattr(exp_state.data, attr)
for attr in attrs
if attr != "metrics"
}
)
queued_data[rev] = ranges
return queued_data
def collect_active(
+11 -6
View File
@@ -21,19 +21,25 @@ def _collect_indexes( # noqa: PLR0913
revs=None,
max_size=None,
types=None,
config=None,
):
indexes = {}
collection_exc = None
config = config or {}
if remote:
core = config.get("core") or {}
core["remote"] = remote
config["core"] = core
for rev in repo.brancher(
revs=revs,
all_branches=all_branches,
all_tags=all_tags,
all_commits=all_commits,
):
saved_remote = repo.config["core"].get("remote")
try:
if remote:
repo.config["core"]["remote"] = remote
repo.config.update(config)
idx = repo.index.targets_view(
targets,
@@ -46,9 +52,6 @@ def _collect_indexes( # noqa: PLR0913
except Exception as exc: # pylint: disable=broad-except
collection_exc = exc
logger.exception("failed to collect '%s'", rev or "workspace")
finally:
if remote:
repo.config["core"]["remote"] = saved_remote
if not indexes and collection_exc:
raise collection_exc
@@ -71,6 +74,7 @@ def fetch( # noqa: C901, PLR0913
revs=None,
max_size=None,
types=None,
config=None,
) -> int:
"""Download data items from a cloud and imported repositories
@@ -114,6 +118,7 @@ def fetch( # noqa: C901, PLR0913
revs=revs,
max_size=max_size,
types=types,
config=config,
)
cache_key = ("fetch", tokenize(sorted(indexes.keys())))
+7 -12
View File
@@ -22,7 +22,6 @@ class GetDVCFileError(DvcException):
def get(url, path, out=None, rev=None, jobs=None, force=False, config=None):
from dvc.config import Config
from dvc.dvcfile import is_valid_filename
from dvc.fs.callbacks import Callback
from dvc.repo import Repo
out = resolve_output(path, out, force=force)
@@ -40,6 +39,7 @@ def get(url, path, out=None, rev=None, jobs=None, force=False, config=None):
uninitialized=True,
config=config,
) as repo:
from dvc.fs import download
from dvc.fs.data import DataFileSystem
fs: Union[DataFileSystem, "DVCFileSystem"]
@@ -49,14 +49,9 @@ def get(url, path, out=None, rev=None, jobs=None, force=False, config=None):
else:
fs = repo.dvcfs
fs_path = fs.from_os_path(path)
with Callback.as_tqdm_callback(
desc=f"Downloading {fs.path.name(path)}",
unit="files",
) as cb:
fs.get(
fs_path,
os.path.abspath(out),
batch_size=jobs,
callback=cb,
)
download(
fs,
fs_path,
os.path.abspath(out),
jobs=jobs,
)
+1 -1
View File
@@ -14,4 +14,4 @@ def get_url(url, out=None, *, config=None, jobs=None, force=False):
src_fs, src_path = parse_external_url(url, config)
if not src_fs.exists(src_path):
raise URLMissingError(url)
download(src_fs, src_path, out, jobs=jobs)
download(src_fs, src_path, out.fs_path, jobs=jobs)
+1 -1
View File
@@ -34,7 +34,7 @@ dependencies = [
"configobj>=5.0.6",
"distro>=1.3",
"dpath<3,>=2.1.0",
"dvc-data>=2.6.0,<2.7.0",
"dvc-data>=2.7.1,<2.8.0",
"dvc-http>=2.29.0",
"dvc-render>=0.3.1,<1",
"dvc-studio-client>=0.9.2,<1",
+4 -2
View File
@@ -10,7 +10,7 @@ from funcy import lsplit
from dvc.cli import main
from dvc.dvcfile import LOCK_FILE, PROJECT_FILE
from dvc.exceptions import CyclicGraphError, ReproductionError
from dvc.fs import LocalFileSystem, system
from dvc.fs import system
from dvc.output import Output
from dvc.stage import PipelineStage, Stage
from dvc.stage.cache import RunCacheNotSupported
@@ -624,12 +624,14 @@ class TestReproAlreadyCached:
assert stage.outs[0].hash_info != saved_stage.outs[0].hash_info
def test_force_import(self, mocker, tmp_dir, dvc):
from dvc.dependency import base
tmp_dir.dvc_gen("foo", "foo")
ret = main(["import-url", "foo", "bar"])
assert ret == 0
spy_get = mocker.spy(LocalFileSystem, "get")
spy_get = mocker.spy(base, "fs_download")
spy_checkout = mocker.spy(Output, "checkout")
assert main(["unfreeze", "bar.dvc"]) == 0
+2 -1
View File
@@ -203,7 +203,8 @@ def test_verify_hashes(tmp_dir, scm, dvc, mocker, tmp_path_factory, local_remote
# Removing cache will invalidate existing state entries
dvc.cache.local.clear()
dvc.config["remote"]["upstream"]["verify"] = True
with dvc.config.edit() as conf:
conf["remote"]["upstream"]["verify"] = True
dvc.pull()
assert hash_spy.call_count == 10
+11 -31
View File
@@ -196,11 +196,7 @@ def test_gc_cloud_positive(tmp_dir, dvc, tmp_path_factory, local_remote):
assert main(["gc", "-vf", flag]) == 0
def test_gc_cloud_remove_order(tmp_dir, scm, dvc, tmp_path_factory, mocker):
storage = os.fspath(tmp_path_factory.mktemp("test_remote_base"))
dvc.config["remote"]["local_remote"] = {"url": storage}
dvc.config["core"]["remote"] = "local_remote"
def test_gc_cloud_remove_order(tmp_dir, scm, dvc, mocker, local_remote):
(standalone, dir1, dir2) = tmp_dir.dvc_gen(
{
"file1": "standalone",
@@ -321,11 +317,7 @@ def test_date(tmp_dir, scm, dvc):
) # "modified, again"
def test_gc_not_in_remote(tmp_dir, scm, dvc, tmp_path_factory, mocker):
storage = os.fspath(tmp_path_factory.mktemp("test_remote_base"))
dvc.config["remote"]["local_remote"] = {"url": storage}
dvc.config["core"]["remote"] = "local_remote"
def test_gc_not_in_remote(tmp_dir, scm, dvc, mocker, local_remote):
(standalone, dir1, dir2) = tmp_dir.dvc_gen(
{
"file1": "standalone",
@@ -358,12 +350,9 @@ def test_gc_not_in_remote(tmp_dir, scm, dvc, tmp_path_factory, mocker):
)
def test_gc_not_in_remote_remote_arg(tmp_dir, scm, dvc, tmp_path_factory, mocker):
storage = os.fspath(tmp_path_factory.mktemp("test_remote_base"))
dvc.config["remote"]["local_remote"] = {"url": storage}
dvc.config["core"]["remote"] = "local_remote"
other_storage = os.fspath(tmp_path_factory.mktemp("test_remote_other"))
dvc.config["remote"]["other_remote"] = {"url": other_storage}
def test_gc_not_in_remote_remote_arg(tmp_dir, scm, dvc, mocker, make_remote):
make_remote("local_remote", typ="local")
make_remote("other_remote", typ="local", default=False)
tmp_dir.dvc_gen(
{
@@ -385,15 +374,9 @@ def test_gc_not_in_remote_remote_arg(tmp_dir, scm, dvc, tmp_path_factory, mocker
assert len(mocked_remove.mock_calls) == 3
def test_gc_not_in_remote_with_remote_field(
tmp_dir, scm, dvc, tmp_path_factory, mocker
):
storage = os.fspath(tmp_path_factory.mktemp("test_remote_base"))
dvc.config["remote"]["local_remote"] = {"url": storage}
dvc.config["core"]["remote"] = "local_remote"
other_storage = os.fspath(tmp_path_factory.mktemp("test_remote_other"))
dvc.config["remote"]["other_remote"] = {"url": other_storage}
def test_gc_not_in_remote_with_remote_field(tmp_dir, scm, dvc, mocker, make_remote):
make_remote("local_remote", typ="local")
make_remote("other_remote", typ="local", default=False)
text = textwrap.dedent(
"""\
@@ -420,12 +403,9 @@ def test_gc_not_in_remote_cloud(tmp_dir, scm, dvc):
dvc.gc(workspace=True, not_in_remote=True, cloud=True)
def test_gc_cloud_remote_field(tmp_dir, scm, dvc, tmp_path_factory, mocker):
storage = os.fspath(tmp_path_factory.mktemp("test_remote_base"))
dvc.config["remote"]["local_remote"] = {"url": storage}
dvc.config["core"]["remote"] = "local_remote"
other_storage = os.fspath(tmp_path_factory.mktemp("test_remote_other"))
dvc.config["remote"]["other_remote"] = {"url": other_storage}
def test_gc_cloud_remote_field(tmp_dir, scm, dvc, mocker, make_remote):
make_remote("local_remote", typ="local")
make_remote("other_remote", typ="local", default=False)
text = textwrap.dedent(
"""\
+1 -9
View File
@@ -1,5 +1,3 @@
from dataclasses import dataclass
import pytest
from dvc_http import HTTPFileSystem, HTTPSFileSystem
from dvc_s3 import S3FileSystem
@@ -63,14 +61,8 @@ def test_remote_url():
}
@dataclass
class FakeRepo:
config: dict
def test_get_cloud_fs():
repo = FakeRepo({})
cls, config, path = get_cloud_fs(repo, url="ssh://example.com:/dir/path")
cls, config, path = get_cloud_fs({}, url="ssh://example.com:/dir/path")
assert cls is SSHFileSystem
assert config == {"host": "example.com", "verify": False}
assert path == "/dir/path"
+7 -7
View File
@@ -13,11 +13,11 @@ def test_get_cloud_fs(tmp_dir, dvc):
first = f"{base}/first"
second = f"{first}/second"
_, _, path = get_cloud_fs(dvc, name="base")
_, _, path = get_cloud_fs(dvc.config, name="base")
assert path == base
_, _, path = get_cloud_fs(dvc, name="first")
_, _, path = get_cloud_fs(dvc.config, name="first")
assert path == first
_, _, path = get_cloud_fs(dvc, name="second")
_, _, path = get_cloud_fs(dvc.config, name="second")
assert path == second
@@ -34,9 +34,9 @@ def test_get_cloud_fs_validate(tmp_dir, dvc):
default=False,
)
assert get_cloud_fs(dvc, name="base")[1]["host"] == "example.com"
assert get_cloud_fs(dvc, name="first")[1]["host"] == "example.com"
assert get_cloud_fs(dvc, name="first")[1]["type"] == ["symlink"]
assert get_cloud_fs(dvc.config, name="base")[1]["host"] == "example.com"
assert get_cloud_fs(dvc.config, name="first")[1]["host"] == "example.com"
assert get_cloud_fs(dvc.config, name="first")[1]["type"] == ["symlink"]
with pytest.raises(ConfigError):
get_cloud_fs(dvc, name="second")
get_cloud_fs(dvc.config, name="second")
+1 -2
View File
@@ -2,7 +2,6 @@ import os
from dvc.output import Output
from dvc.stage import Stage
from dvc.utils import relpath
from dvc_data.hashfile.hash_info import HashInfo
from dvc_data.hashfile.meta import Meta
@@ -11,7 +10,7 @@ def test_str_workdir_outside_repo(tmp_dir, erepo_dir):
stage = Stage(erepo_dir.dvc)
output = Output(stage, "path", cache=False)
assert relpath("path", erepo_dir.dvc.root_dir) == str(output)
assert os.path.abspath("path") == str(output)
def test_str_workdir_inside_repo(dvc):
+4 -4
View File
@@ -12,7 +12,7 @@ def test_remote_with_hash_jobs(dvc):
}
dvc.config["core"]["checksum_jobs"] = 200
cls, config, _ = get_cloud_fs(dvc, name="with_hash_jobs")
cls, config, _ = get_cloud_fs(dvc.config, name="with_hash_jobs")
fs = cls(**config)
assert fs.hash_jobs == 100
@@ -23,7 +23,7 @@ def test_remote_with_jobs(dvc):
"jobs": 100,
}
cls, config, _ = get_cloud_fs(dvc, name="with_jobs")
cls, config, _ = get_cloud_fs(dvc.config, name="with_jobs")
fs = cls(**config)
assert fs.jobs == 100
@@ -32,7 +32,7 @@ def test_remote_without_hash_jobs(dvc):
dvc.config["remote"]["without_hash_jobs"] = {"url": "s3://bucket/name"}
dvc.config["core"]["checksum_jobs"] = 200
cls, config, _ = get_cloud_fs(dvc, name="without_hash_jobs")
cls, config, _ = get_cloud_fs(dvc.config, name="without_hash_jobs")
fs = cls(**config)
assert fs.hash_jobs == 200
@@ -40,7 +40,7 @@ def test_remote_without_hash_jobs(dvc):
def test_remote_without_hash_jobs_default(dvc):
dvc.config["remote"]["without_hash_jobs"] = {"url": "s3://bucket/name"}
cls, config, _ = get_cloud_fs(dvc, name="without_hash_jobs")
cls, config, _ = get_cloud_fs(dvc.config, name="without_hash_jobs")
fs = cls(**config)
assert fs.hash_jobs == fs.HASH_JOBS
+2 -2
View File
@@ -145,14 +145,14 @@ def test_remote_with_jobs(dvc, base_url, fs_cls):
remote_config = {"url": base_url}
dvc.config["remote"]["dav"] = remote_config
cls, config, _ = get_cloud_fs(dvc, name="dav")
cls, config, _ = get_cloud_fs(dvc.config, name="dav")
assert config["user"] == user
assert f"{scheme}://{user}@example.com" in config["host"]
assert cls is fs_cls
# config from remote takes priority
remote_config.update({"user": "admin"})
cls, config, _ = get_cloud_fs(dvc, name="dav")
cls, config, _ = get_cloud_fs(dvc.config, name="dav")
assert config["user"] == "admin"
assert f"{scheme}://{user}@example.com" in config["host"]
assert cls is fs_cls