mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 23:09:28 +01:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a9d0ac2ba9 | ||
|
|
41df96fcdb | ||
|
|
53ec0fa41d |
@@ -37,6 +37,7 @@ class CmdGet(CmdBaseNoRepo):
|
||||
rev=self.args.rev,
|
||||
jobs=self.args.jobs,
|
||||
force=self.args.force,
|
||||
config=self.args.config,
|
||||
)
|
||||
return 0
|
||||
except CloneError:
|
||||
@@ -102,4 +103,12 @@ def add_parser(subparsers, parent_parser):
|
||||
default=False,
|
||||
help="Override local file or folder if exists.",
|
||||
)
|
||||
get_parser.add_argument(
|
||||
"--config",
|
||||
type=str,
|
||||
help=(
|
||||
"Path to a config file that will be merged with the config "
|
||||
"in the target repository."
|
||||
),
|
||||
)
|
||||
get_parser.set_defaults(func=CmdGet)
|
||||
|
||||
@@ -22,6 +22,7 @@ class CmdImport(CmdBase):
|
||||
no_exec=self.args.no_exec,
|
||||
no_download=self.args.no_download,
|
||||
jobs=self.args.jobs,
|
||||
config=self.args.config,
|
||||
)
|
||||
except CloneError:
|
||||
logger.exception("failed to import '%s'", self.args.path)
|
||||
@@ -94,4 +95,12 @@ def add_parser(subparsers, parent_parser):
|
||||
),
|
||||
metavar="<number>",
|
||||
)
|
||||
import_parser.add_argument(
|
||||
"--config",
|
||||
type=str,
|
||||
help=(
|
||||
"Path to a config file that will be merged with the config "
|
||||
"in the target repository."
|
||||
),
|
||||
)
|
||||
import_parser.set_defaults(func=CmdImport)
|
||||
|
||||
@@ -34,6 +34,7 @@ class CmdList(CmdBaseNoRepo):
|
||||
rev=self.args.rev,
|
||||
recursive=self.args.recursive,
|
||||
dvc_only=self.args.dvc_only,
|
||||
config=self.args.config,
|
||||
)
|
||||
if self.args.json:
|
||||
ui.write_json(entries)
|
||||
@@ -80,6 +81,14 @@ def add_parser(subparsers, parent_parser):
|
||||
help="Git revision (e.g. SHA, branch, tag)",
|
||||
metavar="<commit>",
|
||||
)
|
||||
list_parser.add_argument(
|
||||
"--config",
|
||||
type=str,
|
||||
help=(
|
||||
"Path to a config file that will be merged with the config "
|
||||
"in the target repository."
|
||||
),
|
||||
)
|
||||
list_parser.add_argument(
|
||||
"path",
|
||||
nargs="?",
|
||||
|
||||
+21
-12
@@ -176,23 +176,32 @@ class Config(dict):
|
||||
# the repo.
|
||||
return self.fs if level == "repo" else self.wfs
|
||||
|
||||
def _load_config(self, level):
|
||||
@staticmethod
|
||||
def load_file(path, fs=None) -> dict:
|
||||
from configobj import ConfigObj, ConfigObjError
|
||||
|
||||
from dvc.fs import localfs
|
||||
|
||||
fs = fs or localfs
|
||||
|
||||
with fs.open(path) as fobj:
|
||||
try:
|
||||
conf_obj = ConfigObj(fobj)
|
||||
except UnicodeDecodeError as exc:
|
||||
raise ConfigError(str(exc)) from exc
|
||||
except ConfigObjError as exc:
|
||||
raise ConfigError(str(exc)) from exc
|
||||
|
||||
return _parse_named(_lower_keys(conf_obj.dict()))
|
||||
|
||||
def _load_config(self, level):
|
||||
filename = self.files[level]
|
||||
fs = self._get_fs(level)
|
||||
|
||||
if fs.exists(filename):
|
||||
with fs.open(filename) as fobj:
|
||||
try:
|
||||
conf_obj = ConfigObj(fobj)
|
||||
except UnicodeDecodeError as exc:
|
||||
raise ConfigError(str(exc)) from exc
|
||||
except ConfigObjError as exc:
|
||||
raise ConfigError(str(exc)) from exc
|
||||
else:
|
||||
conf_obj = ConfigObj()
|
||||
return _parse_named(_lower_keys(conf_obj.dict()))
|
||||
try:
|
||||
return self.load_file(filename, fs=fs)
|
||||
except FileNotFoundError:
|
||||
return {}
|
||||
|
||||
def _save_config(self, level, conf_dict):
|
||||
from configobj import ConfigObj
|
||||
|
||||
+28
-2
@@ -16,12 +16,14 @@ class RepoDependency(Dependency):
|
||||
PARAM_URL = "url"
|
||||
PARAM_REV = "rev"
|
||||
PARAM_REV_LOCK = "rev_lock"
|
||||
PARAM_CONFIG = "config"
|
||||
|
||||
REPO_SCHEMA = {
|
||||
PARAM_REPO: {
|
||||
Required(PARAM_URL): str,
|
||||
PARAM_REV: str,
|
||||
PARAM_REV_LOCK: str,
|
||||
PARAM_CONFIG: str,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -60,7 +62,24 @@ class RepoDependency(Dependency):
|
||||
self.def_repo[self.PARAM_REV_LOCK] = rev
|
||||
|
||||
def dumpd(self, **kwargs) -> Dict[str, Union[str, Dict[str, str]]]:
|
||||
return {self.PARAM_PATH: self.def_path, self.PARAM_REPO: self.def_repo}
|
||||
repo = {self.PARAM_URL: self.def_repo[self.PARAM_URL]}
|
||||
|
||||
rev = self.def_repo.get(self.PARAM_REV)
|
||||
if rev:
|
||||
repo[self.PARAM_REV] = self.def_repo[self.PARAM_REV]
|
||||
|
||||
rev_lock = self.def_repo.get(self.PARAM_REV_LOCK)
|
||||
if rev_lock:
|
||||
repo[self.PARAM_REV_LOCK] = rev_lock
|
||||
|
||||
config = self.def_repo.get(self.PARAM_CONFIG)
|
||||
if config:
|
||||
repo[self.PARAM_CONFIG] = config
|
||||
|
||||
return {
|
||||
self.PARAM_PATH: self.def_path,
|
||||
self.PARAM_REPO: repo,
|
||||
}
|
||||
|
||||
def update(self, rev: Optional[str] = None):
|
||||
if rev:
|
||||
@@ -77,9 +96,16 @@ class RepoDependency(Dependency):
|
||||
def _make_fs(
|
||||
self, rev: Optional[str] = None, locked: bool = True
|
||||
) -> "DVCFileSystem":
|
||||
from dvc.config import Config
|
||||
from dvc.fs import DVCFileSystem
|
||||
|
||||
config = {"cache": self.repo.config["cache"]}
|
||||
conf = self.def_repo.get("config")
|
||||
if conf:
|
||||
config = Config.load_file(conf)
|
||||
else:
|
||||
config = {}
|
||||
|
||||
config["cache"] = self.repo.config["cache"]
|
||||
config["cache"]["dir"] = self.repo.cache.local_cache_dir
|
||||
|
||||
return DVCFileSystem(
|
||||
|
||||
@@ -17,6 +17,10 @@ class DvcException(Exception):
|
||||
class InvalidArgumentError(ValueError, DvcException):
|
||||
"""Thrown if arguments are invalid."""
|
||||
|
||||
def __init__(self, msg, *args):
|
||||
self.msg = msg
|
||||
super().__init__(msg, *args)
|
||||
|
||||
|
||||
class OutputDuplicationError(DvcException):
|
||||
"""Thrown if a file/directory is specified as an output in more than one
|
||||
|
||||
+6
-1
@@ -19,7 +19,8 @@ class GetDVCFileError(DvcException):
|
||||
)
|
||||
|
||||
|
||||
def get(url, path, out=None, rev=None, jobs=None, force=False):
|
||||
def get(url, path, out=None, rev=None, jobs=None, force=False, config=None):
|
||||
from dvc.config import Config
|
||||
from dvc.dvcfile import is_valid_filename
|
||||
from dvc.fs.callbacks import Callback
|
||||
from dvc.repo import Repo
|
||||
@@ -29,11 +30,15 @@ def get(url, path, out=None, rev=None, jobs=None, force=False):
|
||||
if is_valid_filename(out):
|
||||
raise GetDVCFileError
|
||||
|
||||
if config and not isinstance(config, dict):
|
||||
config = Config.load_file(config)
|
||||
|
||||
with Repo.open(
|
||||
url=url,
|
||||
rev=rev,
|
||||
subrepos=True,
|
||||
uninitialized=True,
|
||||
config=config,
|
||||
) as repo:
|
||||
from dvc.fs.data import DataFileSystem
|
||||
|
||||
|
||||
+4
-1
@@ -1,6 +1,9 @@
|
||||
def imp(self, url, path, out=None, rev=None, **kwargs):
|
||||
def imp(self, url, path, out=None, rev=None, config=None, **kwargs):
|
||||
erepo = {"url": url}
|
||||
if rev is not None:
|
||||
erepo["rev"] = rev
|
||||
|
||||
if config is not None:
|
||||
erepo["config"] = config
|
||||
|
||||
return self.imp_url(path, out=out, erepo=erepo, frozen=True, **kwargs)
|
||||
|
||||
+12
-1
@@ -13,6 +13,7 @@ def ls(
|
||||
rev: Optional[str] = None,
|
||||
recursive: Optional[bool] = None,
|
||||
dvc_only: bool = False,
|
||||
config: Optional[str] = None,
|
||||
):
|
||||
"""Methods for getting files and outputs for the repo.
|
||||
|
||||
@@ -22,6 +23,7 @@ def ls(
|
||||
rev (str, optional): SHA commit, branch or tag name
|
||||
recursive (bool, optional): recursively walk the repo
|
||||
dvc_only (bool, optional): show only DVC-artifacts
|
||||
config (bool, optional): path to config file
|
||||
|
||||
Returns:
|
||||
list of `entry`
|
||||
@@ -35,9 +37,18 @@ def ls(
|
||||
"isexec": bool,
|
||||
}
|
||||
"""
|
||||
from dvc.config import Config
|
||||
|
||||
from . import Repo
|
||||
|
||||
with Repo.open(url, rev=rev, subrepos=True, uninitialized=True) as repo:
|
||||
if config and not isinstance(config, dict):
|
||||
config_dict = Config.load_file(config)
|
||||
else:
|
||||
config_dict = None
|
||||
|
||||
with Repo.open(
|
||||
url, rev=rev, subrepos=True, uninitialized=True, config=config_dict
|
||||
) as repo:
|
||||
path = path or ""
|
||||
|
||||
ret = _ls(repo, path, recursive, dvc_only)
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import logging
|
||||
import os
|
||||
from copy import deepcopy
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -39,6 +41,22 @@ def test_artifacts_read_subdir(tmp_dir, dvc):
|
||||
}
|
||||
|
||||
|
||||
def test_artifacts_read_bad_name(tmp_dir, dvc, caplog):
|
||||
bad_name_dvcyaml = deepcopy(dvcyaml)
|
||||
bad_name_dvcyaml["artifacts"]["bad_name"] = {"type": "model", "path": "bad.pkl"}
|
||||
|
||||
(tmp_dir / "dvc.yaml").dump(bad_name_dvcyaml)
|
||||
|
||||
artifacts = {
|
||||
name: Artifact(**values)
|
||||
for name, values in bad_name_dvcyaml["artifacts"].items()
|
||||
}
|
||||
|
||||
with caplog.at_level(logging.WARNING):
|
||||
assert tmp_dir.dvc.artifacts.read() == {"dvc.yaml": artifacts}
|
||||
assert "Can't use 'bad_name' as artifact name (ID)" in caplog.text
|
||||
|
||||
|
||||
def test_artifacts_add_subdir(tmp_dir, dvc):
|
||||
subdir = tmp_dir / "subdir"
|
||||
subdir.mkdir()
|
||||
|
||||
@@ -18,32 +18,50 @@ def _test_cli(mocker, *args):
|
||||
def test_list(mocker):
|
||||
url = "local_dir"
|
||||
m = _test_cli(mocker, url)
|
||||
m.assert_called_once_with(url, None, recursive=False, rev=None, dvc_only=False)
|
||||
m.assert_called_once_with(
|
||||
url, None, recursive=False, rev=None, dvc_only=False, config=None
|
||||
)
|
||||
|
||||
|
||||
def test_list_recursive(mocker):
|
||||
url = "local_dir"
|
||||
m = _test_cli(mocker, url, "-R")
|
||||
m.assert_called_once_with(url, None, recursive=True, rev=None, dvc_only=False)
|
||||
m.assert_called_once_with(
|
||||
url, None, recursive=True, rev=None, dvc_only=False, config=None
|
||||
)
|
||||
|
||||
|
||||
def test_list_git_ssh_rev(mocker):
|
||||
url = "[email protected]:repo"
|
||||
m = _test_cli(mocker, url, "--rev", "123")
|
||||
m.assert_called_once_with(url, None, recursive=False, rev="123", dvc_only=False)
|
||||
m.assert_called_once_with(
|
||||
url, None, recursive=False, rev="123", dvc_only=False, config=None
|
||||
)
|
||||
|
||||
|
||||
def test_list_targets(mocker):
|
||||
url = "local_dir"
|
||||
target = "subdir"
|
||||
m = _test_cli(mocker, url, target)
|
||||
m.assert_called_once_with(url, target, recursive=False, rev=None, dvc_only=False)
|
||||
m.assert_called_once_with(
|
||||
url, target, recursive=False, rev=None, dvc_only=False, config=None
|
||||
)
|
||||
|
||||
|
||||
def test_list_outputs_only(mocker):
|
||||
url = "local_dir"
|
||||
m = _test_cli(mocker, url, None, "--dvc-only")
|
||||
m.assert_called_once_with(url, None, recursive=False, rev=None, dvc_only=True)
|
||||
m.assert_called_once_with(
|
||||
url, None, recursive=False, rev=None, dvc_only=True, config=None
|
||||
)
|
||||
|
||||
|
||||
def test_list_config(mocker):
|
||||
url = "local_dir"
|
||||
m = _test_cli(mocker, url, None, "--config", "myconfig")
|
||||
m.assert_called_once_with(
|
||||
url, None, recursive=False, rev=None, dvc_only=False, config="myconfig"
|
||||
)
|
||||
|
||||
|
||||
def test_show_json(mocker, capsys):
|
||||
|
||||
@@ -14,6 +14,8 @@ def test_get(mocker):
|
||||
"version",
|
||||
"--jobs",
|
||||
"4",
|
||||
"--config",
|
||||
"myconfig",
|
||||
]
|
||||
)
|
||||
assert cli_args.func == CmdGet
|
||||
@@ -24,7 +26,13 @@ def test_get(mocker):
|
||||
assert cmd.run() == 0
|
||||
|
||||
m.assert_called_once_with(
|
||||
"repo_url", path="src", out="out", rev="version", jobs=4, force=False
|
||||
"repo_url",
|
||||
path="src",
|
||||
out="out",
|
||||
rev="version",
|
||||
jobs=4,
|
||||
config="myconfig",
|
||||
force=False,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -14,6 +14,8 @@ def test_import(mocker, dvc):
|
||||
"version",
|
||||
"--jobs",
|
||||
"3",
|
||||
"--config",
|
||||
"myconfig",
|
||||
]
|
||||
)
|
||||
assert cli_args.func == CmdImport
|
||||
@@ -31,6 +33,7 @@ def test_import(mocker, dvc):
|
||||
no_exec=False,
|
||||
no_download=False,
|
||||
jobs=3,
|
||||
config="myconfig",
|
||||
)
|
||||
|
||||
|
||||
@@ -61,6 +64,7 @@ def test_import_no_exec(mocker, dvc):
|
||||
no_exec=True,
|
||||
no_download=False,
|
||||
jobs=None,
|
||||
config=None,
|
||||
)
|
||||
|
||||
|
||||
@@ -91,4 +95,5 @@ def test_import_no_download(mocker, dvc):
|
||||
no_exec=False,
|
||||
no_download=True,
|
||||
jobs=None,
|
||||
config=None,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user