Compare commits

...
14 Commits
Author SHA1 Message Date
Ruslan Kuprieiev 31efaac4f3 dvc: bump to 1.2.3 2020-07-29 01:25:45 +03:00
Ruslan Kuprieiev 5d1c1c418c gdrive: add workaround for downloading empty files (#4287)
* [WIP] gdrive: add workaround for downloading empty files

Better to solve it on PyDrive's side:
https://github.com/iterative/dvc/issues/4286

Fixes #4286

* deps: limit flatten_json

```
pylint...................................................................Failed
- hook id: pylint
- exit code: 2

Registered custom plugin. Some checks will be disabled for tests.
************* Module dvc.command.experiments
dvc/command/experiments.py:14:4: E0401: Unable to import 'flatten_json'
(import-error)
dvc/command/experiments.py:39:4: E0401: Unable to import 'flatten_json'
(import-error)

-----------------------------------
Your code has been rated at 9.99/10

Registered custom plugin. Some checks will be disabled for tests.
************* Module dvc.utils.diff
dvc/utils/diff.py:38:8: E0401: Unable to import 'flatten_json'
(import-error)
************* Module dvc.command.metrics
dvc/command/metrics.py:17:4: E0401: Unable to import 'flatten_json'
(import-error)

-----------------------------------
```
2020-07-29 01:25:06 +03:00
f4595a02b2 version: redesign output format for the dvc version command #3499 (#4114)
* Version: Redesign output format #3499

Redesigned the output format for the command: dvc version

* version: reformat output format #3499

Done some suggested fixes in code for the dvc command output format issue.

* version: redesign output format #3499

* version: redesign output format for dvc command #3499

* version: redesign output format #3499

* version: redesign output format #3499

* Redesign output format for the dvc version command #3499

* Redesign output format for the dvc version command #3499

* Update dvc/command/version.py

Co-authored-by: Saugat Pachhai <[email protected]>

Co-authored-by: Ruslan Kuprieiev <[email protected]>
Co-authored-by: Saugat Pachhai <[email protected]>
2020-07-28 18:22:26 +03:00
Ruslan Kuprieiev f4b820aa4c dvc: bump to 1.2.2 2020-07-28 16:02:23 +03:00
Ruslan Kuprieiev 0b111d2a10 daemon: use Popen on Mac (#4296)
Turns out macs are not happy with double-forks without exec.

Fixes #4294
2020-07-28 16:01:58 +03:00
Ruslan Kuprieiev b14ad4f8cc dvc: bump to 1.2.1 2020-07-27 20:02:24 +03:00
Peter Rowlands (변기호) 864bc999d1 experiments: temporarily disable styling in show table (#4290) 2020-07-27 20:00:25 +09:00
Ruslan Kuprieiev 2bf6eef53b deps: limit flatten_json
```
pylint...................................................................Failed
- hook id: pylint
- exit code: 2

Registered custom plugin. Some checks will be disabled for tests.
************* Module dvc.command.experiments
dvc/command/experiments.py:14:4: E0401: Unable to import 'flatten_json'
(import-error)
dvc/command/experiments.py:39:4: E0401: Unable to import 'flatten_json'
(import-error)

-----------------------------------
Your code has been rated at 9.99/10

Registered custom plugin. Some checks will be disabled for tests.
************* Module dvc.utils.diff
dvc/utils/diff.py:38:8: E0401: Unable to import 'flatten_json'
(import-error)
************* Module dvc.command.metrics
dvc/command/metrics.py:17:4: E0401: Unable to import 'flatten_json'
(import-error)

-----------------------------------
```
2020-07-27 02:20:46 +03:00
Ruslan Kuprieiev fe9ae2c5d1 ignore: dynamically collect dvcignore (#4284)
This allows us avoid collecting dvcignore for the whole repo if we only
care about particular paths. As a result, in a repo with 2 datasets
(2M + 0.5M files), creating a defunct stage takes ~4sec on 1.2.0, but
~1sec(most of it is actually dvc module initialization) with this PR.

This is also a pre-requisite for dynamic dvcignore and subrepo
collection (https://github.com/iterative/dvc/pull/4247) while walking
the tree.

Also, it is important to clarify that regular `dvc status`(without
arguments) has the same performance after this PR, because when we check
dataset for changes, we call things like `tree.exists()`, which call
dvcignore and make it collect dvcignore in the dataset itself, so we
still endup collecting dvcignore for the whole repo (including walking
into the datasets). This should be solved soon by telling dvcignore that
it shouldn't walk into the datasets searching for `.dvcignore`s.
2020-07-26 23:04:09 +03:00
dtrifiroandDaniele Trifirò e2b1fb1f64 add toml support for ParamsDependency (#4258)
* add toml support for ParamsDependency

* add toml support to dvc show

Co-authored-by: Daniele Trifirò <[email protected]>
2020-07-24 19:09:04 +03:00
Ruslan Kuprieiev 39c0bdbe79 dvc: properly handle relative symlinks (#4275)
Fix #4160
2020-07-24 14:44:04 +03:00
Ruslan Kuprieiev 1fd8b47936 ignore: use use_dvcignore flag (#4274) 2020-07-24 02:53:42 +03:00
Ruslan Kuprieiev fe4f1f4eb7 dvc: add dvc/utils/build.py to gitignore (#4272)
Added by accident in https://github.com/iterative/dvc/pull/4271/files
2020-07-24 02:30:28 +03:00
Ruslan Kuprieiev 0c99fd3277 config: don't switch between trees (#4271)
No longer needed, since we have `use_dvcignore` flag.
2020-07-24 00:28:04 +03:00
19 changed files with 209 additions and 106 deletions
+6 -10
View File
@@ -82,29 +82,25 @@ def _show_experiments(all_experiments, console, precision=None):
metric_names, param_names = _collect_names(all_experiments)
table = Table(row_styles=["white", "bright_white"])
table.add_column("Experiment", header_style="black on grey93")
table = Table()
table.add_column("Experiment")
for name in metric_names:
table.add_column(
name, justify="right", header_style="black on cornsilk1"
)
table.add_column(name, justify="right")
for name in param_names:
table.add_column(
name, justify="left", header_style="black on light_cyan1"
)
table.add_column(name, justify="left")
for base_rev, experiments in all_experiments.items():
if Git.is_sha(base_rev):
base_rev = base_rev[:7]
for row, style, in _collect_rows(
for row, _, in _collect_rows(
base_rev,
experiments,
metric_names,
param_names,
precision=precision,
):
table.add_row(*row, style=style)
table.add_row(*row)
console.print(table)
+30 -17
View File
@@ -10,7 +10,7 @@ from dvc.command.base import CmdBaseNoRepo, append_doc_link
from dvc.exceptions import DvcException, NotDvcRepoError
from dvc.scm.base import SCMError
from dvc.system import System
from dvc.utils import is_binary, relpath
from dvc.utils import relpath
from dvc.utils.pkg import PKG
from dvc.version import __version__
@@ -27,13 +27,18 @@ class CmdVersion(CmdBaseNoRepo):
def run(self):
from dvc.repo import Repo
package = PKG
if PKG is None:
package = ""
else:
package = f"({PKG})"
info = [
f"DVC version: {__version__}",
f"Python version: {platform.python_version()}",
f"Platform: {platform.platform()}",
f"Binary: {is_binary()}",
f"Package: {PKG}",
f"Supported remotes: {self.get_supported_remotes()}",
f"DVC version: {__version__} {package}",
"---------------------------------",
f"Platform: Python {platform.python_version()} on "
f"{platform.platform()}",
f"Supports: {self.get_supported_remotes()}",
]
try:
@@ -46,13 +51,13 @@ class CmdVersion(CmdBaseNoRepo):
# `dvc config cache.shared group`.
if os.path.exists(repo.cache.local.cache_dir):
info.append(
"Cache: {}".format(self.get_linktype_support_info(repo))
"Cache types: {}".format(
self.get_linktype_support_info(repo)
)
)
if psutil:
fs_type = self.get_fs_type(repo.cache.local.cache_dir)
info.append(
f"Filesystem type (cache directory): {fs_type}"
)
info.append(f"Cache directory: {fs_type}")
else:
logger.warning(
"Unable to detect supported link types, as cache "
@@ -68,11 +73,11 @@ class CmdVersion(CmdBaseNoRepo):
root_directory = os.getcwd()
info.append("Repo: dvc, git (broken)")
else:
info.append("Repo: {}".format(_get_dvc_repo_info(repo)))
if psutil:
fs_root = self.get_fs_type(os.path.abspath(root_directory))
info.append(f"Workspace directory: {fs_root}")
if psutil:
fs_root = self.get_fs_type(os.path.abspath(root_directory))
info.append(f"Filesystem type (workspace): {fs_root}")
info.append("Repo: {}".format(_get_dvc_repo_info(repo)))
logger.info("\n".join(info))
return 0
@@ -80,7 +85,7 @@ class CmdVersion(CmdBaseNoRepo):
@staticmethod
def get_fs_type(path):
partition = {
pathlib.Path(part.mountpoint): (part.fstype, part.device)
pathlib.Path(part.mountpoint): (part.fstype + " on " + part.device)
for part in psutil.disk_partitions(all=True)
}
@@ -115,7 +120,9 @@ class CmdVersion(CmdBaseNoRepo):
os.unlink(dst)
except DvcException:
status = "not supported"
cache.append(f"{name} - {status}")
if status == "supported":
cache.append(name)
os.remove(src)
return ", ".join(cache)
@@ -129,6 +136,12 @@ class CmdVersion(CmdBaseNoRepo):
if not tree_cls.get_missing_deps():
supported_remotes.append(tree_cls.scheme)
if len(supported_remotes) == len(TREES):
return "All remotes"
if len(supported_remotes) == 1:
return supported_remotes
return ", ".join(supported_remotes)
+4 -6
View File
@@ -314,10 +314,9 @@ class Config(dict):
def _load_config(self, level):
filename = self.files[level]
tree = self.tree if level == "repo" else self.wtree
if tree.exists(filename, use_dvcignore=False):
with tree.open(filename) as fobj:
if self.tree.exists(filename, use_dvcignore=False):
with self.tree.open(filename) as fobj:
conf_obj = configobj.ConfigObj(fobj)
else:
conf_obj = configobj.ConfigObj()
@@ -325,14 +324,13 @@ class Config(dict):
def _save_config(self, level, conf_dict):
filename = self.files[level]
tree = self.tree if level == "repo" else self.wtree
logger.debug(f"Writing '{filename}'.")
tree.makedirs(os.path.dirname(filename))
self.tree.makedirs(os.path.dirname(filename))
config = configobj.ConfigObj(_pack_remotes(conf_dict))
with tree.open(filename, "wb") as fobj:
with self.tree.open(filename, "wb") as fobj:
config.write(fobj)
config.filename = filename
+17 -12
View File
@@ -3,6 +3,7 @@
import inspect
import logging
import os
import platform
import sys
from subprocess import Popen
@@ -15,25 +16,24 @@ CREATE_NEW_PROCESS_GROUP = 0x00000200
DETACHED_PROCESS = 0x00000008
def _spawn_windows(cmd, env):
from subprocess import STARTUPINFO, STARTF_USESHOWWINDOW
def _popen(cmd, **kwargs):
prefix = [sys.executable]
if not is_binary():
prefix += [sys.argv[0]]
return Popen(prefix + cmd, close_fds=True, shell=False, **kwargs)
def _spawn_windows(cmd, env):
from subprocess import STARTUPINFO, STARTF_USESHOWWINDOW
creationflags = CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS
startupinfo = STARTUPINFO()
startupinfo.dwFlags |= STARTF_USESHOWWINDOW
Popen(
prefix + cmd,
env=env,
close_fds=True,
shell=False,
creationflags=creationflags,
startupinfo=startupinfo,
_popen(
cmd, env=env, creationflags=creationflags, startupinfo=startupinfo,
).communicate()
@@ -65,8 +65,13 @@ def _spawn_posix(cmd, env):
sys.stdout.close()
sys.stderr.close()
os.environ.update(env)
main(cmd)
if platform.system() == "Darwin":
# workaround for MacOS bug
# https://github.com/iterative/dvc/issues/4294
_popen(cmd, env=env).communicate()
else:
os.environ.update(env)
main(cmd)
os._exit(0) # pylint: disable=protected-access
+7 -2
View File
@@ -2,6 +2,7 @@ import os
from collections import defaultdict
import dpath.util
import toml
import yaml
from voluptuous import Any
@@ -21,6 +22,8 @@ class ParamsDependency(LocalDependency):
PARAM_PARAMS = "params"
PARAM_SCHEMA = {PARAM_PARAMS: Any(dict, list, None)}
DEFAULT_PARAMS_FILE = "params.yaml"
PARAMS_FILE_LOADERS = defaultdict(lambda: yaml.safe_load)
PARAMS_FILE_LOADERS.update({".toml": toml.load})
def __init__(self, stage, path, params):
info = {}
@@ -87,8 +90,10 @@ class ParamsDependency(LocalDependency):
with self.repo.tree.open(self.path_info, "r") as fobj:
try:
config = yaml.safe_load(fobj)
except yaml.YAMLError as exc:
config = self.PARAMS_FILE_LOADERS[
self.path_info.suffix.lower()
](fobj)
except (yaml.YAMLError, toml.TomlDecodeError) as exc:
raise BadParamFileError(
f"Unable to read parameters from '{self}'"
) from exc
+38 -10
View File
@@ -1,7 +1,7 @@
import logging
import os
import re
from itertools import groupby
from itertools import groupby, takewhile
from pathspec.patterns import GitWildMatchPattern
from pathspec.util import normalize_file
@@ -131,18 +131,19 @@ class DvcIgnoreFilter:
self.ignores_trie_tree[root_dir] = DvcIgnorePatterns(
default_ignore_patterns, root_dir
)
for root, dirs, _ in self.tree.walk(self.root_dir):
self._update(root)
self._update_sub_repo(root, dirs)
dirs[:], _ = self(root, dirs, [])
self._update(self.root_dir)
def _update(self, dirname):
old_pattern = self.ignores_trie_tree.longest_prefix(dirname).value
matches = old_pattern.matches(dirname, DvcIgnore.DVCIGNORE_FILE, False)
ignore_file_path = os.path.join(dirname, DvcIgnore.DVCIGNORE_FILE)
if self.tree.exists(ignore_file_path):
if not matches and self.tree.exists(
ignore_file_path, use_dvcignore=False
):
new_pattern = DvcIgnorePatterns.from_files(
ignore_file_path, self.tree
)
old_pattern = self._get_trie_pattern(dirname)
if old_pattern:
self.ignores_trie_tree[dirname] = DvcIgnorePatterns(
*merge_patterns(
@@ -154,11 +155,18 @@ class DvcIgnoreFilter:
)
else:
self.ignores_trie_tree[dirname] = new_pattern
elif old_pattern:
self.ignores_trie_tree[dirname] = old_pattern
# NOTE: using `walk` + `break` because tree doesn't have `listdir()`
for root, dirs, _ in self.tree.walk(dirname, use_dvcignore=False):
self._update_sub_repo(root, dirs)
break
def _update_sub_repo(self, root, dirs):
for d in dirs:
if self._is_dvc_repo(root, d):
old_pattern = self._get_trie_pattern(root)
old_pattern = self.ignores_trie_tree.longest_prefix(root).value
if old_pattern:
self.ignores_trie_tree[root] = DvcIgnorePatterns(
*merge_patterns(
@@ -181,8 +189,28 @@ class DvcIgnoreFilter:
return dirs, files
def _get_trie_pattern(self, dirname):
ignore_pattern = self.ignores_trie_tree.longest_prefix(dirname).value
return ignore_pattern
ignore_pattern = self.ignores_trie_tree.get(dirname)
if ignore_pattern:
return ignore_pattern
prefix = self.ignores_trie_tree.longest_prefix(dirname).key
if not prefix:
# outside of the repo
return None
dirs = list(
takewhile(
lambda path: path != prefix,
(parent.fspath for parent in PathInfo(dirname).parents),
)
)
dirs.reverse()
dirs.append(dirname)
for parent in dirs:
self._update(parent)
return self.ignores_trie_tree.get(dirname)
def _is_ignored(self, path, is_dir=False):
if self._outside_repo(path):
+5 -2
View File
@@ -1,5 +1,6 @@
import logging
import toml
import yaml
from dvc.dependency.param import ParamsDependency
@@ -34,8 +35,10 @@ def _read_params(repo, configs, rev):
with repo.tree.open(config, "r") as fobj:
try:
res[str(config)] = yaml.safe_load(fobj)
except yaml.YAMLError:
res[str(config)] = ParamsDependency.PARAMS_FILE_LOADERS[
config.suffix.lower()
](fobj)
except (yaml.YAMLError, toml.TomlDecodeError):
logger.debug(
"failed to read '%s' on '%s'", config, rev, exc_info=True
)
+10 -1
View File
@@ -381,6 +381,9 @@ class GDriveTree(BaseTree):
# it does not create a file on the remote
gdrive_file = self._drive.CreateFile(param)
gdrive_file.FetchMetadata(fields="fileSize")
size = gdrive_file["fileSize"]
with Tqdm(
desc=progress_desc,
disable=no_progress_bar,
@@ -388,7 +391,13 @@ class GDriveTree(BaseTree):
# explicit `bar_format` as `total` will be set by `update_to`
bar_format=Tqdm.BAR_FMT_DEFAULT,
) as pbar:
gdrive_file.GetContentFile(to_file, callback=pbar.update_to)
if size:
gdrive_file.GetContentFile(to_file, callback=pbar.update_to)
else:
# PyDrive doesn't like downloading empty files
# https://github.com/iterative/dvc/issues/4286
with open(to_file, "w"):
pass
@contextmanager
@_gdrive_retry
+7 -10
View File
@@ -46,12 +46,8 @@ class GitTree(BaseTree): # pylint:disable=abstract-method
from dvc.ignore import DvcIgnoreFilter, DvcIgnoreFilterNoop
root = self.dvcignore_root or self.tree_root
if not self.use_dvcignore:
return DvcIgnoreFilterNoop(self, root)
self.use_dvcignore = False
ret = DvcIgnoreFilter(self, root)
self.use_dvcignore = True
return ret
cls = DvcIgnoreFilter if self.use_dvcignore else DvcIgnoreFilterNoop
return cls(self, root)
def open(
self, path, mode="r", encoding="utf-8"
@@ -160,7 +156,7 @@ class GitTree(BaseTree): # pylint:disable=abstract-method
if not topdown:
yield os.path.normpath(tree.abspath), dirs, nondirs
def walk(self, top, topdown=True, onerror=None):
def walk(self, top, topdown=True, onerror=None, use_dvcignore=True):
"""Directory tree generator.
See `os.walk` for the docs. Differences:
@@ -178,9 +174,10 @@ class GitTree(BaseTree): # pylint:disable=abstract-method
return
for root, dirs, files in self._walk(tree, topdown=topdown):
dirs[:], files[:] = self.dvcignore(
os.path.abspath(root), dirs, files
)
if use_dvcignore:
dirs[:], files[:] = self.dvcignore(
os.path.abspath(root), dirs, files
)
yield root, dirs, files
def isexec(self, path):
+7 -10
View File
@@ -58,12 +58,8 @@ class LocalTree(BaseTree):
from dvc.ignore import DvcIgnoreFilter, DvcIgnoreFilterNoop
root = self.dvcignore_root or self.tree_root
if not self.use_dvcignore:
return DvcIgnoreFilterNoop(self, root)
self.use_dvcignore = False
ret = DvcIgnoreFilter(self, root)
self.use_dvcignore = True
return ret
cls = DvcIgnoreFilter if self.use_dvcignore else DvcIgnoreFilterNoop
return cls(self, root)
@staticmethod
def open(path_info, mode="r", encoding=None):
@@ -101,7 +97,7 @@ class LocalTree(BaseTree):
System.is_symlink(path_info) or System.is_hardlink(path_info)
)
def walk(self, top, topdown=True, onerror=None):
def walk(self, top, topdown=True, onerror=None, use_dvcignore=True):
"""Directory tree generator.
See `os.walk` for the docs. Differences:
@@ -110,9 +106,10 @@ class LocalTree(BaseTree):
for root, dirs, files in os.walk(
top, topdown=topdown, onerror=onerror
):
dirs[:], files[:] = self.dvcignore(
os.path.abspath(root), dirs, files
)
if use_dvcignore:
dirs[:], files[:] = self.dvcignore(
os.path.abspath(root), dirs, files
)
yield os.path.normpath(root), dirs, files
+1
View File
@@ -0,0 +1 @@
/build.py
+1 -2
View File
@@ -96,8 +96,7 @@ def move(src, dst, mode=None):
tmp = f"{dst}.{uuid()}"
if os.path.islink(src):
# readlink does not accept path-like obj for Windows in Python <3.8
shutil.copy(os.readlink(os.fspath(src)), tmp)
shutil.copy(src, tmp)
os.unlink(src)
else:
shutil.move(src, tmp)
+1 -1
View File
@@ -6,7 +6,7 @@
import os
import subprocess
_BASE_VERSION = "1.2.0"
_BASE_VERSION = "1.2.3"
def _generate_version(base_version):
+1 -1
View File
@@ -17,7 +17,7 @@ count=true
[isort]
include_trailing_comma=true
known_first_party=dvc,tests
known_third_party=PyInstaller,RangeHTTPServer,boto3,colorama,configobj,distro,dpath,flaky,flufl,funcy,git,grandalf,mock,moto,nanotime,networkx,packaging,pathspec,pygtrie,pylint,pytest,requests,ruamel,setuptools,shortuuid,shtab,tqdm,voluptuous,yaml,zc
known_third_party=PyInstaller,RangeHTTPServer,boto3,colorama,configobj,distro,dpath,flaky,flufl,funcy,git,grandalf,mock,moto,nanotime,networkx,packaging,pathspec,pygtrie,pylint,pytest,requests,ruamel,setuptools,shortuuid,shtab,toml,tqdm,voluptuous,yaml,zc
line_length=79
force_grid_wrap=0
use_parentheses=True
+2 -1
View File
@@ -62,6 +62,7 @@ install_requires = [
"appdirs>=1.4.3",
"PyYAML>=5.1.2,<5.4", # Compatibility with awscli
"ruamel.yaml>=0.16.1",
"toml>=0.10.1",
"funcy>=1.14",
"pathspec>=0.6.0",
"shortuuid>=0.5.0",
@@ -74,7 +75,7 @@ install_requires = [
"networkx>=2.1,<2.5",
"pydot>=1.2.4",
"speedcopy>=2.0.1; python_version < '3.8' and sys_platform == 'win32'",
"flatten_json>=0.1.6",
"flatten_json>=0.1.6,<0.1.8",
"tabulate>=0.8.7",
"pygtrie==2.3.2",
"dpath>=2.0.1,<3",
+10
View File
@@ -14,6 +14,16 @@ def test_show(tmp_dir, dvc):
assert dvc.params.show() == {"": {"params.yaml": {"foo": "bar"}}}
def test_show_toml(tmp_dir, dvc):
tmp_dir.gen("params.toml", "[foo]\nbar = 42\nbaz = [1, 2]\n")
dvc.run(
cmd="echo params.toml", params=["params.toml:foo"], single_stage=True
)
assert dvc.params.show() == {
"": {"params.toml": {"foo": {"bar": 42, "baz": [1, 2]}}}
}
def test_show_multiple(tmp_dir, dvc):
tmp_dir.gen("params.yaml", "foo: bar\nbaz: qux\n")
dvc.run(
+20
View File
@@ -806,3 +806,23 @@ def test_add_pipeline_file(tmp_dir, dvc, run_copy):
with pytest.raises(OutputIsStageFileError):
dvc.add(PIPELINE_FILE)
def test_add_symlink(tmp_dir, dvc):
tmp_dir.gen({"dir": {"bar": "bar"}})
(tmp_dir / "dir" / "foo").symlink_to(os.path.join(".", "bar"))
dvc.add(os.path.join("dir", "foo"))
assert not (tmp_dir / "dir" / "foo").is_symlink()
assert not (tmp_dir / "dir" / "bar").is_symlink()
assert (tmp_dir / "dir" / "foo").read_text() == "bar"
assert (tmp_dir / "dir" / "bar").read_text() == "bar"
assert (tmp_dir / ".dvc" / "cache").read_text() == {
"37": {"b51d194a7513e45b56f6524f2d51f2": "bar"}
}
assert not (
tmp_dir / ".dvc" / "cache" / "37" / "b51d194a7513e45b56f6524f2d51f2"
).is_symlink()
+10 -21
View File
@@ -16,15 +16,10 @@ def test_info_in_repo(scm_init, tmp_dir, caplog):
assert main(["version"]) == 0
assert re.search(r"DVC version: \d+\.\d+\.\d+", caplog.text)
assert re.search(r"Python version: \d\.\d\.\d", caplog.text)
assert re.search(r"Platform: .*", caplog.text)
assert re.search(r"Binary: (True|False)", caplog.text)
assert re.search(r"Package: .*", caplog.text)
assert re.search(r"Supported remotes: .*", caplog.text)
assert re.search(
r"(Cache: (.*link - (not )?supported(,\s)?){3})", caplog.text
)
assert re.search(r"DVC version: \d+\.\d+\.\d+\+.*", caplog.text)
assert re.search(r"Platform: Python \d\.\d\.\d on .*", caplog.text)
assert re.search(r"Supports: .*", caplog.text)
assert re.search(r"Cache types: .*", caplog.text)
if scm_init:
assert "Repo: dvc, git" in caplog.text
@@ -58,26 +53,20 @@ def test_fs_info_in_repo(tmp_dir, dvc, caplog):
os.mkdir(dvc.cache.local.cache_dir)
assert main(["version"]) == 0
assert "Filesystem type (cache directory): " in caplog.text
assert "Filesystem type (workspace): " in caplog.text
assert re.search(r"Cache directory: .* on .*", caplog.text)
assert re.search(r"Workspace directory: .* on .*", caplog.text)
def test_info_outside_of_repo(tmp_dir, caplog):
assert main(["version"]) == 0
assert re.search(r"DVC version: \d+\.\d+\.\d+", caplog.text)
assert re.search(r"Python version: \d\.\d\.\d", caplog.text)
assert re.search(r"Platform: .*", caplog.text)
assert re.search(r"Binary: (True|False)", caplog.text)
assert re.search(r"Package: .*", caplog.text)
assert re.search(r"Supported remotes: .*", caplog.text)
assert not re.search(r"(Cache: (.*link - (not )?(,\s)?){3})", caplog.text)
assert re.search(r"DVC version: \d+\.\d+\.\d+\+.*", caplog.text)
assert re.search(r"Platform: Python \d\.\d\.\d on .*", caplog.text)
assert re.search(r"Supports: .*", caplog.text)
assert not re.search(r"Cache types: .*", caplog.text)
assert "Repo:" not in caplog.text
@pytest.mark.skipif(psutil is None, reason="No psutil.")
def test_fs_info_outside_of_repo(tmp_dir, caplog):
assert main(["version"]) == 0
assert "Filesystem type (cache directory): " not in caplog.text
assert "Filesystem type (workspace): " in caplog.text
+32
View File
@@ -1,4 +1,5 @@
import pytest
import toml
import yaml
from dvc.dependency import ParamsDependency, loadd_from, loads_params
@@ -99,6 +100,37 @@ def test_read_params_nested(tmp_dir, dvc):
assert dep.read_params() == {"some.path.foo": ["val1", "val2"]}
def test_read_params_default_loader(tmp_dir, dvc):
parameters_file = "parameters.foo"
tmp_dir.gen(
parameters_file,
yaml.dump({"some": {"path": {"foo": ["val1", "val2"]}}}),
)
dep = ParamsDependency(Stage(dvc), parameters_file, ["some.path.foo"])
assert dep.read_params() == {"some.path.foo": ["val1", "val2"]}
def test_read_params_wrong_suffix(tmp_dir, dvc):
parameters_file = "parameters.toml"
tmp_dir.gen(
parameters_file,
yaml.dump({"some": {"path": {"foo": ["val1", "val2"]}}}),
)
dep = ParamsDependency(Stage(dvc), parameters_file, ["some.path.foo"])
with pytest.raises(BadParamFileError):
dep.read_params()
def test_read_params_toml(tmp_dir, dvc):
parameters_file = "parameters.toml"
tmp_dir.gen(
parameters_file,
toml.dumps({"some": {"path": {"foo": ["val1", "val2"]}}}),
)
dep = ParamsDependency(Stage(dvc), parameters_file, ["some.path.foo"])
assert dep.read_params() == {"some.path.foo": ["val1", "val2"]}
def test_save_info_missing_config(dvc):
dep = ParamsDependency(Stage(dvc), None, ["foo"])
with pytest.raises(MissingParamsError):