dvc: bump to 1.2.2

Turns out macs are not happy with double-forks without exec.

Fixes #4294

dvc/command/experiments.py

@@ -82,29 +82,25 @@ def _show_experiments(all_experiments, console, precision=None):
 
     metric_names, param_names = _collect_names(all_experiments)
 
-    table = Table(row_styles=["white", "bright_white"])
-    table.add_column("Experiment", header_style="black on grey93")
+    table = Table()
+    table.add_column("Experiment")
     for name in metric_names:
-        table.add_column(
-            name, justify="right", header_style="black on cornsilk1"
-        )
+        table.add_column(name, justify="right")
     for name in param_names:
-        table.add_column(
-            name, justify="left", header_style="black on light_cyan1"
-        )
+        table.add_column(name, justify="left")
 
     for base_rev, experiments in all_experiments.items():
         if Git.is_sha(base_rev):
             base_rev = base_rev[:7]
 
-        for row, style, in _collect_rows(
+        for row, _, in _collect_rows(
             base_rev,
             experiments,
             metric_names,
             param_names,
             precision=precision,
         ):
-            table.add_row(*row, style=style)
+            table.add_row(*row)
 
     console.print(table)
 

dvc/config.py

@@ -314,10 +314,9 @@ class Config(dict):
 
     def _load_config(self, level):
         filename = self.files[level]
-        tree = self.tree if level == "repo" else self.wtree
 
-        if tree.exists(filename, use_dvcignore=False):
-            with tree.open(filename) as fobj:
+        if self.tree.exists(filename, use_dvcignore=False):
+            with self.tree.open(filename) as fobj:
                 conf_obj = configobj.ConfigObj(fobj)
         else:
             conf_obj = configobj.ConfigObj()
@@ -325,14 +324,13 @@ class Config(dict):
 
     def _save_config(self, level, conf_dict):
         filename = self.files[level]
-        tree = self.tree if level == "repo" else self.wtree
 
         logger.debug(f"Writing '{filename}'.")
 
-        tree.makedirs(os.path.dirname(filename))
+        self.tree.makedirs(os.path.dirname(filename))
 
         config = configobj.ConfigObj(_pack_remotes(conf_dict))
-        with tree.open(filename, "wb") as fobj:
+        with self.tree.open(filename, "wb") as fobj:
             config.write(fobj)
         config.filename = filename
 

dvc/daemon.py

@@ -3,6 +3,7 @@
 import inspect
 import logging
 import os
+import platform
 import sys
 from subprocess import Popen
 
@@ -15,25 +16,24 @@ CREATE_NEW_PROCESS_GROUP = 0x00000200
 DETACHED_PROCESS = 0x00000008
 
 
-def _spawn_windows(cmd, env):
-    from subprocess import STARTUPINFO, STARTF_USESHOWWINDOW
-
+def _popen(cmd, **kwargs):
     prefix = [sys.executable]
     if not is_binary():
         prefix += [sys.argv[0]]
 
+    return Popen(prefix + cmd, close_fds=True, shell=False, **kwargs)
+
+
+def _spawn_windows(cmd, env):
+    from subprocess import STARTUPINFO, STARTF_USESHOWWINDOW
+
     creationflags = CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS
 
     startupinfo = STARTUPINFO()
     startupinfo.dwFlags |= STARTF_USESHOWWINDOW
 
-    Popen(
-        prefix + cmd,
-        env=env,
-        close_fds=True,
-        shell=False,
-        creationflags=creationflags,
-        startupinfo=startupinfo,
+    _popen(
+        cmd, env=env, creationflags=creationflags, startupinfo=startupinfo,
     ).communicate()
 
 
@@ -65,8 +65,13 @@ def _spawn_posix(cmd, env):
     sys.stdout.close()
     sys.stderr.close()
 
-    os.environ.update(env)
-    main(cmd)
+    if platform.system() == "Darwin":
+        # workaround for MacOS bug
+        # https://github.com/iterative/dvc/issues/4294
+        _popen(cmd, env=env).communicate()
+    else:
+        os.environ.update(env)
+        main(cmd)
 
     os._exit(0)  # pylint: disable=protected-access
 

dvc/dependency/param.py

@@ -2,6 +2,7 @@ import os
 from collections import defaultdict
 
 import dpath.util
+import toml
 import yaml
 from voluptuous import Any
 
@@ -21,6 +22,8 @@ class ParamsDependency(LocalDependency):
     PARAM_PARAMS = "params"
     PARAM_SCHEMA = {PARAM_PARAMS: Any(dict, list, None)}
     DEFAULT_PARAMS_FILE = "params.yaml"
+    PARAMS_FILE_LOADERS = defaultdict(lambda: yaml.safe_load)
+    PARAMS_FILE_LOADERS.update({".toml": toml.load})
 
     def __init__(self, stage, path, params):
         info = {}
@@ -87,8 +90,10 @@ class ParamsDependency(LocalDependency):
 
         with self.repo.tree.open(self.path_info, "r") as fobj:
             try:
-                config = yaml.safe_load(fobj)
-            except yaml.YAMLError as exc:
+                config = self.PARAMS_FILE_LOADERS[
+                    self.path_info.suffix.lower()
+                ](fobj)
+            except (yaml.YAMLError, toml.TomlDecodeError) as exc:
                 raise BadParamFileError(
                     f"Unable to read parameters from '{self}'"
                 ) from exc

dvc/ignore.py

@@ -1,7 +1,7 @@
 import logging
 import os
 import re
-from itertools import groupby
+from itertools import groupby, takewhile
 
 from pathspec.patterns import GitWildMatchPattern
 from pathspec.util import normalize_file
@@ -131,18 +131,19 @@ class DvcIgnoreFilter:
         self.ignores_trie_tree[root_dir] = DvcIgnorePatterns(
             default_ignore_patterns, root_dir
         )
-        for root, dirs, _ in self.tree.walk(self.root_dir):
-            self._update(root)
-            self._update_sub_repo(root, dirs)
-            dirs[:], _ = self(root, dirs, [])
+        self._update(self.root_dir)
 
     def _update(self, dirname):
+        old_pattern = self.ignores_trie_tree.longest_prefix(dirname).value
+        matches = old_pattern.matches(dirname, DvcIgnore.DVCIGNORE_FILE, False)
+
         ignore_file_path = os.path.join(dirname, DvcIgnore.DVCIGNORE_FILE)
-        if self.tree.exists(ignore_file_path):
+        if not matches and self.tree.exists(
+            ignore_file_path, use_dvcignore=False
+        ):
             new_pattern = DvcIgnorePatterns.from_files(
                 ignore_file_path, self.tree
             )
-            old_pattern = self._get_trie_pattern(dirname)
             if old_pattern:
                 self.ignores_trie_tree[dirname] = DvcIgnorePatterns(
                     *merge_patterns(
@@ -154,11 +155,18 @@ class DvcIgnoreFilter:
                 )
             else:
                 self.ignores_trie_tree[dirname] = new_pattern
+        elif old_pattern:
+            self.ignores_trie_tree[dirname] = old_pattern
+
+        # NOTE: using `walk` + `break` because tree doesn't have `listdir()`
+        for root, dirs, _ in self.tree.walk(dirname, use_dvcignore=False):
+            self._update_sub_repo(root, dirs)
+            break
 
     def _update_sub_repo(self, root, dirs):
         for d in dirs:
             if self._is_dvc_repo(root, d):
-                old_pattern = self._get_trie_pattern(root)
+                old_pattern = self.ignores_trie_tree.longest_prefix(root).value
                 if old_pattern:
                     self.ignores_trie_tree[root] = DvcIgnorePatterns(
                         *merge_patterns(
@@ -181,8 +189,28 @@ class DvcIgnoreFilter:
             return dirs, files
 
     def _get_trie_pattern(self, dirname):
-        ignore_pattern = self.ignores_trie_tree.longest_prefix(dirname).value
-        return ignore_pattern
+        ignore_pattern = self.ignores_trie_tree.get(dirname)
+        if ignore_pattern:
+            return ignore_pattern
+
+        prefix = self.ignores_trie_tree.longest_prefix(dirname).key
+        if not prefix:
+            # outside of the repo
+            return None
+
+        dirs = list(
+            takewhile(
+                lambda path: path != prefix,
+                (parent.fspath for parent in PathInfo(dirname).parents),
+            )
+        )
+        dirs.reverse()
+        dirs.append(dirname)
+
+        for parent in dirs:
+            self._update(parent)
+
+        return self.ignores_trie_tree.get(dirname)
 
     def _is_ignored(self, path, is_dir=False):
         if self._outside_repo(path):

dvc/repo/params/show.py

@@ -1,5 +1,6 @@
 import logging
 
+import toml
 import yaml
 
 from dvc.dependency.param import ParamsDependency
@@ -34,8 +35,10 @@ def _read_params(repo, configs, rev):
 
         with repo.tree.open(config, "r") as fobj:
             try:
-                res[str(config)] = yaml.safe_load(fobj)
-            except yaml.YAMLError:
+                res[str(config)] = ParamsDependency.PARAMS_FILE_LOADERS[
+                    config.suffix.lower()
+                ](fobj)
+            except (yaml.YAMLError, toml.TomlDecodeError):
                 logger.debug(
                     "failed to read '%s' on '%s'", config, rev, exc_info=True
                 )

dvc/tree/git.py

@@ -46,12 +46,8 @@ class GitTree(BaseTree):  # pylint:disable=abstract-method
         from dvc.ignore import DvcIgnoreFilter, DvcIgnoreFilterNoop
 
         root = self.dvcignore_root or self.tree_root
-        if not self.use_dvcignore:
-            return DvcIgnoreFilterNoop(self, root)
-        self.use_dvcignore = False
-        ret = DvcIgnoreFilter(self, root)
-        self.use_dvcignore = True
-        return ret
+        cls = DvcIgnoreFilter if self.use_dvcignore else DvcIgnoreFilterNoop
+        return cls(self, root)
 
     def open(
         self, path, mode="r", encoding="utf-8"
@@ -160,7 +156,7 @@ class GitTree(BaseTree):  # pylint:disable=abstract-method
         if not topdown:
             yield os.path.normpath(tree.abspath), dirs, nondirs
 
-    def walk(self, top, topdown=True, onerror=None):
+    def walk(self, top, topdown=True, onerror=None, use_dvcignore=True):
         """Directory tree generator.
 
         See `os.walk` for the docs. Differences:
@@ -178,9 +174,10 @@ class GitTree(BaseTree):  # pylint:disable=abstract-method
             return
 
         for root, dirs, files in self._walk(tree, topdown=topdown):
-            dirs[:], files[:] = self.dvcignore(
-                os.path.abspath(root), dirs, files
-            )
+            if use_dvcignore:
+                dirs[:], files[:] = self.dvcignore(
+                    os.path.abspath(root), dirs, files
+                )
             yield root, dirs, files
 
     def isexec(self, path):

dvc/tree/local.py

@@ -58,12 +58,8 @@ class LocalTree(BaseTree):
         from dvc.ignore import DvcIgnoreFilter, DvcIgnoreFilterNoop
 
         root = self.dvcignore_root or self.tree_root
-        if not self.use_dvcignore:
-            return DvcIgnoreFilterNoop(self, root)
-        self.use_dvcignore = False
-        ret = DvcIgnoreFilter(self, root)
-        self.use_dvcignore = True
-        return ret
+        cls = DvcIgnoreFilter if self.use_dvcignore else DvcIgnoreFilterNoop
+        return cls(self, root)
 
     @staticmethod
     def open(path_info, mode="r", encoding=None):
@@ -101,7 +97,7 @@ class LocalTree(BaseTree):
             System.is_symlink(path_info) or System.is_hardlink(path_info)
         )
 
-    def walk(self, top, topdown=True, onerror=None):
+    def walk(self, top, topdown=True, onerror=None, use_dvcignore=True):
         """Directory tree generator.
 
         See `os.walk` for the docs. Differences:
@@ -110,9 +106,10 @@ class LocalTree(BaseTree):
         for root, dirs, files in os.walk(
             top, topdown=topdown, onerror=onerror
         ):
-            dirs[:], files[:] = self.dvcignore(
-                os.path.abspath(root), dirs, files
-            )
+            if use_dvcignore:
+                dirs[:], files[:] = self.dvcignore(
+                    os.path.abspath(root), dirs, files
+                )
 
             yield os.path.normpath(root), dirs, files
 

dvc/utils/.gitignore

@@ -0,0 +1 @@
+/build.py

dvc/utils/fs.py

@@ -96,8 +96,7 @@ def move(src, dst, mode=None):
     tmp = f"{dst}.{uuid()}"
 
     if os.path.islink(src):
-        # readlink does not accept path-like obj for Windows in Python <3.8
-        shutil.copy(os.readlink(os.fspath(src)), tmp)
+        shutil.copy(src, tmp)
         os.unlink(src)
     else:
         shutil.move(src, tmp)

dvc/version.py

@@ -6,7 +6,7 @@
 import os
 import subprocess
 
-_BASE_VERSION = "1.2.0"
+_BASE_VERSION = "1.2.2"
 
 
 def _generate_version(base_version):

setup.cfg

@@ -17,7 +17,7 @@ count=true
 [isort]
 include_trailing_comma=true
 known_first_party=dvc,tests
-known_third_party=PyInstaller,RangeHTTPServer,boto3,colorama,configobj,distro,dpath,flaky,flufl,funcy,git,grandalf,mock,moto,nanotime,networkx,packaging,pathspec,pygtrie,pylint,pytest,requests,ruamel,setuptools,shortuuid,shtab,tqdm,voluptuous,yaml,zc
+known_third_party=PyInstaller,RangeHTTPServer,boto3,colorama,configobj,distro,dpath,flaky,flufl,funcy,git,grandalf,mock,moto,nanotime,networkx,packaging,pathspec,pygtrie,pylint,pytest,requests,ruamel,setuptools,shortuuid,shtab,toml,tqdm,voluptuous,yaml,zc
 line_length=79
 force_grid_wrap=0
 use_parentheses=True

setup.py

@@ -62,6 +62,7 @@ install_requires = [
     "appdirs>=1.4.3",
     "PyYAML>=5.1.2,<5.4",  # Compatibility with awscli
     "ruamel.yaml>=0.16.1",
+    "toml>=0.10.1",
     "funcy>=1.14",
     "pathspec>=0.6.0",
     "shortuuid>=0.5.0",
@@ -74,7 +75,7 @@ install_requires = [
     "networkx>=2.1,<2.5",
     "pydot>=1.2.4",
     "speedcopy>=2.0.1; python_version < '3.8' and sys_platform == 'win32'",
-    "flatten_json>=0.1.6",
+    "flatten_json>=0.1.6,<0.1.8",
     "tabulate>=0.8.7",
     "pygtrie==2.3.2",
     "dpath>=2.0.1,<3",

tests/func/params/test_show.py

@@ -14,6 +14,16 @@ def test_show(tmp_dir, dvc):
     assert dvc.params.show() == {"": {"params.yaml": {"foo": "bar"}}}
 
 
+def test_show_toml(tmp_dir, dvc):
+    tmp_dir.gen("params.toml", "[foo]\nbar = 42\nbaz = [1, 2]\n")
+    dvc.run(
+        cmd="echo params.toml", params=["params.toml:foo"], single_stage=True
+    )
+    assert dvc.params.show() == {
+        "": {"params.toml": {"foo": {"bar": 42, "baz": [1, 2]}}}
+    }
+
+
 def test_show_multiple(tmp_dir, dvc):
     tmp_dir.gen("params.yaml", "foo: bar\nbaz: qux\n")
     dvc.run(

tests/func/test_add.py

@@ -806,3 +806,23 @@ def test_add_pipeline_file(tmp_dir, dvc, run_copy):
 
     with pytest.raises(OutputIsStageFileError):
         dvc.add(PIPELINE_FILE)
+
+
+def test_add_symlink(tmp_dir, dvc):
+    tmp_dir.gen({"dir": {"bar": "bar"}})
+
+    (tmp_dir / "dir" / "foo").symlink_to(os.path.join(".", "bar"))
+
+    dvc.add(os.path.join("dir", "foo"))
+
+    assert not (tmp_dir / "dir" / "foo").is_symlink()
+    assert not (tmp_dir / "dir" / "bar").is_symlink()
+    assert (tmp_dir / "dir" / "foo").read_text() == "bar"
+    assert (tmp_dir / "dir" / "bar").read_text() == "bar"
+
+    assert (tmp_dir / ".dvc" / "cache").read_text() == {
+        "37": {"b51d194a7513e45b56f6524f2d51f2": "bar"}
+    }
+    assert not (
+        tmp_dir / ".dvc" / "cache" / "37" / "b51d194a7513e45b56f6524f2d51f2"
+    ).is_symlink()

tests/unit/dependency/test_params.py

@@ -1,4 +1,5 @@
 import pytest
+import toml
 import yaml
 
 from dvc.dependency import ParamsDependency, loadd_from, loads_params
@@ -99,6 +100,37 @@ def test_read_params_nested(tmp_dir, dvc):
     assert dep.read_params() == {"some.path.foo": ["val1", "val2"]}
 
 
+def test_read_params_default_loader(tmp_dir, dvc):
+    parameters_file = "parameters.foo"
+    tmp_dir.gen(
+        parameters_file,
+        yaml.dump({"some": {"path": {"foo": ["val1", "val2"]}}}),
+    )
+    dep = ParamsDependency(Stage(dvc), parameters_file, ["some.path.foo"])
+    assert dep.read_params() == {"some.path.foo": ["val1", "val2"]}
+
+
+def test_read_params_wrong_suffix(tmp_dir, dvc):
+    parameters_file = "parameters.toml"
+    tmp_dir.gen(
+        parameters_file,
+        yaml.dump({"some": {"path": {"foo": ["val1", "val2"]}}}),
+    )
+    dep = ParamsDependency(Stage(dvc), parameters_file, ["some.path.foo"])
+    with pytest.raises(BadParamFileError):
+        dep.read_params()
+
+
+def test_read_params_toml(tmp_dir, dvc):
+    parameters_file = "parameters.toml"
+    tmp_dir.gen(
+        parameters_file,
+        toml.dumps({"some": {"path": {"foo": ["val1", "val2"]}}}),
+    )
+    dep = ParamsDependency(Stage(dvc), parameters_file, ["some.path.foo"])
+    assert dep.read_params() == {"some.path.foo": ["val1", "val2"]}
+
+
 def test_save_info_missing_config(dvc):
     dep = ParamsDependency(Stage(dvc), None, ["foo"])
     with pytest.raises(MissingParamsError):