dvc: bump to 0.78.0

dvc: use ' to mark dynamic command output, et al.

dvc/analytics.py

@@ -164,4 +164,6 @@ def _find_or_create_user_id():
             return user_id
 
     except LockError:
-        logger.debug("Failed to acquire {lockfile}".format(lockfile=lockfile))
+        logger.debug(
+            "Failed to acquire '{lockfile}'".format(lockfile=lockfile)
+        )

dvc/command/checkout.py

@@ -28,19 +28,6 @@ def add_parser(subparsers, parent_parser):
         help=CHECKOUT_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    checkout_parser.add_argument(
-        "-f",
-        "--force",
-        action="store_true",
-        default=False,
-        help="Do not prompt when removing working directory files.",
-    )
-    checkout_parser.add_argument(
-        "--relink",
-        action="store_true",
-        default=False,
-        help="Recreate links or copies from cache to workspace.",
-    )
     checkout_parser.add_argument(
         "-d",
         "--with-deps",
@@ -55,6 +42,19 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Checkout all subdirectories of the specified directory.",
     )
+    checkout_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Do not prompt when removing working directory files.",
+    )
+    checkout_parser.add_argument(
+        "--relink",
+        action="store_true",
+        default=False,
+        help="Recreate links or copies from cache to workspace.",
+    )
     checkout_parser.add_argument(
         "targets",
         nargs="*",

dvc/command/remove.py

@@ -20,7 +20,7 @@ class CmdRemove(CmdBase):
         if self.args.force:
             return False
 
-        msg = "Are you sure you want to remove {} with its outputs?".format(
+        msg = "Are you sure you want to remove '{}' with its outputs?".format(
             target
         )
 
@@ -38,7 +38,7 @@ class CmdRemove(CmdBase):
                 outs_only = self._is_outs_only(target)
                 self.repo.remove(target, outs_only=outs_only)
             except DvcException:
-                logger.exception("failed to remove {}".format(target))
+                logger.exception("failed to remove '{}'".format(target))
                 return 1
         return 0
 

dvc/config.py

@@ -81,7 +81,7 @@ def Choices(*choices):
         *choices: pass allowed values as arguments, or pass a list or
             tuple as a single argument
     """
-    return Any(*choices, msg="expected one of {}".format(",".join(choices)))
+    return Any(*choices, msg="expected one of {}".format(", ".join(choices)))
 
 
 class Config(object):  # pylint: disable=too-many-instance-attributes

dvc/dependency/repo.py

@@ -1,13 +1,18 @@
 from __future__ import unicode_literals
 
 import copy
+import os
 from contextlib import contextmanager
+from dvc.utils.compat import FileNotFoundError
 
 from funcy import merge
 
 from .local import DependencyLOCAL
 from dvc.external_repo import external_repo
 from dvc.utils.compat import str
+from dvc.exceptions import OutputNotFoundError
+from dvc.exceptions import PathMissingError
+from dvc.utils.fs import fs_copy
 
 
 class DependencyREPO(DependencyLOCAL):
@@ -72,10 +77,42 @@ class DependencyREPO(DependencyLOCAL):
 
         return out
 
+    @staticmethod
+    def _is_git_file(repo, path):
+        if not os.path.isabs(path):
+            try:
+                output = repo.find_out_by_relpath(path)
+                if not output.use_cache:
+                    return True
+            except OutputNotFoundError:
+                return True
+        return False
+
+    def _copy_if_git_file(self, to_path):
+        src_path = self.def_path
+        with self._make_repo(
+            cache_dir=self.repo.cache.local.cache_dir
+        ) as repo:
+            if not self._is_git_file(repo, src_path):
+                return False
+
+            src_full_path = os.path.join(repo.root_dir, src_path)
+            dst_full_path = os.path.abspath(to_path)
+            fs_copy(src_full_path, dst_full_path)
+        return True
+
     def download(self, to):
-        out = self.fetch()
-        to.info = copy.copy(out.info)
-        to.checkout()
+        try:
+            if self._copy_if_git_file(to.fspath):
+                return
+
+            out = self.fetch()
+            to.info = copy.copy(out.info)
+            to.checkout()
+        except (FileNotFoundError):
+            raise PathMissingError(
+                self.def_path, self.def_repo[self.PARAM_URL]
+            )
 
     def update(self):
         with self._make_repo(rev_lock=None) as repo:

dvc/exceptions.py

@@ -344,3 +344,12 @@ class NoOutputInExternalRepoError(DvcException):
 class HTTPError(DvcException):
     def __init__(self, code, reason):
         super(HTTPError, self).__init__("'{} {}'".format(code, reason))
+
+
+class PathMissingError(DvcException):
+    def __init__(self, path, repo):
+        msg = (
+            "The path '{}' does not exist in the target repository '{}'"
+            " neighther as an output nor a git-handled file."
+        )
+        super(PathMissingError, self).__init__(msg.format(path, repo))

dvc/main.py

@@ -1,6 +1,7 @@
 """Main entry point for dvc CLI."""
 from __future__ import unicode_literals
 
+import errno
 import logging
 
 from dvc import analytics
@@ -64,6 +65,10 @@ def main(argv=None):
                 "unicode is not supported in DVC for Python 2 "
                 "(end-of-life January 1, 2020), please upgrade to Python 3"
             )
+        elif isinstance(exc, OSError) and exc.errno == errno.EMFILE:
+            logger.exception(
+                "too many open files, please increase your `ulimit`"
+            )
         else:
             logger.exception("unexpected error")
         ret = 255

dvc/remote/base.py

@@ -1,4 +1,7 @@
 from __future__ import unicode_literals
+
+import errno
+
 from dvc.utils.compat import basestring, FileNotFoundError, str, urlparse
 
 import itertools
@@ -55,7 +58,7 @@ class RemoteCmdError(DvcException):
 
 class RemoteActionNotImplemented(DvcException):
     def __init__(self, action, scheme):
-        m = "{} is not supported by {} remote".format(action, scheme)
+        m = "{} is not supported for {} remotes".format(action, scheme)
         super(RemoteActionNotImplemented, self).__init__(m)
 
 
@@ -516,6 +519,16 @@ class RemoteBASE(object):
             return
         self._save_file(path_info, checksum)
 
+    def _handle_transfer_exception(
+        self, from_info, to_info, exception, operation
+    ):
+        if isinstance(exception, OSError) and exception.errno == errno.EMFILE:
+            raise exception
+
+        msg = "failed to {} '{}' to '{}'".format(operation, from_info, to_info)
+        logger.exception(msg)
+        return 1
+
     def upload(self, from_info, to_info, name=None, no_progress_bar=False):
         if not hasattr(self, "_upload"):
             raise RemoteActionNotImplemented("upload", self.scheme)
@@ -537,10 +550,10 @@ class RemoteBASE(object):
                 name=name,
                 no_progress_bar=no_progress_bar,
             )
-        except Exception:
-            msg = "failed to upload '{}' to '{}'"
-            logger.exception(msg.format(from_info, to_info))
-            return 1  # 1 fail
+        except Exception as e:
+            return self._handle_transfer_exception(
+                from_info, to_info, e, "upload"
+            )
 
         return 0
 
@@ -614,10 +627,10 @@ class RemoteBASE(object):
             self._download(
                 from_info, tmp_file, name=name, no_progress_bar=no_progress_bar
             )
-        except Exception:
-            msg = "failed to download '{}' to '{}'"
-            logger.exception(msg.format(from_info, to_info))
-            return 1  # 1 fail
+        except Exception as e:
+            return self._handle_transfer_exception(
+                from_info, to_info, e, "download"
+            )
 
         move(tmp_file, to_info, mode=file_mode)
 

dvc/remote/gdrive.py

@@ -57,7 +57,7 @@ class RemoteGDrive(RemoteBASE):
     path_cls = CloudURLInfo
     REQUIRES = {"pydrive": "pydrive"}
     GDRIVE_USER_CREDENTIALS_DATA = "GDRIVE_USER_CREDENTIALS_DATA"
-    DEFAULT_USER_CREDENTIALS_FILE = ".dvc/tmp/gdrive-user-credentials.json"
+    DEFAULT_USER_CREDENTIALS_FILE = "gdrive-user-credentials.json"
 
     def __init__(self, repo, config):
         super(RemoteGDrive, self).__init__(repo, config)
@@ -82,7 +82,9 @@ class RemoteGDrive(RemoteBASE):
             if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA)
             else self.config.get(
                 Config.SECTION_GDRIVE_USER_CREDENTIALS_FILE,
-                self.DEFAULT_USER_CREDENTIALS_FILE,
+                os.path.join(
+                    self.repo.tmp_dir, self.DEFAULT_USER_CREDENTIALS_FILE
+                ),
             )
         )
 

dvc/remote/oss.py

@@ -69,9 +69,9 @@ class RemoteOSS(RemoteBASE):
     def oss_service(self):
         import oss2
 
-        logger.debug("URL {}".format(self.path_info))
-        logger.debug("key id {}".format(self.key_id))
-        logger.debug("key secret {}".format(self.key_secret))
+        logger.debug("URL: {}".format(self.path_info))
+        logger.debug("key id: {}".format(self.key_id))
+        logger.debug("key secret: {}".format(self.key_secret))
 
         auth = oss2.Auth(self.key_id, self.key_secret)
         bucket = oss2.Bucket(auth, self.endpoint, self.path_info.bucket)

dvc/repo/add.py

@@ -37,7 +37,7 @@ def add(repo, targets, recursive=False, no_commit=False, fname=None):
                     "You are adding a large directory '{target}' recursively,"
                     " consider tracking it as a whole instead.\n"
                     "{purple}HINT:{nc} Remove the generated DVC-file and then"
-                    " run {cyan}dvc add {target}{nc}".format(
+                    " run `{cyan}dvc add {target}{nc}`".format(
                         purple=colorama.Fore.MAGENTA,
                         cyan=colorama.Fore.CYAN,
                         nc=colorama.Style.RESET_ALL,

dvc/repo/gc.py

@@ -12,7 +12,7 @@ logger = logging.getLogger(__name__)
 def _do_gc(typ, func, clist):
     removed = func(clist)
     if not removed:
-        logger.info("No unused {} cache to remove.".format(typ))
+        logger.info("No unused '{}' cache to remove.".format(typ))
 
 
 @locked

dvc/repo/get.py

@@ -1,6 +1,5 @@
 import logging
 import os
-import shutil
 from dvc.utils.compat import FileNotFoundError
 
 import shortuuid
@@ -10,12 +9,14 @@ from dvc.exceptions import (
     NotDvcRepoError,
     OutputNotFoundError,
     UrlNotDvcRepoError,
+    PathMissingError,
 )
 from dvc.external_repo import external_repo
 from dvc.path_info import PathInfo
 from dvc.stage import Stage
 from dvc.utils import resolve_output
 from dvc.utils.fs import remove
+from dvc.utils.fs import fs_copy
 
 logger = logging.getLogger(__name__)
 
@@ -28,15 +29,6 @@ class GetDVCFileError(DvcException):
         )
 
 
-class PathMissingError(DvcException):
-    def __init__(self, path, repo):
-        msg = (
-            "The path '{}' does not exist in the target repository '{}'"
-            " neighther as an output nor a git-handled file."
-        )
-        super(PathMissingError, self).__init__(msg.format(path, repo))
-
-
 @staticmethod
 def get(url, path, out=None, rev=None):
     out = resolve_output(path, out)
@@ -76,7 +68,7 @@ def get(url, path, out=None, rev=None):
                 if os.path.isabs(path):
                     raise FileNotFoundError
 
-                _copy(os.path.join(repo.root_dir, path), out)
+                fs_copy(os.path.join(repo.root_dir, path), out)
 
     except (OutputNotFoundError, FileNotFoundError):
         raise PathMissingError(path, url)
@@ -94,10 +86,3 @@ def _get_cached(repo, output, out):
         # This might happen when pull haven't really pulled all the files
         if failed:
             raise FileNotFoundError
-
-
-def _copy(src, dst):
-    if os.path.isdir(src):
-        shutil.copytree(src, dst)
-    else:
-        shutil.copy2(src, dst)

dvc/stage.py

@@ -116,7 +116,7 @@ class MissingDep(DvcException):
         else:
             dep = "dependency"
 
-        msg = "missing {}: {}".format(dep, ", ".join(map(str, deps)))
+        msg = "missing '{}': {}".format(dep, ", ".join(map(str, deps)))
         super(MissingDep, self).__init__(msg)
 
 
@@ -128,7 +128,7 @@ class MissingDataSource(DvcException):
         if len(missing_files) > 1:
             source += "s"
 
-        msg = "missing data {}: {}".format(source, ", ".join(missing_files))
+        msg = "missing data '{}': {}".format(source, ", ".join(missing_files))
         super(MissingDataSource, self).__init__(msg)
 
 

dvc/updater.py

@@ -124,12 +124,13 @@ class Updater(object):  # pragma: no cover
 
     def _get_update_instructions(self):
         instructions = {
-            "pip": "Run {yellow}pip{reset} install dvc {blue}--upgrade{reset}",
-            "rpm": "Run {yellow}yum{reset} update dvc",
-            "brew": "Run {yellow}brew{reset} upgrade dvc",
+            "pip": "Run `{yellow}pip{reset} install dvc "
+            "{blue}--upgrade{reset}`",
+            "rpm": "Run `{yellow}yum{reset} update dvc`",
+            "brew": "Run `{yellow}brew{reset} upgrade dvc`",
             "deb": (
-                "Run {yellow}apt-get{reset} install"
-                " {blue}--only-upgrade{reset} dvc"
+                "Run `{yellow}apt-get{reset} install"
+                " {blue}--only-upgrade{reset} dvc`"
             ),
             "binary": (
                 "To upgrade follow these steps:\n"
@@ -137,12 +138,11 @@ class Updater(object):  # pragma: no cover
                 "2. Go to {blue}https://dvc.org{reset}\n"
                 "3. Download and install new binary"
             ),
-            "conda": "Run {yellow}conda{reset} update dvc",
-            "choco": "Run {yellow}choco{reset} upgrade dvc",
+            "conda": "Run `{yellow}conda{reset} update dvc`",
+            "choco": "Run `{yellow}choco{reset} upgrade dvc`",
             None: (
-                "Find the latest release at\n{blue}"
-                "https://github.com/iterative/dvc/releases/latest"
-                "{reset}"
+                "Find the latest release at\n"
+                "{blue}https://github.com/iterative/dvc/releases/latest{reset}"
             ),
         }
 

dvc/utils/__init__.py

@@ -50,7 +50,9 @@ def file_md5(fname):
         no_progress_bar = True
         if size >= LARGE_FILE_SIZE:
             no_progress_bar = False
-            msg = "Computing md5 for a large file {}. This is only done once."
+            msg = (
+                "Computing md5 for a large file '{}'. This is only done once."
+            )
             logger.info(msg.format(relpath(fname)))
         name = relpath(fname)
 

dvc/utils/fs.py

@@ -22,6 +22,13 @@ from dvc.utils.compat import str
 logger = logging.getLogger(__name__)
 
 
+def fs_copy(src, dst):
+    if os.path.isdir(src):
+        shutil.copytree(src, dst)
+    else:
+        shutil.copy2(src, dst)
+
+
 def get_inode(path):
     inode = System.inode(path)
     logger.debug("Path {} inode {}".format(path, inode))

dvc/version.py

@@ -7,7 +7,7 @@ import os
 import subprocess
 
 
-_BASE_VERSION = "0.77.2"
+_BASE_VERSION = "0.78.0"
 
 
 def _generate_version(base_version):

scripts/build_posix.sh

@@ -87,6 +87,9 @@ install_dependencies()
 	fi
 
 	gem install --no-document fpm
+	
+	# https://github.com/iterative/dvc/issues/2970
+	gem uninstall -i /Users/travis/.rvm/gems/ruby-2.4.3@global rubygems-bundler
 
 	print_info "Upgrading pip..."
 	pip install --upgrade pip

scripts/ci/before_install.sh

@@ -45,10 +45,10 @@ fi
 
 if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then
     if [[ -n "$PY2" ]]; then
-        choco install python2
+        $scriptdir/retry.sh choco install python2
         echo 'PATH="/c/Python27:/c/Python27/Scripts:$PATH"' >> env.sh
     else
-        choco install python --version 3.7.5
+        $scriptdir/retry.sh choco install python --version 3.7.5
         echo 'PATH="/c/Python37:/c/Python37/Scripts:$PATH"' >> env.sh
     fi
 elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then

tests/conftest.py

@@ -101,41 +101,6 @@ def ssh(ssh_server):
     yield SSHConnection(**ssh_server.test_creds)
 
 
-@pytest.fixture
-def temporary_windows_drive(repo_dir):
-    import string
-    import win32api
-    from ctypes import windll
-    from win32con import DDD_REMOVE_DEFINITION
-
-    drives = [
-        s[0].upper()
-        for s in win32api.GetLogicalDriveStrings().split("\000")
-        if len(s) > 0
-    ]
-
-    new_drive_name = [
-        letter for letter in string.ascii_uppercase if letter not in drives
-    ][0]
-    new_drive = "{}:".format(new_drive_name)
-
-    target_path = repo_dir.mkdtemp()
-
-    set_up_result = windll.kernel32.DefineDosDeviceW(0, new_drive, target_path)
-    if set_up_result == 0:
-        raise RuntimeError("Failed to mount windows drive!")
-
-    # NOTE: new_drive has form of `A:` and joining it with some relative
-    # path might result in non-existing path (A:path\\to)
-    yield os.path.join(new_drive, os.sep)
-
-    tear_down_result = windll.kernel32.DefineDosDeviceW(
-        DDD_REMOVE_DEFINITION, new_drive, target_path
-    )
-    if tear_down_result == 0:
-        raise RuntimeError("Could not unmount windows drive!")
-
-
 @pytest.fixture
 def erepo(repo_dir):
     repo = TestDvcGitFixture()

tests/func/test_add.py

@@ -12,6 +12,7 @@ import pytest
 from mock import patch
 
 import dvc
+from dvc.cache import Cache
 from dvc.exceptions import DvcException
 from dvc.exceptions import RecursiveAddingWhileUsingFilename
 from dvc.exceptions import StageFileCorruptedError
@@ -24,7 +25,8 @@ from dvc.system import System
 from dvc.utils import file_md5
 from dvc.utils import LARGE_DIR_SIZE
 from dvc.utils import relpath
-from dvc.utils.compat import range
+from dvc.utils.compat import range, fspath
+from dvc.utils.fs import path_isin
 from dvc.utils.stage import load_stage_file
 from tests.basic_env import TestDvc
 from tests.utils import get_gitignore_content
@@ -90,7 +92,7 @@ class TestAddCmdDirectoryRecursive(TestDvc):
             "You are adding a large directory 'large-dir' recursively,"
             " consider tracking it as a whole instead.\n"
             "{purple}HINT:{nc} Remove the generated DVC-file and then"
-            " run {cyan}dvc add large-dir{nc}".format(
+            " run `{cyan}dvc add large-dir{nc}`".format(
                 purple=colorama.Fore.MAGENTA,
                 cyan=colorama.Fore.CYAN,
                 nc=colorama.Style.RESET_ALL,
@@ -511,15 +513,56 @@ class TestAddUnprotected(TestDvc):
         self.assertTrue(System.is_hardlink(self.FOO))
 
 
+@pytest.fixture
+def temporary_windows_drive(tmp_path_factory):
+    import string
+    import win32api
+    from ctypes import windll
+    from win32con import DDD_REMOVE_DEFINITION
+
+    drives = [
+        s[0].upper()
+        for s in win32api.GetLogicalDriveStrings().split("\000")
+        if len(s) > 0
+    ]
+
+    new_drive_name = [
+        letter for letter in string.ascii_uppercase if letter not in drives
+    ][0]
+    new_drive = "{}:".format(new_drive_name)
+
+    target_path = tmp_path_factory.mktemp("tmp_windows_drive")
+
+    set_up_result = windll.kernel32.DefineDosDeviceW(
+        0, new_drive, fspath(target_path)
+    )
+    if set_up_result == 0:
+        raise RuntimeError("Failed to mount windows drive!")
+
+    # NOTE: new_drive has form of `A:` and joining it with some relative
+    # path might result in non-existing path (A:path\\to)
+    yield os.path.join(new_drive, os.sep)
+
+    tear_down_result = windll.kernel32.DefineDosDeviceW(
+        DDD_REMOVE_DEFINITION, new_drive, fspath(target_path)
+    )
+    if tear_down_result == 0:
+        raise RuntimeError("Could not unmount windows drive!")
+
+
 @pytest.mark.skipif(os.name != "nt", reason="Windows specific")
 def test_windows_should_add_when_cache_on_different_drive(
-    dvc_repo, repo_dir, temporary_windows_drive
+    tmp_dir, dvc, temporary_windows_drive
 ):
-    ret = main(["config", "cache.dir", temporary_windows_drive])
-    assert ret == 0
+    dvc.config.set("cache", "dir", temporary_windows_drive)
+    dvc.cache = Cache(dvc)
 
-    ret = main(["add", repo_dir.DATA])
-    assert ret == 0
+    stage, = tmp_dir.dvc_gen({"file": "file"})
+    cache_path = stage.outs[0].cache_path
+
+    assert path_isin(cache_path, temporary_windows_drive)
+    assert os.path.isfile(cache_path)
+    filecmp.cmp("file", cache_path)
 
 
 def test_readding_dir_should_not_unprotect_all(tmp_dir, dvc, mocker):

tests/func/test_analytics.py

@@ -14,7 +14,8 @@ def test_daemon_analytics(mock_send, tmp_path):
 
 @mock.patch("dvc.analytics.collect_and_send_report")
 @mock.patch("dvc.analytics.is_enabled", return_value=True)
-def test_main_analytics(mock_is_enabled, mock_report, dvc_repo):
+def test_main_analytics(mock_is_enabled, mock_report, tmp_dir, dvc):
+    tmp_dir.gen("foo", "text")
     assert 0 == main(["add", "foo"])
     assert mock_is_enabled.called
     assert mock_report.called

tests/func/test_destroy.py

@@ -1,28 +0,0 @@
-import os
-
-from dvc.system import System
-
-
-def test_destroy(repo_dir, dvc_repo):
-    # NOTE: using symlink to ensure that data was unprotected after `destroy`
-    dvc_repo.config.set("cache", "type", "symlink")
-
-    foo_stage, = dvc_repo.add(repo_dir.FOO)
-    data_dir_stage, = dvc_repo.add(repo_dir.DATA_DIR)
-
-    dvc_repo.destroy()
-
-    assert not os.path.exists(dvc_repo.dvc_dir)
-    assert not os.path.exists(foo_stage.path)
-    assert not os.path.exists(data_dir_stage.path)
-
-    assert os.path.isfile(repo_dir.FOO)
-    assert os.path.isdir(repo_dir.DATA_DIR)
-    assert os.path.isfile(repo_dir.DATA)
-    assert os.path.isdir(repo_dir.DATA_SUB_DIR)
-    assert os.path.isfile(repo_dir.DATA_SUB)
-
-    assert not System.is_symlink(repo_dir.FOO)
-    assert not System.is_symlink(repo_dir.DATA_DIR)
-    assert not System.is_symlink(repo_dir.DATA)
-    assert not System.is_symlink(repo_dir.DATA_SUB)

tests/func/test_import.py

@@ -7,155 +7,220 @@ import shutil
 import pytest
 from mock import patch
 
+from dvc.cache import Cache
 from dvc.config import Config
-from dvc.exceptions import DownloadError, NoOutputInExternalRepoError
+from dvc.exceptions import DownloadError
+from dvc.exceptions import PathMissingError
+from dvc.exceptions import NoOutputInExternalRepoError
 from dvc.stage import Stage
 from dvc.system import System
 from dvc.utils import makedirs
+from dvc.utils.compat import fspath
 from tests.utils import trees_equal
 
 
-def test_import(git, dvc_repo, erepo):
-    src = erepo.FOO
-    dst = erepo.FOO + "_imported"
+def test_import(tmp_dir, scm, dvc, erepo_dir, monkeypatch):
+    with monkeypatch.context() as m:
+        m.chdir(fspath(erepo_dir))
+        erepo_dir.dvc_gen("foo", "foo content", commit="create foo")
 
-    dvc_repo.imp(erepo.root_dir, src, dst)
+    dvc.imp(fspath(erepo_dir), "foo", "foo_imported")
 
-    assert os.path.exists(dst)
-    assert os.path.isfile(dst)
-    assert filecmp.cmp(erepo.FOO, dst, shallow=False)
-    assert git.git.check_ignore(dst)
+    assert os.path.isfile("foo_imported")
+    assert (tmp_dir / "foo_imported").read_text() == "foo content"
+    assert scm.repo.git.check_ignore("foo_imported")
 
 
-def test_import_dir(git, dvc_repo, erepo):
-    src = erepo.DATA_DIR
-    dst = erepo.DATA_DIR + "_imported"
+def test_import_git_file(erepo_dir, tmp_dir, dvc, scm):
+    src = "some_file"
+    dst = "some_file_imported"
 
-    dvc_repo.imp(erepo.root_dir, src, dst)
+    erepo_dir.scm_gen({src: "hello"}, commit="add a regular file")
 
-    assert os.path.exists(dst)
-    assert os.path.isdir(dst)
-    trees_equal(src, dst)
-    assert git.git.check_ignore(dst)
+    tmp_dir.dvc.imp(fspath(erepo_dir), src, dst)
+
+    assert (tmp_dir / dst).is_file()
+    assert filecmp.cmp(
+        fspath(erepo_dir / src), fspath(tmp_dir / dst), shallow=False
+    )
+    assert tmp_dir.scm.repo.git.check_ignore(fspath(tmp_dir / dst))
 
 
-def test_import_rev(git, dvc_repo, erepo):
-    src = "version"
-    dst = src
+def test_import_git_dir(erepo_dir, tmp_dir, dvc, scm):
+    src = "some_directory"
+    dst = "some_directory_imported"
 
-    dvc_repo.imp(erepo.root_dir, src, dst, rev="branch")
+    erepo_dir.scm_gen({src: {"file.txt": "hello"}}, commit="add a dir")
 
-    assert os.path.exists(dst)
-    assert os.path.isfile(dst)
-    with open(dst, "r+") as fobj:
-        assert fobj.read() == "branch"
-    assert git.git.check_ignore(dst)
+    tmp_dir.dvc.imp(fspath(erepo_dir), src, dst)
+
+    assert (tmp_dir / dst).is_dir()
+    trees_equal(fspath(erepo_dir / src), fspath(tmp_dir / dst))
+    assert tmp_dir.scm.repo.git.check_ignore(fspath(tmp_dir / dst))
 
 
-def test_pull_imported_stage(dvc_repo, erepo):
-    src = erepo.FOO
-    dst = erepo.FOO + "_imported"
+def test_import_dir(tmp_dir, scm, dvc, erepo_dir, monkeypatch):
+    with monkeypatch.context() as m:
+        m.chdir(fspath(erepo_dir))
+        erepo_dir.dvc_gen({"dir": {"foo": "foo content"}}, commit="create dir")
 
-    dvc_repo.imp(erepo.root_dir, src, dst)
+    dvc.imp(fspath(erepo_dir), "dir", "dir_imported")
 
-    dst_stage = Stage.load(dvc_repo, "foo_imported.dvc")
+    assert os.path.isdir("dir_imported")
+    trees_equal(fspath(erepo_dir / "dir"), "dir_imported")
+    assert scm.repo.git.check_ignore("dir_imported")
+
+
+def test_import_non_cached(erepo_dir, tmp_dir, dvc, scm):
+    src = "non_cached_output"
+    dst = src + "_imported"
+
+    erepo_dir.dvc.run(
+        cmd="echo hello > {}".format(src),
+        outs_no_cache=[src],
+        cwd=fspath(erepo_dir),
+    )
+
+    erepo_dir.scm.add([fspath(erepo_dir / src)])
+    erepo_dir.scm.commit("add a non-cached output")
+
+    tmp_dir.dvc.imp(fspath(erepo_dir), src, dst)
+
+    assert (tmp_dir / dst).is_file()
+    assert filecmp.cmp(
+        fspath(erepo_dir / src), fspath(tmp_dir / dst), shallow=False
+    )
+    assert tmp_dir.scm.repo.git.check_ignore(dst)
+
+
+def test_import_rev(tmp_dir, scm, dvc, erepo_dir, monkeypatch):
+    with monkeypatch.context() as m:
+        m.chdir(fspath(erepo_dir))
+        erepo_dir.scm.checkout("new_branch", create_new=True)
+        erepo_dir.dvc_gen("foo", "foo content", commit="create foo on branch")
+        erepo_dir.scm.checkout("master")
+
+    dvc.imp(fspath(erepo_dir), "foo", "foo_imported", rev="new_branch")
+
+    assert (tmp_dir / "foo_imported").read_text() == "foo content"
+    assert scm.repo.git.check_ignore("foo_imported")
+
+
+def test_pull_imported_stage(tmp_dir, dvc, erepo_dir, monkeypatch):
+    with monkeypatch.context() as m:
+        m.chdir(fspath(erepo_dir))
+        erepo_dir.dvc_gen("foo", "foo content", commit="create foo")
+    dvc.imp(fspath(erepo_dir), "foo", "foo_imported")
+
+    dst_stage = Stage.load(dvc, "foo_imported.dvc")
     dst_cache = dst_stage.outs[0].cache_path
 
-    os.remove(dst)
+    os.remove("foo_imported")
     os.remove(dst_cache)
+    dvc.pull(["foo_imported.dvc"])
 
-    dvc_repo.pull(["foo_imported.dvc"])
-
-    assert os.path.isfile(dst)
+    assert os.path.isfile("foo_imported")
     assert os.path.isfile(dst_cache)
 
 
-def test_cache_type_is_properly_overridden(git, dvc_repo, erepo):
-    erepo.dvc.config.set(
-        Config.SECTION_CACHE, Config.SECTION_CACHE_TYPE, "symlink"
-    )
-    erepo.dvc.scm.add([erepo.dvc.config.config_file])
-    erepo.dvc.scm.commit("set source repo cache type to symlinks")
+def test_cache_type_is_properly_overridden(
+    tmp_dir, scm, dvc, erepo_dir, monkeypatch
+):
+    with monkeypatch.context() as m:
+        m.chdir(fspath(erepo_dir))
+        erepo_dir.dvc.config.set(
+            Config.SECTION_CACHE, Config.SECTION_CACHE_TYPE, "symlink"
+        )
+        erepo_dir.dvc.cache = Cache(erepo_dir.dvc)
+        erepo_dir.scm_add(
+            [erepo_dir.dvc.config.config_file],
+            "set source repo cache type to symlink",
+        )
+        erepo_dir.dvc_gen("foo", "foo content", "create foo")
+    assert System.is_symlink(erepo_dir / "foo")
 
-    src = erepo.FOO
-    dst = erepo.FOO + "_imported"
+    dvc.imp(fspath(erepo_dir), "foo", "foo_imported")
 
-    dvc_repo.imp(erepo.root_dir, src, dst)
-
-    assert not System.is_symlink(dst)
-    assert os.path.exists(dst)
-    assert os.path.isfile(dst)
-    assert filecmp.cmp(erepo.FOO, dst, shallow=False)
-    assert git.git.check_ignore(dst)
+    assert not System.is_symlink("foo_imported")
+    assert (tmp_dir / "foo_imported").read_text() == "foo content"
+    assert scm.repo.git.check_ignore("foo_imported")
 
 
-def test_pull_imported_directory_stage(dvc_repo, erepo):
-    src = erepo.DATA_DIR
-    dst = erepo.DATA_DIR + "_imported"
-    stage_file = dst + ".dvc"
+def test_pull_imported_directory_stage(tmp_dir, dvc, erepo_dir, monkeypatch):
+    with monkeypatch.context() as m:
+        m.chdir(fspath(erepo_dir))
+        erepo_dir.dvc_gen({"dir": {"foo": "foo content"}}, commit="create dir")
 
-    dvc_repo.imp(erepo.root_dir, src, dst)
+    dvc.imp(fspath(erepo_dir), "dir", "dir_imported")
 
-    shutil.rmtree(dst)
-    shutil.rmtree(dvc_repo.cache.local.cache_dir)
+    shutil.rmtree("dir_imported")
+    shutil.rmtree(dvc.cache.local.cache_dir)
 
-    dvc_repo.pull([stage_file])
+    dvc.pull(["dir_imported.dvc"])
 
-    assert os.path.exists(dst)
-    assert os.path.isdir(dst)
-    trees_equal(src, dst)
+    assert os.path.isdir("dir_imported")
+    trees_equal(fspath(erepo_dir / "dir"), "dir_imported")
 
 
-def test_download_error_pulling_imported_stage(dvc_repo, erepo):
-    src = erepo.FOO
-    dst = erepo.FOO + "_imported"
+def test_download_error_pulling_imported_stage(
+    tmp_dir, dvc, erepo_dir, monkeypatch
+):
+    with monkeypatch.context() as m:
+        m.chdir(fspath(erepo_dir))
+        erepo_dir.dvc_gen("foo", "foo content", commit="create foo")
+    dvc.imp(fspath(erepo_dir), "foo", "foo_imported")
 
-    dvc_repo.imp(erepo.root_dir, src, dst)
-
-    dst_stage = Stage.load(dvc_repo, "foo_imported.dvc")
+    dst_stage = Stage.load(dvc, "foo_imported.dvc")
     dst_cache = dst_stage.outs[0].cache_path
 
-    os.remove(dst)
+    os.remove("foo_imported")
     os.remove(dst_cache)
 
     with patch(
         "dvc.remote.RemoteLOCAL._download", side_effect=Exception
     ), pytest.raises(DownloadError):
-        dvc_repo.pull(["foo_imported.dvc"])
+        dvc.pull(["foo_imported.dvc"])
 
 
 @pytest.mark.parametrize("dname", [".", "dir", "dir/subdir"])
-def test_import_to_dir(dname, dvc_repo, erepo):
-    src = erepo.FOO
-
+def test_import_to_dir(dname, tmp_dir, dvc, erepo_dir, monkeypatch):
     makedirs(dname, exist_ok=True)
 
-    stage = dvc_repo.imp(erepo.root_dir, src, dname)
+    with monkeypatch.context() as m:
+        m.chdir(fspath(erepo_dir))
+        erepo_dir.dvc_gen("foo", "foo content", commit="create foo")
 
-    dst = os.path.join(dname, os.path.basename(src))
+    stage = dvc.imp(fspath(erepo_dir), "foo", dname)
+
+    dst = os.path.join(dname, "foo")
 
     assert stage.outs[0].fspath == os.path.abspath(dst)
     assert os.path.isdir(dname)
-    assert filecmp.cmp(erepo.FOO, dst, shallow=False)
+    assert (tmp_dir / dst).read_text() == "foo content"
 
 
-def test_pull_non_workspace(git, dvc_repo, erepo):
-    src = "version"
-    dst = src
+def test_pull_non_workspace(tmp_dir, scm, dvc, erepo_dir, monkeypatch):
+    with monkeypatch.context() as m:
+        m.chdir(fspath(erepo_dir))
+        erepo_dir.dvc_gen("foo", "master content", commit="create foo")
+        erepo_dir.scm.checkout("new_branch", create_new=True)
+        erepo_dir.dvc_gen("foo", "branch content", commit="modify foo")
 
-    stage = dvc_repo.imp(erepo.root_dir, src, dst, rev="branch")
-    dvc_repo.scm.add([stage.relpath])
-    dvc_repo.scm.commit("imported branch")
-    dvc_repo.scm.tag("ref-to-branch")
+    stage = dvc.imp(fspath(erepo_dir), "foo", "foo_imported", rev="new_branch")
+    tmp_dir.scm_add([stage.relpath], commit="imported branch")
+    dvc.scm.tag("ref-to-branch")
 
     # Overwrite via import
-    dvc_repo.imp(erepo.root_dir, src, dst, rev="master")
+    dvc.imp(fspath(erepo_dir), "foo", "foo_imported", rev="master")
 
     os.remove(stage.outs[0].cache_path)
-    dvc_repo.fetch(all_tags=True)
+    dvc.fetch(all_tags=True)
     assert os.path.exists(stage.outs[0].cache_path)
 
 
-def test_import_non_existing(dvc_repo, erepo):
+def test_import_non_existing(erepo_dir, tmp_dir, dvc):
+    with pytest.raises(PathMissingError):
+        tmp_dir.dvc.imp(fspath(erepo_dir), "invalid_output")
+    # https://github.com/iterative/dvc/pull/2837#discussion_r352123053
     with pytest.raises(NoOutputInExternalRepoError):
-        dvc_repo.imp(erepo.root_dir, "invalid_output")
+        tmp_dir.dvc.imp(fspath(erepo_dir), "/root/", "root")

tests/func/test_remote.py

@@ -1,14 +1,17 @@
+import errno
 import os
 import shutil
 
 import configobj
+import pytest
 from mock import patch
 
 from dvc.config import Config
 from dvc.main import main
 from dvc.path_info import PathInfo
-from dvc.remote import RemoteLOCAL
+from dvc.remote import RemoteLOCAL, RemoteConfig
 from dvc.remote.base import RemoteBASE
+from dvc.utils.compat import fspath
 from tests.basic_env import TestDvc
 from tests.remotes import get_local_url, get_local_storagepath
 
@@ -253,3 +256,21 @@ def test_partial_push_n_pull(dvc_repo, repo_dir, caplog):
 def get_last_exc(caplog):
     _, exc, _ = caplog.records[-2].exc_info
     return exc
+
+
+def test_raise_on_too_many_open_files(tmp_dir, dvc, tmp_path_factory, mocker):
+    storage = tmp_path_factory.mktemp("test_remote_base")
+    remote_config = RemoteConfig(dvc.config)
+    remote_config.add("local_remote", fspath(storage), default=True)
+
+    tmp_dir.dvc_gen({"file": "file content"})
+
+    mocker.patch.object(
+        RemoteLOCAL,
+        "_upload",
+        side_effect=OSError(errno.EMFILE, "Too many open files"),
+    )
+
+    with pytest.raises(OSError) as e:
+        dvc.push()
+        assert e.errno == errno.EMFILE

tests/func/test_repo.py

@@ -1,71 +1,73 @@
-from dvc.ignore import DvcIgnore
-from dvc.main import main
+import os
+
+from dvc.scm.git.tree import GitTree
+from dvc.cache import Cache
 from dvc.repo import Repo
-from dvc.scm.git import GitTree
-from dvc.scm.tree import WorkingTree
-from dvc.stage import Stage
-from tests.basic_env import TestDvcGit
+from dvc.system import System
+from dvc.utils.compat import fspath
 
 
-class TestCollect(TestDvcGit):
-    def setUp(self):
-        super(TestCollect, self).setUp()
-        self.dvc.add(self.FOO)
-        self.dvc.run(
-            deps=[self.FOO],
-            outs=[self.BAR],
-            cmd="python code.py {} {}".format(self.FOO, self.BAR),
+def test_destroy(tmp_dir, dvc):
+    dvc.config.set("cache", "type", "symlink")
+    dvc.cache = Cache(dvc)
+
+    tmp_dir.dvc_gen("file", "text")
+    tmp_dir.dvc_gen({"dir": {"file": "lorem", "subdir/file": "ipsum"}})
+
+    dvc.destroy()
+
+    # Remove all the files related to DVC
+    for path in [".dvc", "file.dvc", "dir.dvc"]:
+        assert not (tmp_dir / path).exists()
+
+    # Leave the rest of the files
+    for path in ["file", "dir/file", "dir/subdir/file"]:
+        assert (tmp_dir / path).is_file()
+
+    # Make sure that data was unprotected after `destroy`
+    for path in ["file", "dir", "dir/file", "dir/subdir", "dir/subdir/file"]:
+        assert not System.is_symlink(fspath(tmp_dir / path))
+
+
+def test_collect(tmp_dir, scm, dvc, run_copy):
+    def collect_outs(*args, **kwargs):
+        return set(
+            str(out.path_info)
+            for stage in dvc.collect(*args, **kwargs)
+            for out in stage.outs
         )
-        self.dvc.scm.add([".gitignore", self.FOO + ".dvc", self.BAR + ".dvc"])
-        self.dvc.scm.commit("foo.dvc and bar.dvc")
-        self.dvc.scm.checkout("new_branch", True)
-        self.dvc.run(
-            deps=[self.BAR],
-            outs=["buzz"],
-            cmd="python code.py {} {}".format(self.BAR, "buzz"),
-        )
-        self.dvc.scm.add([".gitignore", "buzz.dvc"])
-        self.dvc.scm.commit("add buzz")
-        self.dvc.scm.checkout("master")
 
-    def _check(self, branch, target, with_deps, expected):
-        if branch:
-            self.dvc.tree = GitTree(self.dvc.scm.repo, branch)
-        else:
-            self.dvc.tree = WorkingTree()
-        result = self.dvc.collect(target + ".dvc", with_deps=with_deps)
-        self.assertEqual([[str(j) for j in i.outs] for i in result], expected)
-        return result
+    tmp_dir.dvc_gen("foo", "foo")
+    run_copy("foo", "bar")
+    scm.add([".gitignore", "foo.dvc", "bar.dvc"])
+    scm.commit("Add foo and bar")
 
-    def test(self):
-        self._check("", self.BAR, True, [[self.FOO], [self.BAR]])
-        self._check("master", self.BAR, True, [[self.FOO], [self.BAR]])
-        self._check(
-            "new_branch", "buzz", True, [[self.FOO], [self.BAR], ["buzz"]]
-        )
-        result = self._check("new_branch", "buzz", False, [["buzz"]])
-        self.assertEqual([str(i) for i in result[0].deps], ["bar"])
+    scm.checkout("new-branch", create_new=True)
+
+    run_copy("bar", "buzz")
+    scm.add([".gitignore", "buzz.dvc"])
+    scm.commit("Add buzz")
+
+    assert collect_outs("bar.dvc", with_deps=True) == {"foo", "bar"}
+
+    dvc.tree = GitTree(scm.repo, "new-branch")
+
+    assert collect_outs("buzz.dvc", with_deps=True) == {"foo", "bar", "buzz"}
+    assert collect_outs("buzz.dvc", with_deps=False) == {"buzz"}
 
 
-class TestIgnore(TestDvcGit):
-    def _stage_name(self, file):
-        return file + Stage.STAGE_FILE_SUFFIX
+def test_stages(tmp_dir, dvc):
+    def stages():
+        return set(stage.relpath for stage in Repo(fspath(tmp_dir)).stages)
 
-    def test_should_not_gather_stage_files_from_ignored_dir(self):
-        ret = main(["add", self.FOO, self.BAR, self.DATA, self.DATA_SUB])
-        self.assertEqual(0, ret)
+    tmp_dir.dvc_gen({"file": "a", "dir/file": "b", "dir/subdir/file": "c"})
 
-        stages = self.dvc.stages
-        self.assertEqual(4, len(stages))
+    assert stages() == {
+        "file.dvc",
+        os.path.join("dir", "file.dvc"),
+        os.path.join("dir", "subdir", "file.dvc"),
+    }
 
-        self.create(DvcIgnore.DVCIGNORE_FILE, self.DATA_DIR)
+    tmp_dir.gen(".dvcignore", "dir")
 
-        self.dvc = Repo(self.dvc.root_dir)
-        stages = self.dvc.stages
-        self.assertEqual(2, len(stages))
-
-        stagenames = [s.relpath for s in stages]
-        self.assertIn(self._stage_name(self.FOO), stagenames)
-        self.assertIn(self._stage_name(self.BAR), stagenames)
-        self.assertNotIn(self._stage_name(self.DATA), stagenames)
-        self.assertNotIn(self._stage_name(self.DATA_SUB), stagenames)
+    assert stages() == {"file.dvc"}

tests/func/test_updater.py

@@ -1,52 +0,0 @@
-import os
-
-import mock
-import pytest
-
-from dvc.updater import Updater
-
-
-@pytest.fixture
-def updater(dvc_repo):
-    return Updater(dvc_repo.dvc_dir)
-
-
-def test_updater(updater):
-    # NOTE: only test on travis CRON to avoid generating too much logs
-    travis = os.getenv("TRAVIS") == "true"
-    if not travis:
-        return
-
-    cron = os.getenv("TRAVIS_EVENT_TYPE") == "cron"
-    if not cron:
-        return
-
-    env = os.environ.copy()
-    if env.get("CI"):
-        del env["CI"]
-
-    with mock.patch.dict(os.environ, env):
-        updater.check()
-        updater.check()
-        updater.check()
-
-
-def test_check_version_newer(updater):
-    updater.latest = "0.20.8"
-    updater.current = "0.21.0"
-
-    assert not updater._is_outdated()
-
-
-def test_check_version_equal(updater):
-    updater.latest = "0.20.8"
-    updater.current = "0.20.8"
-
-    assert not updater._is_outdated()
-
-
-def test_check_version_outdated(updater):
-    updater.latest = "0.21.0"
-    updater.current = "0.20.8"
-
-    assert updater._is_outdated()

tests/unit/remote/gdrive/conftest.py

@@ -1,9 +0,0 @@
-import pytest
-
-from dvc.remote.gdrive import RemoteGDrive
-
-
-@pytest.fixture
-def gdrive(repo):
-    ret = RemoteGDrive(None, {"url": "gdrive://root/data"})
-    return ret

tests/unit/remote/test_gdrive.py

@@ -3,7 +3,7 @@ from dvc.remote.gdrive import RemoteGDrive
 
 
 @mock.patch("dvc.remote.gdrive.RemoteGDrive.init_drive")
-def test_init_drive(repo):
+def test_init(repo):
     url = "gdrive://root/data"
     gdrive = RemoteGDrive(repo, {"url": url})
     assert str(gdrive.path_info) == url

tests/unit/test_repo.py

@@ -1,13 +1,6 @@
 import os
 
-from tests.basic_env import TestDvc
 
-
-class TestIsDvcInternal(TestDvc):
-    def test_return_false_on_non_dvc_file(self):
-        path = os.path.join("path", "to-non-.dvc", "file")
-        self.assertFalse(self.dvc.is_dvc_internal(path))
-
-    def test_return_true_on_dvc_internal_file(self):
-        path = os.path.join("path", "to", ".dvc", "file")
-        self.assertTrue(path)
+def test_is_dvc_internal(dvc):
+    assert dvc.is_dvc_internal(os.path.join("path", "to", ".dvc", "file"))
+    assert not dvc.is_dvc_internal(os.path.join("path", "to-non-.dvc", "file"))

tests/unit/test_updater.py

@@ -1,40 +1,61 @@
 import json
 import os
+import mock
+import pytest
 
 from dvc import __version__
 from dvc.updater import Updater
 
 
-class MockResponse(object):
-    def __init__(self, json_data, status_code):
-        self.json_data = json_data
-        self.status_code = status_code
-
-    def json(self):
-        return self.json_data
+@pytest.fixture
+def updater(dvc):
+    return Updater(dvc.dvc_dir)
 
 
-def mocked_requests_get(*args, **kwargs):
-    class MockResponse:
-        def __init__(self, json_data, status_code):
-            self.json_data = json_data
-            self.status_code = status_code
+@mock.patch("requests.get")
+def test_fetch(mock_get, updater):
+    mock_get.return_value.status_code = 200
+    mock_get.return_value.json.return_value = {"version": __version__}
 
-        def json(self):
-            return self.json_data
-
-    return MockResponse({"version": __version__}, 200)
-
-
-def test_fetch(dvc_repo, mocker):
-    updater = Updater(dvc_repo.dvc_dir)
     assert not os.path.exists(updater.updater_file)
 
-    mock_get = mocker.patch("requests.get", side_effect=mocked_requests_get)
     updater.fetch(detach=False)
-    mock_get.assert_called_once_with(Updater.URL, timeout=Updater.TIMEOUT_GET)
 
+    mock_get.assert_called_once_with(Updater.URL, timeout=Updater.TIMEOUT_GET)
     assert os.path.isfile(updater.updater_file)
+
     with open(updater.updater_file, "r") as fobj:
         info = json.load(fobj)
+
     assert info["version"] == __version__
+
+
+@pytest.mark.parametrize(
+    "latest, current, result",
+    [
+        ("0.20.8", "0.21.0", False),
+        ("0.20.8", "0.20.8", False),
+        ("0.20.8", "0.19.0", True),
+    ],
+)
+def test_is_outdated(latest, current, result, updater):
+    updater.latest = latest
+    updater.current = current
+
+    assert updater._is_outdated() == result
+
+
+@pytest.mark.skipif(
+    os.getenv("TRAVIS_EVENT_TYPE") != "cron",
+    reason="Only run on travis CRON to avoid generating too much logs",
+)
+@mock.patch("dvc.updater.Updater._check")
+def test_check(mock_check, updater, monkeypatch):
+    monkeypatch.delenv("CI", None)
+    monkeypatch.setenv("DVC_TEST", False)
+
+    updater.check()
+    updater.check()
+    updater.check()
+
+    assert mock_check.call_count == 3