dvc: bump to 0.50.1

Related https://github.com/iterative/dvc/issues/2191

.appveyor.yml

@@ -1,8 +1,3 @@
-branches:
-  only:
-    - master
-    - devel
-
 cache:
   - C:\ProgramData\chocolatey\bin -> appveyor.yml
   - C:\ProgramData\chocolatey\lib -> appveyor.yml
@@ -27,42 +22,45 @@ environment:
   matrix:
 
     - PYTHON: "C:\\Python27"
-      PYTHON_VERSION: "2.7.x" # currently 2.7.9
+      PYTHON_VERSION: "2.7.x"
       PYTHON_ARCH: "32"
       RUBY: C:\Ruby25\bin
 
-# Disabled as unnecessary
-#    - PYTHON: "C:\\Python27-x64"
-#      PYTHON_VERSION: "2.7.x" # currently 2.7.9
-#      PYTHON_ARCH: "64"
+    - PYTHON: "C:\\Python37"
+      PYTHON_VERSION: "3.7.x"
+      PYTHON_ARCH: "32"
+      RUBY: C:\Ruby25\bin
 
 install:
-  - SET PATH=%RUBY%;%PATH%
-  - git clone -q https://github.com/nanaya-tachibana/oss-emulator.git
+  - cinst awscli
+  - cinst wget
+  - cinst gcloudsdk
+  - cinst gsutil
+  - cinst openssl.light --version 1.1.1
+  - refreshenv
+  - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%RUBY%;%PATH%"
+  - git clone -q https://github.com/iterative/oss-emulator.git
   - gem install thor builder
   - ps: cd oss-emulator
   - ps: $OSSProcess = Start-Process -FilePath ruby -ArgumentList "bin\emulator -r store -p 50004" -PassThru -NoNewWindow
   - ps: cd ..
-  - npm install -g azurite
+  # NOTE: azurite 3 on py3 suffers from bug [1], so we are using an older
+  # tried-and-true version instead.
+  #
+  # [1] https://github.com/Azure/Azurite/issues/215
+  - npm install -g [email protected]
   - ps: $AzuriteProcess = Start-Process azurite-blob -PassThru
-  - cinst wget
-  - cinst awscli
-  - cinst gcloudsdk
-  - refreshenv
-  - cinst gsutil
-  - cinst openssl.light --version 1.1.1
   - python -m pip install -U pip setuptools wheel
-  - pip install -r requirements.txt
-  - python setup.py install
+  - pip install .[all,tests]
 
 build: false
 
 before_test:
+  - pip uninstall -y awscli # Using awscli from chocolatey
   - aws configure set aws_access_key_id "%aws_access_key_id%"
   - aws configure set aws_secret_access_key "%aws_secret_access_key%"
   - aws configure set region us-east-2
   - openssl enc -d -aes-256-cbc -md md5 -k "%GCP_CREDS%" -in scripts\ci\gcp-creds.json.enc -out scripts\ci\gcp-creds.json & exit 0
-  - pip install -r tests/requirements.txt
 
 test_script:
   - python -mtests
@@ -87,3 +85,5 @@ deploy:
     secure: 25c1pNoXWbtTwUkKLcThwcl2A2xG8oXHsPuD73HVbQ7SRy9X6HzkCc2sXzhpfRtF
   on:
     APPVEYOR_REPO_TAG: true
+    PYTHON_VERSION: "3.7.x"
+    PYTHON_ARCH: "32"

.deepsource.toml

@@ -0,0 +1,15 @@
+version = 1
+
+test_patterns = [
+  '*/tests/*'
+]
+
+exclude_patterns = [
+  
+]
+
+[[analyzers]]
+name = 'python'
+enabled = true
+runtime_version = '3.x.x'
+

.dvc/.gitignore

@@ -6,3 +6,4 @@
 /state-journal
 /state-wal
 /cache
+/pkg

.gitignore

@@ -28,3 +28,11 @@ innosetup/config.ini
 
 pip-wheel-metadata/
 .vscode/
+
+azurite
+env.sh
+tests/remotes_env
+scripts/ci/gcp-creds.json
+.gcp-creds.json
+
+*~

.travis.yml

@@ -48,7 +48,7 @@ matrix:
     language: ruby
     rvm: 2.4.0
     env:
-    - PYTHON_VER=3.6.2
+    - PYTHON_VER=3.7.3
     - PYTHON_CONFIGURE_OPTS="--enable-framework"
   - if: type = cron OR tag is present
     os: osx

README.rst

@@ -31,8 +31,8 @@
 
 |
 
-**Data Science Version Control** or **DVC** is an **open-source** tool for data science and
-machine learning projects. Key features:
+**Data Version Control** or **DVC** is an **open-source** tool for data science and machine
+learning projects. Key features:
 
 #. simple **command line** Git-like experience. Does not require installing and maintaining
    any databases. Does not depend on any proprietary online services;

dvc/__init__.py

@@ -19,18 +19,3 @@ dvc.logger.setup()
 # we can simply ignore them so that they don't show up when you are using dvc.
 warnings.filterwarnings("ignore", message="numpy.dtype size changed")
 warnings.filterwarnings("ignore", message="numpy.ufunc size changed")
-
-# Ignore paramiko's warning: https://github.com/paramiko/paramiko/issues/1386.
-# This only affects paramiko 2.4.2 and should be fixed in the next version.
-# Cryptography developers decided that it is a brilliant idea not to inherit
-# from DeprecationWarning [1] because it is invisible by default, and decided
-# to spam everyone instead. So it is their fault and not paramiko's.
-#
-# [1] https://github.com/pyca/cryptography/blob/2.6.1/src/cryptography/
-#     utils.py#L14
-try:
-    from cryptography.utils import CryptographyDeprecationWarning
-
-    warnings.simplefilter("ignore", CryptographyDeprecationWarning)
-except ImportError:
-    pass

dvc/api.py

@@ -0,0 +1,74 @@
+from contextlib import contextmanager
+import os
+import tempfile
+
+try:
+    from contextlib import _GeneratorContextManager as GCM
+except ImportError:
+    from contextlib import GeneratorContextManager as GCM
+
+from dvc.utils import remove
+from dvc.utils.compat import urlparse
+from dvc.repo import Repo
+from dvc.external_repo import ExternalRepo
+
+
+def get_url(path, repo=None, remote=None):
+    """Returns an url of a resource specified by path in repo"""
+    with _make_repo(repo) as _repo:
+        abspath = os.path.join(_repo.root_dir, path)
+        out, = _repo.find_outs_by_path(abspath)
+        remote_obj = _repo.cloud.get_remote(remote)
+        return str(remote_obj.checksum_to_path_info(out.checksum))
+
+
+def open(path, repo=None, remote=None, mode="r", encoding=None):
+    """Opens a specified resource as a file descriptor"""
+    args = (path,)
+    kwargs = {
+        "repo": repo,
+        "remote": remote,
+        "mode": mode,
+        "encoding": encoding,
+    }
+    return _OpenContextManager(_open, args, kwargs)
+
+
+class _OpenContextManager(GCM):
+    def __init__(self, func, args, kwds):
+        self.gen = func(*args, **kwds)
+        self.func, self.args, self.kwds = func, args, kwds
+
+    def __getattr__(self, name):
+        raise AttributeError(
+            "dvc.api.open() should be used in a with statement"
+        )
+
+
+def _open(path, repo=None, remote=None, mode="r", encoding=None):
+    with _make_repo(repo) as _repo:
+        abspath = os.path.join(_repo.root_dir, path)
+        with _repo.open(
+            abspath, remote=remote, mode=mode, encoding=encoding
+        ) as fd:
+            yield fd
+
+
+def read(path, repo=None, remote=None, mode="r", encoding=None):
+    """Read a specified resource into string"""
+    with open(path, repo, remote=remote, mode=mode, encoding=encoding) as fd:
+        return fd.read()
+
+
+@contextmanager
+def _make_repo(repo_url):
+    if not repo_url or urlparse(repo_url).scheme == "":
+        yield Repo(repo_url)
+    else:
+        tmp_dir = tempfile.mkdtemp("dvc-repo")
+        try:
+            ext_repo = ExternalRepo(tmp_dir, url=repo_url)
+            ext_repo.install()
+            yield ext_repo.repo
+        finally:
+            remove(tmp_dir)

dvc/cache.py

@@ -7,6 +7,20 @@ import os
 from dvc.config import Config
 
 
+class CacheConfig(object):
+    def __init__(self, config):
+        self.config = config
+
+    def set_dir(self, dname, level=None):
+        from dvc.remote.config import RemoteConfig
+
+        configobj = self.config.get_configobj(level)
+        path = RemoteConfig.resolve_path(dname, configobj.filename)
+        self.config.set(
+            Config.SECTION_CACHE, Config.SECTION_CACHE_DIR, path, level=level
+        )
+
+
 class Cache(object):
     """Class that manages cache locations of a dvc repo.
 
@@ -42,7 +56,7 @@ class Cache(object):
                 Config.SECTION_CACHE_PROTECTED: protected,
             }
 
-        self.local = Remote(repo, settings)
+        self.local = Remote(repo, **settings)
         self.s3 = self._get_remote(config, Config.SECTION_CACHE_S3)
         self.gs = self._get_remote(config, Config.SECTION_CACHE_GS)
         self.ssh = self._get_remote(config, Config.SECTION_CACHE_SSH)
@@ -87,5 +101,4 @@ class Cache(object):
         if not remote:
             return None
 
-        settings = self.repo.config.get_remote_settings(remote)
-        return Remote(self.repo, settings)
+        return Remote(self.repo, name=remote)

dvc/cli.py

@@ -9,7 +9,8 @@ import logging
 
 from dvc.command.base import fix_subparsers
 import dvc.command.init as init
-import dvc.command.pkg as pkg
+import dvc.command.get as get
+import dvc.command.get_url as get_url
 import dvc.command.destroy as destroy
 import dvc.command.remove as remove
 import dvc.command.move as move
@@ -20,6 +21,7 @@ import dvc.command.data_sync as data_sync
 import dvc.command.gc as gc
 import dvc.command.add as add
 import dvc.command.imp as imp
+import dvc.command.imp_url as imp_url
 import dvc.command.config as config
 import dvc.command.checkout as checkout
 import dvc.command.remote as remote
@@ -41,7 +43,8 @@ logger = logging.getLogger(__name__)
 
 COMMANDS = [
     init,
-    pkg,
+    get,
+    get_url,
     destroy,
     add,
     remove,
@@ -52,6 +55,7 @@ COMMANDS = [
     data_sync,
     gc,
     imp,
+    imp_url,
     config,
     checkout,
     remote,

dvc/command/add.py

@@ -54,7 +54,7 @@ def add_parser(subparsers, parent_parser):
         help="Don't put files/directories into cache.",
     )
     add_parser.add_argument(
-        "-f", "--file", help="Specify name of the DVC file it generates."
+        "-f", "--file", help="Specify name of the DVC-file it generates."
     )
     add_parser.add_argument(
         "targets", nargs="+", help="Input files/directories."

dvc/command/base.py

@@ -38,10 +38,13 @@ def append_doc_link(help_message, path):
 class CmdBase(object):
     def __init__(self, args):
         from dvc.repo import Repo
+        from dvc.updater import Updater
 
         self.repo = Repo()
         self.config = self.repo.config
         self.args = args
+        updater = Updater(self.repo.dvc_dir)
+        updater.check()
 
     @property
     def default_targets(self):

dvc/command/cache.py

@@ -3,18 +3,18 @@ from __future__ import unicode_literals
 import argparse
 
 from dvc.command.base import fix_subparsers, append_doc_link
-from dvc.command.remote import CmdRemoteAdd
 from dvc.command.config import CmdConfig
+from dvc.cache import CacheConfig
 
 
 class CmdCacheDir(CmdConfig):
-    def run(self):
-        self.args.name = "cache.dir"
-        self.args.value = CmdRemoteAdd.resolve_path(
-            self.args.value, self.configobj.filename
-        )
+    def __init__(self, args):
+        super(CmdCacheDir, self).__init__(args)
+        self.cache_config = CacheConfig(self.config)
 
-        return super(CmdCacheDir, self).run()
+    def run(self):
+        self.cache_config.set_dir(self.args.value, level=self.args.level)
+        return 0
 
 
 def add_parser(subparsers, parent_parser):

dvc/command/checkout.py

@@ -30,13 +30,6 @@ def add_parser(subparsers, parent_parser):
         help=CHECKOUT_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    checkout_parser.add_argument(
-        "-d",
-        "--with-deps",
-        action="store_true",
-        default=False,
-        help="Checkout all dependencies of the specified target.",
-    )
     checkout_parser.add_argument(
         "-f",
         "--force",
@@ -44,6 +37,13 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Do not prompt when removing working directory files.",
     )
+    checkout_parser.add_argument(
+        "-d",
+        "--with-deps",
+        action="store_true",
+        default=False,
+        help="Checkout all dependencies of the specified target.",
+    )
     checkout_parser.add_argument(
         "-R",
         "--recursive",
@@ -51,5 +51,5 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Checkout all subdirectories of the specified directory.",
     )
-    checkout_parser.add_argument("targets", nargs="*", help="DVC files.")
+    checkout_parser.add_argument("targets", nargs="*", help="DVC-files.")
     checkout_parser.set_defaults(func=CmdCheckout)

dvc/command/commit.py

@@ -34,7 +34,7 @@ class CmdCommit(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    COMMIT_HELP = "Save changed data to cache and update DVC files."
+    COMMIT_HELP = "Save changed data to cache and update DVC-files."
 
     commit_parser = subparsers.add_parser(
         "commit",
@@ -65,6 +65,6 @@ def add_parser(subparsers, parent_parser):
         help="Commit cache for subdirectories of the specified directory.",
     )
     commit_parser.add_argument(
-        "targets", nargs="*", default=None, help="DVC files."
+        "targets", nargs="*", default=None, help="DVC-files."
     )
     commit_parser.set_defaults(func=CmdCommit)

dvc/command/config.py

@@ -1,12 +1,10 @@
 from __future__ import unicode_literals
 
-import os
 import argparse
 import logging
 
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.config import Config
-from dvc.exceptions import DvcException
 
 
 logger = logging.getLogger(__name__)
@@ -14,87 +12,48 @@ logger = logging.getLogger(__name__)
 
 class CmdConfig(CmdBaseNoRepo):
     def __init__(self, args):
-        from dvc.repo import Repo, NotDvcRepoError
-
         super(CmdConfig, self).__init__(args)
 
-        try:
-            dvc_dir = os.path.join(Repo.find_root(), Repo.DVC_DIR)
-            saved_exc = None
-        except NotDvcRepoError as exc:
-            dvc_dir = None
-            saved_exc = exc
-
-        self.config = Config(dvc_dir, validate=False)
-        if self.args.system:
-            self.configobj = self.config._system_config
-        elif self.args.glob:
-            self.configobj = self.config._global_config
-        elif self.args.local:
-            if dvc_dir is None:
-                raise saved_exc
-            self.configobj = self.config._local_config
-        else:
-            if dvc_dir is None:
-                raise saved_exc
-            self.configobj = self.config._repo_config
-
-    def _unset(self, section, opt=None, configobj=None):
-        if configobj is None:
-            configobj = self.configobj
-
-        try:
-            self.config.unset(configobj, section, opt)
-            self.config.save(configobj)
-        except DvcException:
-            logger.exception("failed to unset '{}'".format(self.args.name))
-            return 1
-        return 0
-
-    def _show(self, section, opt):
-        try:
-            self.config.show(self.configobj, section, opt)
-        except DvcException:
-            logger.exception("failed to show '{}'".format(self.args.name))
-            return 1
-        return 0
-
-    def _set(self, section, opt, value):
-        try:
-            self.config.set(self.configobj, section, opt, value)
-            self.config.save(self.configobj)
-        except DvcException:
-            logger.exception(
-                "failed to set '{}.{}' to '{}'".format(section, opt, value)
-            )
-            return 1
-        return 0
+        self.config = Config(validate=False)
 
     def run(self):
         section, opt = self.args.name.lower().strip().split(".", 1)
 
         if self.args.unset:
-            return self._unset(section, opt)
+            self.config.unset(section, opt, level=self.args.level)
         elif self.args.value is None:
-            return self._show(section, opt)
+            logger.info(self.config.get(section, opt, level=self.args.level))
         else:
-            return self._set(section, opt, self.args.value)
+            self.config.set(
+                section, opt, self.args.value, level=self.args.level
+            )
+
+        return 0
 
 
 parent_config_parser = argparse.ArgumentParser(add_help=False)
 parent_config_parser.add_argument(
     "--global",
-    dest="glob",
-    action="store_true",
-    default=False,
+    dest="level",
+    action="store_const",
+    const=Config.LEVEL_GLOBAL,
     help="Use global config.",
 )
 parent_config_parser.add_argument(
-    "--system", action="store_true", default=False, help="Use system config."
+    "--system",
+    dest="level",
+    action="store_const",
+    const=Config.LEVEL_SYSTEM,
+    help="Use system config.",
 )
 parent_config_parser.add_argument(
-    "--local", action="store_true", default=False, help="Use local config."
+    "--local",
+    dest="level",
+    action="store_const",
+    const=Config.LEVEL_LOCAL,
+    help="Use local config.",
 )
+parent_config_parser.set_defaults(level=Config.LEVEL_REPO)
 
 
 def add_parser(subparsers, parent_parser):

dvc/command/data_sync.py

@@ -4,6 +4,7 @@ import argparse
 import logging
 
 from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException
 
 
 logger = logging.getLogger(__name__)
@@ -45,7 +46,7 @@ class CmdDataPull(CmdDataBase):
                 force=self.args.force,
                 recursive=self.args.recursive,
             )
-        except Exception:
+        except DvcException:
             logger.exception("failed to pull data from the cloud")
             return 1
         self.check_up_to_date(processed_files_count)
@@ -65,7 +66,7 @@ class CmdDataPush(CmdDataBase):
                 with_deps=self.args.with_deps,
                 recursive=self.args.recursive,
             )
-        except Exception:
+        except DvcException:
             logger.exception("failed to push data to the cloud")
             return 1
         self.check_up_to_date(processed_files_count)
@@ -85,7 +86,7 @@ class CmdDataFetch(CmdDataBase):
                 with_deps=self.args.with_deps,
                 recursive=self.args.recursive,
             )
-        except Exception:
+        except DvcException:
             logger.exception("failed to fetch data from the cloud")
             return 1
         self.check_up_to_date(processed_files_count)
@@ -113,7 +114,13 @@ def shared_parent_parser():
         help="Show checksums instead of file names.",
     )
     shared_parent_parser.add_argument(
-        "targets", nargs="*", default=None, help="DVC files."
+        "targets",
+        nargs="*",
+        default=None,
+        help=(
+            "Limit the scope to these DVC-files."
+            " With -R a directory to search DVC-files in can be specified."
+        ),
     )
 
     return shared_parent_parser
@@ -149,13 +156,6 @@ def add_parser(subparsers, _parent_parser):
         default=False,
         help="Fetch cache for all tags.",
     )
-    pull_parser.add_argument(
-        "-d",
-        "--with-deps",
-        action="store_true",
-        default=False,
-        help="Fetch cache for all dependencies of the specified target.",
-    )
     pull_parser.add_argument(
         "-f",
         "--force",
@@ -163,6 +163,13 @@ def add_parser(subparsers, _parent_parser):
         default=False,
         help="Do not prompt when removing working directory files.",
     )
+    pull_parser.add_argument(
+        "-d",
+        "--with-deps",
+        action="store_true",
+        default=False,
+        help="Fetch cache for all dependencies of the specified target.",
+    )
     pull_parser.add_argument(
         "-R",
         "--recursive",

dvc/command/destroy.py

@@ -35,7 +35,7 @@ class CmdDestroy(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    DESTROY_HELP = "Remove DVC files, local DVC config and data cache."
+    DESTROY_HELP = "Remove DVC-files, local DVC config and data cache."
 
     destroy_parser = subparsers.add_parser(
         "destroy",

dvc/command/diff.py

@@ -5,10 +5,10 @@ import argparse
 import humanize
 import inflect
 import logging
+from funcy import compact
 
 from dvc.exceptions import DvcException
 from dvc.command.base import CmdBase, append_doc_link
-from dvc.utils.collections import compact
 import dvc.repo.diff as diff
 
 
@@ -16,7 +16,8 @@ logger = logging.getLogger(__name__)
 
 
 class CmdDiff(CmdBase):
-    def _print_size(self, size):
+    @staticmethod
+    def _print_size(size):
         if size < 0:
             change = "decreased by {}"
         elif size > 0:
@@ -26,14 +27,16 @@ class CmdDiff(CmdBase):
         natur_size = humanize.naturalsize(abs(size))
         return change.format(natur_size)
 
-    def _get_md5_string(self, sign, file_name, checksum):
+    @staticmethod
+    def _get_md5_string(sign, file_name, checksum):
         sample_msg = ""
         if file_name:
             sample_msg = "{}{} with md5 {}\n"
             sample_msg = sample_msg.format(sign, file_name, checksum)
         return sample_msg
 
-    def _get_dir_changes(self, dct):
+    @classmethod
+    def _get_dir_changes(cls, dct):
         engine = inflect.engine()
         changes_msg = (
             "{} {} not changed, {} {} modified, {} {} added, "
@@ -48,11 +51,12 @@ class CmdDiff(CmdBase):
             engine.plural("file", dct[diff.DIFF_NEW]),
             dct[diff.DIFF_DEL],
             engine.plural("file", dct[diff.DIFF_DEL]),
-            self._print_size(dct[diff.DIFF_SIZE]),
+            cls._print_size(dct[diff.DIFF_SIZE]),
         )
         return changes_msg
 
-    def _get_file_changes(self, dct):
+    @classmethod
+    def _get_file_changes(cls, dct):
         if (
             dct.get(diff.DIFF_OLD_FILE)
             and dct.get(diff.DIFF_NEW_FILE)
@@ -69,19 +73,21 @@ class CmdDiff(CmdBase):
             )
         else:
             msg = "file was modified, file size {}".format(
-                self._print_size(dct[diff.DIFF_SIZE])
+                cls._print_size(dct[diff.DIFF_SIZE])
             )
         return msg
 
-    def _get_royal_changes(self, dct):
+    @classmethod
+    def _get_royal_changes(cls, dct):
         if dct[diff.DIFF_SIZE] != diff.DIFF_SIZE_UNKNOWN:
             if dct.get("is_dir"):
-                return self._get_dir_changes(dct)
+                return cls._get_dir_changes(dct)
             else:
-                return self._get_file_changes(dct)
+                return cls._get_file_changes(dct)
         return "size is ?"
 
-    def _show(self, diff_dct):
+    @classmethod
+    def _show(cls, diff_dct):
         msg = "dvc diff from {} to {}".format(
             diff_dct[diff.DIFF_A_REF], diff_dct[diff.DIFF_B_REF]
         )
@@ -90,18 +96,18 @@ class CmdDiff(CmdBase):
             return
         for dct in diff_dct[diff.DIFF_LIST]:
             msg += "\n\ndiff for '{}'\n".format(dct[diff.DIFF_TARGET])
-            msg += self._get_md5_string(
+            msg += cls._get_md5_string(
                 "-",
                 dct.get(diff.DIFF_OLD_FILE),
                 dct.get(diff.DIFF_OLD_CHECKSUM),
             )
-            msg += self._get_md5_string(
+            msg += cls._get_md5_string(
                 "+",
                 dct.get(diff.DIFF_NEW_FILE),
                 dct.get(diff.DIFF_NEW_CHECKSUM),
             )
             msg += "\n"
-            msg += self._get_royal_changes(dct)
+            msg += cls._get_royal_changes(dct)
         logger.info(msg)
         return msg
 

dvc/command/get.py

@@ -0,0 +1,52 @@
+from __future__ import unicode_literals
+
+import argparse
+import logging
+
+from dvc.repo import Repo
+from dvc.exceptions import DvcException
+from .base import CmdBaseNoRepo, append_doc_link
+
+
+logger = logging.getLogger(__name__)
+
+
+class CmdGet(CmdBaseNoRepo):
+    def run(self):
+        try:
+            Repo.get(
+                self.args.url,
+                path=self.args.path,
+                out=self.args.out,
+                rev=self.args.rev,
+            )
+            return 0
+        except DvcException:
+            logger.exception(
+                "failed to get '{}' from '{}'".format(
+                    self.args.path, self.args.url
+                )
+            )
+            return 1
+
+
+def add_parser(subparsers, parent_parser):
+    GET_HELP = "Download data from DVC repository."
+    get_parser = subparsers.add_parser(
+        "get",
+        parents=[parent_parser],
+        description=append_doc_link(GET_HELP, "get"),
+        help=GET_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    get_parser.add_argument(
+        "url", help="DVC repository URL to download data from."
+    )
+    get_parser.add_argument("path", help="Path to data within DVC repository.")
+    get_parser.add_argument(
+        "-o", "--out", nargs="?", help="Destination path to put data to."
+    )
+    get_parser.add_argument(
+        "--rev", nargs="?", help="DVC repository git revision."
+    )
+    get_parser.set_defaults(func=CmdGet)

dvc/command/get_url.py

@@ -0,0 +1,39 @@
+from __future__ import unicode_literals
+
+import argparse
+import logging
+
+from dvc.repo import Repo
+from dvc.exceptions import DvcException
+from .base import CmdBaseNoRepo, append_doc_link
+
+
+logger = logging.getLogger(__name__)
+
+
+class CmdGetUrl(CmdBaseNoRepo):
+    def run(self):
+        try:
+            Repo.get_url(self.args.url, out=self.args.out)
+            return 0
+        except DvcException:
+            logger.exception("failed to get '{}'".format(self.args.url))
+            return 1
+
+
+def add_parser(subparsers, parent_parser):
+    GET_HELP = "Download or copy files from URL."
+    get_parser = subparsers.add_parser(
+        "get-url",
+        parents=[parent_parser],
+        description=append_doc_link(GET_HELP, "get-url"),
+        help=GET_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    get_parser.add_argument(
+        "url", help="See `dvc import-url -h` for full list of supported URLs."
+    )
+    get_parser.add_argument(
+        "out", nargs="?", help="Destination path to put data to."
+    )
+    get_parser.set_defaults(func=CmdGetUrl)

dvc/command/imp.py

@@ -1,10 +1,8 @@
 from __future__ import unicode_literals
 
 import argparse
-import os
 import logging
 
-from dvc.utils.compat import urlparse
 from dvc.exceptions import DvcException
 from dvc.command.base import CmdBase, append_doc_link
 
@@ -14,19 +12,22 @@ logger = logging.getLogger(__name__)
 
 class CmdImport(CmdBase):
     def run(self):
+        logger.warning(
+            "This is a beta feature! Do not use in the production! Old "
+            "`dvc import` is now called `dvc import-url`, please use "
+            "it instead."
+        )
         try:
-            default_out = os.path.basename(urlparse(self.args.url).path)
-
-            out = self.args.out or default_out
-
             self.repo.imp(
-                self.args.url, out, self.args.resume, fname=self.args.file
+                self.args.url,
+                self.args.path,
+                out=self.args.out,
+                rev=self.args.rev,
             )
         except DvcException:
             logger.exception(
-                "failed to import {}. You could also try downloading "
-                "it manually and adding it with `dvc add` command.".format(
-                    self.args.url
+                "failed to import '{}' from '{}'.".format(
+                    self.args.path, self.args.url
                 )
             )
             return 1
@@ -34,7 +35,10 @@ class CmdImport(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    IMPORT_HELP = "Download or copy files from URL and take under DVC control."
+    IMPORT_HELP = (
+        "BETA: Download data from DVC repository and take it under DVC "
+        "control."
+    )
 
     import_parser = subparsers.add_parser(
         "import",
@@ -43,28 +47,14 @@ def add_parser(subparsers, parent_parser):
         help=IMPORT_HELP,
         formatter_class=argparse.RawTextHelpFormatter,
     )
+    import_parser.add_argument("url", help="DVC repository URL.")
     import_parser.add_argument(
-        "url",
-        help="Supported urls:\n"
-        "/path/to/file\n"
-        "C:\\\\path\\to\\file\n"
-        "https://example.com/path/to/file\n"
-        "s3://bucket/path/to/file\n"
-        "gs://bucket/path/to/file\n"
-        "hdfs://example.com/path/to/file\n"
-        "ssh://example.com:/path/to/file\n"
-        "remote://myremote/path/to/file (see `dvc remote`)",
+        "path", help="Path to data within DVC repository."
     )
     import_parser.add_argument(
-        "--resume",
-        action="store_true",
-        default=False,
-        help="Resume previously started download.",
+        "-o", "--out", nargs="?", help="Destination path to put data to."
     )
     import_parser.add_argument(
-        "out", nargs="?", help="Destination path to put files to."
-    )
-    import_parser.add_argument(
-        "-f", "--file", help="Specify name of the DVC file it generates."
+        "--rev", nargs="?", help="DVC repository git revision."
     )
     import_parser.set_defaults(func=CmdImport)

dvc/command/imp_url.py

@@ -0,0 +1,69 @@
+from __future__ import unicode_literals
+
+import argparse
+import logging
+
+from dvc.exceptions import DvcException
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
+
+
+class CmdImportUrl(CmdBase):
+    def run(self):
+        try:
+            self.repo.imp_url(
+                self.args.url,
+                out=self.args.out,
+                resume=self.args.resume,
+                fname=self.args.file,
+            )
+        except DvcException:
+            logger.exception(
+                "failed to import {}. You could also try downloading "
+                "it manually and adding it with `dvc add` command.".format(
+                    self.args.url
+                )
+            )
+            return 1
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    IMPORT_HELP = (
+        "Download or copy file from URL and take it under DVC control."
+    )
+
+    import_parser = subparsers.add_parser(
+        "import-url",
+        parents=[parent_parser],
+        description=append_doc_link(IMPORT_HELP, "import-url"),
+        help=IMPORT_HELP,
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    import_parser.add_argument(
+        "url",
+        help="Supported urls:\n"
+        "/path/to/file\n"
+        "C:\\\\path\\to\\file\n"
+        "https://example.com/path/to/file\n"
+        "s3://bucket/path/to/file\n"
+        "gs://bucket/path/to/file\n"
+        "hdfs://example.com/path/to/file\n"
+        "ssh://example.com:/path/to/file\n"
+        "remote://myremote/path/to/file (see `dvc remote`)",
+    )
+    import_parser.add_argument(
+        "--resume",
+        action="store_true",
+        default=False,
+        help="Resume previously started download.",
+    )
+    import_parser.add_argument(
+        "out", nargs="?", help="Destination path to put files to."
+    )
+    import_parser.add_argument(
+        "-f", "--file", help="Specify name of the DVC-file it generates."
+    )
+    import_parser.set_defaults(func=CmdImportUrl)

dvc/command/lock.py

@@ -37,7 +37,7 @@ class CmdUnlock(CmdLockBase):
 
 
 def add_parser(subparsers, parent_parser):
-    LOCK_HELP = "Lock DVC file."
+    LOCK_HELP = "Lock DVC-file."
     lock_parser = subparsers.add_parser(
         "lock",
         parents=[parent_parser],
@@ -45,10 +45,10 @@ def add_parser(subparsers, parent_parser):
         help=LOCK_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    lock_parser.add_argument("targets", nargs="+", help="DVC files.")
+    lock_parser.add_argument("targets", nargs="+", help="DVC-files.")
     lock_parser.set_defaults(func=CmdLock)
 
-    UNLOCK_HELP = "Unlock DVC file."
+    UNLOCK_HELP = "Unlock DVC-file."
     unlock_parser = subparsers.add_parser(
         "unlock",
         parents=[parent_parser],
@@ -56,5 +56,5 @@ def add_parser(subparsers, parent_parser):
         help=UNLOCK_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    unlock_parser.add_argument("targets", nargs="+", help="DVC files.")
+    unlock_parser.add_argument("targets", nargs="+", help="DVC-files.")
     unlock_parser.set_defaults(func=CmdUnlock)

dvc/command/move.py

@@ -27,7 +27,7 @@ def add_parser(subparsers, parent_parser):
     MOVE_HELP = "Rename or move a DVC controlled data file or a directory."
     MOVE_DESCRIPTION = (
         "Rename or move a DVC controlled data file or a directory.\n"
-        "It renames and modifies the corresponding DVC file to reflect the"
+        "It renames and modifies the corresponding DVC-file to reflect the"
         " changes."
     )
 

dvc/command/pipeline.py

@@ -1,9 +1,9 @@
 from __future__ import unicode_literals
 
+from dvc.utils import relpath
 from dvc.utils.compat import str
 
 import argparse
-import os
 import logging
 
 from dvc.exceptions import DvcException
@@ -21,7 +21,7 @@ class CmdPipelineShow(CmdBase):
         stage = Stage.load(self.repo, target)
         G = self.repo.graph()[0]
         stages = networkx.get_node_attributes(G, "stage")
-        node = os.path.relpath(stage.path, self.repo.root_dir)
+        node = relpath(stage.path, self.repo.root_dir)
         nodes = networkx.dfs_postorder_nodes(G, node)
 
         if locked:
@@ -29,6 +29,8 @@ class CmdPipelineShow(CmdBase):
 
         for n in nodes:
             if commands:
+                if stages[n].cmd is None:
+                    continue
                 logger.info(stages[n].cmd)
             elif outs:
                 for out in stages[n].outs:
@@ -41,7 +43,7 @@ class CmdPipelineShow(CmdBase):
         from dvc.stage import Stage
 
         stage = Stage.load(self.repo, target)
-        node = os.path.relpath(stage.path, self.repo.root_dir)
+        node = relpath(stage.path, self.repo.root_dir)
 
         pipelines = list(
             filter(lambda g: node in g.nodes(), self.repo.pipelines())
@@ -115,7 +117,8 @@ class CmdPipelineShow(CmdBase):
             observe_list.pop(0)
         tree.show()
 
-    def __write_dot(self, target, commands, outs, filename):
+    def __write_dot(self, target, commands, outs):
+        from dvc.utils.compat import StringIO
         import networkx
         from networkx.drawing.nx_pydot import write_dot
 
@@ -124,7 +127,10 @@ class CmdPipelineShow(CmdBase):
 
         simple_g = networkx.DiGraph()
         simple_g.add_edges_from(edges)
-        write_dot(simple_g, filename)
+
+        dot_file = StringIO()
+        write_dot(simple_g, dot_file)
+        logger.info(dot_file.getvalue())
 
     def run(self):
         if not self.args.targets:
@@ -138,10 +144,7 @@ class CmdPipelineShow(CmdBase):
                     )
                 elif self.args.dot:
                     self.__write_dot(
-                        target,
-                        self.args.commands,
-                        self.args.outs,
-                        self.args.dot,
+                        target, self.args.commands, self.args.outs
                     )
                 elif self.args.tree:
                     self._show_dependencies_tree(
@@ -208,14 +211,14 @@ def add_parser(subparsers, parent_parser):
         "--commands",
         action="store_true",
         default=False,
-        help="Print commands instead of paths to DVC files.",
+        help="Print commands instead of paths to DVC-files.",
     )
     pipeline_show_group.add_argument(
         "-o",
         "--outs",
         action="store_true",
         default=False,
-        help="Print output files instead of paths to DVC files.",
+        help="Print output files instead of paths to DVC-files.",
     )
     pipeline_show_parser.add_argument(
         "-l",
@@ -231,7 +234,10 @@ def add_parser(subparsers, parent_parser):
         help="Output DAG as ASCII.",
     )
     pipeline_show_parser.add_argument(
-        "--dot", help="Write DAG in .dot format."
+        "--dot",
+        action="store_true",
+        default=False,
+        help="Print DAG with .dot format.",
     )
     pipeline_show_parser.add_argument(
         "--tree",
@@ -240,7 +246,7 @@ def add_parser(subparsers, parent_parser):
         help="Output DAG as Dependencies Tree.",
     )
     pipeline_show_parser.add_argument(
-        "targets", nargs="*", help="DVC files. 'Dvcfile' by default."
+        "targets", nargs="*", help="DVC-files. 'Dvcfile' by default."
     )
     pipeline_show_parser.set_defaults(func=CmdPipelineShow)
 

dvc/command/pkg.py

@@ -1,89 +0,0 @@
-from __future__ import unicode_literals
-
-import argparse
-import logging
-
-from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, fix_subparsers, append_doc_link
-
-
-logger = logging.getLogger(__name__)
-
-
-class CmdPkgInstall(CmdBase):
-    def run(self):
-        try:
-            self.repo.pkg.install(
-                self.args.address,
-                self.args.target_dir,
-                self.args.select,
-                self.args.file,
-            )
-            return 0
-        except DvcException:
-            logger.exception(
-                "failed to install package '{}'".format(self.args.address)
-            )
-            return 1
-
-
-def add_parser(subparsers, parent_parser):
-    from dvc.command.config import parent_config_parser
-
-    PKG_HELP = "Manage DVC packages."
-    pkg_parser = subparsers.add_parser(
-        "pkg",
-        parents=[parent_parser],
-        description=append_doc_link(PKG_HELP, "pkg"),
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-        add_help=False,
-    )
-
-    pkg_subparsers = pkg_parser.add_subparsers(
-        dest="cmd", help="Use dvc pkg CMD --help for command-specific help."
-    )
-
-    fix_subparsers(pkg_subparsers)
-
-    PKG_INSTALL_HELP = "Install package."
-    pkg_install_parser = pkg_subparsers.add_parser(
-        "install",
-        parents=[parent_config_parser, parent_parser],
-        description=append_doc_link(PKG_INSTALL_HELP, "pkg-install"),
-        help=PKG_INSTALL_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    pkg_install_parser.add_argument(
-        "address",
-        nargs="?",
-        default="",
-        help="Package address: git://<url> or https://github.com/...",
-    )
-    pkg_install_parser.add_argument(
-        "target_dir",
-        metavar="target",
-        nargs="?",
-        default=".",
-        help="Target directory to deploy package outputs. "
-        "Default value is the current dir.",
-    )
-    pkg_install_parser.add_argument(
-        "-s",
-        "--select",
-        metavar="OUT",
-        action="append",
-        default=[],
-        help="Select and persist only specified outputs from a package. "
-        "The parameter can be used multiple times. "
-        "All outputs will be selected by default.",
-    )
-    pkg_install_parser.add_argument(
-        "-f",
-        "--file",
-        help="Specify name of the stage file. It should be "
-        "either 'Dvcfile' or have a '.dvc' suffix (e.g. "
-        "'prepare.dvc', 'clean.dvc', etc). "
-        "By default the file has 'mod_' prefix and imported package name "
-        "followed by .dvc",
-    )
-    pkg_install_parser.set_defaults(func=CmdPkgInstall)

dvc/command/remote.py

@@ -1,130 +1,64 @@
 from __future__ import unicode_literals
 
 import argparse
-import os
-import re
 import logging
 
 from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig
-from dvc.config import Config
-
+from dvc.remote.config import RemoteConfig
 
 logger = logging.getLogger(__name__)
 
 
-class CmdRemoteAdd(CmdConfig):
-    @staticmethod
-    def resolve_path(path, config_file):
-        """Resolve path relative to config file location.
+class CmdRemoteConfig(CmdConfig):
+    def __init__(self, args):
+        super(CmdRemoteConfig, self).__init__(args)
+        self.remote_config = RemoteConfig(self.config)
 
-        Args:
-            path: Path to be resolved.
-            config_file: Path to config file, which `path` is specified
-                relative to.
-
-        Returns:
-            Path relative to the `config_file` location. If `path` is an
-            absolute path then it will be returned without change.
-
-        """
-        if os.path.isabs(path):
-            return path
-        return os.path.relpath(path, os.path.dirname(config_file))
 
+class CmdRemoteAdd(CmdRemoteConfig):
     def run(self):
-        from dvc.remote import _get, RemoteLOCAL
-
-        remote = _get({Config.SECTION_REMOTE_URL: self.args.url})
-        if remote == RemoteLOCAL and not self.args.url.startswith("remote://"):
-            self.args.url = self.resolve_path(
-                self.args.url, self.configobj.filename
-            )
-
-        section = Config.SECTION_REMOTE_FMT.format(self.args.name)
-        if (section in self.configobj.keys()) and not self.args.force:
-            logger.error(
-                "Remote with name {} already exists. "
-                "Use -f (--force) to overwrite remote "
-                "with new value".format(self.args.name)
-            )
-            return 1
-
-        ret = self._set(section, Config.SECTION_REMOTE_URL, self.args.url)
-        if ret != 0:
-            return ret
-
-        if self.args.default:
-            msg = "Setting '{}' as a default remote.".format(self.args.name)
-            logger.info(msg)
-            ret = self._set(
-                Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, self.args.name
-            )
-
-        return ret
-
-
-class CmdRemoteRemove(CmdConfig):
-    def _remove_default(self, config):
-        core = config.get(Config.SECTION_CORE, None)
-        if core is None:
-            return 0
-
-        default = core.get(Config.SECTION_CORE_REMOTE, None)
-        if default is None:
-            return 0
-
-        if default == self.args.name:
-            return self._unset(
-                Config.SECTION_CORE,
-                opt=Config.SECTION_CORE_REMOTE,
-                configobj=config,
-            )
-
-    def run(self):
-        section = Config.SECTION_REMOTE_FMT.format(self.args.name)
-        ret = self._unset(section)
-        if ret != 0:
-            return ret
-
-        for configobj in [
-            self.config._local_config,
-            self.config._repo_config,
-            self.config._global_config,
-            self.config._system_config,
-        ]:
-            self._remove_default(configobj)
-            self.config.save(configobj)
-            if configobj == self.configobj:
-                break
-
+        self.remote_config.add(
+            self.args.name,
+            self.args.url,
+            force=self.args.force,
+            default=self.args.default,
+            level=self.args.level,
+        )
         return 0
 
 
-class CmdRemoteModify(CmdConfig):
+class CmdRemoteRemove(CmdRemoteConfig):
     def run(self):
-        section = Config.SECTION_REMOTE_FMT.format(self.args.name)
-        self.args.name = "{}.{}".format(section, self.args.option)
-        return super(CmdRemoteModify, self).run()
+        self.remote_config.remove(self.args.name, level=self.args.level)
+        return 0
 
 
-class CmdRemoteDefault(CmdConfig):
+class CmdRemoteModify(CmdRemoteConfig):
     def run(self):
-        self.args.value = self.args.name
-        self.args.name = "core.remote"
-        return super(CmdRemoteDefault, self).run()
+        self.remote_config.modify(
+            self.args.name,
+            self.args.option,
+            self.args.value,
+            level=self.args.level,
+        )
+        return 0
 
 
-class CmdRemoteList(CmdConfig):
+class CmdRemoteDefault(CmdRemoteConfig):
     def run(self):
-        for section in self.configobj.keys():
-            r = re.match(Config.SECTION_REMOTE_REGEX, section)
-            if r:
-                name = r.group("name")
-                url = self.configobj[section].get(
-                    Config.SECTION_REMOTE_URL, ""
-                )
-                logger.info("{}\t{}".format(name, url))
+        self.remote_config.set_default(
+            self.args.name, unset=self.args.unset, level=self.args.level
+        )
+        return 0
+
+
+class CmdRemoteList(CmdRemoteConfig):
+    def run(self):
+        for name, url in self.remote_config.list(
+            level=self.args.level
+        ).items():
+            logger.info("{}\t{}".format(name, url))
         return 0
 
 

dvc/command/remove.py

@@ -43,7 +43,7 @@ class CmdRemove(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    REMOVE_HELP = "Remove outputs of DVC file."
+    REMOVE_HELP = "Remove outputs of DVC-file."
     remove_parser = subparsers.add_parser(
         "remove",
         parents=[parent_parser],
@@ -57,14 +57,14 @@ def add_parser(subparsers, parent_parser):
         "--outs",
         action="store_true",
         default=True,
-        help="Only remove DVC file outputs.(default)",
+        help="Only remove DVC-file outputs. (Default)",
     )
     remove_parser_group.add_argument(
         "-p",
         "--purge",
         action="store_true",
         default=False,
-        help="Remove DVC file and all its outputs.",
+        help="Remove DVC-file and all its outputs.",
     )
     remove_parser.add_argument(
         "-f",
@@ -73,5 +73,5 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Force purge.",
     )
-    remove_parser.add_argument("targets", nargs="+", help="DVC files.")
+    remove_parser.add_argument("targets", nargs="+", help="DVC-files.")
     remove_parser.set_defaults(func=CmdRemove)

dvc/command/repro.py

@@ -15,7 +15,6 @@ logger = logging.getLogger(__name__)
 
 class CmdRepro(CmdBase):
     def run(self):
-        recursive = not self.args.single_item
         saved_dir = os.path.realpath(os.curdir)
         if self.args.cwd:
             os.chdir(self.args.cwd)
@@ -31,7 +30,7 @@ class CmdRepro(CmdBase):
             try:
                 stages = self.repo.reproduce(
                     target,
-                    recursive=recursive,
+                    single_item=self.args.single_item,
                     force=self.args.force,
                     dry=self.args.dry,
                     interactive=self.args.interactive,
@@ -40,6 +39,7 @@ class CmdRepro(CmdBase):
                     ignore_build_cache=self.args.ignore_build_cache,
                     no_commit=self.args.no_commit,
                     downstream=self.args.downstream,
+                    recursive=self.args.recursive,
                 )
 
                 if len(stages) == 0:
@@ -58,7 +58,7 @@ class CmdRepro(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    REPRO_HELP = "Check for changes and reproduce DVC file and dependencies."
+    REPRO_HELP = "Check for changes and reproduce DVC-file and dependencies."
     repro_parser = subparsers.add_parser(
         "repro",
         parents=[parent_parser],
@@ -69,7 +69,7 @@ def add_parser(subparsers, parent_parser):
     repro_parser.add_argument(
         "targets",
         nargs="*",
-        help="DVC file to reproduce (default - 'Dvcfile').",
+        help="DVC-file to reproduce (default - 'Dvcfile').",
     )
     repro_parser.add_argument(
         "-f",
@@ -128,6 +128,13 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Reproduce all pipelines in the repo.",
     )
+    repro_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="Reproduce all stages in the specified directory.",
+    )
     repro_parser.add_argument(
         "--ignore-build-cache",
         action="store_true",

dvc/command/root.py

@@ -1,19 +1,18 @@
 from __future__ import unicode_literals
 
 import argparse
-import os
 import logging
 
 from dvc.repo import Repo
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
-
+from dvc.utils import relpath
 
 logger = logging.getLogger(__name__)
 
 
 class CmdRoot(CmdBaseNoRepo):
     def run(self):
-        logger.info(os.path.relpath(Repo.find_root()))
+        logger.info(relpath(Repo.find_root()))
         return 0
 
 

dvc/command/run.py

@@ -127,7 +127,7 @@ def add_parser(subparsers, parent_parser):
         "(do not put into DVC cache).",
     )
     run_parser.add_argument(
-        "-f", "--file", help="Specify name of the DVC file it generates."
+        "-f", "--file", help="Specify name of the DVC-file it generates."
     )
     run_parser.add_argument(
         "-c", "--cwd", default=None, help="Deprecated, use -w and -f instead."
@@ -155,7 +155,7 @@ def add_parser(subparsers, parent_parser):
         "--overwrite-dvcfile",
         action="store_true",
         default=False,
-        help="Overwrite existing dvc file without asking for confirmation.",
+        help="Overwrite existing DVC-file without asking for confirmation.",
     )
     run_parser.add_argument(
         "--ignore-build-cache",

dvc/command/tag.py

@@ -82,14 +82,14 @@ def add_parser(subparsers, parent_parser):
     )
     tag_add_parser.add_argument("tag", help="Dvc tag.")
     tag_add_parser.add_argument(
-        "targets", nargs="*", default=[None], help="Dvc files."
+        "targets", nargs="*", default=[None], help="DVC-files."
     )
     tag_add_parser.add_argument(
         "-d",
         "--with-deps",
         action="store_true",
         default=False,
-        help="Add tag for all dependencies of the specified DVC file.",
+        help="Add tag for all dependencies of the specified DVC-file.",
     )
     tag_add_parser.add_argument(
         "-R",
@@ -109,14 +109,14 @@ def add_parser(subparsers, parent_parser):
     )
     tag_remove_parser.add_argument("tag", help="Dvc tag.")
     tag_remove_parser.add_argument(
-        "targets", nargs="*", default=[None], help="Dvc files."
+        "targets", nargs="*", default=[None], help="DVC-files."
     )
     tag_remove_parser.add_argument(
         "-d",
         "--with-deps",
         action="store_true",
         default=False,
-        help="Remove tag for all dependencies of the specified DVC file.",
+        help="Remove tag for all dependencies of the specified DVC-file.",
     )
     tag_remove_parser.add_argument(
         "-R",
@@ -135,14 +135,14 @@ def add_parser(subparsers, parent_parser):
         help=TAG_LIST_HELP,
     )
     tag_list_parser.add_argument(
-        "targets", nargs="*", default=[None], help="Dvc files."
+        "targets", nargs="*", default=[None], help="DVC-files."
     )
     tag_list_parser.add_argument(
         "-d",
         "--with-deps",
         action="store_true",
         default=False,
-        help="List tags for all dependencies of the specified DVC file.",
+        help="List tags for all dependencies of the specified DVC-file.",
     )
     tag_list_parser.add_argument(
         "-R",

dvc/command/version.py

@@ -1,11 +1,19 @@
 from __future__ import unicode_literals
 
+import os
+import psutil
 import platform
 import argparse
 import logging
+import uuid
 
+from dvc.utils import is_binary
+from dvc.utils.compat import pathlib
+from dvc.repo import Repo
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.version import __version__
+from dvc.exceptions import DvcException, NotDvcRepoError
+from dvc.system import System
 
 
 logger = logging.getLogger(__name__)
@@ -13,18 +21,86 @@ logger = logging.getLogger(__name__)
 
 class CmdVersion(CmdBaseNoRepo):
     def run(self):
+        dvc_version = __version__
+        python_version = platform.python_version()
+        platform_type = platform.platform()
+        binary = is_binary()
+
         info = (
             "DVC version: {dvc_version}\n"
             "Python version: {python_version}\n"
-            "Platform: {platform_type}"
+            "Platform: {platform_type}\n"
+            "Binary: {binary}\n"
         ).format(
-            dvc_version=__version__,
-            python_version=platform.python_version(),
-            platform_type=platform.platform(),
+            dvc_version=dvc_version,
+            python_version=python_version,
+            platform_type=platform_type,
+            binary=binary,
+        )
+
+        try:
+            repo = Repo()
+            root_directory = repo.root_dir
+
+            info += (
+                "Cache: {cache}\n"
+                "Filesystem type (cache directory): {fs_cache}\n"
+            ).format(
+                cache=self.get_linktype_support_info(repo),
+                fs_cache=self.get_fs_type(repo.cache.local.cache_dir),
+            )
+
+        except NotDvcRepoError:
+            root_directory = os.getcwd()
+
+        info += ("Filesystem type (workspace): {fs_root}").format(
+            fs_root=self.get_fs_type(os.path.abspath(root_directory))
         )
         logger.info(info)
         return 0
 
+    @staticmethod
+    def get_fs_type(path):
+        partition = {
+            pathlib.Path(part.mountpoint): (part.fstype, part.device)
+            for part in psutil.disk_partitions()
+        }
+        for parent in pathlib.Path(path).parents:
+            if parent in partition:
+                return partition[parent]
+        return ("unkown", "none")
+
+    @staticmethod
+    def get_linktype_support_info(repo):
+        links = {
+            "reflink": System.reflink,
+            "hardlink": System.hardlink,
+            "symlink": System.symlink,
+        }
+
+        fname = "." + str(uuid.uuid4())
+        src = os.path.join(repo.cache.local.cache_dir, fname)
+        open(src, "w").close()
+        dst = os.path.join(repo.root_dir, fname)
+
+        cache = []
+
+        for name, link in links.items():
+            try:
+                link(src, dst)
+                os.unlink(dst)
+                supported = True
+            except DvcException:
+                supported = False
+            cache.append(
+                "{name} - {supported}".format(
+                    name=name, supported=True if supported else False
+                )
+            )
+        os.remove(src)
+
+        return ", ".join(cache)
+
 
 def add_parser(subparsers, parent_parser):
     VERSION_HELP = "Show DVC version and system/environment informaion."

dvc/config.py

@@ -2,15 +2,17 @@
 
 from __future__ import unicode_literals
 
-from dvc.utils.compat import str, open, urlparse
+from dvc.utils.compat import str, open
 
 import os
+import re
+import copy
 import errno
 import configobj
 import logging
 
-from schema import Schema, Optional, And, Use, Regex
-from dvc.exceptions import DvcException
+from schema import Schema, Optional, And, Use, Regex, SchemaError
+from dvc.exceptions import DvcException, NotDvcRepoError
 
 logger = logging.getLogger(__name__)
 
@@ -20,12 +22,12 @@ class ConfigError(DvcException):
 
     Args:
         msg (str): error message.
-        ex (Exception): optional exception that has caused this error.
+        cause (Exception): optional exception that has caused this error.
     """
 
-    def __init__(self, msg, ex=None):
+    def __init__(self, msg, cause=None):
         super(ConfigError, self).__init__(
-            "config file error: {}".format(msg), ex
+            "config file error: {}".format(msg), cause=cause
         )
 
 
@@ -43,24 +45,6 @@ def supported_cache_type(types):
     return True
 
 
-def supported_loglevel(level):
-    """Checks if log level config option has a valid value.
-
-    Args:
-        level (str): log level name.
-    """
-    return level in ["info", "debug", "warning", "error"]
-
-
-def supported_cloud(cloud):
-    """Checks if obsoleted cloud option has a valid value.
-
-    Args:
-        cloud (str): cloud type name.
-    """
-    return cloud in ["aws", "gcp", "local", ""]
-
-
 def is_bool(val):
     """Checks that value is a boolean.
 
@@ -109,6 +93,23 @@ def is_percent(val):
     return int(val) >= 0 and int(val) <= 100
 
 
+class Choices(object):
+    """Checks that value belongs to the specified set of values
+
+    Args:
+        *choices: pass allowed values as arguments, or pass a list or
+            tuple as a single argument
+    """
+
+    def __init__(self, *choices):
+        if len(choices) == 1 and isinstance(choices[0], (list, tuple)):
+            choices = choices[0]
+        self.choices = choices
+
+    def __call__(self, value):
+        return value in self.choices
+
+
 class Config(object):  # pylint: disable=too-many-instance-attributes
     """Class that manages configuration files for a dvc repo.
 
@@ -134,11 +135,18 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     CONFIG = "config"
     CONFIG_LOCAL = "config.local"
 
+    LEVEL_LOCAL = 0
+    LEVEL_REPO = 1
+    LEVEL_GLOBAL = 2
+    LEVEL_SYSTEM = 3
+
     BOOL_SCHEMA = And(str, is_bool, Use(to_bool))
 
     SECTION_CORE = "core"
     SECTION_CORE_LOGLEVEL = "loglevel"
-    SECTION_CORE_LOGLEVEL_SCHEMA = And(Use(str.lower), supported_loglevel)
+    SECTION_CORE_LOGLEVEL_SCHEMA = And(
+        Use(str.lower), Choices("info", "debug", "warning", "error")
+    )
     SECTION_CORE_REMOTE = "remote"
     SECTION_CORE_INTERACTIVE_SCHEMA = BOOL_SCHEMA
     SECTION_CORE_INTERACTIVE = "interactive"
@@ -173,7 +181,11 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
 
     # backward compatibility
     SECTION_CORE_CLOUD = "cloud"
-    SECTION_CORE_CLOUD_SCHEMA = And(Use(str.lower), supported_cloud)
+    SECTION_CORE_CLOUD_SCHEMA = And(
+        Use(str.lower),
+        # only the clouds which were supported by obsoleted cloud config
+        Choices("aws", "gcp", "local", ""),
+    )
     SECTION_CORE_STORAGEPATH = "storagepath"
 
     SECTION_CORE_SCHEMA = {
@@ -201,6 +213,7 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     SECTION_AWS_REGION = "region"
     SECTION_AWS_PROFILE = "profile"
     SECTION_AWS_USE_SSL = "use_ssl"
+    SECTION_AWS_SSE = "sse"
     SECTION_AWS_SCHEMA = {
         SECTION_AWS_STORAGEPATH: str,
         Optional(SECTION_AWS_REGION): str,
@@ -209,6 +222,7 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         Optional(SECTION_AWS_ENDPOINT_URL): str,
         Optional(SECTION_AWS_LIST_OBJECTS, default=False): BOOL_SCHEMA,
         Optional(SECTION_AWS_USE_SSL, default=True): BOOL_SCHEMA,
+        Optional(SECTION_AWS_SSE): str,
     }
 
     # backward compatibility
@@ -241,6 +255,7 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     SECTION_REMOTE_TIMEOUT = "timeout"
     SECTION_REMOTE_PASSWORD = "password"
     SECTION_REMOTE_ASK_PASSWORD = "ask_password"
+    SECTION_REMOTE_NO_TRAVERSE = "no_traverse"
     SECTION_REMOTE_SCHEMA = {
         SECTION_REMOTE_URL: str,
         Optional(SECTION_AWS_REGION): str,
@@ -249,6 +264,7 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         Optional(SECTION_AWS_ENDPOINT_URL): str,
         Optional(SECTION_AWS_LIST_OBJECTS, default=False): BOOL_SCHEMA,
         Optional(SECTION_AWS_USE_SSL, default=True): BOOL_SCHEMA,
+        Optional(SECTION_AWS_SSE): str,
         Optional(SECTION_GCP_PROJECTNAME): str,
         Optional(SECTION_CACHE_TYPE): SECTION_CACHE_TYPE_SCHEMA,
         Optional(SECTION_CACHE_PROTECTED, default=False): BOOL_SCHEMA,
@@ -263,6 +279,7 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         Optional(SECTION_OSS_ACCESS_KEY_SECRET): str,
         Optional(SECTION_OSS_ENDPOINT): str,
         Optional(PRIVATE_CWD): str,
+        Optional(SECTION_REMOTE_NO_TRAVERSE, default=True): BOOL_SCHEMA,
     }
 
     SECTION_STATE = "state"
@@ -285,30 +302,20 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     }
 
     def __init__(self, dvc_dir=None, validate=True):
-        self.system_config_file = os.path.join(
-            self.get_system_config_dir(), self.CONFIG
-        )
-        self.global_config_file = os.path.join(
-            self.get_global_config_dir(), self.CONFIG
-        )
+        self.dvc_dir = dvc_dir
+        self.validate = validate
 
-        if dvc_dir is not None:
-            self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
-            self.config_file = os.path.join(dvc_dir, self.CONFIG)
-            self.config_local_file = os.path.join(dvc_dir, self.CONFIG_LOCAL)
+        if not dvc_dir:
+            try:
+                from dvc.repo import Repo
+
+                self.dvc_dir = os.path.join(Repo.find_dvc_dir())
+            except NotDvcRepoError:
+                self.dvc_dir = None
         else:
-            self.dvc_dir = None
-            self.config_file = None
-            self.config_local_file = None
+            self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
 
-        self._system_config = None
-        self._global_config = None
-        self._repo_config = None
-        self._local_config = None
-
-        self.config = None
-
-        self.load(validate=validate)
+        self.load()
 
     @staticmethod
     def get_global_config_dir():
@@ -350,35 +357,7 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         open(config_file, "w+").close()
         return Config(dvc_dir)
 
-    def _load(self):
-        self._system_config = configobj.ConfigObj(self.system_config_file)
-        self._global_config = configobj.ConfigObj(self.global_config_file)
-
-        if self.config_file is not None:
-            self._repo_config = configobj.ConfigObj(self.config_file)
-        else:
-            self._repo_config = configobj.ConfigObj()
-
-        if self.config_local_file is not None:
-            self._local_config = configobj.ConfigObj(self.config_local_file)
-        else:
-            self._local_config = configobj.ConfigObj()
-
-        self.config = None
-
-    def _load_config(self, path):
-        config = configobj.ConfigObj(path)
-        config = self._lower(config)
-        self._resolve_paths(config, path)
-        return config
-
-    @staticmethod
-    def _resolve_path(path, config_file):
-        assert os.path.isabs(config_file)
-        config_dir = os.path.dirname(config_file)
-        return os.path.abspath(os.path.join(config_dir, path))
-
-    def _resolve_cache_path(self, config, fname):
+    def _resolve_cache_path(self, config):
         cache = config.get(self.SECTION_CACHE)
         if cache is None:
             return
@@ -387,72 +366,86 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         if cache_dir is None:
             return
 
-        cache[self.PRIVATE_CWD] = os.path.dirname(fname)
+        cache[self.PRIVATE_CWD] = os.path.dirname(config.filename)
 
-    def _resolve_paths(self, config, fname):
-        if fname is None:
-            return
+    def _resolve_paths(self, config):
+        if config.filename is None:
+            return config
 
-        self._resolve_cache_path(config, fname)
-        for section in config.values():
+        ret = copy.deepcopy(config)
+        self._resolve_cache_path(ret)
+
+        for section in ret.values():
             if self.SECTION_REMOTE_URL not in section.keys():
                 continue
 
-            section[self.PRIVATE_CWD] = os.path.dirname(fname)
+            section[self.PRIVATE_CWD] = os.path.dirname(ret.filename)
 
-    def load(self, validate=True):
+        return ret
+
+    def _load_configs(self):
+        system_config_file = os.path.join(
+            self.get_system_config_dir(), self.CONFIG
+        )
+
+        global_config_file = os.path.join(
+            self.get_global_config_dir(), self.CONFIG
+        )
+
+        self._system_config = configobj.ConfigObj(system_config_file)
+        self._global_config = configobj.ConfigObj(global_config_file)
+        self._repo_config = configobj.ConfigObj()
+        self._local_config = configobj.ConfigObj()
+
+        if not self.dvc_dir:
+            return
+
+        config_file = os.path.join(self.dvc_dir, self.CONFIG)
+        config_local_file = os.path.join(self.dvc_dir, self.CONFIG_LOCAL)
+
+        self._repo_config = configobj.ConfigObj(config_file)
+        self._local_config = configobj.ConfigObj(config_local_file)
+
+    @property
+    def config_local_file(self):
+        return self._local_config.filename
+
+    @property
+    def config_file(self):
+        return self._repo_config.filename
+
+    def load(self):
         """Loads config from all the config files.
 
-        Args:
-            validate (bool): optional flag to tell dvc if it should validate
-                the config or just load it as is. 'True' by default.
-
-
         Raises:
             dvc.config.ConfigError: thrown if config has invalid format.
         """
-        self._load()
+        self._load_configs()
+
+        self.config = configobj.ConfigObj()
+        for c in [
+            self._system_config,
+            self._global_config,
+            self._repo_config,
+            self._local_config,
+        ]:
+            c = self._resolve_paths(c)
+            c = self._lower(c)
+            self.config.merge(c)
+
+        if not self.validate:
+            return
+
+        d = self.config.dict()
         try:
-            self.config = self._load_config(self.system_config_file)
-            user = self._load_config(self.global_config_file)
-            config = self._load_config(self.config_file)
-            local = self._load_config(self.config_local_file)
-
-            # NOTE: schema doesn't support ConfigObj.Section validation, so we
-            # need to convert our config to dict before passing it to
-            for conf in [user, config, local]:
-                self.config = self._merge(self.config, conf)
-
-            if validate:
-                self.config = Schema(self.SCHEMA).validate(self.config)
-
-            # NOTE: now converting back to ConfigObj
-            self.config = configobj.ConfigObj(
-                self.config, write_empty_values=True
-            )
-            self.config.filename = self.config_file
-            self._resolve_paths(self.config, self.config_file)
-        except Exception as ex:
-            raise ConfigError(ex)
-
-    @staticmethod
-    def _get_key(conf, name, add=False):
-        for k in conf.keys():
-            if k.lower() == name.lower():
-                return k
-
-        if add:
-            conf[name] = {}
-            return name
-
-        return None
+            d = Schema(self.SCHEMA).validate(d)
+        except SchemaError as exc:
+            raise ConfigError("config format error", cause=exc)
+        self.config = configobj.ConfigObj(d, write_empty_values=True)
 
     def save(self, config=None):
         """Saves config to config files.
 
-        Args:
-            config (configobj.ConfigObj): optional config object to save.
-
         Raises:
             dvc.config.ConfigError: thrown if failed to write config file.
         """
@@ -467,121 +460,103 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
             ]
 
         for conf in clist:
-            if conf.filename is None:
-                continue
+            self._save(conf)
 
-            try:
-                logger.debug("Writing '{}'.".format(conf.filename))
-                dname = os.path.dirname(os.path.abspath(conf.filename))
-                try:
-                    os.makedirs(dname)
-                except OSError as exc:
-                    if exc.errno != errno.EEXIST:
-                        raise
-                conf.write()
-            except Exception as exc:
-                msg = "failed to write config '{}'".format(conf.filename)
-                raise ConfigError(msg, exc)
-
-    def get_remote_settings(self, name):
-        import posixpath
-
-        """
-        Args:
-            name (str): The name of the remote that we want to retrieve
-
-        Returns:
-            dict: The content beneath the given remote name.
-
-        Example:
-            >>> config = {'remote "server"': {'url': 'ssh://localhost/'}}
-            >>> get_remote_settings("server")
-            {'url': 'ssh://localhost/'}
-        """
-        settings = self.config[self.SECTION_REMOTE_FMT.format(name)]
-        parsed = urlparse(settings["url"])
-
-        # Support for cross referenced remotes.
-        # This will merge the settings, giving priority to the outer reference.
-        # For example, having:
-        #
-        #       dvc remote add server ssh://localhost
-        #       dvc remote modify server user root
-        #       dvc remote modify server ask_password true
-        #
-        #       dvc remote add images remote://server/tmp/pictures
-        #       dvc remote modify images user alice
-        #       dvc remote modify images ask_password false
-        #       dvc remote modify images password asdf1234
-        #
-        # Results on a config dictionary like:
-        #
-        #       {
-        #           "url": "ssh://localhost/tmp/pictures",
-        #           "user": "alice",
-        #           "password": "asdf1234",
-        #           "ask_password": False,
-        #       }
-        #
-        if parsed.scheme == "remote":
-            reference = self.get_remote_settings(parsed.netloc)
-            url = posixpath.join(reference["url"], parsed.path.lstrip("/"))
-            merged = reference.copy()
-            merged.update(settings)
-            merged["url"] = url
-            return merged
-
-        return settings
+        self.load()
 
     @staticmethod
-    def unset(config, section, opt=None):
+    def _save(config):
+        if config.filename is None:
+            return
+
+        logger.debug("Writing '{}'.".format(config.filename))
+        dname = os.path.dirname(os.path.abspath(config.filename))
+        try:
+            os.makedirs(dname)
+        except OSError as exc:
+            if exc.errno != errno.EEXIST:
+                raise
+        config.write()
+
+    def unset(self, section, opt=None, level=None, force=False):
         """Unsets specified option and/or section in the config.
 
         Args:
-            config (configobj.ConfigObj): config to work on.
             section (str): section name.
             opt (str): optional option name.
+            level (int): config level to use.
+            force (bool): don't error-out even if section doesn't exist. False
+                by default.
+
+        Raises:
+            dvc.config.ConfigError: thrown if section doesn't exist and
+                `force != True`.
         """
+        config = self.get_configobj(level)
+
         if section not in config.keys():
+            if force:
+                return
             raise ConfigError("section '{}' doesn't exist".format(section))
 
-        if opt is None:
-            del config[section]
-            return
+        if opt:
+            if opt not in config[section].keys():
+                if force:
+                    return
+                raise ConfigError(
+                    "option '{}.{}' doesn't exist".format(section, opt)
+                )
+            del config[section][opt]
 
-        if opt not in config[section].keys():
-            raise ConfigError(
-                "option '{}.{}' doesn't exist".format(section, opt)
-            )
-        del config[section][opt]
-
-        if not config[section]:
+            if not config[section]:
+                del config[section]
+        else:
             del config[section]
 
-    @staticmethod
-    def set(config, section, opt, value):
+        self.save(config)
+
+    def set(self, section, opt, value, level=None, force=True):
         """Sets specified option in the config.
 
         Args:
-            config (configobj.ConfigObj): config to work on.
             section (str): section name.
             opt (str): option name.
             value: value to set option to.
+            level (int): config level to use.
+            force (bool): set option even if section already exists. True by
+                default.
+
+        Raises:
+            dvc.config.ConfigError: thrown if section already exists and
+                `force != True`.
+
         """
+        config = self.get_configobj(level)
+
         if section not in config.keys():
             config[section] = {}
+        elif not force:
+            raise ConfigError(
+                "Section '{}' already exists. Use `-f|--force` to overwrite "
+                "section with new value.".format(section)
+            )
 
         config[section][opt] = value
+        self.save(config)
 
-    @staticmethod
-    def show(config, section, opt):
-        """Prints option value from the config.
+    def get(self, section, opt=None, level=None):
+        """Return option value from the config.
 
         Args:
-            config (configobj.ConfigObj): config to work on.
             section (str): section name.
             opt (str): option name.
+            level (int): config level to use.
+
+        Returns:
+            value (str, int): option value.
         """
+        config = self.get_configobj(level)
+
         if section not in config.keys():
             raise ConfigError("section '{}' doesn't exist".format(section))
 
@@ -590,25 +565,35 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
                 "option '{}.{}' doesn't exist".format(section, opt)
             )
 
-        logger.info(config[section][opt])
-
-    @staticmethod
-    def _merge(first, second):
-        res = {}
-        sections = list(first.keys()) + list(second.keys())
-        for section in sections:
-            first_copy = first.get(section, {}).copy()
-            second_copy = second.get(section, {}).copy()
-            first_copy.update(second_copy)
-            res[section] = first_copy
-        return res
+        return config[section][opt]
 
     @staticmethod
     def _lower(config):
-        new_config = {}
+        new_config = configobj.ConfigObj()
         for s_key, s_value in config.items():
             new_s = {}
             for key, value in s_value.items():
                 new_s[key.lower()] = str(value)
             new_config[s_key.lower()] = new_s
         return new_config
+
+    def get_configobj(self, level):
+        configs = {
+            self.LEVEL_LOCAL: self._local_config,
+            self.LEVEL_REPO: self._repo_config,
+            self.LEVEL_GLOBAL: self._global_config,
+            self.LEVEL_SYSTEM: self._system_config,
+        }
+
+        return configs.get(level, self._repo_config)
+
+    def list_options(self, section_regex, option, level=None):
+        ret = {}
+        config = self.get_configobj(level)
+        for section in config.keys():
+            r = re.match(section_regex, section)
+            if r:
+                name = r.group("name")
+                value = config[section].get(option, "")
+                ret[name] = value
+        return ret

dvc/data_cloud.py

@@ -25,7 +25,6 @@ class DataCloud(object):
     Args:
         repo (dvc.repo.Repo): repo instance that belongs to the repo that
             we are working on.
-        config (dict): config of the repo.
 
     Raises:
         config.ConfigError: thrown when config has invalid format.
@@ -43,40 +42,43 @@ class DataCloud(object):
         "https": RemoteHTTP,
     }
 
-    def __init__(self, repo, config=None):
+    def __init__(self, repo):
         self.repo = repo
-        self._config = config
-        self._core = self._config[Config.SECTION_CORE]
 
     @property
-    def _cloud(self):
-        """Returns a Remote instance using the `core.remote` config"""
-        remote = self._core.get(Config.SECTION_CORE_REMOTE, "")
+    def _config(self):
+        return self.repo.config.config
+
+    @property
+    def _core(self):
+        return self._config.get(Config.SECTION_CORE, {})
+
+    def get_remote(self, remote=None, command="<command>"):
+        if not remote:
+            remote = self._core.get(Config.SECTION_CORE_REMOTE)
 
         if remote:
             return self._init_remote(remote)
 
-        if self._core.get(Config.SECTION_CORE_CLOUD, None):
-            # backward compatibility
+        # Old config format support for backward compatibility
+        if Config.SECTION_CORE_CLOUD in self._core:
             msg = "using obsoleted config format. Consider updating."
             logger.warning(msg)
             return self._init_compat()
 
-        return None
+        raise ConfigError(
+            "No remote repository specified. Setup default repository with\n"
+            "    dvc config core.remote <name>\n"
+            "or use:\n"
+            "    dvc {} -r <name>\n".format(command)
+        )
 
     def _init_remote(self, remote):
-        section = Config.SECTION_REMOTE_FMT.format(remote).lower()
-        cloud_config = self._config.get(section, None)
-        if not cloud_config:
-            msg = "can't find remote section '{}' in config"
-            raise ConfigError(msg.format(section))
-
-        return Remote(self.repo, cloud_config)
+        return Remote(self.repo, name=remote)
 
     def _init_compat(self):
         name = self._core.get(Config.SECTION_CORE_CLOUD, "").strip().lower()
         if name == "":
-            self._cloud = None
             return None
 
         cloud_type = self.CLOUD_MAP.get(name, None)
@@ -104,20 +106,6 @@ class DataCloud(object):
         cloud = cloud_type(self.repo, cloud_config)
         return cloud
 
-    def _get_cloud(self, remote, cmd):
-        if remote:
-            return self._init_remote(remote)
-
-        if self._cloud:
-            return self._cloud
-
-        raise ConfigError(
-            "No remote repository specified. Setup default repository with\n"
-            "    dvc config core.remote <name>\n"
-            "or use:\n"
-            "    dvc {} -r <name>\n".format(cmd)
-        )
-
     def push(self, targets, jobs=None, remote=None, show_checksums=False):
         """Push data items in a cloud-agnostic way.
 
@@ -132,7 +120,7 @@ class DataCloud(object):
         return self.repo.cache.local.push(
             targets,
             jobs=jobs,
-            remote=self._get_cloud(remote, "push"),
+            remote=self.get_remote(remote, "push"),
             show_checksums=show_checksums,
         )
 
@@ -150,7 +138,7 @@ class DataCloud(object):
         return self.repo.cache.local.pull(
             targets,
             jobs=jobs,
-            remote=self._get_cloud(remote, "pull"),
+            remote=self.get_remote(remote, "pull"),
             show_checksums=show_checksums,
         )
 
@@ -166,7 +154,7 @@ class DataCloud(object):
             show_checksums (bool): show checksums instead of file names in
                 information messages.
         """
-        cloud = self._get_cloud(remote, "status")
+        remote = self.get_remote(remote, "status")
         return self.repo.cache.local.status(
-            targets, jobs=jobs, remote=cloud, show_checksums=show_checksums
+            targets, jobs=jobs, remote=remote, show_checksums=show_checksums
         )

dvc/dependency/__init__.py

@@ -13,8 +13,11 @@ from dvc.dependency.hdfs import DependencyHDFS
 from dvc.dependency.ssh import DependencySSH
 from dvc.dependency.http import DependencyHTTP
 from dvc.dependency.https import DependencyHTTPS
+from .repo import DependencyREPO
 
 from dvc.remote import Remote
+from dvc.external_repo import ExternalRepo
+
 
 DEPS = [
     DependencyGS,
@@ -44,6 +47,7 @@ DEP_MAP = {
 SCHEMA = output.SCHEMA.copy()
 del SCHEMA[schema.Optional(OutputBase.PARAM_CACHE)]
 del SCHEMA[schema.Optional(OutputBase.PARAM_METRIC)]
+SCHEMA[schema.Optional(DependencyREPO.PARAM_REPO)] = ExternalRepo.SCHEMA
 
 
 def _get(stage, p, info):
@@ -52,10 +56,13 @@ def _get(stage, p, info):
     parsed = urlparse(p)
 
     if parsed.scheme == "remote":
-        settings = stage.repo.config.get_remote_settings(parsed.netloc)
-        remote = Remote(stage.repo, settings)
+        remote = Remote(stage.repo, name=parsed.netloc)
         return DEP_MAP[remote.scheme](stage, p, info, remote=remote)
 
+    if info and info.get(DependencyREPO.PARAM_REPO):
+        repo = info.pop(DependencyREPO.PARAM_REPO)
+        return DependencyREPO(repo, stage, p, info)
+
     for d in DEPS:
         if d.supported(p):
             return d(stage, p, info)
@@ -70,8 +77,9 @@ def loadd_from(stage, d_list):
     return ret
 
 
-def loads_from(stage, s_list):
+def loads_from(stage, s_list, erepo=None):
     ret = []
     for s in s_list:
-        ret.append(_get(stage, s, {}))
+        info = {DependencyREPO.PARAM_REPO: erepo} if erepo else {}
+        ret.append(_get(stage, s, info))
     return ret

dvc/dependency/http.py

@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-from dvc.path import Path
-from dvc.utils.compat import urlparse, urljoin
 from dvc.output.base import OutputBase
 from dvc.remote.http import RemoteHTTP
 from dvc.dependency.base import DependencyBase
@@ -9,12 +7,3 @@ from dvc.dependency.base import DependencyBase
 
 class DependencyHTTP(DependencyBase, OutputBase):
     REMOTE = RemoteHTTP
-
-    def __init__(self, stage, path, info=None, remote=None):
-        super(DependencyHTTP, self).__init__(
-            stage, path, info=info, remote=remote
-        )
-        if path.startswith("remote"):
-            path = urljoin(self.remote.cache_dir, urlparse(path).path)
-
-        self.path_info = Path(self.scheme, url=self.url, path=path)

dvc/dependency/repo.py

@@ -0,0 +1,43 @@
+from __future__ import unicode_literals
+
+import os
+import copy
+
+from dvc.utils.compat import urlparse
+from dvc.external_repo import ExternalRepo
+
+from .local import DependencyLOCAL
+
+
+class DependencyREPO(DependencyLOCAL):
+    PARAM_REPO = "repo"
+
+    def __init__(self, erepo, stage, *args, **kwargs):
+        self.erepo = ExternalRepo(stage.repo.dvc_dir, **erepo)
+        super(DependencyLOCAL, self).__init__(stage, *args, **kwargs)
+
+    def _parse_path(self, remote, path):
+        self.erepo.install(self.repo.cache.local.cache_dir)
+
+        out_path = os.path.join(
+            self.erepo.repo.root_dir, urlparse(path).path.lstrip("/")
+        )
+
+        out, = self.erepo.repo.find_outs_by_path(out_path)
+        self.info = copy.copy(out.info)
+        self._erepo_stage = copy.copy(out.stage.path)
+        return self.REMOTE.path_cls(out.cache_path)
+
+    @property
+    def is_in_repo(self):
+        return False
+
+    def dumpd(self):
+        ret = super(DependencyLOCAL, self).dumpd()
+        ret[self.PARAM_REPO] = self.erepo.dumpd()
+        return ret
+
+    def download(self, to, resume=False):
+        self.erepo.repo.fetch(self._erepo_stage)
+        to.info = copy.copy(self.info)
+        to.checkout()

dvc/exceptions.py

@@ -4,9 +4,10 @@ from __future__ import unicode_literals
 
 from dvc.utils.compat import str, builtin_str
 
-import os
 import traceback
 
+from dvc.utils import relpath
+
 
 class DvcException(Exception):
     """Base class for all dvc exceptions.
@@ -62,7 +63,7 @@ class OutputNotFoundError(DvcException):
     def __init__(self, output):
         super(OutputNotFoundError, self).__init__(
             "unable to find stage file with output '{path}'".format(
-                path=os.path.relpath(output)
+                path=relpath(output)
             )
         )
 
@@ -98,9 +99,7 @@ class CircularDependencyError(DvcException):
     """
 
     def __init__(self, dependency):
-        assert isinstance(dependency, str) or isinstance(
-            dependency, builtin_str
-        )
+        assert isinstance(dependency, (str, builtin_str))
 
         msg = (
             "file/directory '{}' is specified as an output and as a "
@@ -118,7 +117,7 @@ class ArgumentDuplicationError(DvcException):
     """
 
     def __init__(self, path):
-        assert isinstance(path, str) or isinstance(path, builtin_str)
+        assert isinstance(path, (str, builtin_str))
         msg = "file '{}' is specified more than once."
         super(ArgumentDuplicationError, self).__init__(msg.format(path))
 
@@ -200,7 +199,7 @@ class BadMetricError(DvcException):
     def __init__(self, path):
         super(BadMetricError, self).__init__(
             "'{}' does not exist, not a metric or is malformed".format(
-                os.path.relpath(path)
+                relpath(path)
             )
         )
 
@@ -215,7 +214,7 @@ class NoMetricsError(DvcException):
 
 class StageFileCorruptedError(DvcException):
     def __init__(self, path, cause=None):
-        path = os.path.relpath(path)
+        path = relpath(path)
         super(StageFileCorruptedError, self).__init__(
             "unable to read stage file: {} "
             "YAML file structure is corrupted".format(path),

dvc/external_repo.py

@@ -0,0 +1,152 @@
+from __future__ import unicode_literals
+
+import os
+import shutil
+import logging
+import shortuuid
+
+from funcy import cached_property
+from schema import Optional
+
+from dvc.config import Config
+from dvc.cache import CacheConfig
+from dvc.exceptions import DvcException
+from dvc.utils.compat import makedirs, str
+from dvc.utils import remove
+
+
+logger = logging.getLogger(__name__)
+
+
+class ExternalRepoError(DvcException):
+    pass
+
+
+class NotInstalledError(ExternalRepoError):
+    def __init__(self, name):
+        super(NotInstalledError, self).__init__(
+            "Repo '{}' is not installed".format(name)
+        )
+
+
+class InstallError(ExternalRepoError):
+    def __init__(self, url, path, cause):
+        super(InstallError, self).__init__(
+            "Failed to install repo '{}' to '{}'".format(url, path),
+            cause=cause,
+        )
+
+
+class RevError(ExternalRepoError):
+    def __init__(self, url, rev, cause):
+        super(RevError, self).__init__(
+            "Failed to access revision '{}' for repo '{}'".format(rev, url),
+            cause=cause,
+        )
+
+
+class ExternalRepo(object):
+    REPOS_DIR = "repos"
+
+    PARAM_URL = "url"
+    PARAM_VERSION = "rev"
+
+    SCHEMA = {Optional(PARAM_URL): str, Optional(PARAM_VERSION): str}
+
+    def __init__(self, dvc_dir, **kwargs):
+        self.repos_dir = os.path.join(dvc_dir, self.REPOS_DIR)
+        self.url = kwargs[self.PARAM_URL]
+
+        self.name = "{}-{}".format(os.path.basename(self.url), hash(self.url))
+
+        self.rev = kwargs.get(self.PARAM_VERSION)
+        self.path = os.path.join(self.repos_dir, self.name)
+
+    @cached_property
+    def repo(self):
+        from dvc.repo import Repo
+
+        if not self.installed:
+            raise NotInstalledError(self.name)
+
+        return Repo(self.path, rev=self.rev)
+
+    @property
+    def installed(self):
+        return os.path.exists(self.path)
+
+    def _install_to(self, tmp_dir, cache_dir):
+        import git
+
+        try:
+            git.Repo.clone_from(
+                self.url, tmp_dir, depth=1, no_single_branch=True
+            )
+        except git.exc.GitCommandError as exc:
+            raise InstallError(self.url, tmp_dir, exc)
+
+        if self.rev:
+            try:
+                repo = git.Repo(tmp_dir)
+                repo.git.checkout(self.rev)
+                repo.close()
+            except git.exc.GitCommandError as exc:
+                raise RevError(self.url, self.rev, exc)
+
+        if cache_dir:
+            from dvc.repo import Repo
+
+            repo = Repo(tmp_dir)
+            cache_config = CacheConfig(repo.config)
+            cache_config.set_dir(cache_dir, level=Config.LEVEL_LOCAL)
+            repo.scm.git.close()
+
+    def install(self, cache_dir=None, force=False):
+        if self.installed and not force:
+            logger.info(
+                "Skipping installing '{}'('{}') as it is already "
+                "installed.".format(self.name, self.url)
+            )
+            return
+
+        makedirs(self.repos_dir, exist_ok=True)
+
+        # installing package to a temporary directory until we are sure that
+        # it has been installed correctly.
+        #
+        # Note that we can't use tempfile.TemporaryDirectory is using symlinks
+        # to tmpfs, so we won't be able to use move properly.
+        tmp_dir = os.path.join(self.repos_dir, "." + str(shortuuid.uuid()))
+        try:
+            self._install_to(tmp_dir, cache_dir)
+        except ExternalRepoError:
+            if os.path.exists(tmp_dir):
+                remove(tmp_dir)
+            raise
+
+        if self.installed:
+            self.uninstall()
+
+        shutil.move(tmp_dir, self.path)
+
+    def uninstall(self):
+        if not self.installed:
+            logger.info(
+                "Skipping uninstalling '{}' as it is not installed.".format(
+                    self.name
+                )
+            )
+            return
+
+        remove(self.path)
+
+    def update(self):
+        self.repo.scm.fetch(self.rev)
+
+    def dumpd(self):
+        ret = {self.PARAM_URL: self.url}
+
+        if self.rev:
+            ret[self.PARAM_VERSION] = self.rev
+
+        return ret

dvc/ignore.py

@@ -1,7 +1,10 @@
-import os
+from __future__ import unicode_literals
 
-from dulwich.ignore import match_pattern, read_ignore_patterns
-from dvc.utils.compat import cast_bytes
+import os
+from pathspec import PathSpec
+from pathspec.patterns import GitWildMatchPattern
+
+from dvc.utils import relpath
 from dvc.utils.fs import get_parent_dirs_up_to
 
 
@@ -10,23 +13,12 @@ class DvcIgnoreFileHandler(object):
         self.tree = tree
 
     def read_patterns(self, path):
-        with self.tree.open(path, binary=True) as stream:
-            return self._read_patterns(stream)
+        with self.tree.open(path) as fobj:
+            return PathSpec.from_lines(GitWildMatchPattern, fobj)
 
     def get_repo_root(self):
         return self.tree.tree_root
 
-    def _read_patterns(self, binary_stream):
-        negate_patterns = []
-        patterns = []
-        for pattern in read_ignore_patterns(binary_stream):
-            if pattern.lstrip().startswith(b"!"):
-                negate_patterns.append(pattern)
-            else:
-                patterns.append(pattern)
-
-        return negate_patterns, patterns
-
 
 class DvcIgnore(object):
     DVCIGNORE_FILE = ".dvcignore"
@@ -39,12 +31,8 @@ class DvcIgnoreFromFile(DvcIgnore):
     def __init__(self, ignore_file_path, ignore_handler):
         self.ignore_file_path = ignore_file_path
         self.dirname = os.path.normpath(os.path.dirname(ignore_file_path))
-        self.patterns = []
-        self.negate_patterns = []
 
-        self.negate_patterns, self.patterns = ignore_handler.read_patterns(
-            ignore_file_path
-        )
+        self.ignore_spec = ignore_handler.read_patterns(ignore_file_path)
 
     def __call__(self, root, dirs, files):
         files = [f for f in files if not self.matches(root, f)]
@@ -52,26 +40,13 @@ class DvcIgnoreFromFile(DvcIgnore):
 
         return dirs, files
 
-    def get_match(self, abs_path):
-        rel_path = os.path.relpath(abs_path, self.dirname)
-        if os.name == "nt":
-            rel_path = rel_path.replace("\\", "/")
-        rel_path = cast_bytes(rel_path, "utf-8")
-
-        for pattern in self.patterns:
-            if match_pattern(
-                rel_path, pattern
-            ) and self._no_negate_pattern_matches(rel_path):
-                return (abs_path, pattern, self.ignore_file_path)
-        return None
-
     def matches(self, dirname, basename):
-        if self.get_match(os.path.join(dirname, basename)):
-            return True
-        return False
+        abs_path = os.path.join(dirname, basename)
+        relative_path = relpath(abs_path, self.dirname)
+        if os.name == "nt":
+            relative_path = relative_path.replace("\\", "/")
 
-    def _no_negate_pattern_matches(self, path):
-        return all([not match_pattern(path, p) for p in self.negate_patterns])
+        return self.ignore_spec.match_file(relative_path)
 
     def __hash__(self):
         return hash(self.ignore_file_path)

dvc/logger.py

@@ -10,6 +10,12 @@ import logging.config
 import colorama
 
 
+class LoggingException(Exception):
+    def __init__(self, record):
+        msg = "failed to log {}".format(str(record))
+        super(LoggingException, self).__init__(msg)
+
+
 class ExcludeErrorsFilter(logging.Filter):
     def filter(self, record):
         return record.levelno < logging.ERROR
@@ -149,6 +155,12 @@ class ColorFormatter(logging.Formatter):
         progress.clearln()
 
 
+class LoggerHandler(logging.StreamHandler):
+    def handleError(self, record):
+        super(LoggerHandler, self).handleError(record)
+        raise LoggingException(record)
+
+
 def setup(level=logging.INFO):
     colorama.init()
 
@@ -159,14 +171,14 @@ def setup(level=logging.INFO):
             "formatters": {"color": {"()": ColorFormatter}},
             "handlers": {
                 "console": {
-                    "class": "logging.StreamHandler",
+                    "class": "dvc.logger.LoggerHandler",
                     "level": "DEBUG",
                     "formatter": "color",
                     "stream": "ext://sys.stdout",
                     "filters": ["exclude_errors"],
                 },
                 "console_errors": {
-                    "class": "logging.StreamHandler",
+                    "class": "dvc.logger.LoggerHandler",
                     "level": "ERROR",
                     "formatter": "color",
                     "stream": "ext://sys.stderr",
@@ -176,7 +188,11 @@ def setup(level=logging.INFO):
                 "dvc": {
                     "level": level,
                     "handlers": ["console", "console_errors"],
-                }
+                },
+                "paramiko": {
+                    "level": logging.CRITICAL,
+                    "handlers": ["console", "console_errors"],
+                },
             },
         }
     )

dvc/main.py

@@ -5,6 +5,7 @@ from __future__ import unicode_literals
 import logging
 
 from dvc.cli import parse_args
+from dvc.config import ConfigError
 from dvc.analytics import Analytics
 from dvc.exceptions import NotDvcRepoError, DvcParserError
 
@@ -33,9 +34,13 @@ def main(argv=None):
 
         elif args.verbose:
             logger.setLevel(logging.DEBUG)
+            logging.getLogger("paramiko").setLevel(logging.DEBUG)
 
         cmd = args.func(args)
         ret = cmd.run_cmd()
+    except ConfigError:
+        logger.exception("configuration error")
+        ret = 251
     except KeyboardInterrupt:
         logger.exception("interrupted by the user")
         ret = 252

dvc/output/__init__.py

@@ -61,8 +61,7 @@ def _get(stage, p, info, cache, metric, persist=False, tags=None):
     parsed = urlparse(p)
 
     if parsed.scheme == "remote":
-        settings = stage.repo.config.get_remote_settings(parsed.netloc)
-        remote = Remote(stage.repo, settings)
+        remote = Remote(stage.repo, name=parsed.netloc)
         return OUTS_MAP[remote.scheme](
             stage,
             p,

dvc/output/base.py

@@ -1,13 +1,13 @@
 from __future__ import unicode_literals
 
-import re
 import logging
 from copy import copy
 
 from schema import Or, Optional
 
+import dvc.prompt as prompt
 from dvc.exceptions import DvcException
-from dvc.utils.compat import str
+from dvc.utils.compat import str, urlparse
 from dvc.remote.base import RemoteBASE
 
 
@@ -58,6 +58,8 @@ class OutputBase(object):
     DoesNotExistError = OutputDoesNotExistError
     IsNotFileOrDirError = OutputIsNotFileOrDirError
 
+    sep = "/"
+
     def __init__(
         self,
         stage,
@@ -69,9 +71,19 @@ class OutputBase(object):
         persist=False,
         tags=None,
     ):
+        # This output (and dependency) objects have too many paths/urls
+        # here is a list and comments:
+        #
+        #   .def_path - path from definition in stage file
+        #   .path_info - PathInfo/URLInfo structured resolved path
+        #   .fspath - local only, resolved
+        #   .__str__ - for presentation purposes, def_path/relpath
+        #
+        # By resolved path, which contains actual location,
+        # should be absolute and don't contain remote:// refs.
         self.stage = stage
-        self.repo = stage.repo
-        self.url = path
+        self.repo = stage.repo if stage else None
+        self.def_path = path
         self.info = info
         self.remote = remote or self.REMOTE(self.repo, {})
         self.use_cache = False if self.IS_DEPENDENCY else cache
@@ -86,13 +98,26 @@ class OutputBase(object):
                 )
             )
 
+        self.path_info = self._parse_path(remote, path)
+
+    def _parse_path(self, remote, path):
+        if remote:
+            parsed = urlparse(path)
+            return remote.path_info / parsed.path.lstrip("/")
+        else:
+            return self.REMOTE.path_cls(path)
+
     def __repr__(self):
-        return "{class_name}: '{url}'".format(
-            class_name=type(self).__name__, url=(self.url or "No url")
+        return "{class_name}: '{def_path}'".format(
+            class_name=type(self).__name__, def_path=self.def_path
         )
 
     def __str__(self):
-        return self.url
+        return self.def_path
+
+    @property
+    def scheme(self):
+        return self.REMOTE.scheme
 
     @property
     def is_in_repo(self):
@@ -111,35 +136,13 @@ class OutputBase(object):
             "change repo is not supported for {}".format(self.scheme)
         )
 
-    @classmethod
-    def match(cls, url):
-        return re.match(cls.REMOTE.REGEX, url)
-
-    def group(self, name):
-        match = self.match(self.url)
-        if not match:
-            return None
-        return match.group(name)
-
     @classmethod
     def supported(cls, url):
-        return cls.match(url) is not None
-
-    @property
-    def scheme(self):
-        return self.REMOTE.scheme
-
-    @property
-    def path(self):
-        return self.path_info.path
+        return cls.REMOTE.supported(url)
 
     @property
     def cache_path(self):
-        return self.cache.checksum_to_path(self.checksum)
-
-    @property
-    def sep(self):
-        return "/"
+        return self.cache.checksum_to_path_info(self.checksum).url
 
     @property
     def checksum(self):
@@ -228,11 +231,11 @@ class OutputBase(object):
             return
 
         if self.is_in_repo:
-            if self.repo.scm.is_tracked(self.path):
+            if self.repo.scm.is_tracked(self.fspath):
                 raise OutputAlreadyTrackedError(self)
 
             if self.use_cache:
-                self.repo.scm.ignore(self.path)
+                self.repo.scm.ignore(self.fspath)
 
         self.info = self.remote.save_info(self.path_info)
 
@@ -242,7 +245,7 @@ class OutputBase(object):
 
     def dumpd(self):
         ret = copy(self.info)
-        ret[self.PARAM_PATH] = self.url
+        ret[self.PARAM_PATH] = self.def_path
 
         if self.IS_DEPENDENCY:
             return ret
@@ -269,8 +272,8 @@ class OutputBase(object):
             "verify metric is not supported for {}".format(self.scheme)
         )
 
-    def download(self, to_info, resume=False):
-        self.remote.download([self.path_info], [to_info], resume=resume)
+    def download(self, to, resume=False):
+        self.remote.download([self.path_info], [to.path_info], resume=resume)
 
     def checkout(self, force=False, progress_callback=None, tag=None):
         if not self.use_cache:
@@ -294,20 +297,20 @@ class OutputBase(object):
             return
 
         if ignore_remove and self.use_cache and self.is_in_repo:
-            self.repo.scm.ignore_remove(self.path)
+            self.repo.scm.ignore_remove(self.fspath)
 
     def move(self, out):
         if self.scheme == "local" and self.use_cache and self.is_in_repo:
-            self.repo.scm.ignore_remove(self.path)
+            self.repo.scm.ignore_remove(self.fspath)
 
         self.remote.move(self.path_info, out.path_info)
-        self.url = out.url
-        self.path_info = copy(out.path_info)
+        self.def_path = out.def_path
+        self.path_info = out.path_info
         self.save()
         self.commit()
 
         if self.scheme == "local" and self.use_cache and self.is_in_repo:
-            self.repo.scm.ignore(self.path)
+            self.repo.scm.ignore(self.fspath)
 
     def get_files_number(self):
         if not self.use_cache or not self.checksum:
@@ -321,3 +324,96 @@ class OutputBase(object):
     def unprotect(self):
         if self.exists:
             self.remote.unprotect(self.path_info)
+
+    def _collect_used_dir_cache(self, remote=None, force=False, jobs=None):
+        """Get a list of `info`s retaled to the given directory.
+
+        - Pull the directory entry from the remote cache if it was changed.
+
+        Example:
+
+            Given the following commands:
+
+            $ echo "foo" > directory/foo
+            $ echo "bar" > directory/bar
+            $ dvc add directory
+
+            It will return something similar to the following list:
+
+            [
+                { 'path': 'directory/foo', 'md5': 'c157a79031e1', ... },
+                { 'path': 'directory/bar', 'md5': 'd3b07384d113', ... },
+            ]
+        """
+
+        ret = []
+
+        if self.cache.changed_cache_file(self.checksum):
+            try:
+                self.repo.cloud.pull(
+                    [
+                        {
+                            self.remote.PARAM_CHECKSUM: self.checksum,
+                            "name": str(self),
+                        }
+                    ],
+                    jobs=jobs,
+                    remote=remote,
+                    show_checksums=False,
+                )
+            except DvcException:
+                logger.debug("failed to pull cache for '{}'".format(self))
+
+        if self.cache.changed_cache_file(self.checksum):
+            msg = (
+                "Missing cache for directory '{}'. "
+                "Cache for files inside will be lost. "
+                "Would you like to continue? Use '-f' to force."
+            )
+            if not force and not prompt.confirm(msg):
+                raise DvcException(
+                    "unable to fully collect used cache"
+                    " without cache for directory '{}'".format(self)
+                )
+            else:
+                return ret
+
+        for entry in self.dir_cache:
+            info = copy(entry)
+            path_info = self.path_info / entry[self.remote.PARAM_RELPATH]
+            info["name"] = str(path_info)
+            ret.append(info)
+
+        return ret
+
+    def get_used_cache(self, **kwargs):
+        """Get a dumpd of the given `out`, with an entry including the branch.
+
+        The `used_cache` of an output is no more than its `info`.
+
+        In case that the given output is a directory, it will also
+        include the `info` of its files.
+        """
+
+        if self.stage.is_repo_import:
+            return []
+
+        if not self.use_cache:
+            return []
+
+        if not self.info:
+            logger.warning(
+                "Output '{}'({}) is missing version info. Cache for it will "
+                "not be collected. Use dvc repro to get your pipeline up to "
+                "date.".format(self, self.stage)
+            )
+            return []
+
+        ret = [{self.remote.PARAM_CHECKSUM: self.checksum, "name": str(self)}]
+
+        if not self.is_dir_checksum:
+            return ret
+
+        ret.extend(self._collect_used_dir_cache(**kwargs))
+
+        return ret

dvc/output/hdfs.py

@@ -1,38 +1,8 @@
 from __future__ import unicode_literals
 
-import posixpath
-
-from dvc.path.hdfs import PathHDFS
-from dvc.utils.compat import urlparse
 from dvc.output.base import OutputBase
 from dvc.remote.hdfs import RemoteHDFS
 
 
 class OutputHDFS(OutputBase):
     REMOTE = RemoteHDFS
-
-    def __init__(
-        self,
-        stage,
-        path,
-        info=None,
-        remote=None,
-        cache=True,
-        metric=False,
-        persist=False,
-        tags=None,
-    ):
-        super(OutputHDFS, self).__init__(
-            stage,
-            path,
-            info=info,
-            remote=remote,
-            cache=cache,
-            metric=metric,
-            persist=persist,
-            tags=tags,
-        )
-        if remote:
-            path = posixpath.join(remote.url, urlparse(path).path.lstrip("/"))
-        user = remote.user if remote else self.group("user")
-        self.path_info = PathHDFS(user=user, url=self.url, path=path)

dvc/output/local.py

@@ -3,8 +3,7 @@ from __future__ import unicode_literals
 import os
 import logging
 
-from dvc.path.local import PathLOCAL
-from dvc.utils.compat import urlparse
+from dvc.utils.compat import urlparse, fspath_py35, str
 from dvc.istextfile import istextfile
 from dvc.exceptions import DvcException
 from dvc.remote.local import RemoteLOCAL
@@ -16,82 +15,58 @@ logger = logging.getLogger(__name__)
 
 class OutputLOCAL(OutputBase):
     REMOTE = RemoteLOCAL
+    sep = os.sep
 
-    def __init__(
-        self,
-        stage,
-        path,
-        info=None,
-        remote=None,
-        cache=True,
-        metric=False,
-        persist=False,
-        tags=None,
-    ):
-        super(OutputLOCAL, self).__init__(
-            stage,
-            path,
-            info,
-            remote=remote,
-            cache=cache,
-            metric=metric,
-            persist=persist,
-            tags=tags,
-        )
-        if remote:
-            p = os.path.join(
-                remote.prefix, urlparse(self.url).path.lstrip("/")
-            )
+    def _parse_path(self, remote, path):
+        parsed = urlparse(path)
+        if parsed.scheme == "remote":
+            p = remote.path_info / parsed.path.lstrip("/")
         else:
-            p = path
+            # NOTE: we can path either from command line or .dvc file,
+            # so we should expect both posix and windows style paths.
+            # PathInfo accepts both, i.e. / works everywhere, \ only on win.
+            #
+            # FIXME: if we have Windows path containig / or posix one with \
+            # then we have #2059 bug and can't really handle that.
+            p = self.REMOTE.path_cls(path)
+            if not p.is_absolute():
+                p = self.stage.wdir / p
 
-        if not os.path.isabs(p):
-            p = self.remote.to_ospath(p)
-            p = os.path.join(stage.wdir, p)
-        p = os.path.abspath(os.path.normpath(p))
-
-        self.path_info = PathLOCAL(url=self.url, path=p)
+        abs_p = os.path.abspath(os.path.normpath(fspath_py35(p)))
+        return self.REMOTE.path_cls(abs_p)
 
     def __str__(self):
-        return self.rel_path
+        return str(self.path_info)
+
+    @property
+    def fspath(self):
+        return self.path_info.fspath
 
     @property
     def is_in_repo(self):
-        return urlparse(self.url).scheme != "remote" and not os.path.isabs(
-            self.url
-        )
+        def_scheme = urlparse(self.def_path).scheme
+        return def_scheme != "remote" and not os.path.isabs(self.def_path)
 
     def assign_to_stage_file(self, stage):
         from dvc.repo import Repo
 
-        fullpath = os.path.abspath(stage.wdir)
-        self.path_info.path = os.path.join(fullpath, self.stage_path)
+        # Make path info the same but relative to passed stage
+        rel_path = self.path_info.relpath(self.stage.wdir)
+        self.path_info = (
+            self.REMOTE.path_cls(os.path.abspath(stage.wdir)) / rel_path
+        )
 
-        self.repo = Repo(self.path)
+        self.repo = Repo(self.fspath)
 
         self.stage = stage
         return self
 
-    @property
-    def sep(self):
-        return os.sep
-
-    @property
-    def rel_path(self):
-        return os.path.relpath(self.path)
-
-    @property
-    def stage_path(self):
-        return os.path.relpath(self.path, self.stage.wdir)
-
     def dumpd(self):
         ret = super(OutputLOCAL, self).dumpd()
         if self.is_in_repo:
-            path = self.remote.to_posixpath(
-                os.path.relpath(self.path, self.stage.wdir)
-            )
+            path = self.path_info.relpath(self.stage.wdir).as_posix()
         else:
-            path = self.url
+            path = self.def_path
 
         ret[self.PARAM_PATH] = path
 
@@ -101,13 +76,14 @@ class OutputLOCAL(OutputBase):
         if not self.metric:
             return
 
-        if not os.path.exists(self.path):
+        path = fspath_py35(self.path_info)
+        if not os.path.exists(path):
             return
 
-        if os.path.isdir(self.path):
+        if os.path.isdir(path):
             msg = "directory '{}' cannot be used as metrics."
-            raise DvcException(msg.format(self.rel_path))
+            raise DvcException(msg.format(self.path_info))
 
-        if not istextfile(self.path):
+        if not istextfile(path):
             msg = "binary file '{}' cannot be used as metrics."
-            raise DvcException(msg.format(self.rel_path))
+            raise DvcException(msg.format(self.path_info))

dvc/output/s3.py

@@ -1,42 +1,8 @@
 from __future__ import unicode_literals
 
-import posixpath
-
-from dvc.path import Path
 from dvc.remote.s3 import RemoteS3
-from dvc.utils.compat import urlparse
 from dvc.output.base import OutputBase
 
 
 class OutputS3(OutputBase):
     REMOTE = RemoteS3
-
-    def __init__(
-        self,
-        stage,
-        path,
-        info=None,
-        remote=None,
-        cache=True,
-        metric=False,
-        persist=False,
-        tags=None,
-    ):
-        super(OutputS3, self).__init__(
-            stage,
-            path,
-            info=info,
-            remote=remote,
-            cache=cache,
-            metric=metric,
-            persist=persist,
-            tags=tags,
-        )
-        bucket = remote.bucket if remote else urlparse(path).netloc
-        path = urlparse(path).path.lstrip("/")
-        if remote:
-            path = posixpath.join(remote.prefix, path)
-
-        self.path_info = Path(
-            self.scheme, bucket=bucket, path=path, url=self.url
-        )

dvc/output/ssh.py

@@ -1,54 +1,8 @@
 from __future__ import unicode_literals
 
-import getpass
-import posixpath
-
-from dvc.path.ssh import PathSSH
-from dvc.utils.compat import urlparse
 from dvc.output.base import OutputBase
 from dvc.remote.ssh import RemoteSSH
 
 
 class OutputSSH(OutputBase):
     REMOTE = RemoteSSH
-
-    def __init__(
-        self,
-        stage,
-        path,
-        info=None,
-        remote=None,
-        cache=True,
-        metric=False,
-        persist=False,
-        tags=None,
-    ):
-        super(OutputSSH, self).__init__(
-            stage,
-            path,
-            info=info,
-            remote=remote,
-            cache=cache,
-            metric=metric,
-            persist=persist,
-            tags=tags,
-        )
-        parsed = urlparse(path)
-        host = remote.host if remote else parsed.hostname
-        port = (
-            remote.port if remote else (parsed.port or RemoteSSH.DEFAULT_PORT)
-        )
-        user = (
-            remote.user if remote else (parsed.username or getpass.getuser())
-        )
-
-        if remote:
-            path = posixpath.join(
-                remote.prefix, urlparse(path).path.lstrip("/")
-            )
-        else:
-            path = parsed.path
-
-        self.path_info = PathSSH(
-            host=host, user=user, port=port, url=self.url, path=path
-        )

dvc/path/__init__.py

@@ -1,29 +0,0 @@
-from dvc.scheme import Schemes
-
-from dvc.path.azure import PathAZURE
-from dvc.path.gs import PathGS
-from dvc.path.hdfs import PathHDFS
-from dvc.path.http import PathHTTP
-from dvc.path.https import PathHTTPS
-from dvc.path.local import PathLOCAL
-from dvc.path.oss import PathOSS
-from dvc.path.s3 import PathS3
-from dvc.path.ssh import PathSSH
-
-
-PATH_MAP = {
-    Schemes.SSH: PathSSH,
-    Schemes.HDFS: PathHDFS,
-    Schemes.S3: PathS3,
-    Schemes.AZURE: PathAZURE,
-    Schemes.HTTP: PathHTTP,
-    Schemes.HTTPS: PathHTTPS,
-    Schemes.GS: PathGS,
-    Schemes.LOCAL: PathLOCAL,
-    Schemes.OSS: PathOSS,
-}
-
-
-def Path(scheme, *args, **kwargs):
-    cls = PATH_MAP[scheme]
-    return cls(*args, **kwargs)

dvc/path/azure.py

@@ -1,6 +0,0 @@
-from dvc.scheme import Schemes
-from .base import PathCloudBASE
-
-
-class PathAZURE(PathCloudBASE):
-    scheme = Schemes.AZURE

dvc/path/base.py

@@ -1,23 +0,0 @@
-from dvc.utils.compat import urlunsplit
-
-
-class PathBASE(object):
-    scheme = None
-
-    def __init__(self, url=None, path=None):
-        self.url = url
-        self.path = path
-
-    def __str__(self):
-        return self.url
-
-
-class PathCloudBASE(PathBASE):
-    def __init__(self, bucket, url=None, path=None):
-        super(PathCloudBASE, self).__init__(url, path)
-        self.bucket = bucket
-
-    def __str__(self):
-        if not self.url:
-            return urlunsplit((self.scheme, self.bucket, self.path, "", ""))
-        return self.url

dvc/path/gs.py

@@ -1,6 +0,0 @@
-from dvc.scheme import Schemes
-from .base import PathCloudBASE
-
-
-class PathGS(PathCloudBASE):
-    scheme = Schemes.GS

dvc/path/hdfs.py

@@ -1,16 +0,0 @@
-from dvc.utils.compat import urlunsplit
-from dvc.scheme import Schemes
-from .base import PathBASE
-
-
-class PathHDFS(PathBASE):
-    scheme = Schemes.HDFS
-
-    def __init__(self, user, url=None, path=None):
-        super(PathHDFS, self).__init__(url, path)
-        self.user = user
-
-    def __str__(self):
-        if not self.url:
-            return urlunsplit((self.scheme, self.user, self.path, "", ""))
-        return self.url

dvc/path/http.py

@@ -1,13 +0,0 @@
-from dvc.scheme import Schemes
-from dvc.utils.compat import urlunsplit
-
-from .base import PathBASE
-
-
-class PathHTTP(PathBASE):
-    scheme = Schemes.HTTP
-
-    def __str__(self):
-        if not self.url:
-            return urlunsplit((self.scheme, self.path, "", "", ""))
-        return self.url

dvc/path/https.py

@@ -1,6 +0,0 @@
-from dvc.scheme import Schemes
-from .http import PathHTTP
-
-
-class PathHTTPS(PathHTTP):
-    scheme = Schemes.HTTPS

dvc/path/local.py

@@ -1,17 +0,0 @@
-import os
-
-from dvc.scheme import Schemes
-from .base import PathBASE
-
-
-class PathLOCAL(PathBASE):
-    scheme = Schemes.LOCAL
-
-    def __str__(self):
-        if os.name == "nt" and not os.path.commonprefix(
-            [os.getcwd(), self.path]
-        ):
-            # In case of windows, when cache is on different drive than
-            # workspace, we will get ValueError when trying to get relpath
-            return self.path
-        return os.path.relpath(self.path)

dvc/path/oss.py

@@ -1,6 +0,0 @@
-from dvc.scheme import Schemes
-from .base import PathCloudBASE
-
-
-class PathOSS(PathCloudBASE):
-    scheme = Schemes.OSS

dvc/path/s3.py

@@ -1,6 +0,0 @@
-from dvc.scheme import Schemes
-from .base import PathCloudBASE
-
-
-class PathS3(PathCloudBASE):
-    scheme = Schemes.S3

dvc/path/ssh.py

@@ -1,26 +0,0 @@
-from dvc.utils.compat import urlunsplit
-from dvc.scheme import Schemes
-from .base import PathBASE
-
-
-class PathSSH(PathBASE):
-    scheme = Schemes.SSH
-
-    def __init__(self, host, user, port, url=None, path=None):
-        super(PathSSH, self).__init__(url, path)
-        self.host = host
-        self.user = user
-        self.port = port
-
-    def __str__(self):
-        if not self.url:
-            return urlunsplit(
-                (
-                    self.scheme,
-                    "{}@{}:{}".format(self.user, self.host, self.port),
-                    self.path,
-                    "",
-                    "",
-                )
-            )
-        return self.url

dvc/path_info.py

@@ -0,0 +1,210 @@
+from __future__ import unicode_literals, absolute_import
+
+import sys
+import os
+import posixpath
+from funcy import cached_property
+
+from dvc.utils.compat import str, builtin_str, basestring, is_py2
+from dvc.utils.compat import pathlib, urlparse
+
+
+# On Python 2.7/Windows sys.getfilesystemencoding() is set to mbcs,
+# which is lossy, thus we can't use that,
+# see https://github.com/mcmtroffaes/pathlib2/issues/56.
+from dvc.utils import relpath
+
+if is_py2:
+    fs_encoding = "utf-8"
+
+
+class PathInfo(pathlib.PurePath):
+    scheme = "local"
+
+    def __new__(cls, *args):
+        # Construct a proper subclass depending on current os
+        if cls is PathInfo:
+            cls = WindowsPathInfo if os.name == "nt" else PosixPathInfo
+        return cls._from_parts(args)
+
+    @classmethod
+    def from_posix(cls, s):
+        return cls(PosixPathInfo(s))
+
+    def __str__(self):
+        path = self.__fspath__()
+        return relpath(path)
+
+    def __repr__(self):
+        return builtin_str("{}: '{}'").format(type(self).__name__, self)
+
+    # This permits passing it to file utils directly in Python 3.6+
+    # With Python 2.7, Python 3.5+ we are stuck with path_info.fspath for now
+    def __fspath__(self):
+        return pathlib.PurePath.__str__(self)
+
+    @property
+    def fspath(self):
+        return self.__fspath__()
+
+    url = fspath
+
+    def relpath(self, other):
+        return self.__class__(relpath(self, other))
+
+    def isin(self, other):
+        if isinstance(other, basestring):
+            other = self.__class__(other)
+        elif self.__class__ != other.__class__:
+            return False
+        return any(p == other for p in self.parents)
+
+    # pathlib2 uses bytes internally in Python 2, and we use unicode everywhere
+    # for paths in both pythons, thus we need this glue.
+    if is_py2:
+        __unicode__ = __str__
+
+        def __str__(self):
+            return self.__unicode__().encode(sys.getfilesystemencoding())
+
+        @classmethod
+        def _parse_args(cls, args):
+            args = [
+                a.encode(fs_encoding)
+                if isinstance(a, unicode)  # noqa: F821
+                else a
+                for a in args
+            ]
+            return super(PathInfo, cls)._parse_args(args)
+
+        @property
+        def name(self):
+            return super(PathInfo, self).name.decode(fs_encoding)
+
+        def __fspath__(self):  # noqa: F811
+            return pathlib.PurePath.__str__(self).decode(fs_encoding)
+
+        def with_name(self, name):
+            return pathlib.PurePath.with_name(self, name.encode(fs_encoding))
+
+        def as_posix(self):
+            f = self._flavour
+            return str(self).replace(f.sep, "/")
+
+
+class WindowsPathInfo(PathInfo, pathlib.PureWindowsPath):
+    pass
+
+
+class PosixPathInfo(PathInfo, pathlib.PurePosixPath):
+    pass
+
+
+class URLInfo(object):
+    DEFAULT_PORTS = {"http": 80, "https": 443, "ssh": 22}
+
+    def __init__(self, url):
+        self.parsed = urlparse(url)
+        assert self.parsed.scheme != "remote"
+
+    @classmethod
+    def from_parts(
+        cls, scheme=None, netloc=None, host=None, user=None, port=None, path=""
+    ):
+        assert scheme and (bool(host) ^ bool(netloc))
+
+        if netloc is None:
+            netloc = host
+            if user:
+                netloc = user + "@" + host
+            if port:
+                netloc += ":" + str(port)
+        return cls("{}://{}{}".format(scheme, netloc, path))
+
+    @cached_property
+    def url(self):
+        p = self.parsed
+        netloc = p.hostname
+        if p.username:
+            netloc = p.username + "@" + netloc
+        if p.port and int(p.port) != self.DEFAULT_PORTS.get(p.scheme):
+            netloc += ":" + str(p.port)
+        return "{}://{}{}".format(p.scheme, netloc, p.path)
+
+    def __str__(self):
+        return self.url
+
+    def __repr__(self):
+        return "{}: '{}'".format(type(self).__name__, self)
+
+    def __eq__(self, other):
+        if isinstance(other, basestring):
+            other = self.__class__(other)
+        return str(self) == str(other)
+
+    def __hash__(self):
+        return hash(self.url)
+
+    def __div__(self, other):
+        p = self.parsed
+        new_path = posixpath.join(p.path, str(other))
+        if not new_path.startswith("/"):
+            new_path = "/" + new_path
+        new_url = "{}://{}{}".format(p.scheme, p.netloc, new_path)
+        return self.__class__(new_url)
+
+    __truediv__ = __div__
+
+    def __getattr__(self, name):
+        return getattr(self.parsed, name)
+
+    @property
+    def port(self):
+        return self.parsed.port or self.DEFAULT_PORTS.get(self.parsed.scheme)
+
+    @property
+    def host(self):
+        return self.parsed.hostname
+
+    @property
+    def user(self):
+        return self.parsed.username
+
+    @cached_property
+    def _path(self):
+        return pathlib.PurePosixPath(self.parsed.path)
+
+    @property
+    def name(self):
+        return self._path.name
+
+    @property
+    def parts(self):
+        return (self.scheme, self.netloc) + self._path.parts
+
+    @property
+    def bucket(self):
+        return self.netloc
+
+    @property
+    def parent(self):
+        return self.from_parts(
+            scheme=self.scheme, netloc=self.netloc, path=self._path.parent
+        )
+
+    def isin(self, other):
+        if isinstance(other, basestring):
+            other = self.__class__(other)
+        elif self.__class__ != other.__class__:
+            return False
+        return (
+            self.scheme == other.scheme
+            and self.netloc == other.netloc
+            and PathInfo(self.path).isin(PathInfo(other.path))
+        )
+
+
+class CloudURLInfo(URLInfo):
+    @property
+    def path(self):
+        return self.parsed.path.lstrip("/")

dvc/progress.py

@@ -111,7 +111,7 @@ class Progress(object):
 
         logger = logging.getLogger(__name__)
 
-        if logger.getEffectiveLevel() == "CRITICAL":
+        if logger.getEffectiveLevel() == logging.CRITICAL:
             return
 
         print(*args, **kwargs)

dvc/remote/__init__.py

@@ -10,6 +10,8 @@ from dvc.remote.http import RemoteHTTP
 from dvc.remote.https import RemoteHTTPS
 from dvc.remote.oss import RemoteOSS
 
+from .config import RemoteConfig
+
 
 REMOTES = [
     RemoteAZURE,
@@ -31,5 +33,11 @@ def _get(config):
     return RemoteLOCAL
 
 
-def Remote(repo, config):
-    return _get(config)(repo, config)
+def Remote(repo, **kwargs):
+    name = kwargs.get("name")
+    if name:
+        remote_config = RemoteConfig(repo.config)
+        settings = remote_config.get_settings(name)
+    else:
+        settings = kwargs
+    return _get(settings)(repo, settings)

dvc/remote/azure.py

@@ -6,7 +6,6 @@ import re
 import logging
 
 from dvc.scheme import Schemes
-from dvc.path.azure import PathAZURE
 
 try:
     from azure.storage.blob import BlockBlobService
@@ -14,11 +13,11 @@ try:
 except ImportError:
     BlockBlobService = None
 
-from dvc.utils import tmp_fname, move
-from dvc.utils.compat import urlparse, makedirs
+from dvc.utils.compat import urlparse
 from dvc.progress import progress
 from dvc.config import Config
 from dvc.remote.base import RemoteBASE
+from dvc.path_info import CloudURLInfo
 
 
 logger = logging.getLogger(__name__)
@@ -34,6 +33,7 @@ class Callback(object):
 
 class RemoteAZURE(RemoteBASE):
     scheme = Schemes.AZURE
+    path_cls = CloudURLInfo
     REGEX = (
         r"azure://((?P<path>[^=;]*)?|("
         # backward compatibility
@@ -47,58 +47,57 @@ class RemoteAZURE(RemoteBASE):
     def __init__(self, repo, config):
         super(RemoteAZURE, self).__init__(repo, config)
 
-        self.url = config.get(Config.SECTION_REMOTE_URL, "azure://")
-        match = re.match(self.REGEX, self.url)  # backward compatibility
+        url = config.get(Config.SECTION_REMOTE_URL, "azure://")
 
+        match = re.match(self.REGEX, url)  # backward compatibility
         path = match.group("path")
-        self.bucket = (
-            urlparse(self.url if path else "").netloc
+        bucket = (
+            urlparse(url if path else "").netloc
             or match.group("container_name")  # backward compatibility
             or os.getenv("AZURE_STORAGE_CONTAINER_NAME")
         )
 
-        self.prefix = urlparse(self.url).path.lstrip("/") if path else ""
-
         self.connection_string = (
             config.get(Config.SECTION_AZURE_CONNECTION_STRING)
             or match.group("connection_string")  # backward compatibility
             or os.getenv("AZURE_STORAGE_CONNECTION_STRING")
         )
 
-        if not self.bucket:
+        if not bucket:
             raise ValueError("azure storage container name missing")
 
         if not self.connection_string:
             raise ValueError("azure storage connection string missing")
 
         self.__blob_service = None
-        self.path_info = PathAZURE(bucket=self.bucket)
+        self.path_info = (
+            self.path_cls(url)
+            if path
+            else self.path_cls.from_parts(scheme=self.scheme, netloc=bucket)
+        )
 
     @property
     def blob_service(self):
         if self.__blob_service is None:
-            logger.debug("URL {}".format(self.url))
+            logger.debug("URL {}".format(self.path_info))
             logger.debug("Connection string {}".format(self.connection_string))
             self.__blob_service = BlockBlobService(
                 connection_string=self.connection_string
             )
-            logger.debug("Container name {}".format(self.bucket))
+            logger.debug("Container name {}".format(self.path_info.bucket))
             try:  # verify that container exists
                 self.__blob_service.list_blobs(
-                    self.bucket, delimiter="/", num_results=1
+                    self.path_info.bucket, delimiter="/", num_results=1
                 )
             except AzureMissingResourceHttpError:
-                self.__blob_service.create_container(self.bucket)
+                self.__blob_service.create_container(self.path_info.bucket)
         return self.__blob_service
 
     def remove(self, path_info):
         if path_info.scheme != self.scheme:
             raise NotImplementedError
 
-        logger.debug(
-            "Removing azure://{}/{}".format(path_info.bucket, path_info.path)
-        )
-
+        logger.debug("Removing {}".format(path_info))
         self.blob_service.delete_blob(path_info.bucket, path_info.path)
 
     def _list_paths(self, bucket, prefix):
@@ -118,85 +117,20 @@ class RemoteAZURE(RemoteBASE):
             next_marker = blobs.next_marker
 
     def list_cache_paths(self):
-        return self._list_paths(self.bucket, self.prefix)
+        return self._list_paths(self.path_info.bucket, self.path_info.path)
 
-    def upload(self, from_infos, to_infos, names=None, no_progress_bar=False):
-        names = self._verify_path_args(to_infos, from_infos, names)
-
-        for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info.scheme != self.scheme:
-                raise NotImplementedError
-
-            if from_info.scheme != "local":
-                raise NotImplementedError
-
-            bucket = to_info.bucket
-            path = to_info.path
-
-            logger.debug(
-                "Uploading '{}' to '{}/{}'".format(
-                    from_info.path, bucket, path
-                )
-            )
-
-            if not name:
-                name = os.path.basename(from_info.path)
-
-            cb = None if no_progress_bar else Callback(name)
-
-            try:
-                self.blob_service.create_blob_from_path(
-                    bucket, path, from_info.path, progress_callback=cb
-                )
-            except Exception:
-                msg = "failed to upload '{}'".format(from_info.path)
-                logger.warning(msg)
-            else:
-                progress.finish_target(name)
-
-    def download(
-        self,
-        from_infos,
-        to_infos,
-        no_progress_bar=False,
-        names=None,
-        resume=False,
+    def _upload(
+        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
     ):
-        names = self._verify_path_args(from_infos, to_infos, names)
+        cb = None if no_progress_bar else Callback(name)
+        self.blob_service.create_blob_from_path(
+            to_info.bucket, to_info.path, from_file, progress_callback=cb
+        )
 
-        for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info.scheme != self.scheme:
-                raise NotImplementedError
-
-            if to_info.scheme != "local":
-                raise NotImplementedError
-
-            bucket = from_info.bucket
-            path = from_info.path
-
-            logger.debug(
-                "Downloading '{}/{}' to '{}'".format(
-                    bucket, path, to_info.path
-                )
-            )
-
-            tmp_file = tmp_fname(to_info.path)
-            if not name:
-                name = os.path.basename(to_info.path)
-
-            cb = None if no_progress_bar else Callback(name)
-
-            makedirs(os.path.dirname(to_info.path), exist_ok=True)
-
-            try:
-                self.blob_service.get_blob_to_path(
-                    bucket, path, tmp_file, progress_callback=cb
-                )
-            except Exception:
-                msg = "failed to download '{}/{}'".format(bucket, path)
-                logger.warning(msg)
-            else:
-                move(tmp_file, to_info.path)
-
-                if not no_progress_bar:
-                    progress.finish_target(name)
+    def _download(
+        self, from_info, to_file, name=None, no_progress_bar=False, **_kwargs
+    ):
+        cb = None if no_progress_bar else Callback(name)
+        self.blob_service.get_blob_to_path(
+            from_info.bucket, from_info.path, to_file, progress_callback=cb
+        )

dvc/remote/base.py

@@ -1,26 +1,24 @@
 from __future__ import unicode_literals
 
-from copy import copy
-
-from dvc.path.local import PathLOCAL
-from dvc.utils.compat import str
+from dvc.utils.compat import str, basestring, urlparse, fspath_py35, makedirs
 
 import os
-import re
 import json
-import ntpath
 import logging
 import tempfile
-import posixpath
+import itertools
+from contextlib import contextmanager
 from operator import itemgetter
 from multiprocessing import cpu_count
+from concurrent.futures import ThreadPoolExecutor
 
 import dvc.prompt as prompt
 from dvc.config import Config
 from dvc.exceptions import DvcException, ConfirmRemoveError
-from dvc.progress import progress
-from dvc.utils import LARGE_DIR_SIZE, tmp_fname
+from dvc.progress import progress, ProgressCallback
+from dvc.utils import LARGE_DIR_SIZE, tmp_fname, to_chunks, move, relpath
 from dvc.state import StateBase
+from dvc.path_info import PathInfo, URLInfo
 
 
 logger = logging.getLogger(__name__)
@@ -67,8 +65,8 @@ class RemoteMissingDepsError(DvcException):
 
 
 class RemoteBASE(object):
-    scheme = None
-    REGEX = None
+    scheme = "base"
+    path_cls = URLInfo
     REQUIRES = {}
     JOBS = 4 * cpu_count()
 
@@ -104,13 +102,14 @@ class RemoteBASE(object):
             raise RemoteMissingDepsError(msg)
 
         self.protected = False
+        self.no_traverse = config.get(Config.SECTION_REMOTE_NO_TRAVERSE)
         self.state = StateBase()
-
         self._dir_info = {}
 
     def __repr__(self):
-        return "{class_name}: '{url}'".format(
-            class_name=type(self).__name__, url=(self.url or "No url")
+        return "{class_name}: '{path_info}'".format(
+            class_name=type(self).__name__,
+            path_info=self.path_info or "No path",
         )
 
     def compat_config(config):
@@ -118,39 +117,14 @@ class RemoteBASE(object):
 
     @classmethod
     def supported(cls, config):
-        url = config[Config.SECTION_REMOTE_URL]
-        return cls.match(url) is not None
+        if isinstance(config, basestring):
+            url = config
+        else:
+            url = config[Config.SECTION_REMOTE_URL]
 
-    @staticmethod
-    def _replace_sep(path, from_sep, to_sep):
-        assert not ntpath.isabs(path)
-        assert not posixpath.isabs(path)
-        return path.replace(from_sep, to_sep)
-
-    @classmethod
-    def to_posixpath(cls, path):
-        return cls._replace_sep(path, "\\", "/")
-
-    @classmethod
-    def to_ntpath(cls, path):
-        return cls._replace_sep(path, "/", "\\")
-
-    def to_ospath(self, path):
-        if self.ospath == ntpath:
-            return self.to_ntpath(path)
-        elif self.ospath == posixpath:
-            return self.to_posixpath(path)
-        assert False
-
-    @classmethod
-    def match(cls, url):
-        return re.match(cls.REGEX, url)
-
-    def group(self, name):
-        m = self.match(self.url)
-        if not m:
-            return None
-        return m.group(name)
+        # NOTE: silently skipping remote, calling code should handle that
+        parsed = urlparse(url)
+        return parsed.scheme == cls.scheme
 
     @property
     def cache(self):
@@ -162,28 +136,31 @@ class RemoteBASE(object):
     def _collect_dir(self, path_info):
         dir_info = []
 
-        p_info = copy(path_info)
-        dpath = p_info.path
         for root, dirs, files in self.walk(path_info):
             if len(files) > LARGE_DIR_SIZE:
                 msg = (
                     "Computing md5 for a large directory {}. "
                     "This is only done once."
                 )
-                relpath = self.ospath.relpath(root)
-                logger.info(msg.format(relpath))
-                files = progress(files, name=relpath)
+                title = str(self.path_cls(root))
+                logger.info(msg.format(title))
+                files = progress(files, name=title)
 
             for fname in files:
-                path = self.ospath.join(root, fname)
-                p_info.path = path
-                relpath = self.to_posixpath(self.ospath.relpath(path, dpath))
-
-                checksum = self.get_file_checksum(p_info)
+                file_info = self.path_cls(root) / fname
+                relative_path = file_info.relative_to(path_info)
                 dir_info.append(
                     {
-                        self.PARAM_RELPATH: relpath,
-                        self.PARAM_CHECKSUM: checksum,
+                        # NOTE: this is lossy transformation:
+                        #   "hey\there" -> "hey/there"
+                        #   "hey/there" -> "hey/there"
+                        # The latter is fine filename on Windows,
+                        # which will transform to dir/file on back transform.
+                        #
+                        # Yes, this is a BUG, as long as we permit "/" in
+                        # filenames on Windows and "\" on Unix
+                        self.PARAM_RELPATH: relative_path.as_posix(),
+                        self.PARAM_CHECKSUM: self.get_file_checksum(file_info),
                     }
                 )
 
@@ -192,32 +169,27 @@ class RemoteBASE(object):
 
     def get_dir_checksum(self, path_info):
         dir_info = self._collect_dir(path_info)
-        checksum, from_info, to_info = self._get_dir_info_checksum(
-            dir_info, path_info
-        )
+        checksum, tmp_info = self._get_dir_info_checksum(dir_info)
+        new_info = self.cache.checksum_to_path_info(checksum)
         if self.cache.changed_cache_file(checksum):
-            self.cache.move(from_info, to_info)
+            self.cache.move(tmp_info, new_info)
 
         self.state.save(path_info, checksum)
-        self.state.save(to_info, checksum)
+        self.state.save(new_info, checksum)
 
         return checksum
 
-    def _get_dir_info_checksum(self, dir_info, path_info):
-        to_info = copy(path_info)
-        to_info.path = self.cache.ospath.join(self.cache.prefix, tmp_fname(""))
-
+    def _get_dir_info_checksum(self, dir_info):
         tmp = tempfile.NamedTemporaryFile(delete=False).name
         with open(tmp, "w+") as fobj:
             json.dump(dir_info, fobj, sort_keys=True)
 
-        from_info = PathLOCAL(path=tmp)
+        from_info = PathInfo(tmp)
+        to_info = self.cache.path_info / tmp_fname("")
         self.cache.upload([from_info], [to_info], no_progress_bar=True)
 
         checksum = self.get_file_checksum(to_info) + self.CHECKSUM_DIR_SUFFIX
-        from_info = copy(to_info)
-        to_info.path = self.cache.checksum_to_path(checksum)
-        return checksum, from_info, to_info
+        return checksum, to_info
 
     def get_dir_cache(self, checksum):
         assert checksum
@@ -235,7 +207,7 @@ class RemoteBASE(object):
 
         fobj = tempfile.NamedTemporaryFile(delete=False)
         path = fobj.name
-        to_info = PathLOCAL(path=path)
+        to_info = self.path_cls(path)
         self.cache.download([path_info], [to_info], no_progress_bar=True)
 
         try:
@@ -249,11 +221,13 @@ class RemoteBASE(object):
 
         if not isinstance(d, list):
             msg = "dir cache file format error '{}' [skipping the file]"
-            logger.error(msg.format(os.path.relpath(path)))
+            logger.error(msg.format(relpath(path)))
             return []
 
         for info in d:
-            info[self.PARAM_RELPATH] = self.to_ospath(info[self.PARAM_RELPATH])
+            # NOTE: here is a BUG, see comment to .as_posix() below
+            relative_path = self.path_cls.from_posix(info[self.PARAM_RELPATH])
+            info[self.PARAM_RELPATH] = relative_path.fspath
 
         return d
 
@@ -333,7 +307,7 @@ class RemoteBASE(object):
         logger.debug("'{}' hasn't changed.".format(path_info))
         return False
 
-    def link(self, from_info, to_info):
+    def link(self, from_info, to_info, link_type=None):
         self.copy(from_info, to_info)
 
     def _save_file(self, path_info, checksum, save_link=True):
@@ -360,13 +334,9 @@ class RemoteBASE(object):
         cache_info = self.checksum_to_path_info(checksum)
         dir_info = self.get_dir_cache(checksum)
 
-        entry_info = copy(path_info)
         for entry in dir_info:
+            entry_info = path_info / entry[self.PARAM_RELPATH]
             entry_checksum = entry[self.PARAM_CHECKSUM]
-            entry_info.path = self.ospath.join(
-                path_info.path, entry[self.PARAM_RELPATH]
-            )
-
             self._save_file(entry_info, entry_checksum, save_link=False)
 
         self.state.save_link(path_info)
@@ -385,7 +355,8 @@ class RemoteBASE(object):
     def walk(self, path_info):
         raise NotImplementedError
 
-    def protect(self, path_info):
+    @staticmethod
+    def protect(path_info):
         pass
 
     def save(self, path_info, checksum_info):
@@ -410,18 +381,113 @@ class RemoteBASE(object):
             return
         self._save_file(path_info, checksum)
 
+    @contextmanager
+    def transfer_context(self):
+        yield None
+
+    def upload(self, from_infos, to_infos, names=None, no_progress_bar=False):
+        if not hasattr(self, "_upload"):
+            raise RemoteActionNotImplemented("upload", self.scheme)
+        names = self._verify_path_args(to_infos, from_infos, names)
+        fails = 0
+
+        with self.transfer_context() as ctx:
+            for from_info, to_info, name in zip(from_infos, to_infos, names):
+                if to_info.scheme != self.scheme:
+                    raise NotImplementedError
+
+                if from_info.scheme != "local":
+                    raise NotImplementedError
+
+                msg = "Uploading '{}' to '{}'"
+                logger.debug(msg.format(from_info, to_info))
+
+                if not name:
+                    name = from_info.name
+
+                if not no_progress_bar:
+                    progress.update_target(name, 0, None)
+
+                try:
+                    self._upload(
+                        from_info.fspath,
+                        to_info,
+                        name=name,
+                        ctx=ctx,
+                        no_progress_bar=no_progress_bar,
+                    )
+                except Exception:
+                    fails += 1
+                    msg = "failed to upload '{}' to '{}'"
+                    logger.exception(msg.format(from_info, to_info))
+                    continue
+
+                if not no_progress_bar:
+                    progress.finish_target(name)
+
+        return fails
+
     def download(
         self,
         from_infos,
         to_infos,
+        names=None,
         no_progress_bar=False,
-        name=None,
         resume=False,
     ):
-        raise RemoteActionNotImplemented("download", self.scheme)
+        if not hasattr(self, "_download"):
+            raise RemoteActionNotImplemented("download", self.scheme)
 
-    def upload(self, from_infos, to_infos, names=None, no_progress_bar=False):
-        raise RemoteActionNotImplemented("upload", self.scheme)
+        names = self._verify_path_args(from_infos, to_infos, names)
+        fails = 0
+
+        with self.transfer_context() as ctx:
+            for to_info, from_info, name in zip(to_infos, from_infos, names):
+                if from_info.scheme != self.scheme:
+                    raise NotImplementedError
+
+                if to_info.scheme == self.scheme != "local":
+                    self.copy(from_info, to_info, ctx=ctx)
+                    continue
+
+                if to_info.scheme != "local":
+                    raise NotImplementedError
+
+                msg = "Downloading '{}' to '{}'".format(from_info, to_info)
+                logger.debug(msg)
+
+                tmp_file = tmp_fname(to_info)
+                if not name:
+                    name = to_info.name
+
+                if not no_progress_bar:
+                    # real progress is not always available,
+                    # lets at least show start and finish
+                    progress.update_target(name, 0, None)
+
+                makedirs(fspath_py35(to_info.parent), exist_ok=True)
+
+                try:
+                    self._download(
+                        from_info,
+                        tmp_file,
+                        name=name,
+                        ctx=ctx,
+                        resume=resume,
+                        no_progress_bar=no_progress_bar,
+                    )
+                except Exception:
+                    fails += 1
+                    msg = "failed to download '{}' to '{}'"
+                    logger.exception(msg.format(from_info, to_info))
+                    continue
+
+                move(tmp_file, fspath_py35(to_info))
+
+                if not no_progress_bar:
+                    progress.finish_target(name)
+
+        return fails
 
     def remove(self, path_info):
         raise RemoteActionNotImplemented("remove", self.scheme)
@@ -430,10 +496,10 @@ class RemoteBASE(object):
         self.copy(from_info, to_info)
         self.remove(from_info)
 
-    def copy(self, from_info, to_info):
+    def copy(self, from_info, to_info, ctx=None):
         raise RemoteActionNotImplemented("copy", self.scheme)
 
-    def exists(self, path_infos):
+    def exists(self, path_info):
         raise NotImplementedError
 
     @classmethod
@@ -450,25 +516,11 @@ class RemoteBASE(object):
 
         return names
 
-    @property
-    def ospath(self):
-        return posixpath
-
-    def checksum_to_path(self, checksum):
-        assert checksum
-        return self.ospath.join(self.prefix, checksum[0:2], checksum[2:])
-
     def path_to_checksum(self, path):
-        relpath = self.ospath.relpath(path, self.prefix)
-        return self.ospath.dirname(relpath) + self.ospath.basename(relpath)
+        return "".join(self.path_cls(path).parts[-2:])
 
     def checksum_to_path_info(self, checksum):
-        path_info = copy(self.path_info)
-        path_info.path = self.checksum_to_path(checksum)
-        return path_info
-
-    def md5s_to_path_infos(self, md5s):
-        return [self.checksum_to_path_info(md5) for md5 in md5s]
+        return self.path_info / checksum[0:2] / checksum[2:]
 
     def list_cache_paths(self):
         raise NotImplementedError
@@ -483,9 +535,7 @@ class RemoteBASE(object):
         )
 
     def gc(self, cinfos):
-        from dvc.remote.local import RemoteLOCAL
-
-        used = {info[RemoteLOCAL.PARAM_CHECKSUM] for info in cinfos["local"]}
+        used = self.extract_used_local_checksums(cinfos)
 
         if self.scheme != "":
             used |= {
@@ -503,6 +553,17 @@ class RemoteBASE(object):
         return removed
 
     def changed_cache_file(self, checksum):
+        """Compare the given checksum with the (corresponding) actual one.
+
+        - Use `State` as a cache for computed checksums
+            + The entries are invalidated by taking into account the following:
+                * mtime
+                * inode
+                * size
+                * checksum
+
+        - Remove the file from cache if it doesn't match the actual checksum
+        """
         cache_info = self.checksum_to_path_info(checksum)
         actual = self.get_checksum(cache_info)
 
@@ -528,11 +589,15 @@ class RemoteBASE(object):
         if self.changed_cache_file(checksum):
             return True
 
+        if not self._changed_unpacked_dir(checksum):
+            return False
+
         for entry in self.get_dir_cache(checksum):
-            checksum = entry[self.PARAM_CHECKSUM]
-            if self.changed_cache_file(checksum):
+            entry_checksum = entry[self.PARAM_CHECKSUM]
+            if self.changed_cache_file(entry_checksum):
                 return True
 
+        self._update_unpacked_dir(checksum)
         return False
 
     def changed_cache(self, checksum):
@@ -541,15 +606,44 @@ class RemoteBASE(object):
         return self.changed_cache_file(checksum)
 
     def cache_exists(self, checksums):
-        # NOTE: The reason for such an odd logic is that most of the remotes
-        # take much shorter time to just retrieve everything they have under
-        # a certain prefix(e.g. s3, gs, ssh, hdfs). Other remotes that can
-        # check if particular file exists much quicker, use their own
-        # implementation of cache_exists(see http, local).
-        #
-        # Result of all() might be way too big, so we should walk through
-        # it in one pass.
-        return list(filter(lambda checksum: checksum in checksums, self.all()))
+        """Check if the given checksums are stored in the remote.
+
+        There are two ways of performing this check:
+
+        - Traverse: Get a list of all the files in the remote
+            (traversing the cache directory) and compare it with
+            the given checksums.
+
+        - No traverse: For each given checksum, run the `exists`
+            method and filter the checksums that aren't on the remote.
+            This is done in parallel threads.
+            It also shows a progress bar when performing the check.
+
+        The reason for such an odd logic is that most of the remotes
+        take much shorter time to just retrieve everything they have under
+        a certain prefix (e.g. s3, gs, ssh, hdfs). Other remotes that can
+        check if particular file exists much quicker, use their own
+        implementation of cache_exists (see http, local).
+
+        Returns:
+            A list with checksums that were found in the remote
+        """
+        progress_callback = ProgressCallback(len(checksums))
+
+        def exists_with_progress(chunks):
+            return self.batch_exists(chunks, callback=progress_callback)
+
+        if self.no_traverse and hasattr(self, "batch_exists"):
+            with ThreadPoolExecutor(max_workers=self.JOBS) as executor:
+                path_infos = [self.checksum_to_path_info(x) for x in checksums]
+                chunks = to_chunks(path_infos, self.JOBS)
+                results = executor.map(exists_with_progress, chunks)
+                in_remote = itertools.chain.from_iterable(results)
+                ret = list(itertools.compress(checksums, in_remote))
+                progress_callback.finish("")
+                return ret
+
+        return list(set(checksums) & set(self.all()))
 
     def already_cached(self, path_info):
         current = self.get_checksum(path_info)
@@ -587,7 +681,7 @@ class RemoteBASE(object):
         self.state.save_link(path_info)
         self.state.save(path_info, checksum)
         if progress_callback:
-            progress_callback.update(path_info.url)
+            progress_callback.update(str(path_info))
 
     def makedirs(self, path_info):
         raise NotImplementedError
@@ -604,13 +698,11 @@ class RemoteBASE(object):
 
         logger.debug("Linking directory '{}'.".format(path_info))
 
-        entry_info = copy(path_info)
         for entry in dir_info:
-            relpath = entry[self.PARAM_RELPATH]
+            relative_path = entry[self.PARAM_RELPATH]
             entry_checksum = entry[self.PARAM_CHECKSUM]
             entry_cache_info = self.checksum_to_path_info(entry_checksum)
-            entry_info.url = self.ospath.join(path_info.url, relpath)
-            entry_info.path = self.ospath.join(path_info.path, relpath)
+            entry_info = path_info / relative_path
 
             entry_checksum_info = {self.PARAM_CHECKSUM: entry_checksum}
             if self.changed(entry_info, entry_checksum_info):
@@ -619,7 +711,7 @@ class RemoteBASE(object):
                 self.link(entry_cache_info, entry_info)
                 self.state.save(entry_info, entry_checksum)
             if progress_callback:
-                progress_callback.update(entry_info.url)
+                progress_callback.update(str(entry_info))
 
         self._remove_redundant_files(path_info, dir_info, force)
 
@@ -628,30 +720,22 @@ class RemoteBASE(object):
 
     def _remove_redundant_files(self, path_info, dir_info, force):
         existing_files = set(
-            self.ospath.join(root, fname)
+            self.path_cls(root) / fname
             for root, _, files in self.walk(path_info)
             for fname in files
         )
 
-        needed_files = set(
-            self.ospath.join(path_info.path, entry[self.PARAM_RELPATH])
-            for entry in dir_info
-        )
+        needed_files = {
+            path_info / entry[self.PARAM_RELPATH] for entry in dir_info
+        }
 
-        delta = existing_files - needed_files
-
-        d_info = copy(path_info)
-        for path in delta:
-            d_info.path = path
-            self.safe_remove(d_info, force)
+        for path in existing_files - needed_files:
+            self.safe_remove(path, force)
 
     def checkout(
         self, path_info, checksum_info, force=False, progress_callback=None
     ):
-        if (
-            path_info.scheme not in ["local"]
-            and path_info.scheme != self.scheme
-        ):
+        if path_info.scheme not in ["local", self.scheme]:
             raise NotImplementedError
 
         checksum = checksum_info.get(self.PARAM_CHECKSUM)
@@ -690,3 +774,19 @@ class RemoteBASE(object):
     @staticmethod
     def unprotect(path_info):
         pass
+
+    def _get_unpacked_dir_names(self, checksums):
+        return set()
+
+    def extract_used_local_checksums(self, cinfos):
+        from dvc.remote import RemoteLOCAL
+
+        used = {info[RemoteLOCAL.PARAM_CHECKSUM] for info in cinfos["local"]}
+        unpacked = self._get_unpacked_dir_names(used)
+        return used | unpacked
+
+    def _changed_unpacked_dir(self, checksum):
+        return True
+
+    def _update_unpacked_dir(self, checksum, progress_callback=None):
+        pass

dvc/remote/config.py

@@ -0,0 +1,152 @@
+import os
+import logging
+import posixpath
+
+from dvc.utils import relpath
+from dvc.utils.compat import urlparse
+from dvc.config import Config, ConfigError
+
+
+logger = logging.getLogger(__name__)
+
+
+class RemoteConfig(object):
+    def __init__(self, config):
+        self.config = config
+
+    def get_settings(self, name):
+        """
+        Args:
+            name (str): The name of the remote that we want to retrieve
+
+        Returns:
+            dict: The content beneath the given remote name.
+
+        Example:
+            >>> config = {'remote "server"': {'url': 'ssh://localhost/'}}
+            >>> get_settings("server")
+            {'url': 'ssh://localhost/'}
+        """
+        settings = self.config.config.get(
+            Config.SECTION_REMOTE_FMT.format(name.lower())
+        )
+
+        if settings is None:
+            raise ConfigError(
+                "unable to find remote section '{}'".format(name)
+            )
+
+        parsed = urlparse(settings["url"])
+
+        # Support for cross referenced remotes.
+        # This will merge the settings, giving priority to the outer reference.
+        # For example, having:
+        #
+        #       dvc remote add server ssh://localhost
+        #       dvc remote modify server user root
+        #       dvc remote modify server ask_password true
+        #
+        #       dvc remote add images remote://server/tmp/pictures
+        #       dvc remote modify images user alice
+        #       dvc remote modify images ask_password false
+        #       dvc remote modify images password asdf1234
+        #
+        # Results on a config dictionary like:
+        #
+        #       {
+        #           "url": "ssh://localhost/tmp/pictures",
+        #           "user": "alice",
+        #           "password": "asdf1234",
+        #           "ask_password": False,
+        #       }
+        #
+        if parsed.scheme == "remote":
+            reference = self.get_settings(parsed.netloc)
+            url = posixpath.join(reference["url"], parsed.path.lstrip("/"))
+            merged = reference.copy()
+            merged.update(settings)
+            merged["url"] = url
+            return merged
+
+        return settings
+
+    @staticmethod
+    def resolve_path(path, config_file):
+        """Resolve path relative to config file location.
+
+        Args:
+            path: Path to be resolved.
+            config_file: Path to config file, which `path` is specified
+                relative to.
+
+        Returns:
+            Path relative to the `config_file` location. If `path` is an
+            absolute path then it will be returned without change.
+
+        """
+        if os.path.isabs(path):
+            return path
+        return relpath(path, os.path.dirname(config_file))
+
+    def add(self, name, url, default=False, force=False, level=None):
+        from dvc.remote import _get, RemoteLOCAL
+
+        configobj = self.config.get_configobj(level)
+        remote = _get({Config.SECTION_REMOTE_URL: url})
+        if remote == RemoteLOCAL and not url.startswith("remote://"):
+            url = self.resolve_path(url, configobj.filename)
+
+        self.config.set(
+            Config.SECTION_REMOTE_FMT.format(name),
+            Config.SECTION_REMOTE_URL,
+            url,
+            force=force,
+            level=level,
+        )
+        if default:
+            logger.info("Setting '{}' as a default remote.".format(name))
+            self.config.set(
+                Config.SECTION_CORE,
+                Config.SECTION_CORE_REMOTE,
+                name,
+                level=level,
+            )
+
+    def remove(self, name, level=None):
+        self.config.unset(Config.SECTION_REMOTE_FMT.format(name), level=level)
+
+        if level is None:
+            level = Config.LEVEL_REPO
+
+        for lev in [
+            Config.LEVEL_LOCAL,
+            Config.LEVEL_REPO,
+            Config.LEVEL_GLOBAL,
+            Config.LEVEL_SYSTEM,
+        ]:
+            self.config.unset(
+                Config.SECTION_CORE,
+                Config.SECTION_CORE_REMOTE,
+                level=lev,
+                force=True,
+            )
+            if lev == level:
+                break
+
+    def modify(self, name, option, value, level=None):
+        self.config.set(
+            Config.SECTION_REMOTE_FMT.format(name), option, value, level=level
+        )
+
+    def list(self, level=None):
+        return self.config.list_options(
+            Config.SECTION_REMOTE_REGEX, Config.SECTION_REMOTE_URL, level=level
+        )
+
+    def set_default(self, name, unset=False, level=None):
+        if unset:
+            self.config.unset(Config.SECTION_CORE, Config.SECTION_CORE_REMOTE)
+            return
+        self.config.set(
+            Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, name, level=level
+        )

dvc/remote/gs.py

@@ -1,22 +1,19 @@
 from __future__ import unicode_literals
 
-import os
 import logging
-
-from dvc.scheme import Schemes
-from dvc.path.gs import PathGS
+import itertools
+from contextlib import contextmanager
 
 try:
     from google.cloud import storage
 except ImportError:
     storage = None
 
-from dvc.utils import tmp_fname, move
-from dvc.utils.compat import urlparse, makedirs
 from dvc.remote.base import RemoteBASE
 from dvc.config import Config
-from dvc.progress import progress
 from dvc.exceptions import DvcException
+from dvc.path_info import CloudURLInfo
+from dvc.scheme import Schemes
 
 
 logger = logging.getLogger(__name__)
@@ -24,29 +21,24 @@ logger = logging.getLogger(__name__)
 
 class RemoteGS(RemoteBASE):
     scheme = Schemes.GS
-    REGEX = r"^gs://(?P<path>.*)$"
+    path_cls = CloudURLInfo
     REQUIRES = {"google.cloud.storage": storage}
     PARAM_CHECKSUM = "md5"
 
     def __init__(self, repo, config):
         super(RemoteGS, self).__init__(repo, config)
-        storagepath = "gs://"
-        storagepath += config.get(Config.SECTION_AWS_STORAGEPATH, "/")
-        storagepath.lstrip("/")
-        self.url = config.get(Config.SECTION_REMOTE_URL, storagepath)
+
+        storagepath = "gs://" + config.get(Config.SECTION_GCP_STORAGEPATH, "/")
+        url = config.get(Config.SECTION_REMOTE_URL, storagepath)
+        self.path_info = self.path_cls(url)
+
         self.projectname = config.get(Config.SECTION_GCP_PROJECTNAME, None)
         self.credentialpath = config.get(Config.SECTION_GCP_CREDENTIALPATH)
 
-        parsed = urlparse(self.url)
-        self.bucket = parsed.netloc
-        self.prefix = parsed.path.lstrip("/")
-
-        self.path_info = PathGS(bucket=self.bucket)
-
     @staticmethod
     def compat_config(config):
         ret = config.copy()
-        url = "gs://" + ret.pop(Config.SECTION_AWS_STORAGEPATH, "").lstrip("/")
+        url = "gs://" + ret.pop(Config.SECTION_GCP_STORAGEPATH, "").lstrip("/")
         ret[Config.SECTION_REMOTE_URL] = url
         return ret
 
@@ -72,135 +64,66 @@ class RemoteGS(RemoteBASE):
         md5 = base64.b64decode(b64_md5)
         return codecs.getencoder("hex")(md5)[0].decode("utf-8")
 
-    def copy(self, from_info, to_info, gs=None):
-        gs = gs if gs else self.gs
+    def copy(self, from_info, to_info, ctx=None):
+        gs = ctx or self.gs
 
-        blob = gs.bucket(from_info.bucket).get_blob(from_info.path)
+        from_bucket = gs.bucket(from_info.bucket)
+        blob = from_bucket.get_blob(from_info.path)
         if not blob:
             msg = "'{}' doesn't exist in the cloud".format(from_info.path)
             raise DvcException(msg)
 
-        bucket = self.gs.bucket(to_info.bucket)
-        bucket.copy_blob(
-            blob, self.gs.bucket(to_info.bucket), new_name=to_info.path
-        )
+        to_bucket = gs.bucket(to_info.bucket)
+        from_bucket.copy_blob(blob, to_bucket, new_name=to_info.path)
 
     def remove(self, path_info):
         if path_info.scheme != "gs":
             raise NotImplementedError
 
-        logger.debug(
-            "Removing gs://{}/{}".format(path_info.bucket, path_info.path)
-        )
-
+        logger.debug("Removing gs://{}".format(path_info))
         blob = self.gs.bucket(path_info.bucket).get_blob(path_info.path)
         if not blob:
             return
 
         blob.delete()
 
-    def _list_paths(self, bucket, prefix):
-        for blob in self.gs.bucket(bucket).list_blobs(prefix=prefix):
+    def _list_paths(self, bucket, prefix, gs=None):
+        gs = gs or self.gs
+
+        for blob in gs.bucket(bucket).list_blobs(prefix=prefix):
             yield blob.name
 
     def list_cache_paths(self):
-        return self._list_paths(self.bucket, self.prefix)
+        return self._list_paths(self.path_info.bucket, self.path_info.path)
 
     def exists(self, path_info):
-        assert not isinstance(path_info, list)
-        assert path_info.scheme == "gs"
-
-        paths = self._list_paths(path_info.bucket, path_info.path)
+        paths = set(self._list_paths(path_info.bucket, path_info.path))
         return any(path_info.path == path for path in paths)
 
-    def upload(self, from_infos, to_infos, names=None, no_progress_bar=False):
-        names = self._verify_path_args(to_infos, from_infos, names)
-
+    def batch_exists(self, path_infos, callback):
+        paths = []
         gs = self.gs
 
-        for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info.scheme != "gs":
-                raise NotImplementedError
-
-            if from_info.scheme != "local":
-                raise NotImplementedError
-
-            logger.debug(
-                "Uploading '{}' to '{}/{}'".format(
-                    from_info.path, to_info.bucket, to_info.path
-                )
+        for path_info in path_infos:
+            paths.append(
+                self._list_paths(path_info.bucket, path_info.path, gs)
             )
+            callback.update(str(path_info))
 
-            if not name:
-                name = os.path.basename(from_info.path)
+        paths = set(itertools.chain.from_iterable(paths))
 
-            if not no_progress_bar:
-                progress.update_target(name, 0, None)
+        return [path_info.path in paths for path_info in path_infos]
 
-            try:
-                bucket = gs.bucket(to_info.bucket)
-                blob = bucket.blob(to_info.path)
-                blob.upload_from_filename(from_info.path)
-            except Exception:
-                msg = "failed to upload '{}' to '{}/{}'"
-                logger.exception(
-                    msg.format(from_info.path, to_info.bucket, to_info.path)
-                )
-                continue
+    @contextmanager
+    def transfer_context(self):
+        yield self.gs
 
-            progress.finish_target(name)
+    def _upload(self, from_file, to_info, ctx=None, **_kwargs):
+        bucket = ctx.bucket(to_info.bucket)
+        blob = bucket.blob(to_info.path)
+        blob.upload_from_filename(from_file)
 
-    def download(
-        self,
-        from_infos,
-        to_infos,
-        no_progress_bar=False,
-        names=None,
-        resume=False,
-    ):
-        names = self._verify_path_args(from_infos, to_infos, names)
-
-        gs = self.gs
-
-        for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info.scheme != "gs":
-                raise NotImplementedError
-
-            if to_info.scheme == "gs":
-                self.copy(from_info, to_info, gs=gs)
-                continue
-
-            if to_info.scheme != "local":
-                raise NotImplementedError
-
-            msg = "Downloading '{}/{}' to '{}'".format(
-                from_info.bucket, from_info.path, to_info.path
-            )
-            logger.debug(msg)
-
-            tmp_file = tmp_fname(to_info.path)
-            if not name:
-                name = os.path.basename(to_info.path)
-
-            if not no_progress_bar:
-                # percent_cb is not available for download_to_filename, so
-                # lets at least update progress at pathpoints(start, finish)
-                progress.update_target(name, 0, None)
-
-            makedirs(os.path.dirname(to_info.path), exist_ok=True)
-
-            try:
-                bucket = gs.bucket(from_info.bucket)
-                blob = bucket.get_blob(from_info.path)
-                blob.download_to_filename(tmp_file)
-            except Exception:
-                msg = "failed to download '{}/{}' to '{}'"
-                logger.exception(
-                    msg.format(from_info.bucket, from_info.path, to_info.path)
-                )
-                continue
-
-            move(tmp_file, to_info.path)
-
-            if not no_progress_bar:
-                progress.finish_target(name)
+    def _download(self, from_info, to_file, ctx=None, **_kwargs):
+        bucket = ctx.bucket(from_info.bucket)
+        blob = bucket.get_blob(from_info.path)
+        blob.download_to_filename(to_file)

dvc/remote/hdfs.py

@@ -9,7 +9,7 @@ from subprocess import Popen, PIPE
 
 from dvc.config import Config
 from dvc.scheme import Schemes
-from dvc.path.hdfs import PathHDFS
+
 from dvc.remote.base import RemoteBASE, RemoteCmdError
 from dvc.utils import fix_env, tmp_fname
 
@@ -24,16 +24,15 @@ class RemoteHDFS(RemoteBASE):
 
     def __init__(self, repo, config):
         super(RemoteHDFS, self).__init__(repo, config)
-        self.url = config.get(Config.SECTION_REMOTE_URL, "/")
-        self.prefix = self.url
-        self.user = self.group("user")
+        url = config.get(Config.SECTION_REMOTE_URL, "/")
+        self.path_info = self.path_cls(url)
+
+        self.user = self.path_info.user
         if not self.user:
             self.user = config.get(
                 Config.SECTION_REMOTE_USER, getpass.getuser()
             )
 
-        self.path_info = PathHDFS(user=self.user)
-
     def hadoop_fs(self, cmd, user=None):
         cmd = "hadoop fs -" + cmd
         if user:
@@ -72,7 +71,7 @@ class RemoteHDFS(RemoteBASE):
         )
         return self._group(regex, stdout, "checksum")
 
-    def copy(self, from_info, to_info):
+    def copy(self, from_info, to_info, **_kwargs):
         dname = posixpath.dirname(to_info.path)
         self.hadoop_fs("mkdir -p {}".format(dname), user=to_info.user)
         self.hadoop_fs(
@@ -103,73 +102,35 @@ class RemoteHDFS(RemoteBASE):
         except RemoteCmdError:
             return False
 
-    def upload(self, from_infos, to_infos, names=None, no_progress_bar=False):
-        names = self._verify_path_args(to_infos, from_infos, names)
+    def _upload(self, from_file, to_info, **_kwargs):
+        self.hadoop_fs(
+            "mkdir -p {}".format(to_info.parent.url), user=to_info.user
+        )
 
-        for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info.scheme != "hdfs":
-                raise NotImplementedError
+        tmp_file = tmp_fname(to_info.url)
 
-            if from_info.scheme != "local":
-                raise NotImplementedError
+        self.hadoop_fs(
+            "copyFromLocal {} {}".format(from_file, tmp_file),
+            user=to_info.user,
+        )
 
-            self.hadoop_fs(
-                "mkdir -p {}".format(posixpath.dirname(to_info.path)),
-                user=to_info.user,
-            )
+        self.hadoop_fs(
+            "mv {} {}".format(tmp_file, to_info.url), user=to_info.user
+        )
 
-            tmp_file = tmp_fname(to_info.path)
-
-            self.hadoop_fs(
-                "copyFromLocal {} {}".format(from_info.path, tmp_file),
-                user=to_info.user,
-            )
-
-            self.hadoop_fs(
-                "mv {} {}".format(tmp_file, to_info.path), user=to_info.user
-            )
-
-    def download(
-        self,
-        from_infos,
-        to_infos,
-        no_progress_bar=False,
-        names=None,
-        resume=False,
-    ):
-        names = self._verify_path_args(from_infos, to_infos, names)
-
-        for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info.scheme != "hdfs":
-                raise NotImplementedError
-
-            if to_info.scheme == "hdfs":
-                self.copy(from_info, to_info)
-                continue
-
-            if to_info.scheme != "local":
-                raise NotImplementedError
-
-            dname = os.path.dirname(to_info.path)
-            if not os.path.exists(dname):
-                os.makedirs(dname)
-
-            tmp_file = tmp_fname(to_info.path)
-
-            self.hadoop_fs(
-                "copyToLocal {} {}".format(from_info.path, tmp_file),
-                user=from_info.user,
-            )
-
-            os.rename(tmp_file, to_info.path)
+    def _download(self, from_info, to_file, **_kwargs):
+        self.hadoop_fs(
+            "copyToLocal {} {}".format(from_info.url, to_file),
+            user=from_info.user,
+        )
 
     def list_cache_paths(self):
         try:
-            self.hadoop_fs("test -e {}".format(self.prefix))
+            self.hadoop_fs("test -e {}".format(self.path_info.url))
         except RemoteCmdError:
             return []
 
-        stdout = self.hadoop_fs("ls -R {}".format(self.prefix))
+        stdout = self.hadoop_fs("ls -R {}".format(self.path_info.url))
         lines = stdout.split("\n")
         flist = []
         for line in lines:

dvc/remote/http.py

@@ -1,8 +1,8 @@
 from __future__ import unicode_literals
 
 from dvc.scheme import Schemes
-from dvc.path import Path
-from dvc.utils.compat import open, makedirs
+
+from dvc.utils.compat import open, makedirs, fspath_py35
 
 import os
 import threading
@@ -34,31 +34,26 @@ class ProgressBarCallback(object):
 
 class RemoteHTTP(RemoteBASE):
     scheme = Schemes.HTTP
-    REGEX = r"^http://.*$"
     REQUEST_TIMEOUT = 10
     CHUNK_SIZE = 2 ** 16
     PARAM_CHECKSUM = "etag"
 
     def __init__(self, repo, config):
         super(RemoteHTTP, self).__init__(repo, config)
-        self.cache_dir = config.get(Config.SECTION_REMOTE_URL)
-        self.url = self.cache_dir
 
-        self.path_info = Path(self.scheme)
-
-    @property
-    def prefix(self):
-        return self.cache_dir
+        url = config.get(Config.SECTION_REMOTE_URL)
+        self.path_info = self.path_cls(url) if url else None
 
     def download(
         self,
         from_infos,
         to_infos,
-        no_progress_bar=False,
         names=None,
+        no_progress_bar=False,
         resume=False,
     ):
         names = self._verify_path_args(to_infos, from_infos, names)
+        fails = 0
 
         for to_info, from_info, name in zip(to_infos, from_infos, names):
             if from_info.scheme != self.scheme:
@@ -67,17 +62,15 @@ class RemoteHTTP(RemoteBASE):
             if to_info.scheme != "local":
                 raise NotImplementedError
 
-            msg = "Downloading '{}' to '{}'".format(
-                from_info.path, to_info.path
-            )
+            msg = "Downloading '{}' to '{}'".format(from_info, to_info)
             logger.debug(msg)
 
             if not name:
-                name = os.path.basename(to_info.path)
+                name = to_info.name
 
-            makedirs(os.path.dirname(to_info.path), exist_ok=True)
+            makedirs(fspath_py35(to_info.parent), exist_ok=True)
 
-            total = self._content_length(from_info.path)
+            total = self._content_length(from_info.url)
 
             if no_progress_bar or not total:
                 cb = None
@@ -86,35 +79,37 @@ class RemoteHTTP(RemoteBASE):
 
             try:
                 self._download_to(
-                    from_info.path, to_info.path, callback=cb, resume=resume
+                    from_info.url, to_info.fspath, callback=cb, resume=resume
                 )
 
             except Exception:
-                msg = "failed to download '{}'".format(from_info.path)
+                fails += 1
+                msg = "failed to download '{}'".format(from_info)
                 logger.exception(msg)
                 continue
 
             if not no_progress_bar:
                 progress.finish_target(name)
 
+        return fails
+
     def exists(self, path_info):
-        assert not isinstance(path_info, list)
-        assert path_info.scheme == self.scheme
-        return bool(self._request("HEAD", path_info.path))
+        return bool(self._request("HEAD", path_info.url))
 
-    def cache_exists(self, md5s):
-        assert isinstance(md5s, list)
+    def batch_exists(self, path_infos, callback):
+        results = []
 
-        def func(md5):
-            return bool(self._request("HEAD", self.checksum_to_path(md5)))
+        for path_info in path_infos:
+            results.append(self.exists(path_info))
+            callback.update(str(path_info))
 
-        return list(filter(func, md5s))
+        return results
 
     def _content_length(self, url):
         return self._request("HEAD", url).headers.get("Content-Length")
 
     def get_file_checksum(self, path_info):
-        url = path_info.path
+        url = path_info.url
         etag = self._request("HEAD", url).headers.get("ETag") or self._request(
             "HEAD", url
         ).headers.get("Content-MD5")

dvc/remote/https.py

@@ -7,4 +7,3 @@ from .http import RemoteHTTP
 
 class RemoteHTTPS(RemoteHTTP):
     scheme = Schemes.HTTPS
-    REGEX = r"^https://.*$"

dvc/remote/local/__init__.py

@@ -3,17 +3,13 @@ from __future__ import unicode_literals
 from copy import copy
 
 from dvc.scheme import Schemes
-from dvc.path.base import PathBASE
-from dvc.path.local import PathLOCAL
 from dvc.remote.local.slow_link_detection import slow_link_guard
-from dvc.utils.compat import str, makedirs
+from dvc.utils.compat import str, makedirs, fspath_py35
 
 import os
 import stat
-import uuid
-import ntpath
+from shortuuid import uuid
 import shutil
-import posixpath
 import logging
 
 from dvc.system import System
@@ -32,24 +28,26 @@ from dvc.utils import (
     tmp_fname,
     file_md5,
     walk_files,
+    relpath,
 )
 from dvc.config import Config
 from dvc.exceptions import DvcException
 from dvc.progress import progress
 from concurrent.futures import ThreadPoolExecutor
 
-from dvc.utils.fs import get_mtime_and_size, get_inode
-
+from dvc.path_info import PathInfo
 
 logger = logging.getLogger(__name__)
 
 
 class RemoteLOCAL(RemoteBASE):
     scheme = Schemes.LOCAL
-    REGEX = r"^(?P<path>.*)$"
+    path_cls = PathInfo
     PARAM_CHECKSUM = "md5"
     PARAM_PATH = "path"
 
+    UNPACKED_DIR_SUFFIX = ".unpacked"
+
     DEFAULT_CACHE_TYPES = ["reflink", "copy"]
     CACHE_TYPE_MAP = {
         "copy": shutil.copyfile,
@@ -62,12 +60,6 @@ class RemoteLOCAL(RemoteBASE):
         super(RemoteLOCAL, self).__init__(repo, config)
         self.state = self.repo.state if self.repo else None
         self.protected = config.get(Config.SECTION_CACHE_PROTECTED, False)
-        storagepath = config.get(Config.SECTION_AWS_STORAGEPATH, None)
-        self.cache_dir = config.get(Config.SECTION_REMOTE_URL, storagepath)
-
-        if self.cache_dir is not None and not os.path.isabs(self.cache_dir):
-            cwd = config[Config.PRIVATE_CWD]
-            self.cache_dir = os.path.abspath(os.path.join(cwd, self.cache_dir))
 
         types = config.get(Config.SECTION_CACHE_TYPE, None)
         if types:
@@ -77,36 +69,46 @@ class RemoteLOCAL(RemoteBASE):
         else:
             self.cache_types = copy(self.DEFAULT_CACHE_TYPES)
 
-        if self.cache_dir is not None and not os.path.exists(self.cache_dir):
-            os.mkdir(self.cache_dir)
+        # A clunky way to detect cache dir
+        storagepath = config.get(Config.SECTION_LOCAL_STORAGEPATH, None)
+        cache_dir = config.get(Config.SECTION_REMOTE_URL, storagepath)
 
+        if cache_dir is not None and not os.path.isabs(cache_dir):
+            cwd = config[Config.PRIVATE_CWD]
+            cache_dir = os.path.abspath(os.path.join(cwd, cache_dir))
+
+        if cache_dir is not None and not os.path.exists(cache_dir):
+            os.mkdir(cache_dir)
+
+        self.path_info = PathInfo(cache_dir) if cache_dir else None
         self._dir_info = {}
-        self.path_info = PathLOCAL()
+
+    @property
+    def cache_dir(self):
+        return self.path_info.fspath if self.path_info else None
+
+    @classmethod
+    def supported(cls, config):
+        return True
 
     @staticmethod
     def compat_config(config):
         ret = config.copy()
-        url = ret.pop(Config.SECTION_AWS_STORAGEPATH, "")
+        url = ret.pop(Config.SECTION_LOCAL_STORAGEPATH, "")
         ret[Config.SECTION_REMOTE_URL] = url
         return ret
 
-    @property
-    def url(self):
-        return self.cache_dir
-
-    @property
-    def prefix(self):
-        return self.cache_dir
-
     def list_cache_paths(self):
-        clist = []
-        for entry in os.listdir(self.cache_dir):
-            subdir = os.path.join(self.cache_dir, entry)
-            if not os.path.isdir(subdir):
-                continue
+        assert self.path_info is not None
 
-            for cache in os.listdir(subdir):
-                clist.append(os.path.join(subdir, cache))
+        clist = []
+        for entry in os.listdir(fspath_py35(self.path_info)):
+            subdir = self.path_info / entry
+            if not os.path.isdir(fspath_py35(subdir)):
+                continue
+            clist.extend(
+                subdir / cache for cache in os.listdir(fspath_py35(subdir))
+            )
 
         return clist
 
@@ -114,68 +116,84 @@ class RemoteLOCAL(RemoteBASE):
         if not md5:
             return None
 
-        return self.checksum_to_path(md5)
+        return self.checksum_to_path_info(md5).url
 
     def exists(self, path_info):
-        assert isinstance(path_info, PathBASE)
         assert path_info.scheme == "local"
-        return os.path.lexists(path_info.path)
+        return os.path.lexists(fspath_py35(path_info))
 
     def makedirs(self, path_info):
         if not self.exists(path_info):
-            os.makedirs(path_info.path)
+            os.makedirs(fspath_py35(path_info))
 
-    @slow_link_guard
-    def link(self, cache_info, path_info):
-        cache = cache_info.path
-        path = path_info.path
+    def link(self, from_info, to_info, link_type=None):
+        from_path = from_info.fspath
+        to_path = to_info.fspath
 
-        assert os.path.isfile(cache)
+        assert os.path.isfile(from_path)
 
-        dname = os.path.dirname(path)
+        dname = os.path.dirname(to_path)
         if not os.path.exists(dname):
             os.makedirs(dname)
 
         # NOTE: just create an empty file for an empty cache
-        if os.path.getsize(cache) == 0:
-            open(path, "w+").close()
+        if os.path.getsize(from_path) == 0:
+            open(to_path, "w+").close()
 
-            msg = "Created empty file: {} -> {}".format(cache, path)
+            msg = "Created empty file: {} -> {}".format(from_path, to_path)
             logger.debug(msg)
             return
 
-        i = len(self.cache_types)
+        if not link_type:
+            link_types = self.cache_types
+        else:
+            link_types = [link_type]
+
+        self._try_links(from_info, to_info, link_types)
+
+    @classmethod
+    def _get_link_method(cls, link_type):
+        try:
+            return cls.CACHE_TYPE_MAP[link_type]
+        except KeyError:
+            raise DvcException(
+                "Cache type: '{}' not supported!".format(link_type)
+            )
+
+    def _link(self, from_info, to_info, link_method):
+        if self.exists(to_info):
+            raise DvcException("Link '{}' already exists!".format(to_info))
+        else:
+            link_method(from_info.fspath, to_info.fspath)
+
+        if self.protected:
+            self.protect(to_info)
+
+        msg = "Created {}'{}': {} -> {}".format(
+            "protected " if self.protected else "",
+            self.cache_types[0],
+            from_info,
+            to_info,
+        )
+        logger.debug(msg)
+
+    @slow_link_guard
+    def _try_links(self, from_info, to_info, link_types):
+        i = len(link_types)
         while i > 0:
+            link_method = self._get_link_method(link_types[0])
             try:
-                self.CACHE_TYPE_MAP[self.cache_types[0]](cache, path)
-
-                if self.protected:
-                    self.protect(path_info)
-
-                msg = "Created {}'{}': {} -> {}".format(
-                    "protected " if self.protected else "",
-                    self.cache_types[0],
-                    cache,
-                    path,
-                )
-
-                logger.debug(msg)
+                self._link(from_info, to_info, link_method)
                 return
 
             except DvcException as exc:
                 msg = "Cache type '{}' is not supported: {}"
-                logger.debug(msg.format(self.cache_types[0], str(exc)))
-                del self.cache_types[0]
+                logger.debug(msg.format(link_types[0], str(exc)))
+                del link_types[0]
                 i -= 1
 
         raise DvcException("no possible cache types left to try out.")
 
-    @property
-    def ospath(self):
-        if os.name == "nt":
-            return ntpath
-        return posixpath
-
     def already_cached(self, path_info):
         assert path_info.scheme in ["", "local"]
 
@@ -187,7 +205,7 @@ class RemoteLOCAL(RemoteBASE):
         return not self.changed_cache(current_md5)
 
     def is_empty(self, path_info):
-        path = path_info.path
+        path = path_info.fspath
 
         if self.isfile(path_info) and os.path.getsize(path) == 0:
             return True
@@ -198,119 +216,65 @@ class RemoteLOCAL(RemoteBASE):
         return False
 
     def isfile(self, path_info):
-        return os.path.isfile(path_info.path)
+        return os.path.isfile(fspath_py35(path_info))
 
     def isdir(self, path_info):
-        return os.path.isdir(path_info.path)
+        return os.path.isdir(fspath_py35(path_info))
 
     def walk(self, path_info):
-        return os.walk(path_info.path)
+        return os.walk(fspath_py35(path_info))
 
     def get_file_checksum(self, path_info):
-        return file_md5(path_info.path)[0]
+        return file_md5(fspath_py35(path_info))[0]
 
     def remove(self, path_info):
         if path_info.scheme != "local":
             raise NotImplementedError
 
-        remove(path_info.path)
+        if self.exists(path_info):
+            remove(path_info.fspath)
 
     def move(self, from_info, to_info):
         if from_info.scheme != "local" or to_info.scheme != "local":
             raise NotImplementedError
 
-        inp = from_info.path
-        outp = to_info.path
+        inp = from_info.fspath
+        outp = to_info.fspath
 
         # moving in two stages to make the whole operation atomic in
         # case inp and outp are in different filesystems and actual
         # physical copying of data is happening
-        tmp = "{}.{}".format(outp, str(uuid.uuid4()))
+        tmp = "{}.{}".format(outp, str(uuid()))
         move(inp, tmp)
         move(tmp, outp)
 
     def cache_exists(self, md5s):
-        assert isinstance(md5s, list)
-        return list(filter(lambda md5: not self.changed_cache_file(md5), md5s))
+        return [
+            checksum
+            for checksum in progress(md5s)
+            if not self.changed_cache_file(checksum)
+        ]
 
-    def upload(self, from_infos, to_infos, names=None, no_progress_bar=False):
-        names = self._verify_path_args(to_infos, from_infos, names)
-
-        for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info.scheme != "local":
-                raise NotImplementedError
-
-            if from_info.scheme != "local":
-                raise NotImplementedError
-
-            logger.debug(
-                "Uploading '{}' to '{}'".format(from_info.path, to_info.path)
-            )
-
-            if not name:
-                name = os.path.basename(from_info.path)
-
-            makedirs(os.path.dirname(to_info.path), exist_ok=True)
-            tmp_file = tmp_fname(to_info.path)
-
-            try:
-                copyfile(
-                    from_info.path,
-                    tmp_file,
-                    name=name,
-                    no_progress_bar=no_progress_bar,
-                )
-                os.rename(tmp_file, to_info.path)
-            except Exception:
-                logger.exception(
-                    "failed to upload '{}' to '{}'".format(
-                        from_info.path, to_info.path
-                    )
-                )
-
-    def download(
-        self,
-        from_infos,
-        to_infos,
-        no_progress_bar=False,
-        names=None,
-        resume=False,
+    def _upload(
+        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
     ):
-        names = self._verify_path_args(from_infos, to_infos, names)
+        makedirs(fspath_py35(to_info.parent), exist_ok=True)
 
-        for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info.scheme != "local":
-                raise NotImplementedError
+        tmp_file = tmp_fname(to_info)
+        copyfile(
+            from_file, tmp_file, name=name, no_progress_bar=no_progress_bar
+        )
+        os.rename(tmp_file, fspath_py35(to_info))
 
-            if to_info.scheme != "local":
-                raise NotImplementedError
-
-            logger.debug(
-                "Downloading '{}' to '{}'".format(from_info.path, to_info.path)
-            )
-
-            if not name:
-                name = os.path.basename(to_info.path)
-
-            makedirs(os.path.dirname(to_info.path), exist_ok=True)
-            tmp_file = tmp_fname(to_info.path)
-            try:
-                copyfile(
-                    from_info.path,
-                    tmp_file,
-                    no_progress_bar=no_progress_bar,
-                    name=name,
-                )
-
-                move(tmp_file, to_info.path)
-            except Exception:
-                logger.exception(
-                    "failed to download '{}' to '{}'".format(
-                        from_info.path, to_info.path
-                    )
-                )
-
-                continue
+    def _download(
+        self, from_info, to_file, name=None, no_progress_bar=False, **_kwargs
+    ):
+        copyfile(
+            fspath_py35(from_info),
+            to_file,
+            no_progress_bar=no_progress_bar,
+            name=name,
+        )
 
     def _group(self, checksum_infos, show_checksums=False):
         by_md5 = {}
@@ -322,7 +286,7 @@ class RemoteLOCAL(RemoteBASE):
                 by_md5[md5] = {"name": md5}
                 continue
 
-            name = info[self.PARAM_PATH]
+            name = info["name"]
             branch = info.get("branch")
             if branch:
                 name += "({})".format(branch)
@@ -342,25 +306,15 @@ class RemoteLOCAL(RemoteBASE):
         show_checksums=False,
         download=False,
     ):
-        logger.info("Preparing to collect status from {}".format(remote.url))
-        title = "Collecting information"
-
-        ret = {}
-
-        progress.set_n_total(1)
-        progress.update_target(title, 0, 100)
-
-        progress.update_target(title, 10, 100)
-
-        ret = self._group(checksum_infos, show_checksums=show_checksums)
-        md5s = list(ret.keys())
-
-        progress.update_target(title, 30, 100)
+        logger.info(
+            "Preparing to collect status from {}".format(remote.path_info)
+        )
+        ret = self._group(checksum_infos, show_checksums=show_checksums) or {}
+        md5s = list(ret)
 
+        logger.info("Collecting information from local cache...")
         local_exists = self.cache_exists(md5s)
 
-        progress.update_target(title, 40, 100)
-
         # This is a performance optimization. We can safely assume that,
         # if the resources that we want to fetch are already cached,
         # there's no need to check the remote storage for the existance of
@@ -368,12 +322,9 @@ class RemoteLOCAL(RemoteBASE):
         if download and sorted(local_exists) == sorted(md5s):
             remote_exists = local_exists
         else:
+            logger.info("Collecting information from remote cache...")
             remote_exists = list(remote.cache_exists(md5s))
 
-        progress.update_target(title, 90, 100)
-
-        progress.finish_target(title)
-
         self._fill_statuses(ret, local_exists, remote_exists)
 
         self._log_missing_caches(ret)
@@ -390,26 +341,16 @@ class RemoteLOCAL(RemoteBASE):
             info["status"] = status
 
     def _get_chunks(self, download, remote, status_info, status, jobs):
-        title = "Analysing status."
-
-        progress.set_n_total(1)
-        total = len(status_info)
-        current = 0
-
         cache = []
         path_infos = []
         names = []
-        for md5, info in status_info.items():
+        for md5, info in progress(
+            status_info.items(), name="Analysing status"
+        ):
             if info["status"] == status:
                 cache.append(self.checksum_to_path_info(md5))
                 path_infos.append(remote.checksum_to_path_info(md5))
                 names.append(info["name"])
-            current += 1
-            progress.update_target(title, current, total)
-
-        progress.finish_target(title)
-
-        progress.set_n_total(len(names))
 
         if download:
             to_infos = cache
@@ -418,12 +359,10 @@ class RemoteLOCAL(RemoteBASE):
             to_infos = path_infos
             from_infos = cache
 
-        return list(
-            zip(
-                to_chunks(from_infos, jobs),
-                to_chunks(to_infos, jobs),
-                to_chunks(names, jobs),
-            )
+        return (
+            to_chunks(from_infos, jobs),
+            to_chunks(to_infos, jobs),
+            to_chunks(names, jobs),
         )
 
     def _process(
@@ -434,12 +373,10 @@ class RemoteLOCAL(RemoteBASE):
         show_checksums=False,
         download=False,
     ):
-        msg = "Preparing to {} data {} '{}'"
         logger.info(
-            msg.format(
-                "download" if download else "upload",
-                "from" if download else "to",
-                remote.url,
+            "Preparing to {} '{}'".format(
+                "download data from" if download else "upload data to",
+                remote.path_info,
             )
         )
 
@@ -463,19 +400,22 @@ class RemoteLOCAL(RemoteBASE):
 
         chunks = self._get_chunks(download, remote, status_info, status, jobs)
 
-        if len(chunks) == 0:
+        if len(chunks[0]) == 0:
             return 0
 
-        futures = []
-        with ThreadPoolExecutor(max_workers=jobs) as executor:
-            for from_infos, to_infos, names in chunks:
-                res = executor.submit(func, from_infos, to_infos, names=names)
-                futures.append(res)
+        if jobs > 1:
+            with ThreadPoolExecutor(max_workers=jobs) as executor:
+                fails = sum(executor.map(func, *chunks))
+        else:
+            fails = sum(map(func, *chunks))
 
-        for f in futures:
-            f.result()
+        if fails:
+            msg = "{} file(s) failed to {}"
+            raise DvcException(
+                msg.format(fails, "download" if download else "upload")
+            )
 
-        return len(chunks)
+        return len(chunks[0])
 
     def push(self, checksum_infos, remote, jobs=None, show_checksums=False):
         return self._process(
@@ -495,22 +435,6 @@ class RemoteLOCAL(RemoteBASE):
             download=True,
         )
 
-    def _changed_dir_cache(self, checksum):
-        mtime, size = get_mtime_and_size(self.cache_dir)
-        inode = get_inode(self.cache_dir)
-
-        existing_record = self.state.get_state_record_for_inode(inode)
-        if existing_record:
-            cached_mtime, cached_size, _, _ = existing_record
-            changed = not (mtime == cached_mtime and size == cached_size)
-        else:
-            changed = True
-
-        if not changed:
-            return False
-
-        return super(RemoteLOCAL, self)._changed_dir_cache(checksum)
-
     def _log_missing_caches(self, checksum_info_dict):
         missing_caches = [
             (md5, info)
@@ -534,18 +458,14 @@ class RemoteLOCAL(RemoteBASE):
     def _unprotect_file(path):
         if System.is_symlink(path) or System.is_hardlink(path):
             logger.debug("Unprotecting '{}'".format(path))
-            tmp = os.path.join(os.path.dirname(path), "." + str(uuid.uuid4()))
+            tmp = os.path.join(os.path.dirname(path), "." + str(uuid()))
 
             # The operations order is important here - if some application
             # would access the file during the process of copyfile then it
             # would get only the part of file. So, at first, the file should be
             # copied with the temporary name, and then original file should be
             # replaced by new.
-            copyfile(
-                path,
-                tmp,
-                name="Unprotecting '{}'".format(os.path.relpath(path)),
-            )
+            copyfile(path, tmp, name="Unprotecting '{}'".format(relpath(path)))
             remove(path)
             os.rename(tmp, path)
 
@@ -564,7 +484,7 @@ class RemoteLOCAL(RemoteBASE):
 
     @staticmethod
     def unprotect(path_info):
-        path = path_info.path
+        path = path_info.fspath
         if not os.path.exists(path):
             raise DvcException(
                 "can't unprotect non-existing data '{}'".format(path)
@@ -577,4 +497,57 @@ class RemoteLOCAL(RemoteBASE):
 
     @staticmethod
     def protect(path_info):
-        os.chmod(path_info.path, stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH)
+        os.chmod(
+            fspath_py35(path_info), stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH
+        )
+
+    def _get_unpacked_dir_path_info(self, checksum):
+        info = self.checksum_to_path_info(checksum)
+        return info.with_name(info.name + self.UNPACKED_DIR_SUFFIX)
+
+    def _path_info_changed(self, path_info):
+        if self.exists(path_info) and self.state.get(path_info):
+            return False
+        return True
+
+    def _update_unpacked_dir(self, checksum):
+        unpacked_dir_info = self._get_unpacked_dir_path_info(checksum)
+
+        if not self._path_info_changed(unpacked_dir_info):
+            return
+
+        self.remove(unpacked_dir_info)
+
+        try:
+            dir_info = self.get_dir_cache(checksum)
+            self._create_unpacked_dir(checksum, dir_info, unpacked_dir_info)
+        except DvcException:
+            logger.warning("Could not create '{}'".format(unpacked_dir_info))
+
+            self.remove(unpacked_dir_info)
+
+    def _create_unpacked_dir(self, checksum, dir_info, unpacked_dir_info):
+        self.makedirs(unpacked_dir_info)
+
+        for entry in progress(dir_info, name="Created unpacked dir"):
+            entry_cache_info = self.checksum_to_path_info(
+                entry[self.PARAM_CHECKSUM]
+            )
+            relative_path = entry[self.PARAM_RELPATH]
+            self.link(
+                entry_cache_info, unpacked_dir_info / relative_path, "hardlink"
+            )
+
+        self.state.save(unpacked_dir_info, checksum)
+
+    def _changed_unpacked_dir(self, checksum):
+        status_unpacked_dir_info = self._get_unpacked_dir_path_info(checksum)
+
+        return not self.state.get(status_unpacked_dir_info)
+
+    def _get_unpacked_dir_names(self, checksums):
+        unpacked = set()
+        for c in checksums:
+            if self.is_dir_checksum(c):
+                unpacked.add(c + self.UNPACKED_DIR_SUFFIX)
+        return unpacked

dvc/remote/local/slow_link_detection.py

@@ -31,13 +31,12 @@ class SlowLinkDetectorDecorator(object):
             and self.should_display()
         ):
             msg = (
-                "You can cut execution time considerably. Check:\n"
+                "You can cut execution time considerably by using a different"
+                " `cache.type` configuration.\nSee "
                 "{blue}https://dvc.org/doc/commands-reference/config#cache{"
-                "reset}"
-                "\nfor "
-                "more information.\nTo disable this message, run:\n'dvc "
-                "config "
-                "cache.slow_link_warning False'".format(
+                "reset} for more information.\n"
+                "To disable this message, run:\n"
+                "'dvc config cache.slow_link_warning false'".format(
                     blue=colorama.Fore.BLUE, reset=colorama.Fore.RESET
                 )
             )
@@ -49,7 +48,7 @@ class SlowLinkDetectorDecorator(object):
 def slow_link_guard(method):
     def call(remote_local, *args, **kwargs):
         cache_config = remote_local.repo.config.config.get(
-            Config.SECTION_CACHE
+            Config.SECTION_CACHE, {}
         )
         should_warn = cache_config.get(
             Config.SECTION_CACHE_SLOW_LINK_WARNING, True

dvc/remote/oss.py

@@ -5,19 +5,16 @@ import os
 import logging
 
 from dvc.scheme import Schemes
-from dvc.path.oss import PathOSS
 
 try:
     import oss2
 except ImportError:
     oss2 = None
 
-from dvc.utils import tmp_fname, move
-from dvc.utils.compat import urlparse, makedirs
-from dvc.progress import progress
 from dvc.config import Config
 from dvc.remote.base import RemoteBASE
 from dvc.remote.azure import Callback
+from dvc.path_info import CloudURLInfo
 
 
 logger = logging.getLogger(__name__)
@@ -43,7 +40,7 @@ class RemoteOSS(RemoteBASE):
     """
 
     scheme = Schemes.OSS
-    REGEX = r"^oss://(?P<path>.*)?$"
+    path_cls = CloudURLInfo
     REQUIRES = {"oss2": oss2}
     PARAM_CHECKSUM = "etag"
     COPY_POLL_SECONDS = 5
@@ -51,10 +48,8 @@ class RemoteOSS(RemoteBASE):
     def __init__(self, repo, config):
         super(RemoteOSS, self).__init__(repo, config)
 
-        self.url = config.get(Config.SECTION_REMOTE_URL)
-        parsed = urlparse(self.url)
-        self.bucket = parsed.netloc
-        self.prefix = parsed.path.lstrip("/")
+        url = config.get(Config.SECTION_REMOTE_URL)
+        self.path_info = self.path_cls(url) if url else None
 
         self.endpoint = config.get(Config.SECTION_OSS_ENDPOINT) or os.getenv(
             "OSS_ENDPOINT"
@@ -73,17 +68,17 @@ class RemoteOSS(RemoteBASE):
         )
 
         self._bucket = None
-        self.path_info = PathOSS(bucket=self.bucket)
 
     @property
     def oss_service(self):
         if self._bucket is None:
-            logger.debug("URL {}".format(self.url))
+            logger.debug("URL {}".format(self.path_info))
             logger.debug("key id {}".format(self.key_id))
             logger.debug("key secret {}".format(self.key_secret))
             auth = oss2.Auth(self.key_id, self.key_secret)
-            logger.debug("bucket name {}".format(self.bucket))
-            self._bucket = oss2.Bucket(auth, self.endpoint, self.bucket)
+            self._bucket = oss2.Bucket(
+                auth, self.endpoint, self.path_info.bucket
+            )
             try:  # verify that bucket exists
                 self._bucket.get_bucket_info()
             except oss2.exceptions.NoSuchBucket:
@@ -99,10 +94,7 @@ class RemoteOSS(RemoteBASE):
         if path_info.scheme != self.scheme:
             raise NotImplementedError
 
-        logger.debug(
-            "Removing oss://{}/{}".format(path_info.bucket, path_info.path)
-        )
-
+        logger.debug("Removing oss://{}".format(path_info))
         self.oss_service.delete_object(path_info.path)
 
     def _list_paths(self, prefix):
@@ -110,83 +102,20 @@ class RemoteOSS(RemoteBASE):
             yield blob.key
 
     def list_cache_paths(self):
-        return self._list_paths(self.prefix)
+        return self._list_paths(self.path_info.path)
 
-    def upload(self, from_infos, to_infos, names=None, no_progress_bar=False):
-        names = self._verify_path_args(to_infos, from_infos, names)
-
-        for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info.scheme != self.scheme:
-                raise NotImplementedError
-
-            if from_info.scheme != "local":
-                raise NotImplementedError
-
-            bucket = to_info.bucket
-            path = to_info.path
-
-            logger.debug(
-                "Uploading '{}' to 'oss://{}/{}'".format(
-                    from_info.path, bucket, path
-                )
-            )
-
-            if not name:
-                name = os.path.basename(from_info.path)
-
-            cb = None if no_progress_bar else Callback(name)
-
-            try:
-                self.oss_service.put_object_from_file(
-                    path, from_info.path, progress_callback=cb
-                )
-            except Exception:
-                msg = "failed to upload '{}'".format(from_info.path)
-                logger.warning(msg)
-            else:
-                progress.finish_target(name)
-
-    def download(
-        self,
-        from_infos,
-        to_infos,
-        names=None,
-        no_progress_bar=False,
-        resume=False,
+    def _upload(
+        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
     ):
-        names = self._verify_path_args(from_infos, to_infos, names)
-        for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info.scheme != self.scheme:
-                raise NotImplementedError
-            if to_info.scheme != "local":
-                raise NotImplementedError
+        cb = None if no_progress_bar else Callback(name)
+        self.oss_service.put_object_from_file(
+            to_info.path, from_file, progress_callback=cb
+        )
 
-            bucket = from_info.bucket
-            path = from_info.path
-
-            logger.debug(
-                "Downloading 'oss://{}/{}' to '{}'".format(
-                    bucket, path, to_info.path
-                )
-            )
-
-            tmp_file = tmp_fname(to_info.path)
-            if not name:
-                name = os.path.basename(to_info.path)
-
-            cb = None if no_progress_bar else Callback(name)
-
-            makedirs(os.path.dirname(to_info.path), exist_ok=True)
-
-            try:
-                self.oss_service.get_object_to_file(
-                    path, tmp_file, progress_callback=cb
-                )
-            except Exception:
-                msg = "failed to download 'oss://{}/{}'".format(bucket, path)
-                logger.warning(msg)
-            else:
-                move(tmp_file, to_info.path)
-
-                if not no_progress_bar:
-                    progress.finish_target(name)
+    def _download(
+        self, from_info, to_file, name=None, no_progress_bar=False, **_kwargs
+    ):
+        cb = None if no_progress_bar else Callback(name)
+        self.oss_service.get_object_to_file(
+            from_info.path, to_file, progress_callback=cb
+        )

dvc/remote/s3.py

@@ -3,21 +3,20 @@ from __future__ import unicode_literals
 import os
 import threading
 import logging
-
-from dvc.scheme import Schemes
-from dvc.path.s3 import PathS3
+import itertools
+from contextlib import contextmanager
 
 try:
     import boto3
 except ImportError:
     boto3 = None
 
-from dvc.utils import tmp_fname, move
-from dvc.utils.compat import urlparse, makedirs
 from dvc.progress import progress
 from dvc.config import Config
 from dvc.remote.base import RemoteBASE
 from dvc.exceptions import DvcException, ETagMismatchError
+from dvc.path_info import CloudURLInfo
+from dvc.scheme import Schemes
 
 logger = logging.getLogger(__name__)
 
@@ -37,7 +36,7 @@ class Callback(object):
 
 class RemoteS3(RemoteBASE):
     scheme = Schemes.S3
-    REGEX = r"^s3://(?P<path>.*)$"
+    path_cls = CloudURLInfo
     REQUIRES = {"boto3": boto3}
     PARAM_CHECKSUM = "etag"
 
@@ -48,15 +47,12 @@ class RemoteS3(RemoteBASE):
             config.get(Config.SECTION_AWS_STORAGEPATH, "").lstrip("/")
         )
 
-        self.url = config.get(Config.SECTION_REMOTE_URL, storagepath)
+        url = config.get(Config.SECTION_REMOTE_URL, storagepath)
+        self.path_info = self.path_cls(url)
 
-        self.region = os.environ.get("AWS_DEFAULT_REGION") or config.get(
-            Config.SECTION_AWS_REGION
-        )
+        self.region = config.get(Config.SECTION_AWS_REGION)
 
-        self.profile = os.environ.get("AWS_PROFILE") or config.get(
-            Config.SECTION_AWS_PROFILE
-        )
+        self.profile = config.get(Config.SECTION_AWS_PROFILE)
 
         self.endpoint_url = config.get(Config.SECTION_AWS_ENDPOINT_URL)
 
@@ -64,16 +60,16 @@ class RemoteS3(RemoteBASE):
 
         self.use_ssl = config.get(Config.SECTION_AWS_USE_SSL, True)
 
+        self.extra_args = {}
+
+        self.sse = config.get(Config.SECTION_AWS_SSE, "")
+        if self.sse:
+            self.extra_args["ServerSideEncryption"] = self.sse
+
         shared_creds = config.get(Config.SECTION_AWS_CREDENTIALPATH)
         if shared_creds:
             os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", shared_creds)
 
-        parsed = urlparse(self.url)
-        self.bucket = parsed.netloc
-        self.prefix = parsed.path.lstrip("/")
-
-        self.path_info = PathS3(bucket=self.bucket)
-
     @staticmethod
     def compat_config(config):
         ret = config.copy()
@@ -112,9 +108,11 @@ class RemoteS3(RemoteBASE):
         return obj
 
     @classmethod
-    def _copy_multipart(cls, s3, from_info, to_info, size, n_parts):
+    def _copy_multipart(
+        cls, s3, from_info, to_info, size, n_parts, extra_args
+    ):
         mpu = s3.create_multipart_upload(
-            Bucket=to_info.bucket, Key=to_info.path
+            Bucket=to_info.bucket, Key=to_info.path, **extra_args
         )
         mpu_id = mpu["UploadId"]
 
@@ -154,7 +152,7 @@ class RemoteS3(RemoteBASE):
         )
 
     @classmethod
-    def _copy(cls, s3, from_info, to_info):
+    def _copy(cls, s3, from_info, to_info, extra_args):
         # NOTE: object's etag depends on the way it was uploaded to s3 or the
         # way it was copied within the s3. More specifically, it depends on
         # the chunk size that was used to transfer it, which would affect
@@ -182,32 +180,30 @@ class RemoteS3(RemoteBASE):
         _, _, parts_suffix = etag.partition("-")
         if parts_suffix:
             n_parts = int(parts_suffix)
-            cls._copy_multipart(s3, from_info, to_info, size, n_parts)
+            cls._copy_multipart(
+                s3, from_info, to_info, size, n_parts, extra_args=extra_args
+            )
         else:
             source = {"Bucket": from_info.bucket, "Key": from_info.path}
-            s3.copy(source, to_info.bucket, to_info.path)
+            s3.copy(source, to_info.bucket, to_info.path, ExtraArgs=extra_args)
 
         cached_etag = cls.get_etag(s3, to_info.bucket, to_info.path)
         if etag != cached_etag:
             raise ETagMismatchError(etag, cached_etag)
 
-    def copy(self, from_info, to_info, s3=None):
-        s3 = s3 if s3 else self.s3
-        self._copy(s3, from_info, to_info)
+    def copy(self, from_info, to_info, ctx=None):
+        self._copy(ctx or self.s3, from_info, to_info, self.extra_args)
 
     def remove(self, path_info):
         if path_info.scheme != "s3":
             raise NotImplementedError
 
-        logger.debug(
-            "Removing s3://{}/{}".format(path_info.bucket, path_info.path)
-        )
-
+        logger.debug("Removing {}".format(path_info))
         self.s3.delete_object(Bucket=path_info.bucket, Key=path_info.path)
 
-    def _list_paths(self, bucket, prefix):
+    def _list_paths(self, bucket, prefix, s3=None):
         """ Read config for list object api, paginate through list objects."""
-        s3 = self.s3
+        s3 = s3 or self.s3
         kwargs = {"Bucket": bucket, "Prefix": prefix}
         if self.list_objects:
             list_objects_api = "list_objects"
@@ -222,104 +218,61 @@ class RemoteS3(RemoteBASE):
                 yield item["Key"]
 
     def list_cache_paths(self):
-        return self._list_paths(self.bucket, self.prefix)
+        return self._list_paths(self.path_info.bucket, self.path_info.path)
 
     def exists(self, path_info):
-        assert not isinstance(path_info, list)
-        assert path_info.scheme == "s3"
-
         paths = self._list_paths(path_info.bucket, path_info.path)
         return any(path_info.path == path for path in paths)
 
-    def upload(self, from_infos, to_infos, names=None, no_progress_bar=False):
-        names = self._verify_path_args(to_infos, from_infos, names)
-
+    def batch_exists(self, path_infos, callback):
+        paths = []
         s3 = self.s3
 
-        for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info.scheme != "s3":
-                raise NotImplementedError
-
-            if from_info.scheme != "local":
-                raise NotImplementedError
-
-            logger.debug(
-                "Uploading '{}' to '{}/{}'".format(
-                    from_info.path, to_info.bucket, to_info.path
-                )
+        for path_info in path_infos:
+            paths.append(
+                self._list_paths(path_info.bucket, path_info.path, s3)
             )
+            callback.update(str(path_info))
 
-            if not name:
-                name = os.path.basename(from_info.path)
+        paths = set(itertools.chain.from_iterable(paths))
+        return [path_info.path in paths for path_info in path_infos]
 
-            total = os.path.getsize(from_info.path)
-            cb = None if no_progress_bar else Callback(name, total)
+    @contextmanager
+    def transfer_context(self):
+        yield self.s3
 
-            try:
-                s3.upload_file(
-                    from_info.path, to_info.bucket, to_info.path, Callback=cb
-                )
-            except Exception:
-                msg = "failed to upload '{}'".format(from_info.path)
-                logger.exception(msg)
-                continue
+    def _upload(
+        self, from_file, to_info, name=None, ctx=None, no_progress_bar=False
+    ):
+        total = os.path.getsize(from_file)
+        cb = None if no_progress_bar else Callback(name, total)
+        ctx.upload_file(
+            from_file,
+            to_info.bucket,
+            to_info.path,
+            Callback=cb,
+            ExtraArgs=self.extra_args,
+        )
 
-            progress.finish_target(name)
-
-    def download(
+    def _download(
         self,
-        from_infos,
-        to_infos,
+        from_info,
+        to_file,
+        name=None,
+        ctx=None,
         no_progress_bar=False,
-        names=None,
         resume=False,
     ):
-        names = self._verify_path_args(from_infos, to_infos, names)
+        s3 = ctx
 
-        s3 = self.s3
+        if no_progress_bar:
+            cb = None
+        else:
+            total = s3.head_object(
+                Bucket=from_info.bucket, Key=from_info.path
+            )["ContentLength"]
+            cb = Callback(name, total)
 
-        for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info.scheme != "s3":
-                raise NotImplementedError
-
-            if to_info.scheme == "s3":
-                self.copy(from_info, to_info, s3=s3)
-                continue
-
-            if to_info.scheme != "local":
-                raise NotImplementedError
-
-            msg = "Downloading '{}/{}' to '{}'".format(
-                from_info.bucket, from_info.path, to_info.path
-            )
-            logger.debug(msg)
-
-            tmp_file = tmp_fname(to_info.path)
-            if not name:
-                name = os.path.basename(to_info.path)
-
-            makedirs(os.path.dirname(to_info.path), exist_ok=True)
-
-            try:
-                if no_progress_bar:
-                    cb = None
-                else:
-                    total = s3.head_object(
-                        Bucket=from_info.bucket, Key=from_info.path
-                    )["ContentLength"]
-                    cb = Callback(name, total)
-
-                s3.download_file(
-                    from_info.bucket, from_info.path, tmp_file, Callback=cb
-                )
-            except Exception:
-                msg = "failed to download '{}/{}'".format(
-                    from_info.bucket, from_info.path
-                )
-                logger.exception(msg)
-                continue
-
-            move(tmp_file, to_info.path)
-
-            if not no_progress_bar:
-                progress.finish_target(name)
+        s3.download_file(
+            from_info.bucket, from_info.path, to_file, Callback=cb
+        )

dvc/remote/ssh/__init__.py

@@ -3,9 +3,9 @@ from __future__ import unicode_literals
 import os
 import getpass
 import logging
-
-from dvc.path import Schemes
-from dvc.path.ssh import PathSSH
+import itertools
+import errno
+from concurrent.futures import ThreadPoolExecutor
 
 try:
     import paramiko
@@ -15,8 +15,10 @@ except ImportError:
 import dvc.prompt as prompt
 from dvc.remote.ssh.connection import SSHConnection
 from dvc.config import Config
-from dvc.utils.compat import urlparse
+from dvc.utils import to_chunks
+from dvc.utils.compat import urlparse, StringIO
 from dvc.remote.base import RemoteBASE
+from dvc.scheme import Schemes
 
 
 logger = logging.getLogger(__name__)
@@ -24,11 +26,6 @@ logger = logging.getLogger(__name__)
 
 class RemoteSSH(RemoteBASE):
     scheme = Schemes.SSH
-
-    # NOTE: we support both URL-like (ssh://[user@]host.xz[:port]/path) and
-    # SCP-like (ssh://[user@]host.xz:/absolute/path) urls.
-    REGEX = r"^ssh://.*$"
-
     REQUIRES = {"paramiko": paramiko}
 
     JOBS = 4
@@ -38,27 +35,36 @@ class RemoteSSH(RemoteBASE):
 
     def __init__(self, repo, config):
         super(RemoteSSH, self).__init__(repo, config)
-        self.url = config.get(Config.SECTION_REMOTE_URL, "ssh://")
 
-        parsed = urlparse(self.url)
-        self.host = parsed.hostname
+        url = config.get(Config.SECTION_REMOTE_URL)
+        if url:
+            parsed = urlparse(url)
+            user_ssh_config = self._load_user_ssh_config(parsed.hostname)
 
-        user_ssh_config = self._load_user_ssh_config(self.host)
+            host = user_ssh_config.get("hostname", parsed.hostname)
+            user = (
+                config.get(Config.SECTION_REMOTE_USER)
+                or parsed.username
+                or user_ssh_config.get("user")
+                or getpass.getuser()
+            )
+            port = (
+                config.get(Config.SECTION_REMOTE_PORT)
+                or parsed.port
+                or self._try_get_ssh_config_port(user_ssh_config)
+                or self.DEFAULT_PORT
+            )
+            self.path_info = self.path_cls.from_parts(
+                scheme=self.scheme,
+                host=host,
+                user=user,
+                port=port,
+                path=parsed.path,
+            )
+        else:
+            self.path_info = None
+            user_ssh_config = {}
 
-        self.host = user_ssh_config.get("hostname", self.host)
-        self.user = (
-            config.get(Config.SECTION_REMOTE_USER)
-            or parsed.username
-            or user_ssh_config.get("user")
-            or getpass.getuser()
-        )
-        self.prefix = parsed.path or "/"
-        self.port = (
-            config.get(Config.SECTION_REMOTE_PORT)
-            or parsed.port
-            or self._try_get_ssh_config_port(user_ssh_config)
-            or self.DEFAULT_PORT
-        )
         self.keyfile = config.get(
             Config.SECTION_REMOTE_KEY_FILE
         ) or self._try_get_ssh_config_keyfile(user_ssh_config)
@@ -68,10 +74,6 @@ class RemoteSSH(RemoteBASE):
             Config.SECTION_REMOTE_ASK_PASSWORD, False
         )
 
-        self.path_info = PathSSH(
-            host=self.host, user=self.user, port=self.port
-        )
-
     @staticmethod
     def ssh_config_filename():
         return os.path.expanduser(os.path.join("~", ".ssh", "config"))
@@ -81,10 +83,12 @@ class RemoteSSH(RemoteBASE):
         user_config_file = RemoteSSH.ssh_config_filename()
         user_ssh_config = dict()
         if hostname and os.path.exists(user_config_file):
+            ssh_config = paramiko.SSHConfig()
             with open(user_config_file) as f:
-                ssh_config = paramiko.SSHConfig()
-                ssh_config.parse(f)
-                user_ssh_config = ssh_config.lookup(hostname)
+                # For whatever reason parsing directly from f is unreliable
+                f_copy = StringIO(f.read())
+                ssh_config.parse(f_copy)
+            user_ssh_config = ssh_config.lookup(hostname)
         return user_ssh_config
 
     @staticmethod
@@ -129,12 +133,37 @@ class RemoteSSH(RemoteBASE):
         )
 
     def exists(self, path_info):
-        assert not isinstance(path_info, list)
-        assert path_info.scheme == self.scheme
-
         with self.ssh(path_info) as ssh:
             return ssh.exists(path_info.path)
 
+    def batch_exists(self, path_infos, callback):
+        def _exists(chunk_and_channel):
+            chunk, channel = chunk_and_channel
+            ret = []
+            for path in chunk:
+                try:
+                    channel.stat(path)
+                    ret.append(True)
+                except IOError as exc:
+                    if exc.errno != errno.ENOENT:
+                        raise
+                    ret.append(False)
+                callback.update(path)
+            return ret
+
+        with self.ssh(path_infos[0]) as ssh:
+            with ssh.open_max_sftp_channels() as channels:
+                max_workers = len(channels)
+
+                with ThreadPoolExecutor(max_workers=max_workers) as executor:
+                    paths = [path_info.path for path_info in path_infos]
+                    chunks = to_chunks(paths, max_workers)
+                    chunks_and_channels = zip(chunks, channels)
+                    outcome = executor.map(_exists, chunks_and_channels)
+                    results = list(itertools.chain.from_iterable(outcome))
+
+            return results
+
     def get_file_checksum(self, path_info):
         if path_info.scheme != self.scheme:
             raise NotImplementedError
@@ -146,12 +175,15 @@ class RemoteSSH(RemoteBASE):
         with self.ssh(path_info) as ssh:
             return ssh.isdir(path_info.path)
 
-    def copy(self, from_info, to_info):
+    def copy(self, from_info, to_info, ctx=None):
         if from_info.scheme != self.scheme or to_info.scheme != self.scheme:
             raise NotImplementedError
 
-        with self.ssh(from_info) as ssh:
-            ssh.cp(from_info.path, to_info.path)
+        if ctx:
+            ctx.cp(from_info.path, to_info.path)
+        else:
+            with self.ssh(from_info) as ssh:
+                ssh.cp(from_info.path, to_info.path)
 
     def remove(self, path_info):
         if path_info.scheme != self.scheme:
@@ -167,81 +199,40 @@ class RemoteSSH(RemoteBASE):
         with self.ssh(from_info) as ssh:
             ssh.move(from_info.path, to_info.path)
 
-    def download(
+    def transfer_context(self):
+        return self.ssh(self.path_info)
+
+    def _download(
         self,
-        from_infos,
-        to_infos,
+        from_info,
+        to_file,
+        name=None,
+        ctx=None,
         no_progress_bar=False,
-        names=None,
         resume=False,
     ):
-        names = self._verify_path_args(from_infos, to_infos, names)
-        ssh = self.ssh(from_infos[0])
+        assert from_info.isin(self.path_info)
+        ctx.download(
+            from_info.path,
+            to_file,
+            progress_title=name,
+            no_progress_bar=no_progress_bar,
+        )
 
-        for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info.scheme != self.scheme:
-                raise NotImplementedError
-
-            if to_info.scheme == self.scheme:
-                ssh.cp(from_info.path, to_info.path)
-                continue
-
-            if to_info.scheme != "local":
-                raise NotImplementedError
-
-            logger.debug(
-                "Downloading '{host}/{path}' to '{dest}'".format(
-                    host=from_info.host, path=from_info.path, dest=to_info.path
-                )
-            )
-
-            try:
-                ssh.download(
-                    from_info.path,
-                    to_info.path,
-                    progress_title=name,
-                    no_progress_bar=no_progress_bar,
-                )
-            except Exception:
-                logger.exception(
-                    "failed to download '{host}/{path}' to '{dest}'".format(
-                        host=from_info.host,
-                        path=from_info.path,
-                        dest=to_info.path,
-                    )
-                )
-                continue
-
-        ssh.close()
-
-    def upload(self, from_infos, to_infos, names=None, no_progress_bar=False):
-        names = self._verify_path_args(to_infos, from_infos, names)
-
-        with self.ssh(to_infos[0]) as ssh:
-            for from_info, to_info, name in zip(from_infos, to_infos, names):
-                if to_info.scheme != self.scheme:
-                    raise NotImplementedError
-
-                if from_info.scheme != "local":
-                    raise NotImplementedError
-
-                try:
-                    ssh.upload(
-                        from_info.path,
-                        to_info.path,
-                        progress_title=name,
-                        no_progress_bar=no_progress_bar,
-                    )
-                except Exception:
-                    logger.exception(
-                        "failed to upload '{}' to '{}'".format(
-                            from_info, to_info
-                        )
-                    )
+    def _upload(
+        self, from_file, to_info, name=None, ctx=None, no_progress_bar=False
+    ):
+        assert to_info.isin(self.path_info)
+        ctx.upload(
+            from_file,
+            to_info.path,
+            progress_title=name,
+            no_progress_bar=no_progress_bar,
+        )
 
     def list_cache_paths(self):
         with self.ssh(self.path_info) as ssh:
-            return list(ssh.walk_files(self.prefix))
+            return list(ssh.walk_files(self.path_info.path))
 
     def walk(self, path_info):
         with self.ssh(path_info) as ssh:

dvc/remote/ssh/connection.py

@@ -1,8 +1,8 @@
-import errno
 import os
 import posixpath
 import logging
-from stat import S_ISDIR
+import stat
+from contextlib import contextmanager
 
 try:
     import paramiko
@@ -10,7 +10,7 @@ except ImportError:
     paramiko = None
 
 from dvc.utils import tmp_fname
-from dvc.utils.compat import makedirs
+from dvc.utils.compat import ignore_file_not_found
 from dvc.progress import progress
 from dvc.exceptions import DvcException
 from dvc.remote.base import RemoteCmdError
@@ -73,40 +73,25 @@ class SSHConnection:
 
         self._ssh.close()
 
-    def exists(self, path):
+    def st_mode(self, path):
         self._sftp_connect()
-        try:
-            return self._sftp.stat(path)
-        except IOError:
-            return False
-        pass
+
+        with ignore_file_not_found():
+            return self._sftp.stat(path).st_mode
+
+        return 0
+
+    def exists(self, path, sftp=None):
+        return bool(self.st_mode(path))
 
     def isdir(self, path):
-        from stat import S_ISDIR
-
-        self._sftp_connect()
-        try:
-            return S_ISDIR(self._sftp.stat(path).st_mode)
-        except IOError:
-            return False
+        return stat.S_ISDIR(self.st_mode(path))
 
     def isfile(self, path):
-        from stat import S_ISREG
-
-        self._sftp_connect()
-        try:
-            return S_ISREG(self._sftp.stat(path).st_mode)
-        except IOError:
-            return False
+        return stat.S_ISREG(self.st_mode(path))
 
     def islink(self, path):
-        from stat import S_ISLNK
-
-        self._sftp_connect()
-        try:
-            return S_ISLNK(self._sftp.stat(path).st_mode)
-        except IOError:
-            return False
+        return stat.S_ISLNK(self.st_mode(path))
 
     def makedirs(self, path):
         self._sftp_connect()
@@ -149,7 +134,7 @@ class SSHConnection:
         nondirs = []
         for entry in dir_entries:
             name = entry.filename
-            if S_ISDIR(entry.st_mode):
+            if stat.S_ISDIR(entry.st_mode):
                 dirs.append(name)
             else:
                 nondirs.append(name)
@@ -171,11 +156,8 @@ class SSHConnection:
                 yield posixpath.join(root, fname)
 
     def _remove_file(self, path):
-        try:
+        with ignore_file_not_found():
             self._sftp.remove(path)
-        except IOError as exc:
-            if exc.errno != errno.ENOENT:
-                raise
 
     def _remove_dir(self, path):
         for root, dirs, files in self.walk(path, topdown=False):
@@ -186,11 +168,9 @@ class SSHConnection:
             for dname in dirs:
                 path = posixpath.join(root, dname)
                 self._sftp.rmdir(dname)
-        try:
+
+        with ignore_file_not_found():
             self._sftp.rmdir(path)
-        except IOError as exc:
-            if exc.errno != errno.ENOENT:
-                raise
 
     def remove(self, path):
         self._sftp_connect()
@@ -203,23 +183,15 @@ class SSHConnection:
     def download(self, src, dest, no_progress_bar=False, progress_title=None):
         self._sftp_connect()
 
-        makedirs(os.path.dirname(dest), exist_ok=True)
-        tmp_file = tmp_fname(dest)
-
         if no_progress_bar:
-            self._sftp.get(src, tmp_file)
+            self._sftp.get(src, dest)
         else:
             if not progress_title:
-                progress_title = os.path.basename(dest)
+                progress_title = os.path.basename(src)
 
-            self._sftp.get(src, tmp_file, callback=create_cb(progress_title))
+            self._sftp.get(src, dest, callback=create_cb(progress_title))
             progress.finish_target(progress_title)
 
-        if os.path.exists(dest):
-            os.remove(dest)
-
-        os.rename(tmp_file, dest)
-
     def move(self, src, dst):
         self.makedirs(posixpath.dirname(dst))
         self._sftp_connect()
@@ -320,3 +292,19 @@ class SSHConnection:
     def cp(self, src, dest):
         self.makedirs(posixpath.dirname(dest))
         self.execute("cp {} {}".format(src, dest))
+
+    @contextmanager
+    def open_max_sftp_channels(self):
+        try:
+            channels = []
+            while True:
+                try:
+                    channels.append(self._ssh.open_sftp())
+                except paramiko.ssh_exception.ChannelException:
+                    if not channels:
+                        raise
+                    break
+            yield channels
+        finally:
+            for channel in channels:
+                channel.close()

dvc/repo/__init__.py

@@ -3,16 +3,16 @@ from __future__ import unicode_literals
 import os
 import logging
 
-import dvc.prompt as prompt
-
+from dvc.config import Config
 from dvc.exceptions import (
-    DvcException,
     NotDvcRepoError,
     OutputNotFoundError,
     TargetNotDirectoryError,
 )
 from dvc.ignore import DvcIgnoreFileHandler
-from dvc.path.local import PathLOCAL
+from dvc.path_info import PathInfo
+from dvc.utils.compat import open as _open
+from dvc.utils import relpath
 
 logger = logging.getLogger(__name__)
 
@@ -28,6 +28,7 @@ class Repo(object):
     from dvc.repo.move import move
     from dvc.repo.run import run
     from dvc.repo.imp import imp
+    from dvc.repo.imp_url import imp_url
     from dvc.repo.reproduce import reproduce
     from dvc.repo.checkout import checkout
     from dvc.repo.push import push
@@ -38,19 +39,18 @@ class Repo(object):
     from dvc.repo.commit import commit
     from dvc.repo.diff import diff
     from dvc.repo.brancher import brancher
+    from dvc.repo.get import get
+    from dvc.repo.get_url import get_url
 
-    def __init__(self, root_dir=None):
-        from dvc.config import Config
+    def __init__(self, root_dir=None, rev=None):
         from dvc.state import State
         from dvc.lock import Lock
         from dvc.scm import SCM
         from dvc.cache import Cache
         from dvc.data_cloud import DataCloud
-        from dvc.updater import Updater
         from dvc.repo.metrics import Metrics
         from dvc.scm.tree import WorkingTree
         from dvc.repo.tag import Tag
-        from dvc.repo.pkg import Pkg
 
         root_dir = self.find_root(root_dir)
 
@@ -59,9 +59,13 @@ class Repo(object):
 
         self.config = Config(self.dvc_dir)
 
-        self.tree = WorkingTree(self.root_dir)
-
         self.scm = SCM(self.root_dir, repo=self)
+
+        if rev:
+            self.tree = self.scm.get_tree(rev)
+        else:
+            self.tree = WorkingTree(self.root_dir)
+
         self.lock = Lock(self.dvc_dir)
         # NOTE: storing state and link_state in the repository itself to avoid
         # any possible state corruption in 'shared cache dir' scenario.
@@ -74,29 +78,25 @@ class Repo(object):
             logger.setLevel(level.upper())
 
         self.cache = Cache(self)
-        self.cloud = DataCloud(self, config=self.config.config)
-        self.updater = Updater(self.dvc_dir)
+        self.cloud = DataCloud(self)
 
         self.metrics = Metrics(self)
         self.tag = Tag(self)
-        self.pkg = Pkg(self)
 
         self._ignore()
 
-        self.updater.check()
-
     def __repr__(self):
         return "Repo: '{root_dir}'".format(root_dir=self.root_dir)
 
-    @staticmethod
-    def find_root(root=None):
+    @classmethod
+    def find_root(cls, root=None):
         if root is None:
             root = os.getcwd()
         else:
             root = os.path.abspath(os.path.realpath(root))
 
         while True:
-            dvc_dir = os.path.join(root, Repo.DVC_DIR)
+            dvc_dir = os.path.join(root, cls.DVC_DIR)
             if os.path.isdir(dvc_dir):
                 return root
             if os.path.ismount(root):
@@ -104,10 +104,10 @@ class Repo(object):
             root = os.path.dirname(root)
         raise NotDvcRepoError(root)
 
-    @staticmethod
-    def find_dvc_dir(root=None):
-        root_dir = Repo.find_root(root)
-        return os.path.join(root_dir, Repo.DVC_DIR)
+    @classmethod
+    def find_dvc_dir(cls, root=None):
+        root_dir = cls.find_root(root)
+        return os.path.join(root_dir, cls.DVC_DIR)
 
     @staticmethod
     def init(root_dir=os.curdir, no_scm=False, force=False):
@@ -117,16 +117,21 @@ class Repo(object):
         return Repo(root_dir)
 
     def unprotect(self, target):
-        path_info = PathLOCAL(path=target)
-        return self.cache.local.unprotect(path_info)
+        return self.cache.local.unprotect(PathInfo(target))
 
     def _ignore(self):
+        from dvc.updater import Updater
+        from dvc.external_repo import ExternalRepo
+
+        updater = Updater(self.dvc_dir)
+
         flist = [
             self.state.state_file,
             self.lock.lock_file,
             self.config.config_local_file,
-            self.updater.updater_file,
-            self.updater.lock.lock_file,
+            updater.updater_file,
+            updater.lock.lock_file,
+            os.path.join(self.dvc_dir, ExternalRepo.REPOS_DIR),
         ] + self.state.temp_files
 
         if self.cache.local.cache_dir.startswith(self.root_dir):
@@ -157,14 +162,14 @@ class Repo(object):
         import networkx as nx
         from dvc.stage import Stage
 
-        if not target or recursive:
+        if not target or (recursive and os.path.isdir(target)):
             return self.active_stages(target)
 
         stage = Stage.load(self, target)
         if not with_deps:
             return [stage]
 
-        node = os.path.relpath(stage.path, self.root_dir)
+        node = relpath(stage.path, self.root_dir)
         G = self._get_pipeline(node)
 
         ret = []
@@ -173,73 +178,6 @@ class Repo(object):
 
         return ret
 
-    def _collect_dir_cache(
-        self, out, branch=None, remote=None, force=False, jobs=None
-    ):
-        info = out.dumpd()
-        ret = [info]
-        r = out.remote
-        md5 = info[r.PARAM_CHECKSUM]
-
-        if self.cache.local.changed_cache_file(md5):
-            try:
-                self.cloud.pull(
-                    ret, jobs=jobs, remote=remote, show_checksums=False
-                )
-            except DvcException as exc:
-                msg = "Failed to pull cache for '{}': {}"
-                logger.debug(msg.format(out, exc))
-
-        if self.cache.local.changed_cache_file(md5):
-            msg = (
-                "Missing cache for directory '{}'. "
-                "Cache for files inside will be lost. "
-                "Would you like to continue? Use '-f' to force. "
-            )
-            if not force and not prompt.confirm(msg):
-                raise DvcException(
-                    "unable to fully collect used cache"
-                    " without cache for directory '{}'".format(out)
-                )
-            else:
-                return ret
-
-        for i in out.dir_cache:
-            i["branch"] = branch
-            i[r.PARAM_PATH] = os.path.join(
-                info[r.PARAM_PATH], i[r.PARAM_RELPATH]
-            )
-            ret.append(i)
-
-        return ret
-
-    def _collect_used_cache(
-        self, out, branch=None, remote=None, force=False, jobs=None
-    ):
-        if not out.use_cache or not out.info:
-            if not out.info:
-                logger.warning(
-                    "Output '{}'({}) is missing version "
-                    "info. Cache for it will not be collected. "
-                    "Use dvc repro to get your pipeline up to "
-                    "date.".format(out, out.stage)
-                )
-            return []
-
-        info = out.dumpd()
-        info["branch"] = branch
-        ret = [info]
-
-        if out.scheme != "local":
-            return ret
-
-        if not out.is_dir_checksum:
-            return ret
-
-        return self._collect_dir_cache(
-            out, branch=branch, remote=remote, force=force, jobs=jobs
-        )
-
     def used_cache(
         self,
         target=None,
@@ -252,6 +190,20 @@ class Repo(object):
         jobs=None,
         recursive=False,
     ):
+        """Get the stages related to the given target and collect
+        the `info` of its outputs.
+
+        This is useful to know what files from the cache are _in use_
+        (namely, a file described as an output on a stage).
+
+        The scope is, by default, the working directory, but you can use
+        `all_branches` or `all_tags` to expand scope.
+
+        Returns:
+            A dictionary with Schemes (representing output's location) as keys,
+            and a list with the outputs' `dumpd` as values.
+        """
+
         cache = {}
         cache["local"] = []
         cache["s3"] = []
@@ -264,7 +216,7 @@ class Repo(object):
             all_branches=all_branches, all_tags=all_tags
         ):
             if target:
-                if recursive:
+                if recursive and os.path.isdir(target):
                     stages = self.stages(target)
                 else:
                     stages = self.collect(target, with_deps=with_deps)
@@ -276,7 +228,7 @@ class Repo(object):
             for stage in stages:
                 if active and not target and stage.locked:
                     logger.warning(
-                        "DVC file '{path}' is locked. Its dependencies are"
+                        "DVC-file '{path}' is locked. Its dependencies are"
                         " not going to be pushed/pulled/fetched.".format(
                             path=stage.relpath
                         )
@@ -284,14 +236,12 @@ class Repo(object):
 
                 for out in stage.outs:
                     scheme = out.path_info.scheme
+                    used_cache = out.get_used_cache(
+                        remote=remote, force=force, jobs=jobs
+                    )
+
                     cache[scheme].extend(
-                        self._collect_used_cache(
-                            out,
-                            branch=branch,
-                            remote=remote,
-                            force=force,
-                            jobs=jobs,
-                        )
+                        dict(entry, branch=branch) for entry in used_cache
                     )
 
         return cache
@@ -355,28 +305,28 @@ class Repo(object):
             for out in stage.outs:
                 existing = []
                 for o in outs:
-                    if o.path == out.path:
+                    if o.path_info == out.path_info:
                         existing.append(o.stage)
 
-                    in_o_dir = out.path.startswith(o.path + o.sep)
-                    in_out_dir = o.path.startswith(out.path + out.sep)
+                    in_o_dir = out.path_info.isin(o.path_info)
+                    in_out_dir = o.path_info.isin(out.path_info)
                     if in_o_dir or in_out_dir:
                         raise OverlappingOutputPathsError(o, out)
 
                 if existing:
                     stages = [stage.relpath, existing[0].relpath]
-                    raise OutputDuplicationError(out.path, stages)
+                    raise OutputDuplicationError(str(out), stages)
 
                 outs.append(out)
 
         for stage in stages:
-            path_dir = os.path.dirname(stage.path) + os.sep
+            stage_path_info = PathInfo(stage.path)
             for out in outs:
-                if path_dir.startswith(out.path + os.sep):
+                if stage_path_info.isin(out.path_info):
                     raise StagePathAsOutputError(stage.wdir, stage.relpath)
 
         for stage in stages:
-            node = os.path.relpath(stage.path, self.root_dir)
+            node = relpath(stage.path, self.root_dir)
 
             G.add_node(node, stage=stage)
             G_active.add_node(node, stage=stage)
@@ -384,14 +334,14 @@ class Repo(object):
             for dep in stage.deps:
                 for out in outs:
                     if (
-                        out.path != dep.path
-                        and not dep.path.startswith(out.path + out.sep)
-                        and not out.path.startswith(dep.path + dep.sep)
+                        out.path_info != dep.path_info
+                        and not dep.path_info.isin(out.path_info)
+                        and not out.path_info.isin(dep.path_info)
                     ):
                         continue
 
                     dep_stage = out.stage
-                    dep_node = os.path.relpath(dep_stage.path, self.root_dir)
+                    dep_node = relpath(dep_stage.path, self.root_dir)
                     G.add_node(dep_node, stage=dep_stage)
                     G.add_edge(node, dep_node)
                     if not stage.locked:
@@ -441,7 +391,8 @@ class Repo(object):
                     continue
                 stage = Stage.load(self, path)
                 for out in stage.outs:
-                    outs.append(out.path + out.sep)
+                    if out.scheme == "local":
+                        outs.append(out.fspath + out.sep)
                 stages.append(stage)
 
             def filter_dirs(dname, root=root):
@@ -480,10 +431,10 @@ class Repo(object):
         is_dir = self.tree.isdir(abs_path)
 
         def func(out):
-            if out.path == abs_path:
+            if out.scheme == "local" and out.fspath == abs_path:
                 return True
 
-            if is_dir and recursive and out.path.startswith(abs_path + os.sep):
+            if is_dir and recursive and out.path_info.isin(abs_path):
                 return True
 
             return False
@@ -497,3 +448,16 @@ class Repo(object):
     def is_dvc_internal(self, path):
         path_parts = os.path.normpath(path).split(os.path.sep)
         return self.DVC_DIR in path_parts
+
+    def open(self, path, remote=None, mode="r", encoding=None):
+        """Opens a specified resource as a file descriptor"""
+        out, = self.find_outs_by_path(path)
+        if out.isdir():
+            raise ValueError("Can't open a dir")
+
+        with self.state:
+            cache_info = out.get_used_cache(remote=remote)
+            self.cloud.pull(cache_info, remote=remote)
+
+        cache_file = self.cache.local.checksum_to_path_info(out.checksum)
+        return _open(cache_file.fspath, mode=mode, encoding=encoding)

dvc/repo/brancher.py

@@ -1,3 +1,6 @@
+from dvc.scm.tree import WorkingTree
+
+
 def brancher(  # noqa: E302
     self, branches=None, all_branches=False, tags=None, all_tags=False
 ):
@@ -24,11 +27,8 @@ def brancher(  # noqa: E302
 
     scm = self.scm
 
-    if self.scm.is_dirty():
-        from dvc.scm.tree import WorkingTree
-
-        self.tree = WorkingTree(self.root_dir)
-        yield "Working Tree"
+    self.tree = WorkingTree(self.root_dir)
+    yield "working tree"
 
     if all_branches:
         branches = scm.list_branches()
@@ -36,9 +36,7 @@ def brancher(  # noqa: E302
     if all_tags:
         tags = scm.list_tags()
 
-    if branches is None:
-        revs.extend([scm.active_branch()])
-    else:
+    if branches is not None:
         revs.extend(branches)
 
     if tags is not None:

dvc/repo/checkout.py

@@ -10,10 +10,12 @@ logger = logging.getLogger(__name__)
 
 
 def _cleanup_unused_links(self, all_stages):
-    used = []
-    for stage in all_stages:
-        for out in stage.outs:
-            used.append(out.path)
+    used = [
+        out.fspath
+        for stage in all_stages
+        for out in stage.outs
+        if out.scheme == "local"
+    ]
     self.state.remove_unused_links(used)
 
 
@@ -47,7 +49,7 @@ def checkout(self, target=None, with_deps=False, force=False, recursive=False):
         for stage in stages:
             if stage.locked:
                 logger.warning(
-                    "DVC file '{path}' is locked. Its dependencies are"
+                    "DVC-file '{path}' is locked. Its dependencies are"
                     " not going to be checked out.".format(path=stage.relpath)
                 )
 

dvc/repo/destroy.py

@@ -1,8 +1,8 @@
-import shutil
+from dvc.utils import remove
 
 
 def destroy(self):
     for stage in self.stages():
-        stage.remove()
+        stage.remove(remove_outs=False)
 
-    shutil.rmtree(self.dvc_dir)
+    remove(self.dvc_dir)

dvc/repo/gc.py

@@ -110,4 +110,4 @@ def gc(
             _do_gc("azure", self.cache.azure.gc, clist)
 
         if cloud:
-            _do_gc("remote", self.cloud._get_cloud(remote, "gc -c").gc, clist)
+            _do_gc("remote", self.cloud.get_remote(remote, "gc -c").gc, clist)

dvc/repo/get.py

@@ -0,0 +1,48 @@
+import os
+import shortuuid
+
+from dvc.config import Config
+from dvc.path_info import PathInfo
+from dvc.external_repo import ExternalRepo
+from dvc.utils.compat import urlparse
+from dvc.utils import remove
+
+
+@staticmethod
+def get(url, path, out=None, rev=None):
+    out = out or os.path.basename(urlparse(path).path)
+
+    # Creating a directory right beside the output to make sure that they
+    # are on the same filesystem, so we could take the advantage of
+    # reflink and/or hardlink. Not using tempfile.TemporaryDirectory
+    # because it will create a symlink to tmpfs, which defeats the purpose
+    # and won't work with reflink/hardlink.
+    dpath = os.path.dirname(os.path.abspath(out))
+    tmp_dir = os.path.join(dpath, "." + str(shortuuid.uuid()))
+    try:
+        erepo = ExternalRepo(tmp_dir, url=url, rev=rev)
+        erepo.install()
+        # Try any links possible to avoid data duplication.
+        #
+        # Not using symlink, because we need to remove cache after we are
+        # done, and to make that work we would have to copy data over
+        # anyway before removing the cache, so we might just copy it
+        # right away.
+        #
+        # Also, we can't use theoretical "move" link type here, because
+        # the same cache file might be used a few times in a directory.
+        erepo.repo.config.set(
+            Config.SECTION_CACHE,
+            Config.SECTION_CACHE_TYPE,
+            "reflink,hardlink,copy",
+        )
+        src = os.path.join(erepo.path, urlparse(path).path.lstrip("/"))
+        o, = erepo.repo.find_outs_by_path(src)
+        erepo.repo.fetch(o.stage.path)
+        o.path_info = PathInfo(os.path.abspath(out))
+        with o.repo.state:
+            o.checkout()
+        erepo.repo.scm.git.close()
+    finally:
+        if os.path.exists(tmp_dir):
+            remove(tmp_dir)

dvc/repo/get_url.py

@@ -0,0 +1,19 @@
+import os
+
+import dvc.output as output
+import dvc.dependency as dependency
+
+from dvc.utils.compat import urlparse
+
+
+@staticmethod
+def get_url(url, out=None):
+    out = out or os.path.basename(urlparse(url).path)
+
+    if os.path.exists(url):
+        url = os.path.abspath(url)
+        out = os.path.abspath(out)
+
+    dep, = dependency.loads_from(None, [url])
+    out, = output.loads_from(None, [out], use_cache=False)
+    dep.download(out)

dvc/repo/imp.py

@@ -1,23 +1,9 @@
-from dvc.repo.scm_context import scm_context
+from dvc.external_repo import ExternalRepo
 
 
-@scm_context
-def imp(self, url, out, resume=False, fname=None):
-    from dvc.stage import Stage
+def imp(self, url, path, out=None, rev=None):
+    erepo = {ExternalRepo.PARAM_URL: url}
+    if rev is not None:
+        erepo[ExternalRepo.PARAM_VERSION] = rev
 
-    with self.state:
-        stage = Stage.create(
-            repo=self, cmd=None, deps=[url], outs=[out], fname=fname
-        )
-
-    if stage is None:
-        return None
-
-    self.check_dag(self.stages() + [stage])
-
-    with self.state:
-        stage.run(resume=resume)
-
-    stage.dump()
-
-    return stage
+    return self.imp_url(path, out=out, erepo=erepo)

dvc/repo/imp_url.py

@@ -0,0 +1,34 @@
+import os
+
+from dvc.utils.compat import urlparse
+from dvc.repo.scm_context import scm_context
+
+
+@scm_context
+def imp_url(self, url, out=None, resume=False, fname=None, erepo=None):
+    from dvc.stage import Stage
+
+    default_out = os.path.basename(urlparse(url).path)
+    out = out or default_out
+
+    with self.state:
+        stage = Stage.create(
+            repo=self,
+            cmd=None,
+            deps=[url],
+            outs=[out],
+            fname=fname,
+            erepo=erepo,
+        )
+
+    if stage is None:
+        return None
+
+    self.check_dag(self.stages() + [stage])
+
+    with self.state:
+        stage.run(resume=resume)
+
+    stage.dump()
+
+    return stage

dvc/repo/init.py

@@ -1,5 +1,4 @@
 import os
-import shutil
 import colorama
 import logging
 
@@ -7,8 +6,7 @@ from dvc.repo import Repo
 from dvc.scm import SCM, NoSCM
 from dvc.config import Config
 from dvc.exceptions import InitError
-from dvc.utils import boxify
-
+from dvc.utils import boxify, relpath, remove
 
 logger = logging.getLogger(__name__)
 
@@ -59,7 +57,7 @@ def init(root_dir=os.curdir, no_scm=False, force=False):
     Raises:
         KeyError: Raises an exception.
     """
-    root_dir = os.path.abspath(root_dir)
+    root_dir = os.path.realpath(root_dir)
     dvc_dir = os.path.join(root_dir, Repo.DVC_DIR)
     scm = SCM(root_dir)
     if isinstance(scm, NoSCM) and not no_scm:
@@ -74,11 +72,11 @@ def init(root_dir=os.curdir, no_scm=False, force=False):
         if not force:
             raise InitError(
                 "'{repo}' exists. Use '-f' to force.".format(
-                    repo=os.path.relpath(dvc_dir)
+                    repo=relpath(dvc_dir)
                 )
             )
 
-        shutil.rmtree(dvc_dir)
+        remove(dvc_dir)
 
     os.mkdir(dvc_dir)
 

dvc/repo/metrics/show.py

@@ -134,7 +134,7 @@ def _format_output(content, typ):
     return content
 
 
-def _read_metric(fd, typ=None, xpath=None, rel_path=None, branch=None):
+def _read_metric(fd, typ=None, xpath=None, fname=None, branch=None):
     typ = typ.lower().strip() if typ else typ
     try:
         if xpath:
@@ -146,7 +146,7 @@ def _read_metric(fd, typ=None, xpath=None, rel_path=None, branch=None):
     except Exception:
         logger.exception(
             "unable to read metric in '{}' in branch '{}'".format(
-                rel_path, branch
+                fname, branch
             )
         )
         return None
@@ -220,28 +220,24 @@ def _read_metrics(repo, metrics, branch):
     for out, typ, xpath in metrics:
         assert out.scheme == "local"
         if not typ:
-            typ = os.path.splitext(out.path.lower())[1].replace(".", "")
+            typ = os.path.splitext(out.fspath.lower())[1].replace(".", "")
         if out.use_cache:
             open_fun = open
             path = repo.cache.local.get(out.checksum)
         else:
             open_fun = repo.tree.open
-            path = out.path
+            path = out.fspath
         try:
 
             with open_fun(path) as fd:
                 metric = _read_metric(
-                    fd,
-                    typ=typ,
-                    xpath=xpath,
-                    rel_path=out.rel_path,
-                    branch=branch,
+                    fd, typ=typ, xpath=xpath, fname=str(out), branch=branch
                 )
         except IOError as e:
             if e.errno == errno.ENOENT:
                 logger.warning(
                     NO_METRICS_FILE_AT_REFERENCE_WARNING.format(
-                        out.rel_path, branch
+                        out.path_info, branch
                     )
                 )
                 metric = None
@@ -251,7 +247,7 @@ def _read_metrics(repo, metrics, branch):
         if not metric:
             continue
 
-        res[out.rel_path] = metric
+        res[str(out)] = metric
 
     return res
 

dvc/repo/move.py

@@ -33,10 +33,11 @@ def move(self, from_path, to_path):
     from dvc.stage import Stage
 
     from_out = Output.loads_from(Stage(self), [from_path])[0]
+    assert from_out.scheme == "local"
 
     to_path = _expand_target_path(from_path, to_path)
 
-    outs = self.find_outs_by_path(from_out.path)
+    outs = self.find_outs_by_path(from_out.fspath)
     assert len(outs) == 1
     out = outs[0]
     stage = out.stage
@@ -45,7 +46,7 @@ def move(self, from_path, to_path):
         raise MoveNotDataSourceError(stage.relpath)
 
     stage_name = os.path.splitext(os.path.basename(stage.path))[0]
-    from_name = os.path.basename(from_out.path)
+    from_name = os.path.basename(from_out.fspath)
     if stage_name == from_name:
         os.unlink(stage.path)
 

dvc/repo/pkg/__init__.py

@@ -1,8 +0,0 @@
-class Pkg(object):
-    def __init__(self, repo):
-        self.repo = repo
-
-    def install(self, *args, **kwargs):
-        from dvc.repo.pkg.install import install
-
-        return install(self.repo, *args, **kwargs)

dvc/repo/pkg/install.py

@@ -1,173 +0,0 @@
-from __future__ import unicode_literals
-
-import os
-import shutil
-import logging
-
-from dvc.exceptions import DvcException
-from dvc.stage import Stage
-from dvc.scm.git.temp_git_repo import TempGitRepo
-
-
-logger = logging.getLogger(__name__)
-
-
-class PackageManager(object):
-    PACKAGE_FILE = "package.yaml"
-
-    @staticmethod
-    def read_packages():
-        return []
-
-    @staticmethod
-    def get_package(addr):
-        for pkg_class in [GitPackage]:
-            try:
-                return pkg_class()
-            except Exception:
-                pass
-        return None
-
-    def __init__(self, addr):
-        self._addr = addr
-
-
-class Package(object):
-    MODULES_DIR = "dvc_mod"
-
-    def install_or_update(
-        self, parent_repo, address, target_dir, select=[], fname=None
-    ):
-        raise NotImplementedError(
-            "A method of abstract Package class was called"
-        )
-
-    def is_in_root(self):
-        return True
-
-
-class GitPackage(Package):
-    DEF_DVC_FILE_PREFIX = "mod_"
-
-    def install_or_update(
-        self, parent_repo, address, target_dir, select=[], fname=None
-    ):
-        from git.cmd import Git
-
-        if not self.is_in_root():
-            raise DvcException(
-                "This command can be run only from a repository root"
-            )
-
-        if not os.path.exists(self.MODULES_DIR):
-            logger.debug("Creating modules dir {}".format(self.MODULES_DIR))
-            os.makedirs(self.MODULES_DIR)
-            parent_repo.scm.ignore(os.path.abspath(self.MODULES_DIR))
-
-        module_name = Git.polish_url(address).strip("/").split("/")[-1]
-        if not module_name:
-            raise DvcException(
-                "Package address error: unable to extract package name"
-            )
-
-        with TempGitRepo(
-            address, module_name, Package.MODULES_DIR
-        ) as tmp_repo:
-            outputs_to_copy = tmp_repo.outs
-            if select:
-                outputs_to_copy = list(
-                    filter(lambda out: out.dvc_path in select, outputs_to_copy)
-                )
-
-            fetched_stage_files = set(
-                map(lambda o: o.stage.path, outputs_to_copy)
-            )
-            tmp_repo.fetch(fetched_stage_files)
-
-            module_dir = self.create_module_dir(module_name)
-            tmp_repo.persist_to(module_dir, parent_repo)
-
-            dvc_file = self.get_dvc_file_name(fname, target_dir, module_name)
-            try:
-                self.persist_stage_and_scm_state(
-                    parent_repo, outputs_to_copy, target_dir, dvc_file
-                )
-            except Exception as ex:
-                raise DvcException(
-                    "Package '{}' was installed "
-                    "but stage file '{}' "
-                    "was not created properly: {}".format(
-                        address, dvc_file, ex
-                    )
-                )
-
-        parent_repo.checkout(dvc_file)
-
-    @staticmethod
-    def persist_stage_and_scm_state(
-        parent_repo, outputs_to_copy, target_dir, dvc_file
-    ):
-        stage = Stage.create(
-            repo=parent_repo,
-            fname=dvc_file,
-            validate_state=False,
-            wdir=target_dir,
-        )
-        stage.outs = list(
-            map(lambda o: o.assign_to_stage_file(stage), outputs_to_copy)
-        )
-
-        for out in stage.outs:
-            parent_repo.scm.ignore(out.path, in_curr_dir=True)
-
-        stage.dump()
-
-    @staticmethod
-    def create_module_dir(module_name):
-        module_dir = os.path.join(GitPackage.MODULES_DIR, module_name)
-        if os.path.exists(module_dir):
-            logger.info("Updating package {}".format(module_name))
-            shutil.rmtree(module_dir)
-        else:
-            logger.info("Adding package {}".format(module_name))
-        return module_dir
-
-    def get_dvc_file_name(self, stage_file, target_dir, module_name):
-        if stage_file:
-            dvc_file_path = stage_file
-        else:
-            dvc_file_name = self.DEF_DVC_FILE_PREFIX + module_name + ".dvc"
-            dvc_file_path = os.path.join(target_dir, dvc_file_name)
-        return dvc_file_path
-
-
-def install(self, address, target_dir, select=[], fname=None):
-    """
-    Install package.
-
-    The command can be run only from DVC project root.
-
-    E.g.
-          Having: DVC package in https://github.com/dmpetrov/tag_classifier
-
-          $ dvc pkg install https://github.com/dmpetrov/tag_classifier
-
-          Result: tag_classifier package in dvc_mod/ directory
-    """
-
-    if not os.path.isdir(target_dir):
-        raise DvcException(
-            "target directory '{}' does not exist".format(target_dir)
-        )
-
-    curr_dir = os.path.realpath(os.curdir)
-    if not os.path.realpath(target_dir).startswith(curr_dir):
-        raise DvcException(
-            "the current directory should be a subdirectory of the target "
-            "dir '{}'".format(target_dir)
-        )
-
-    addresses = [address] if address else PackageManager.read_packages()
-    for addr in addresses:
-        mgr = PackageManager.get_package(addr)
-        mgr.install_or_update(self, address, target_dir, select, fname)

dvc/repo/reproduce.py

@@ -5,7 +5,7 @@ import logging
 
 from dvc.exceptions import ReproductionError
 from dvc.repo.scm_context import scm_context
-
+from dvc.utils import relpath
 
 logger = logging.getLogger(__name__)
 
@@ -15,7 +15,7 @@ def _reproduce_stage(stages, node, force, dry, interactive, no_commit):
 
     if stage.locked:
         logger.warning(
-            "DVC file '{path}' is locked. Its dependencies are"
+            "DVC-file '{path}' is locked. Its dependencies are"
             " not going to be reproduced.".format(path=stage.relpath)
         )
 
@@ -35,7 +35,7 @@ def _reproduce_stage(stages, node, force, dry, interactive, no_commit):
 def reproduce(
     self,
     target=None,
-    recursive=True,
+    single_item=False,
     force=False,
     dry=False,
     interactive=False,
@@ -44,7 +44,9 @@ def reproduce(
     ignore_build_cache=False,
     no_commit=False,
     downstream=False,
+    recursive=False,
 ):
+    import networkx as nx
     from dvc.stage import Stage
 
     if not target and not all_pipelines:
@@ -56,10 +58,16 @@ def reproduce(
         interactive = core.get(config.SECTION_CORE_INTERACTIVE, False)
 
     targets = []
-    if pipeline or all_pipelines:
+    if recursive and os.path.isdir(target):
+        G = self.graph(from_directory=target)[1]
+        dir_targets = [
+            os.path.join(self.root_dir, n) for n in nx.dfs_postorder_nodes(G)
+        ]
+        targets.extend(dir_targets)
+    elif pipeline or all_pipelines:
         if pipeline:
             stage = Stage.load(self, target)
-            node = os.path.relpath(stage.path, self.root_dir)
+            node = relpath(stage.path, self.root_dir)
             pipelines = [self._get_pipeline(node)]
         else:
             pipelines = self.pipelines()
@@ -77,7 +85,7 @@ def reproduce(
             stages = _reproduce(
                 self,
                 target,
-                recursive=recursive,
+                single_item=single_item,
                 force=force,
                 dry=dry,
                 interactive=interactive,
@@ -93,7 +101,7 @@ def reproduce(
 def _reproduce(
     self,
     target,
-    recursive=True,
+    single_item=False,
     force=False,
     dry=False,
     interactive=False,
@@ -107,9 +115,13 @@ def _reproduce(
     stage = Stage.load(self, target)
     G = self.graph()[1]
     stages = nx.get_node_attributes(G, "stage")
-    node = os.path.relpath(stage.path, self.root_dir)
+    node = relpath(stage.path, self.root_dir)
 
-    if recursive:
+    if single_item:
+        ret = _reproduce_stage(
+            stages, node, force, dry, interactive, no_commit
+        )
+    else:
         ret = _reproduce_stages(
             G,
             stages,
@@ -121,10 +133,6 @@ def _reproduce(
             no_commit,
             downstream,
         )
-    else:
-        ret = _reproduce_stage(
-            stages, node, force, dry, interactive, no_commit
-        )
 
     return ret
 

dvc/repo/status.py

@@ -14,7 +14,7 @@ def _local_status(self, target=None, with_deps=False):
     for stage in stages:
         if stage.locked:
             logger.warning(
-                "DVC file '{path}' is locked. Its dependencies are"
+                "DVC-file '{path}' is locked. Its dependencies are"
                 " not going to be shown in the status output.".format(
                     path=stage.relpath
                 )
@@ -35,6 +35,32 @@ def _cloud_status(
     with_deps=False,
     all_tags=False,
 ):
+    """Returns a dictionary with the files that are new or deleted.
+
+    - new: Remote doesn't have the file
+    - deleted: File is no longer in the local cache
+
+    Example:
+            Given the following commands:
+
+            $ echo "foo" > foo
+            $ echo "bar" > bar
+            $ dvc add foo bar
+            $ dvc status -c
+
+            It will return something like:
+
+            { "foo": "new", "bar": "new" }
+
+            Now, after pushing and removing "bar" from the local cache:
+
+            $ dvc push
+            $ rm .dvc/cache/c1/57a79031e1c40f85931829bc5fc552
+
+            The result would be:
+
+            { "bar": "deleted" }
+    """
     import dvc.remote.base as cloud
 
     used = self.used_cache(

dvc/scm/git/__init__.py

@@ -6,7 +6,7 @@ import os
 import logging
 
 from dvc.utils.compat import str, open
-from dvc.utils import fix_env
+from dvc.utils import fix_env, relpath
 from dvc.scm.base import (
     Base,
     SCMError,
@@ -42,10 +42,10 @@ class Git(Base):
         from git.exc import InvalidGitRepositoryError
 
         try:
-            self.git = git.Repo(root_dir)
+            self.git = git.Repo(self.root_dir)
         except InvalidGitRepositoryError:
             msg = "{} is not a git repository"
-            raise SCMError(msg.format(root_dir))
+            raise SCMError(msg.format(self.root_dir))
 
         # NOTE: fixing LD_LIBRARY_PATH for binary built by PyInstaller.
         # http://pyinstaller.readthedocs.io/en/stable/runtime-information.html
@@ -92,7 +92,7 @@ class Git(Base):
                 msg.format(self.GITIGNORE, path, ignore_file_dir)
             )
 
-        entry = os.path.relpath(path, ignore_file_dir).replace(os.sep, "/")
+        entry = relpath(path, ignore_file_dir).replace(os.sep, "/")
         # NOTE: using '/' prefix to make path unambiguous
         if len(entry) > 0 and entry[0] != "/":
             entry = "/" + entry
@@ -104,6 +104,16 @@ class Git(Base):
 
         return entry, gitignore
 
+    @staticmethod
+    def _ignored(entry, gitignore_path):
+        if os.path.exists(gitignore_path):
+            with open(gitignore_path, "r") as fobj:
+                ignore_list = fobj.readlines()
+            return any(
+                filter(lambda x: x.strip() == entry.strip(), ignore_list)
+            )
+        return False
+
     def ignore(self, path, in_curr_dir=False):
         base_dir = (
             os.path.realpath(os.curdir)
@@ -112,31 +122,30 @@ class Git(Base):
         )
         entry, gitignore = self._get_gitignore(path, base_dir)
 
-        ignore_list = []
-        if os.path.exists(gitignore):
-            with open(gitignore, "r") as f:
-                ignore_list = f.readlines()
-            if any(filter(lambda x: x.strip() == entry.strip(), ignore_list)):
-                return
+        if self._ignored(entry, gitignore):
+            return
 
-        msg = "Adding '{}' to '{}'.".format(
-            os.path.relpath(path), os.path.relpath(gitignore)
-        )
+        msg = "Adding '{}' to '{}'.".format(relpath(path), relpath(gitignore))
         logger.info(msg)
 
-        self._add_entry_to_gitignore(entry, gitignore, ignore_list)
+        self._add_entry_to_gitignore(entry, gitignore)
 
-        self.track_file(os.path.relpath(gitignore))
+        self.track_file(relpath(gitignore))
 
         self.ignored_paths.append(path)
 
     @staticmethod
-    def _add_entry_to_gitignore(entry, gitignore, ignore_list):
-        content = entry
-        if ignore_list:
-            content = "\n" + content
-        with open(gitignore, "a", encoding="utf-8") as fobj:
-            fobj.write(content)
+    def _add_entry_to_gitignore(entry, gitignore):
+        with open(gitignore, "a+", encoding="utf-8") as fobj:
+            fobj.seek(0, os.SEEK_END)
+            if fobj.tell() == 0:
+                # Empty file
+                prefix = ""
+            else:
+                fobj.seek(fobj.tell() - 1, os.SEEK_SET)
+                last = fobj.read(1)
+                prefix = "" if last == "\n" else "\n"
+            fobj.write("{}{}\n".format(prefix, entry))
 
     def ignore_remove(self, path):
         entry, gitignore = self._get_gitignore(path)
@@ -152,7 +161,7 @@ class Git(Base):
         with open(gitignore, "w") as fobj:
             fobj.writelines(filtered)
 
-        self.track_file(os.path.relpath(gitignore))
+        self.track_file(relpath(gitignore))
 
     def add(self, paths):
         # NOTE: GitPython is not currently able to handle index version >= 3.
@@ -191,7 +200,7 @@ class Git(Base):
     def is_tracked(self, path):
         # it is equivalent to `bool(self.git.git.ls_files(path))` by
         # functionality, but ls_files fails on unicode filenames
-        path = os.path.relpath(path, self.root_dir)
+        path = relpath(path, self.root_dir)
         return path in [i[0] for i in self.git.index.entries]
 
     def is_dirty(self):
@@ -242,7 +251,7 @@ class Git(Base):
 
         logger.info(
             "\n"
-            "To track the changes with git run:\n"
+            "To track the changes with git, run:\n"
             "\n"
             "\tgit add {files}".format(files=" ".join(self.files_to_track))
         )
@@ -260,15 +269,14 @@ class Git(Base):
 
     def _get_diff_trees(self, a_ref, b_ref):
         """Private method for getting the trees and commit hashes of 2 git
-        references.
-        Requires `gitdb` module (from gitpython package).
+        references. Requires `gitdb` module (from gitpython package).
 
         Args:
-            a_ref(str) - git reference
-            b_ref(str) - second git reference. If None, uses HEAD
+            a_ref (str): git reference
+            b_ref (str): second git reference. If None, uses HEAD
 
         Returns:
-            tuple - tuple with elements: (trees, commits)
+            tuple: tuple with elements: (trees, commits)
         """
         from gitdb.exc import BadObject, BadName
 
@@ -294,11 +302,12 @@ class Git(Base):
         Returns the dvc hash names of changed file/directory
 
         Args:
-            a_ref(str) - git reference
-            b_ref(str) - optional second git reference, default None
+            a_ref (str): git reference
+            b_ref (str): optional second git reference, default None
 
         Returns:
-            dict - dictionary with keys: (a_tree, b_tree, a_ref, b_ref, equal)
+            dict: dictionary with keys: {a_ref, b_ref, equal}
+                or {a_ref, b_ref, a_tree, b_tree}
         """
         diff_dct = {DIFF_EQUAL: False}
         trees, commits = self._get_diff_trees(a_ref, b_ref)

dvc/scm/git/temp_git_repo.py

@@ -1,148 +0,0 @@
-import os
-import shutil
-import tempfile
-import logging
-
-from dvc.exceptions import DvcException
-
-
-logger = logging.getLogger(__name__)
-
-
-class TempRepoException(DvcException):
-    """Raise when temporary package repository has not properly created"""
-
-    def __init__(self, temp_repo, msg, cause=None):
-        m = "temp repository '{}' error: {}".format(temp_repo.addr, msg)
-        super(TempRepoException, self).__init__(m, cause)
-
-
-class TempGitRepo(object):
-    GIT_DIR_TO_REMOVE = ".git"
-    GIT_FILES_TO_REMOVE = [".gitignore", ".gitmodules"]
-    SUFFIX = "tmp_DVC_mod"
-
-    def __init__(self, addr, module_name, modules_dir):
-        self.addr = addr
-        self.modules_dir = modules_dir
-        self._tmp_mod_prefix = module_name + "_" + self.SUFFIX + "_"
-        self._reset_state()
-
-    def _reset_state(self):
-        self._set_state(None, [])
-
-    def _set_state(self, cloned_tmp_dir, outs):
-        from dvc.repo import Repo
-
-        self.repo = Repo(cloned_tmp_dir) if cloned_tmp_dir else None
-
-        self._cloned_tmp_dir = cloned_tmp_dir
-        self.outs = outs
-        self._moved_files = []
-
-    def fetch(self, targets):
-        for target in targets:
-            try:
-                self.repo.fetch(target)
-            except Exception as ex:
-                msg = "error in fetching data from {}: {}".format(
-                    os.path.basename(target), ex
-                )
-                raise TempRepoException(self, msg)
-
-    @property
-    def is_state_set(self):
-        return self._cloned_tmp_dir is not None
-
-    def __enter__(self):
-        if self.is_state_set:
-            raise TempRepoException(self, "Git repo cloning duplication")
-
-        module_temp_dir = None
-        try:
-            module_temp_dir = self._clone_to_temp_dir()
-            self._remove_git_files_and_dirs(module_temp_dir)
-
-            outputs = self._read_outputs(module_temp_dir)
-            self._set_state(module_temp_dir, outputs)
-        finally:
-            self._clean_mod_temp_dir(module_temp_dir)
-
-        return self
-
-    def _clone_to_temp_dir(self):
-        import git
-
-        result = tempfile.mktemp(
-            prefix=self._tmp_mod_prefix, dir=self.modules_dir
-        )
-        logger.debug(
-            "Cloning git repository {} to temp dir {}".format(
-                self.addr, result
-            )
-        )
-        git.Repo.clone_from(self.addr, result, depth=1)
-        return result
-
-    def _remove_git_files_and_dirs(self, module_temp_dir):
-        logger.debug(
-            "Removing git meta files from {}: {} dif and {}".format(
-                module_temp_dir,
-                TempGitRepo.GIT_DIR_TO_REMOVE,
-                ", ".join(TempGitRepo.GIT_FILES_TO_REMOVE),
-            )
-        )
-        shutil.rmtree(
-            os.path.join(module_temp_dir, TempGitRepo.GIT_DIR_TO_REMOVE)
-        )
-        for item in TempGitRepo.GIT_FILES_TO_REMOVE:
-            fname = os.path.join(module_temp_dir, item)
-            if os.path.exists(fname):
-                os.remove(fname)
-
-    def _clean_mod_temp_dir(self, module_temp_dir):
-        if not self.is_state_set:
-            if module_temp_dir and os.path.exists(module_temp_dir):
-                shutil.rmtree(module_temp_dir, ignore_errors=True)
-
-    def _read_outputs(self, module_temp_dir):
-        from dvc.repo import Repo
-
-        pkg_repo = Repo(module_temp_dir)
-        stages = pkg_repo.stages()
-        return [out for s in stages for out in s.outs]
-
-    def persist_to(self, module_dir, parent_repo):
-        if not self.is_state_set:
-            raise TempRepoException(self, "cannot persist")
-
-        tmp_repo_cache = self.repo.cache.local.url
-
-        for prefix in os.listdir(tmp_repo_cache):
-            if len(prefix) != 2:
-                logger.warning(
-                    "wrong dir format in cache {}: dir {}".format(
-                        tmp_repo_cache, prefix
-                    )
-                )
-            self._move_all_cache_files(parent_repo, prefix, tmp_repo_cache)
-
-        shutil.move(self._cloned_tmp_dir, module_dir)
-        self._reset_state()
-
-    @staticmethod
-    def _move_all_cache_files(parent_repo, prefix, tmp_repo_cache):
-        obj_name = os.path.join(tmp_repo_cache, prefix)
-        for suffix in os.listdir(obj_name):
-            src_path = os.path.join(tmp_repo_cache, prefix, suffix)
-            pre = os.path.join(parent_repo.cache.local.url, prefix)
-            if not os.path.exists(pre):
-                os.mkdir(pre)
-            dest = os.path.join(pre, suffix)
-            shutil.move(src_path, dest)
-
-    def __exit__(self, exc_type, exc_value, traceback):
-        if self.is_state_set:
-            shutil.rmtree(self._cloned_tmp_dir)
-            self._reset_state()
-        pass

dvc/scm/git/tree.py

@@ -2,7 +2,9 @@ import errno
 import os
 
 from dvc.ignore import DvcIgnoreFilter
+from dvc.utils import relpath
 from dvc.utils.compat import StringIO, BytesIO
+from dvc.exceptions import DvcException
 
 from dvc.scm.tree import BaseTree
 
@@ -31,14 +33,14 @@ class GitTree(BaseTree):
 
     def open(self, path, binary=False):
 
-        relpath = os.path.relpath(path, self.git.working_dir)
+        relative_path = relpath(path, self.git.working_dir)
 
         obj = self.git_object_by_path(path)
         if obj is None:
             msg = "No such file in branch '{}'".format(self.rev)
-            raise IOError(errno.ENOENT, msg, relpath)
+            raise IOError(errno.ENOENT, msg, relative_path)
         if obj.mode == GIT_MODE_DIR:
-            raise IOError(errno.EISDIR, "Is a directory", relpath)
+            raise IOError(errno.EISDIR, "Is a directory", relative_path)
 
         # GitPython's obj.data_stream is a fragile thing, it is better to
         # read it immediately, also it needs to be to decoded if we follow
@@ -76,10 +78,42 @@ class GitTree(BaseTree):
                 return True
         return False
 
+    def _try_fetch_from_remote(self):
+        import git
+
+        try:
+            # checking if tag/branch exists locally
+            self.git.git.show_ref(self.rev, verify=True)
+            return
+        except git.exc.GitCommandError:
+            pass
+
+        try:
+            # checking if it exists on the remote
+            self.git.git.ls_remote("origin", self.rev, exit_code=True)
+            # fetching remote tag/branch so we can reference it locally
+            self.git.git.fetch("origin", "{rev}:{rev}".format(rev=self.rev))
+        except git.exc.GitCommandError:
+            pass
+
     def git_object_by_path(self, path):
-        path = os.path.relpath(os.path.realpath(path), self.git.working_dir)
+        import git
+
+        path = relpath(os.path.realpath(path), self.git.working_dir)
         assert path.split(os.sep, 1)[0] != ".."
-        tree = self.git.tree(self.rev)
+
+        self._try_fetch_from_remote()
+
+        try:
+            tree = self.git.tree(self.rev)
+        except git.exc.BadName as exc:
+            raise DvcException(
+                "revision '{}' not found in git '{}'".format(
+                    self.rev, os.path.relpath(self.git.working_dir)
+                ),
+                cause=exc,
+            )
+
         if not path or path == ".":
             return tree
         for i in path.split(os.sep):

dvc/stage.py

@@ -7,7 +7,11 @@ import re
 import os
 import subprocess
 import logging
+import signal
+import threading
 
+from dvc.utils import relpath
+from dvc.utils.compat import pathlib
 from dvc.utils.fs import contains_symlink_up_to
 from schema import Schema, SchemaError, Optional, Or, And
 
@@ -54,7 +58,7 @@ class StageFileAlreadyExistsError(DvcException):
 
 class StageFileIsNotDvcFileError(DvcException):
     def __init__(self, fname):
-        msg = "'{}' is not a dvc file".format(fname)
+        msg = "'{}' is not a DVC-file".format(fname)
 
         sname = fname + Stage.STAGE_FILE_SUFFIX
         if Stage.is_stage_file(sname):
@@ -175,7 +179,7 @@ class Stage(object):
 
     @property
     def relpath(self):
-        return os.path.relpath(self.path)
+        return relpath(self.path)
 
     @property
     def is_data_source(self):
@@ -212,13 +216,20 @@ class Stage(object):
         """Whether the stage file was created with `dvc import`."""
         return not self.cmd and len(self.deps) == 1 and len(self.outs) == 1
 
+    @property
+    def is_repo_import(self):
+        if not self.is_import:
+            return False
+
+        return isinstance(self.deps[0], dependency.DependencyREPO)
+
     def _changed_deps(self):
         if self.locked:
             return False
 
         if self.is_callback:
             logger.warning(
-                "Dvc file '{fname}' is a 'callback' stage "
+                "DVC-file '{fname}' is a 'callback' stage "
                 "(has a command and no dependencies) and thus always "
                 "considered as changed.".format(fname=self.relpath)
             )
@@ -253,7 +264,7 @@ class Stage(object):
 
     def _changed_md5(self):
         if self.changed_md5():
-            logger.warning("Dvc file '{}' changed.".format(self.relpath))
+            logger.warning("DVC-file '{}' changed.".format(self.relpath))
             return True
         return False
 
@@ -286,8 +297,11 @@ class Stage(object):
         for out in self.outs:
             out.unprotect()
 
-    def remove(self, force=False):
-        self.remove_outs(ignore_remove=True, force=force)
+    def remove(self, force=False, remove_outs=True):
+        if remove_outs:
+            self.remove_outs(ignore_remove=True, force=force)
+        else:
+            self.unprotect_outs()
         os.unlink(self.path)
 
     def reproduce(
@@ -322,32 +336,20 @@ class Stage(object):
             raise StageFileFormatError(fname, exc)
 
     @classmethod
-    def _stage_fname(cls, fname, outs, add):
-        if fname:
-            return fname
-
+    def _stage_fname(cls, outs, add):
         if not outs:
             return cls.STAGE_FILE
 
         out = outs[0]
-        path_handler = out.remote.ospath
+        fname = out.path_info.name + cls.STAGE_FILE_SUFFIX
 
-        fname = path_handler.basename(out.path) + cls.STAGE_FILE_SUFFIX
-
-        fname = Stage._expand_to_path_on_add_local(
-            add, fname, out, path_handler
-        )
-
-        return fname
-
-    @staticmethod
-    def _expand_to_path_on_add_local(add, fname, out, path_handler):
         if (
             add
             and out.is_in_repo
-            and not contains_symlink_up_to(out.path, out.repo.root_dir)
+            and not contains_symlink_up_to(out.fspath, out.repo.root_dir)
         ):
-            fname = path_handler.join(path_handler.dirname(out.path), fname)
+            fname = out.path_info.with_name(fname).fspath
+
         return fname
 
     @staticmethod
@@ -430,6 +432,7 @@ class Stage(object):
         validate_state=True,
         outs_persist=None,
         outs_persist_no_cache=None,
+        erepo=None,
     ):
         if outs is None:
             outs = []
@@ -470,12 +473,15 @@ class Stage(object):
             outs_persist,
             outs_persist_no_cache,
         )
-        stage.deps = dependency.loads_from(stage, deps)
+        stage.deps = dependency.loads_from(stage, deps, erepo=erepo)
 
         stage._check_circular_dependency()
         stage._check_duplicated_arguments()
 
-        fname = Stage._stage_fname(fname, stage.outs, add=add)
+        if not fname:
+            fname = Stage._stage_fname(stage.outs, add=add)
+        stage._check_dvc_filename(fname)
+
         wdir = os.path.abspath(wdir)
 
         if cwd is not None:
@@ -555,7 +561,7 @@ class Stage(object):
             raise StageFileBadNameError(
                 "bad stage filename '{}'. Stage files should be named"
                 " 'Dvcfile' or have a '.dvc' suffix (e.g. '{}.dvc').".format(
-                    os.path.relpath(fname), os.path.basename(fname)
+                    relpath(fname), os.path.basename(fname)
                 )
             )
 
@@ -583,7 +589,7 @@ class Stage(object):
 
         # it raises the proper exceptions by priority:
         # 1. when the file doesn't exists
-        # 2. filename is not a dvc filename
+        # 2. filename is not a DVC-file
         # 3. path doesn't represent a regular file
         Stage._check_file_exists(repo, fname)
         Stage._check_dvc_filename(fname)
@@ -595,7 +601,7 @@ class Stage(object):
         # looses keys in deps and outs load
         state = copy.deepcopy(d)
 
-        Stage.validate(d, fname=os.path.relpath(fname))
+        Stage.validate(d, fname=relpath(fname))
         path = os.path.abspath(fname)
 
         stage = Stage(
@@ -619,16 +625,13 @@ class Stage(object):
         return stage
 
     def dumpd(self):
-        from dvc.remote.base import RemoteBASE
-
+        rel_wdir = relpath(self.wdir, os.path.dirname(self.path))
         return {
             key: value
             for key, value in {
                 Stage.PARAM_MD5: self.md5,
                 Stage.PARAM_CMD: self.cmd,
-                Stage.PARAM_WDIR: RemoteBASE.to_posixpath(
-                    os.path.relpath(self.wdir, os.path.dirname(self.path))
-                ),
+                Stage.PARAM_WDIR: pathlib.PurePath(rel_wdir).as_posix(),
                 Stage.PARAM_LOCKED: self.locked,
                 Stage.PARAM_DEPS: [d.dumpd() for d in self.deps],
                 Stage.PARAM_OUTS: [o.dumpd() for o in self.outs],
@@ -643,15 +646,13 @@ class Stage(object):
         self._check_dvc_filename(fname)
 
         logger.info(
-            "Saving information to '{file}'.".format(
-                file=os.path.relpath(fname)
-            )
+            "Saving information to '{file}'.".format(file=relpath(fname))
         )
         d = self.dumpd()
         apply_diff(d, self._state)
         dump_stage_file(fname, self._state)
 
-        self.repo.scm.track_file(os.path.relpath(fname))
+        self.repo.scm.track_file(relpath(fname))
 
     def _compute_md5(self):
         from dvc.output.base import OutputBase
@@ -695,11 +696,11 @@ class Stage(object):
 
     @staticmethod
     def _changed_entries(entries):
-        ret = []
-        for entry in entries:
-            if entry.checksum and entry.changed_checksum():
-                ret.append(entry.rel_path)
-        return ret
+        return [
+            str(entry)
+            for entry in entries
+            if entry.checksum and entry.changed_checksum()
+        ]
 
     def check_can_commit(self, force):
         changed_deps = self._changed_entries(self.deps)
@@ -751,38 +752,51 @@ class Stage(object):
     def _check_circular_dependency(self):
         from dvc.exceptions import CircularDependencyError
 
-        circular_dependencies = set(d.path for d in self.deps) & set(
-            o.path for o in self.outs
+        circular_dependencies = set(d.path_info for d in self.deps) & set(
+            o.path_info for o in self.outs
         )
 
         if circular_dependencies:
-            raise CircularDependencyError(circular_dependencies.pop())
+            raise CircularDependencyError(str(circular_dependencies.pop()))
 
     def _check_duplicated_arguments(self):
         from dvc.exceptions import ArgumentDuplicationError
         from collections import Counter
 
-        path_counts = Counter(edge.path for edge in self.deps + self.outs)
+        path_counts = Counter(edge.path_info for edge in self.deps + self.outs)
 
         for path, occurrence in path_counts.items():
             if occurrence > 1:
-                raise ArgumentDuplicationError(path)
+                raise ArgumentDuplicationError(str(path))
 
     def _run(self):
         self._check_missing_deps()
         executable = os.getenv("SHELL") if os.name != "nt" else None
         self._warn_if_fish(executable)
 
-        p = subprocess.Popen(
-            self.cmd,
-            cwd=self.wdir,
-            shell=True,
-            env=fix_env(os.environ),
-            executable=executable,
+        main_thread = isinstance(
+            threading.current_thread(), threading._MainThread
         )
-        p.communicate()
+        if main_thread:
+            old_handler = signal.signal(signal.SIGINT, signal.SIG_IGN)
 
-        if p.returncode != 0:
+        p = None
+
+        try:
+            p = subprocess.Popen(
+                self.cmd,
+                cwd=self.wdir,
+                shell=True,
+                env=fix_env(os.environ),
+                executable=executable,
+                close_fds=True,
+            )
+            p.communicate()
+        finally:
+            if main_thread:
+                signal.signal(signal.SIGINT, old_handler)
+
+        if (p is None) or (p.returncode != 0):
             raise StageCmdFailedError(self)
 
     def run(self, dry=False, resume=False, no_commit=False, force=False):
@@ -801,16 +815,14 @@ class Stage(object):
         elif self.is_import:
             logger.info(
                 "Importing '{dep}' -> '{out}'".format(
-                    dep=self.deps[0].path, out=self.outs[0].path
+                    dep=self.deps[0], out=self.outs[0]
                 )
             )
             if not dry:
                 if self._already_cached() and not force:
                     self.outs[0].checkout()
                 else:
-                    self.deps[0].download(
-                        self.outs[0].path_info, resume=resume
-                    )
+                    self.deps[0].download(self.outs[0], resume=resume)
 
         elif self.is_data_source:
             msg = "Verifying data sources in '{}'".format(self.relpath)
@@ -836,12 +848,7 @@ class Stage(object):
                 self.commit()
 
     def check_missing_outputs(self):
-        paths = [
-            out.path if out.scheme != "local" else out.rel_path
-            for out in self.outs
-            if not out.exists
-        ]
-
+        paths = [str(out) for out in self.outs if not out.exists]
         if paths:
             raise MissingDataSource(paths)
 

dvc/state.py

@@ -7,9 +7,10 @@ import sqlite3
 import logging
 
 from dvc.config import Config
-from dvc.utils import remove, current_timestamp
+from dvc.utils import remove, current_timestamp, relpath
 from dvc.exceptions import DvcException
 from dvc.utils.fs import get_mtime_and_size, get_inode
+from dvc.utils.compat import fspath_py35
 
 
 logger = logging.getLogger(__name__)
@@ -116,29 +117,9 @@ class State(StateBase):  # pylint: disable=too-many-instance-attributes
     def __exit__(self, typ, value, tbck):
         self.dump()
 
-    def changed(self, path, md5):
-        """Check if file/directory has the expected md5.
-
-        Args:
-            path (str): path to the file/directory to check.
-            md5 (str): expected md5.
-
-        Returns:
-            bool: True if path has the expected md5, False otherwise.
-        """
-        actual = self.update(path)
-
-        msg = "File '{}', md5 '{}', actual '{}'"
-        logger.debug(msg.format(path, md5, actual))
-
-        if not md5 or not actual:
-            return True
-
-        return actual.split(".")[0] != md5.split(".")[0]
-
-    def _execute(self, cmd):
+    def _execute(self, cmd, parameters=()):
         logger.debug(cmd)
-        return self.cursor.execute(cmd)
+        return self.cursor.execute(cmd, parameters)
 
     def _fetchall(self):
         ret = self.cursor.fetchall()
@@ -249,8 +230,10 @@ class State(StateBase):  # pylint: disable=too-many-instance-attributes
         """Saves state database."""
         assert self.database is not None
 
-        cmd = "SELECT count from {} WHERE rowid={}"
-        self._execute(cmd.format(self.STATE_INFO_TABLE, self.STATE_INFO_ROW))
+        cmd = "SELECT count from {} WHERE rowid=?".format(
+            self.STATE_INFO_TABLE
+        )
+        self._execute(cmd, (self.STATE_INFO_ROW,))
         ret = self._fetchall()
         assert len(ret) == 1
         assert len(ret[0]) == 1
@@ -280,16 +263,10 @@ class State(StateBase):  # pylint: disable=too-many-instance-attributes
             assert len(ret[0]) == 1
             count = ret[0][0]
 
-        cmd = "UPDATE {} SET count = {} WHERE rowid = {}"
-        self._execute(
-            cmd.format(
-                self.STATE_INFO_TABLE,
-                self._to_sqlite(count),
-                self.STATE_INFO_ROW,
-            )
+        cmd = "UPDATE {} SET count = ? WHERE rowid = ?".format(
+            self.STATE_INFO_TABLE
         )
-
-        self._update_cache_directory_state()
+        self._execute(cmd, (self._to_sqlite(count), self.STATE_INFO_ROW))
 
         self.database.commit()
         self.cursor.close()
@@ -303,60 +280,60 @@ class State(StateBase):  # pylint: disable=too-many-instance-attributes
         return actual_mtime != mtime or actual_size != size
 
     def _update_state_record_timestamp_for_inode(self, actual_inode):
-        cmd = 'UPDATE {} SET timestamp = "{}" WHERE inode = {}'
+        cmd = "UPDATE {} SET timestamp = ? WHERE inode = ?".format(
+            self.STATE_TABLE
+        )
         self._execute(
-            cmd.format(
-                self.STATE_TABLE,
-                current_timestamp(),
-                self._to_sqlite(actual_inode),
-            )
+            cmd, (current_timestamp(), self._to_sqlite(actual_inode))
         )
 
     def _update_state_for_path_changed(
-        self, path, actual_inode, actual_mtime, actual_size, checksum
+        self, actual_inode, actual_mtime, actual_size, checksum
     ):
         cmd = (
             "UPDATE {} SET "
-            'mtime = "{}", size = "{}", '
-            'md5 = "{}", timestamp = "{}" '
-            "WHERE inode = {}"
-        )
+            "mtime = ?, size = ?, "
+            "md5 = ?, timestamp = ? "
+            "WHERE inode = ?"
+        ).format(self.STATE_TABLE)
         self._execute(
-            cmd.format(
-                self.STATE_TABLE,
+            cmd,
+            (
                 actual_mtime,
                 actual_size,
                 checksum,
                 current_timestamp(),
                 self._to_sqlite(actual_inode),
-            )
+            ),
         )
 
     def _insert_new_state_record(
-        self, path, actual_inode, actual_mtime, actual_size, checksum
+        self, actual_inode, actual_mtime, actual_size, checksum
     ):
         assert checksum is not None
 
         cmd = (
             "INSERT INTO {}(inode, mtime, size, md5, timestamp) "
-            'VALUES ({}, "{}", "{}", "{}", "{}")'
-        )
+            "VALUES (?, ?, ?, ?, ?)"
+        ).format(self.STATE_TABLE)
         self._execute(
-            cmd.format(
-                self.STATE_TABLE,
+            cmd,
+            (
                 self._to_sqlite(actual_inode),
                 actual_mtime,
                 actual_size,
                 checksum,
                 current_timestamp(),
-            )
+            ),
         )
         self.inserts += 1
 
     def get_state_record_for_inode(self, inode):
-        cmd = "SELECT mtime, size, md5, timestamp from {} " "WHERE inode={}"
-        cmd = cmd.format(self.STATE_TABLE, self._to_sqlite(inode))
-        self._execute(cmd)
+        cmd = (
+            "SELECT mtime, size, md5, timestamp from {} WHERE "
+            "inode=?".format(self.STATE_TABLE)
+        )
+        self._execute(cmd, (self._to_sqlite(inode),))
         results = self._fetchall()
         if results:
             # uniquness constrain on inode
@@ -374,7 +351,7 @@ class State(StateBase):  # pylint: disable=too-many-instance-attributes
         assert path_info.scheme == "local"
         assert checksum is not None
 
-        path = path_info.path
+        path = fspath_py35(path_info)
         assert os.path.exists(path)
 
         actual_mtime, actual_size = get_mtime_and_size(path)
@@ -383,12 +360,12 @@ class State(StateBase):  # pylint: disable=too-many-instance-attributes
         existing_record = self.get_state_record_for_inode(actual_inode)
         if not existing_record:
             self._insert_new_state_record(
-                path, actual_inode, actual_mtime, actual_size, checksum
+                actual_inode, actual_mtime, actual_size, checksum
             )
             return
 
         self._update_state_for_path_changed(
-            path, actual_inode, actual_mtime, actual_size, checksum
+            actual_inode, actual_mtime, actual_size, checksum
         )
 
     def get(self, path_info):
@@ -403,7 +380,7 @@ class State(StateBase):  # pylint: disable=too-many-instance-attributes
             doesn't exist in the state database.
         """
         assert path_info.scheme == "local"
-        path = path_info.path
+        path = fspath_py35(path_info)
 
         if not os.path.exists(path):
             return None
@@ -430,22 +407,19 @@ class State(StateBase):  # pylint: disable=too-many-instance-attributes
             path_info (dict): path info to add to the list of links.
         """
         assert path_info.scheme == "local"
-        path = path_info.path
+        path = fspath_py35(path_info)
 
         if not os.path.exists(path):
             return
 
         mtime, _ = get_mtime_and_size(path)
         inode = get_inode(path)
-        relpath = os.path.relpath(path, self.root_dir)
+        relative_path = relpath(path, self.root_dir)
 
-        cmd = (
-            "REPLACE INTO {}(path, inode, mtime) "
-            'VALUES ("{}", {}, "{}")'.format(
-                self.LINK_STATE_TABLE, relpath, self._to_sqlite(inode), mtime
-            )
+        cmd = "REPLACE INTO {}(path, inode, mtime) " "VALUES (?, ?, ?)".format(
+            self.LINK_STATE_TABLE
         )
-        self._execute(cmd)
+        self._execute(cmd, (relative_path, self._to_sqlite(inode), mtime))
 
     def remove_unused_links(self, used):
         """Removes all saved links except the ones that are used.
@@ -476,22 +450,5 @@ class State(StateBase):  # pylint: disable=too-many-instance-attributes
                 unused.append(relpath)
 
         for relpath in unused:
-            cmd = 'DELETE FROM {} WHERE path = "{}"'
-            self._execute(cmd.format(self.LINK_STATE_TABLE, relpath))
-
-    def _update_cache_directory_state(self):
-        cache_path = self.repo.cache.local.cache_dir
-        mtime, size = get_mtime_and_size(cache_path)
-        inode = get_inode(cache_path)
-
-        cmd = (
-            "INSERT OR REPLACE INTO {}(inode, size, mtime, timestamp, md5) "
-            'VALUES ({}, "{}", "{}", "{}", "")'.format(
-                self.STATE_TABLE,
-                self._to_sqlite(inode),
-                size,
-                mtime,
-                current_timestamp(),
-            )
-        )
-        self._execute(cmd)
+            cmd = "DELETE FROM {} WHERE path = ?".format(self.LINK_STATE_TABLE)
+            self._execute(cmd, (relpath,))

dvc/system.py

@@ -1,6 +1,6 @@
 from __future__ import unicode_literals
 
-from dvc.utils.compat import str, open
+from dvc.utils.compat import str, open, fspath
 
 import os
 import errno
@@ -16,6 +16,8 @@ class System(object):
         import ctypes
         from dvc.exceptions import DvcException
 
+        source, link_name = fspath(source), fspath(link_name)
+
         if System.is_unix():
             try:
                 os.link(source, link_name)
@@ -40,6 +42,8 @@ class System(object):
         import ctypes
         from dvc.exceptions import DvcException
 
+        source, link_name = fspath(source), fspath(link_name)
+
         if System.is_unix():
             try:
                 os.symlink(source, link_name)
@@ -117,6 +121,8 @@ class System(object):
         import platform
         from dvc.exceptions import DvcException
 
+        source, link_name = fspath(source), fspath(link_name)
+
         system = platform.system()
         try:
             if system == "Windows":
@@ -134,7 +140,7 @@ class System(object):
             raise DvcException("reflink is not supported")
 
     @staticmethod
-    def getdirinfo(path):
+    def _getdirinfo(path):
         import ctypes
         from ctypes import c_void_p, c_wchar_p, Structure, WinError, POINTER
         from ctypes.wintypes import DWORD, HANDLE, BOOL
@@ -206,6 +212,8 @@ class System(object):
 
     @staticmethod
     def inode(path):
+        path = fspath(path)
+
         if System.is_unix():
             import ctypes
 
@@ -216,7 +224,7 @@ class System(object):
             inode = ctypes.c_ulong(inode).value
         else:
             # getdirinfo from ntfsutils works on both files and dirs
-            info = System.getdirinfo(path)
+            info = System._getdirinfo(path)
             inode = abs(
                 hash(
                     (
@@ -267,6 +275,8 @@ class System(object):
 
     @staticmethod
     def is_symlink(path):
+        path = fspath(path)
+
         if System.is_unix():
             return os.path.islink(path)
 
@@ -275,14 +285,16 @@ class System(object):
         from winnt import FILE_ATTRIBUTE_REPARSE_POINT
 
         if os.path.lexists(path):
-            info = System.getdirinfo(path)
+            info = System._getdirinfo(path)
             return info.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT
         return False
 
     @staticmethod
     def is_hardlink(path):
+        path = fspath(path)
+
         if System.is_unix():
             return os.stat(path).st_nlink > 1
 
-        info = System.getdirinfo(path)
+        info = System._getdirinfo(path)
         return info.nNumberOfLinks > 1

dvc/utils/__init__.py

@@ -3,6 +3,7 @@
 from __future__ import unicode_literals
 
 from dvc.utils.compat import str, builtin_str, open, cast_bytes_py2, StringIO
+from dvc.utils.compat import fspath
 
 import os
 import sys
@@ -45,8 +46,8 @@ def file_md5(fname):
         if size >= LARGE_FILE_SIZE:
             bar = True
             msg = "Computing md5 for a large file {}. This is only done once."
-            logger.info(msg.format(os.path.relpath(fname)))
-            name = os.path.relpath(fname)
+            logger.info(msg.format(relpath(fname)))
+            name = relpath(fname)
             total = 0
 
         with open(fname, "rb") as fobj:
@@ -113,7 +114,9 @@ def dict_md5(d, exclude=[]):
 
 def copyfile(src, dest, no_progress_bar=False, name=None):
     """Copy file with progress bar"""
+    from dvc.exceptions import DvcException
     from dvc.progress import progress
+    from dvc.system import System
 
     copied = 0
     name = name if name else os.path.basename(dest)
@@ -122,15 +125,18 @@ def copyfile(src, dest, no_progress_bar=False, name=None):
     if os.path.isdir(dest):
         dest = os.path.join(dest, os.path.basename(src))
 
-    with open(src, "rb") as fsrc, open(dest, "wb+") as fdest:
-        while True:
-            buf = fsrc.read(LOCAL_CHUNK_SIZE)
-            if not buf:
-                break
-            fdest.write(buf)
-            copied += len(buf)
-            if not no_progress_bar:
-                progress.update_target(name, copied, total)
+    try:
+        System.reflink(src, dest)
+    except DvcException:
+        with open(src, "rb") as fsrc, open(dest, "wb+") as fdest:
+            while True:
+                buf = fsrc.read(LOCAL_CHUNK_SIZE)
+                if not buf:
+                    break
+                fdest.write(buf)
+                copied += len(buf)
+                if not no_progress_bar:
+                    progress.update_target(name, copied, total)
 
     if not no_progress_bar:
         progress.finish_target(name)
@@ -151,12 +157,13 @@ def move(src, dst):
 
 
 def _chmod(func, p, excinfo):
+    perm = os.lstat(p).st_mode
+    perm |= stat.S_IWRITE
+
     try:
-        perm = os.stat(p).st_mode
-        perm |= stat.S_IWRITE
         os.chmod(p, perm)
     except OSError as exc:
-        # NOTE: broken symlink case.
+        # broken symlink
         if exc.errno != errno.ENOENT:
             raise
 
@@ -164,7 +171,7 @@ def _chmod(func, p, excinfo):
 
 
 def remove(path):
-    logger.debug("Removing '{}'".format(os.path.relpath(path)))
+    logger.debug("Removing '{}'".format(relpath(path)))
 
     try:
         if os.path.isdir(path):
@@ -227,9 +234,9 @@ def convert_to_unicode(data):
 
 def tmp_fname(fname):
     """ Temporary name for a partial download """
-    from uuid import uuid4
+    from shortuuid import uuid
 
-    return fname + "." + str(uuid4()) + ".tmp"
+    return fspath(fname) + "." + str(uuid()) + ".tmp"
 
 
 def current_timestamp():
@@ -279,7 +286,7 @@ def dvc_walk(
 
 def walk_files(directory, ignore_file_handler=None):
     for root, _, files in dvc_walk(
-        str(directory), ignore_file_handler=ignore_file_handler
+        directory, ignore_file_handler=ignore_file_handler
     ):
         for f in files:
             yield os.path.join(root, f)
@@ -363,3 +370,15 @@ def _visual_center(line, width):
     right_padding = spaces - left_padding
 
     return (left_padding * " ") + line + (right_padding * " ")
+
+
+def relpath(path, start=os.curdir):
+    path = fspath(path)
+    start = os.path.abspath(fspath(start))
+
+    # Windows path on different drive than curdir doesn't have relpath
+    if os.name == "nt" and not os.path.commonprefix(
+        [start, os.path.abspath(path)]
+    ):
+        return path
+    return os.path.relpath(path, start)