dvc: bump to 0.65.0

We are incorrectly using `simple_cycle` here, `find_cycle` is much more
suitable for this application and much faster too. For example:

```
$ cat test_networkx.py
import networkx as nx
from dvc.repo.graph import check_acyclic

G = nx.scale_free_graph(1200)

list(nx.find_cycle(G, orientation='ignore'))

list(nx.simple_cycles(G))

$ python -mcProfile -scumtime test_networkx.py &> log.log
```

shows ~100s for `simple_cycles` and ~0s for `find_cycle`.

Fixes #2666

.appveyor.yml

@@ -1,84 +0,0 @@
-branches:
-  only:
-    - master
-    - devel
-
-cache:
-  - C:\ProgramData\chocolatey\bin -> appveyor.yml
-  - C:\ProgramData\chocolatey\lib -> appveyor.yml
-
-platform: x86
-
-environment:
-  aws_access_key_id:
-    secure: veTfhOrd4wUpG7NlRsvS3tufuaFfST8i7SZa2HVH6Vc=
-  aws_secret_access_key:
-    secure: nCQcN/Mzce0erwcNOU3yAuWgF6ZuTvPBEqTF3XFp1poGcDbSIg+s67eVxsxWCXQj
-  GCP_CREDS:
-    secure: 96fJ3r2i2GohbXHwnSs5N4EplQ7q8YmLpPWM0AC+f4s=
-  CODECOV_TOKEN:
-    secure: XN4jRtmGE5Bqg8pPZkwNs7kn3UEI73Rkldqc0MGsQISZBm5TNJZOPofDMc1QnUsf
-  AZURE_STORAGE_CONTAINER_NAME: appveyor-tests
-  AZURE_STORAGE_CONNECTION_STRING: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;
-  OSS_ENDPOINT: localhost:50004
-  OSS_ACCESS_KEY_ID: AccessKeyID
-  OSS_ACCESS_KEY_SECRET: AccessKeySecret
-
-  matrix:
-
-    - PYTHON: "C:\\Python37"
-      PYTHON_VERSION: "3.7.x"
-      PYTHON_ARCH: "32"
-      RUBY: C:\Ruby25\bin
-
-install:
-  - SET PATH=%RUBY%;%PATH%
-  - git clone -q https://github.com/nanaya-tachibana/oss-emulator.git
-  - gem install thor builder
-  - ps: cd oss-emulator
-  - ps: $OSSProcess = Start-Process -FilePath ruby -ArgumentList "bin\emulator -r store -p 50004" -PassThru -NoNewWindow
-  - ps: cd ..
-  - npm install -g azurite
-  - ps: $AzuriteProcess = Start-Process azurite-blob -PassThru
-  - cinst wget
-  - cinst awscli
-  - cinst gcloudsdk
-  - refreshenv
-  - cinst gsutil
-  - cinst openssl.light --version 1.1.1
-  - python -m pip install -U pip setuptools wheel
-  - pip install -e .[all]
-  - python setup.py install
-
-build: false
-
-before_test:
-  - aws configure set aws_access_key_id "%aws_access_key_id%"
-  - aws configure set aws_secret_access_key "%aws_secret_access_key%"
-  - aws configure set region us-east-2
-  - openssl enc -d -aes-256-cbc -md md5 -k "%GCP_CREDS%" -in scripts\ci\gcp-creds.json.enc -out scripts\ci\gcp-creds.json & exit 0
-  - pip install -e .[tests]
-
-test_script:
-  - python -mtests
-
-after_test:
-  - if "%APPVEYOR_PULL_REQUEST_NUMBER%"=="" pip install codecov
-  - if "%APPVEYOR_PULL_REQUEST_NUMBER%"=="" codecov
-  - if "%APPVEYOR_REPO_TAG%"=="true" python -m dvc pull
-  - if "%APPVEYOR_REPO_TAG%"=="true" .\scripts\build_windows.cmd
-
-on_finish:
-  - ps: Stop-Process $AzuriteProcess
-  - ps: Stop-Process $OSSProcess
-
-artifacts:
-  - path: dvc*.exe
-    name: DVC Installer
-
-deploy:
-  provider: GitHub
-  auth_token:
-    secure: 25c1pNoXWbtTwUkKLcThwcl2A2xG8oXHsPuD73HVbQ7SRy9X6HzkCc2sXzhpfRtF
-  on:
-    APPVEYOR_REPO_TAG: true

.deepsource.toml

@@ -0,0 +1,15 @@
+version = 1
+
+test_patterns = [
+  'tests/**'
+]
+
+exclude_patterns = [
+  
+]
+
+[[analyzers]]
+name = 'python'
+enabled = true
+runtime_version = '3.x.x'
+

.dvc/.gitignore

@@ -6,3 +6,6 @@
 /state-journal
 /state-wal
 /cache
+/pkg
+/repos
+/tmp

.dvc/config

@@ -1,4 +1,4 @@
 ['remote "dvc"']
-url = https://dvc.org/s3/dvc
+url = https://remote.dvc.org/dvc
 [core]
 remote = dvc

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,5 +1,5 @@
 * [ ] Have you followed the guidelines in our
-      [Contributing document](https://dvc.org/doc/user-guide/contributing)?
+      [Contributing document](https://dvc.org/doc/user-guide/contributing/core)?
 
 * [ ] Does your PR affect documented changes or does it add new functionality
       that should be documented? If yes, have you created a PR for

.gitignore

@@ -23,6 +23,7 @@ innosetup/config.ini
 *.exe
 
 .coverage
+.coverage.*
 
 *.swp
 
@@ -34,3 +35,5 @@ env.sh
 tests/remotes_env
 scripts/ci/gcp-creds.json
 .gcp-creds.json
+
+*~

.mailmap

@@ -0,0 +1,6 @@
+Paweł Redzyński <[email protected]>
+Dmitry Petrov <[email protected]>
+Earl Hathaway <[email protected]>
+Nabanita Dash <[email protected]>
+Kurian Benoy <[email protected]>
+Sritanu Chakraborty <[email protected]>

.pylintrc

@@ -139,7 +139,9 @@ disable=print-statement,
         deprecated-sys-function,
         exception-escape,
         comprehension-escape,
-        useless-object-inheritance
+        useless-object-inheritance,
+        missing-docstring,
+        bad-continuation
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option
@@ -314,7 +316,7 @@ indent-after-paren=4
 indent-string='    '
 
 # Maximum number of characters on a single line.
-max-line-length=100
+max-line-length=79
 
 # Maximum number of lines in a module.
 max-module-lines=1000

.travis.yml

@@ -1,92 +1,134 @@
-language: python
 cache:
-  - pip
-  - directories:
-    - /opt/python
+  directories:
+    - $HOME/.cache/pip
     - $HOME/.pyenv
 sudo: true
 dist: xenial
-python: "3.7"
 env:
   global:
+    - HOMEBREW_NO_AUTO_UPDATE=1
     - PYENV_ROOT="$HOME/.pyenv"
     - PATH="$PYENV_ROOT/bin:$PATH"
+    - PIP_CACHE_DIR="$HOME/.cache/pip"  # unify pip cache location for all platforms
+stages:
+  - check
+  - test
+  - build
+  - name: deploy
+    if: tag is present
 # Travis doesn't support testing python on Mac yet, so we need
 # to workaround it by installing it directly with brew.
 #
-# Note that we specify language as ruby-2.4.0 to help us satisfy
+# Note that we specify language as ruby-2.4.3 to help us satisfy
 # fpm dependencies, as travis runs el capitan with an old ruby
 # version, that causes fpm to crash.
-matrix:
+jobs:
   include:
-  - name: "2.7"
+  # patch formatting checks
+  - name: "Check patch formatting"
+    stage: "check"
+    os: linux
+    language: python
+    python: 3.7
+    before_install:
+    install:
+    script: ./scripts/ci/check_patch.sh
+  # test jobs
+  - name: "2.7 on Windows"
+    stage: test
+    os: windows           # Windows 10.0.17134 N/A Build 17134
+    language: shell       # 'language: python' is an error on Travis CI Windows
     env:
-    - PYTHON_VER=2.7.13
-  - name: "3.5"
+    - PY2=1
+  - name: "3.7 on Windows"
+    stage: test
+    os: windows           # Windows 10.0.17134 N/A Build 17134
+    language: shell       # 'language: python' is an error on Travis CI Windows
+  - name: "2.7 on Linux"
+    stage: test
+    language: python
+    python: 2.7
     env:
-    - PYTHON_VER=3.5.3
-  - name: "3.6"
-    env:
-    - PYTHON_VER=3.6.2
-  - name: "3.7"
-    env:
-    # NOTE: using default python
-    - PYTHON_VER=
-  - if: type = cron OR tag is present
+    - PY2=1
+  - name: "3.5 on Linux"
+    stage: test
+    language: python
+    python: 3.5
+  - name: "3.6 on Linux"
+    stage: test
+    language: python
+    python: 3.6
+  - name: "3.7 on Linux"
+    stage: test
+    language: python
+    python: 3.7
+  - name: "2.7 on Mac"
+    stage: test
     os: osx
-    osx_image: xcode7.3
-    language: ruby
-    rvm: 2.4.0
+    osx_image: xcode9.4
     env:
-    - PYTHON_VER=2.7.10
-    - CPPFLAGS="-I/usr/local/opt/openssl/include $CPPFLAGS"
-    - LDFLAGS="-L/usr/local/opt/openssl/lib $LDFLAGS"
-    - PYTHON_CONFIGURE_OPTS="--enable-framework"
-  - if: type = cron OR tag is present
+    - PY2=1
+  - name: "3.6 on Mac"
+    stage: test
     os: osx
-    osx_image: xcode7.3
-    language: ruby
-    rvm: 2.4.0
-    env:
-    - PYTHON_VER=3.7.3
-    - PYTHON_CONFIGURE_OPTS="--enable-framework"
-  - if: type = cron OR tag is present
+    osx_image: xcode9.4
+  # build jobs
+  - name: "Mac pkg"
+    stage: build
     os: osx
-    osx_image: xcode7.3
+    osx_image: xcode9.4
     language: ruby
-    rvm: 2.4.0
-    env:
-    - PYTHON_VER=3.5.0
-    - PYTHON_CONFIGURE_OPTS="--enable-framework"
-  - if: type = cron OR tag is present
-    os: osx
-    osx_image: xcode10.1
-    language: ruby
-    rvm: 2.4.0
-    env:
-    - PYTHON_VER=2.7.10
-    - CPPFLAGS="-I/usr/local/opt/openssl/include $CPPFLAGS"
-    - LDFLAGS="-L/usr/local/opt/openssl/lib $LDFLAGS"
-    - PYTHON_CONFIGURE_OPTS="--enable-framework"
-  - if: type = cron OR tag is present
-    os: osx
-    osx_image: xcode10.1
-    language: ruby
-    rvm: 2.4.0
-    env:
-    - PYTHON_VER=3.5.0
-    - PYTHON_CONFIGURE_OPTS="--enable-framework"
+    rvm: 2.4.3
+    install:
+    script: ./scripts/build_posix.sh
+  - name: "Windows pkg"
+    stage: build
+    os: windows
+    language: shell
+    install:
+    script: ./scripts/build_windows.cmd
+  - name: "Linux pkgs"
+    stage: build
+    language: python
+    python: 3.7
+    before_install:
+    install:
+    script: ./scripts/build_posix.sh
+  - name: "Pypi pkgs"
+    stage: build
+    language: python
+    python: 3.7
+    script: ./scripts/build_package.sh
+  # deploy jobs
+  - name: "Pypi pkgs"
+    stage: deploy
+    language: python
+    python: 3.7
+    script: ./scripts/build_package.sh
+    deploy:
+    - provider: pypi
+      user:
+        secure: "IOzNfgPoG3ICg3+s4AuZCCnawgd/KqMzeiAB9R0B9Xb6ProPQJp9O3GBZrtg5OJMHdWQQtwULeOh/LWk4sFdXMcX1iNEgm5Tb7DwO7TDtltW14jUgN9AqfY2twj4UIsW/ON8w0P8v36ZoQq1kO6Pfgjijdm3w+9h6rpps8rJ6jXYpVZVO+SSzzijWddanMkEz4GgctZ2r7U5MWlmm1UogF60kEZIZCQykz0yJnhTpP9nMFDKw6Maki/mcfVwTQ/v3RqjzttHei/aaEMGQXYr3TOX8NPYJXerBpjShDZqcgcCfqPzhi7uCwVYaPvRsMMYH5jgtMz/SO3J6ToeGKT0f1F1ihRyCdusg325TMK7Vv8yHJzL4ngeUbb/oqxWtxhLY7f77Lkx6hBwe6ROpTqiQHIfZad5qyIHihKcDcIHo4FwLHVzxR4y+Ds2js5dXwBkUBaEAx0Dmc6ZdgXqxd815GDwv+VVpzXJhtO6qXmKE36F/Oe6eCNG3dk5MdOK4C2DrM9M8MXllqeQtX6JrM2stWlBalAc9Jn4b0xxAoghX3x3G3hNsLc9Uist1VinY2+IcPozusqj66c+Vbi5YngWSbICOjlIcq2GaOlBi3xioGts396442ztf1eLjMVuvykCI3b4Yz/vJYZOZjUvG/6nLfaFQTJLSXkTvI4do3foeY4="
+      password:
+        secure: "ts/3FSPL+w4euXryip9wCJWUqUwuJ2CqHi0nEpYVM10KL0cp9aMkQXN/7C8ih+j/JZ25kaO+tis1nx0JERVnKrg/1F/fJhxHTpakCP4OSIlhKRXDt7yUUHcKKt5QdM2XCKhApptP6RTpWL1vlc36CMrhdvTcMYp7hlDOiWjdhhim24FGQzabL42Ko59ttyyCROqgcnjHqj+WlhJl7iYO+lv2PLOtslpkIYY5EMjgBLuGyOvxTqZsAXtworsASdB2CrsTE6iMRcrM1afgkyBlUqDezoiv9TwfZrZ85oRoAxPB7su4QWK7lEP0wGQV/SDz/RL7c+cNfRaauxUKVcC8I2LfaEpudv8vi205+v/08wzPbS4xSOnf89irUJnyGxUy+bYLoXPPVPMyzcQRAUnrQl+ENCmN62HU+/2pkIyiUJvt0wYYP0vp+3pIbXbYQGmAIMq7nZJd7K3rikfnQIGKI03xZsFe3PEdjzhDI9Ej/tRUbJx4pgg084QuYYca6MSYn/zJ5pqIG4A7ngCSxoxoWZT0duo8PrUWQeT17L73Q86HeEv3OzZhWfEF2fNDN3Qbsu4CllSZnbE7DXgAuysTwTYHS/na6lp7wejOqdRUXrHbidVla9Mwxyc9JdWNWzUDxALvNxAdNbV6l85mO8izy8wjQF6M1O7idqTfjQeR4pc="
+      skip_existing: true
+      file_glob: true
+      file:
+        - dist/dvc*.whl
+        - dist/dvc*.tar.gz
+      skip_cleanup: true
+      on:
+        tags: true
+        repo: iterative/dvc
+
 before_install:
   - bash ./scripts/ci/before_install.sh
   - source env.sh
-  - python --version
+  - bash ./scripts/ci/check_python.sh
 install:
   - bash ./scripts/ci/install.sh
 script:
   - bash ./scripts/ci/script.sh
-before_deploy:
-  - bash ./scripts/ci/before_deploy.sh
-  - pip install twine
 deploy:
   - provider: releases
     condition: "[[ $(./scripts/ci/deploy_condition.sh) == 'true' ]]"
@@ -97,21 +139,9 @@ deploy:
       - dvc*.rpm
       - dvc*.deb
       - dvc*.pkg
+      - dvc*.exe
     skip_cleanup: true
     on:
       tags: true
       repo: iterative/dvc
-  - provider: pypi
-    user:
-      secure: "IOzNfgPoG3ICg3+s4AuZCCnawgd/KqMzeiAB9R0B9Xb6ProPQJp9O3GBZrtg5OJMHdWQQtwULeOh/LWk4sFdXMcX1iNEgm5Tb7DwO7TDtltW14jUgN9AqfY2twj4UIsW/ON8w0P8v36ZoQq1kO6Pfgjijdm3w+9h6rpps8rJ6jXYpVZVO+SSzzijWddanMkEz4GgctZ2r7U5MWlmm1UogF60kEZIZCQykz0yJnhTpP9nMFDKw6Maki/mcfVwTQ/v3RqjzttHei/aaEMGQXYr3TOX8NPYJXerBpjShDZqcgcCfqPzhi7uCwVYaPvRsMMYH5jgtMz/SO3J6ToeGKT0f1F1ihRyCdusg325TMK7Vv8yHJzL4ngeUbb/oqxWtxhLY7f77Lkx6hBwe6ROpTqiQHIfZad5qyIHihKcDcIHo4FwLHVzxR4y+Ds2js5dXwBkUBaEAx0Dmc6ZdgXqxd815GDwv+VVpzXJhtO6qXmKE36F/Oe6eCNG3dk5MdOK4C2DrM9M8MXllqeQtX6JrM2stWlBalAc9Jn4b0xxAoghX3x3G3hNsLc9Uist1VinY2+IcPozusqj66c+Vbi5YngWSbICOjlIcq2GaOlBi3xioGts396442ztf1eLjMVuvykCI3b4Yz/vJYZOZjUvG/6nLfaFQTJLSXkTvI4do3foeY4="
-    password:
-      secure: "ts/3FSPL+w4euXryip9wCJWUqUwuJ2CqHi0nEpYVM10KL0cp9aMkQXN/7C8ih+j/JZ25kaO+tis1nx0JERVnKrg/1F/fJhxHTpakCP4OSIlhKRXDt7yUUHcKKt5QdM2XCKhApptP6RTpWL1vlc36CMrhdvTcMYp7hlDOiWjdhhim24FGQzabL42Ko59ttyyCROqgcnjHqj+WlhJl7iYO+lv2PLOtslpkIYY5EMjgBLuGyOvxTqZsAXtworsASdB2CrsTE6iMRcrM1afgkyBlUqDezoiv9TwfZrZ85oRoAxPB7su4QWK7lEP0wGQV/SDz/RL7c+cNfRaauxUKVcC8I2LfaEpudv8vi205+v/08wzPbS4xSOnf89irUJnyGxUy+bYLoXPPVPMyzcQRAUnrQl+ENCmN62HU+/2pkIyiUJvt0wYYP0vp+3pIbXbYQGmAIMq7nZJd7K3rikfnQIGKI03xZsFe3PEdjzhDI9Ej/tRUbJx4pgg084QuYYca6MSYn/zJ5pqIG4A7ngCSxoxoWZT0duo8PrUWQeT17L73Q86HeEv3OzZhWfEF2fNDN3Qbsu4CllSZnbE7DXgAuysTwTYHS/na6lp7wejOqdRUXrHbidVla9Mwxyc9JdWNWzUDxALvNxAdNbV6l85mO8izy8wjQF6M1O7idqTfjQeR4pc="
-    skip_existing: true
-    file_glob: true
-    file:
-      - dist/dvc*.whl
-      - dist/dvc*.tar.gz
-    skip_cleanup: true
-    on:
-      tags: true
-      repo: iterative/dvc
+      stage: build

CONTRIBUTING.md

@@ -1 +1 @@
-### See our contribution guide at [dvc.org](https://dvc.org/doc/user-guide/contributing#contributing).
+### See our contribution guide at [dvc.org](https://dvc.org/doc/user-guide/contributing/core).

MANIFEST.in

@@ -0,0 +1,2 @@
+include fastentrypoints.py
+include LICENSE

README.rst

@@ -4,7 +4,8 @@
 
 `Website <https://dvc.org>`_
 • `Docs <https://dvc.org/doc>`_
-• `Twitter <https://twitter.com/iterativeai>`_
+• `Blog <http://blog.dataversioncontrol.com>`_
+• `Twitter <https://twitter.com/DVCorg>`_
 • `Chat (Community & Support) <https://dvc.org/chat>`_
 • `Tutorial <https://dvc.org/doc/get-started>`_
 • `Mailing List <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_
@@ -13,10 +14,6 @@
   :target: https://travis-ci.com/iterative/dvc
   :alt: Travis
 
-.. image:: https://ci.appveyor.com/api/projects/status/github/iterative/dvc?branch=master&svg=true
-  :target: https://ci.appveyor.com/project/iterative/dvc/branch/master
-  :alt: Windows Build
-
 .. image:: https://codeclimate.com/github/iterative/dvc/badges/gpa.svg
   :target: https://codeclimate.com/github/iterative/dvc
   :alt: Code Climate
@@ -29,10 +26,14 @@
   :target: https://www.patreon.com/DVCorg/overview
   :alt: Donate
 
+.. image:: https://anaconda.org/conda-forge/dvc/badges/version.svg
+  :target: https://anaconda.org/conda-forge/dvc
+  :alt: Conda-forge
+
 |
 
-**Data Science Version Control** or **DVC** is an **open-source** tool for data science and
-machine learning projects. Key features:
+**Data Version Control** or **DVC** is an **open-source** tool for data science and machine
+learning projects. Key features:
 
 #. simple **command line** Git-like experience. Does not require installing and maintaining
    any databases. Does not depend on any proprietary online services;
@@ -45,7 +46,7 @@ machine learning projects. Key features:
 
 #. it helps manage experiments with Git tags or branches and **metrics** tracking;
 
-It aims to replace tools like Excel and Docs that are being commonly used as a knowledge repo and
+**DVC** aims to replace tools like Excel and Google Docs that are being commonly used as a knowledge repo and
 a ledger for the team, ad-hoc scripts to track and move deploy different model versions, ad-hoc
 data file suffixes and prefixes.
 
@@ -107,8 +108,8 @@ Please read `Get Started <https://dvc.org/doc/get-started>`_ for the full versio
 Installation
 ============
 
-Read this `instruction <https://dvc.org/doc/get-started/install>`_ to get more details. There are three
-options to install DVC: ``pip``, Homebrew, or an OS-specific package:
+Read this `instruction <https://dvc.org/doc/get-started/install>`_ to get more details. There are four
+options to install DVC: ``pip``, Homebrew, Conda (Anaconda) or an OS-specific package:
 
 pip (PyPI)
 ----------
@@ -118,7 +119,7 @@ pip (PyPI)
    pip install dvc
 
 Depending on the remote storage type you plan to use to keep and share your data, you might need to specify
-one of the optional dependencies: ``s3``, ``gs``, ``azure``, ``oss``, ``ssh``. Or ``all_remotes`` to include them all.
+one of the optional dependencies: ``s3``, ``gs``, ``azure``, ``oss``, ``ssh``. Or ``all`` to include them all.
 The command should look like this: ``pip install dvc[s3]`` - it installs the ``boto3`` library along with
 DVC to support the AWS S3 storage.
 
@@ -140,6 +141,15 @@ or:
 .. code-block:: bash
 
    brew cask install iterative/homebrew-dvc/dvc
+   
+Conda (Anaconda)
+----------------
+
+.. code-block:: bash
+
+   conda install -c conda-forge dvc
+
+Currently, it supports only python version 2.7, 3.6 and 3.7.
 
 Package
 -------
@@ -192,7 +202,7 @@ Related technologies
 
 Contributing
 ============
-Contributions are welcome! Please see our `Contributing Guide <https://dvc.org/doc/user-guide/contributing/>`_ for more
+Contributions are welcome! Please see our `Contributing Guide <https://dvc.org/doc/user-guide/contributing/core>`_ for more
 details.
 
 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/0

dvc/__init__.py

@@ -6,10 +6,12 @@ Make your data science projects reproducible and shareable.
 
 from __future__ import unicode_literals
 
-from dvc.version import __version__  # noqa: F401
 import warnings
+
 import dvc.logger
 
+from dvc.version import __version__  # noqa: F401
+
 
 dvc.logger.setup()
 
@@ -19,18 +21,3 @@ dvc.logger.setup()
 # we can simply ignore them so that they don't show up when you are using dvc.
 warnings.filterwarnings("ignore", message="numpy.dtype size changed")
 warnings.filterwarnings("ignore", message="numpy.ufunc size changed")
-
-# Ignore paramiko's warning: https://github.com/paramiko/paramiko/issues/1386.
-# This only affects paramiko 2.4.2 and should be fixed in the next version.
-# Cryptography developers decided that it is a brilliant idea not to inherit
-# from DeprecationWarning [1] because it is invisible by default, and decided
-# to spam everyone instead. So it is their fault and not paramiko's.
-#
-# [1] https://github.com/pyca/cryptography/blob/2.6.1/src/cryptography/
-#     utils.py#L14
-try:
-    from cryptography.utils import CryptographyDeprecationWarning
-
-    warnings.simplefilter("ignore", CryptographyDeprecationWarning)
-except ImportError:
-    pass

dvc/analytics.py

@@ -10,6 +10,7 @@ import errno
 import logging
 
 from dvc import __version__
+from dvc.utils import env2bool
 
 
 logger = logging.getLogger(__name__)
@@ -66,7 +67,7 @@ class Analytics(object):
                 raise
 
         self.user_id_file = os.path.join(cdir, self.USER_ID_FILE)
-        self.user_id_file_lock = Lock(cdir, self.USER_ID_FILE + ".lock")
+        self.user_id_file_lock = Lock(self.user_id_file + ".lock")
 
     @staticmethod
     def load(path):
@@ -113,7 +114,7 @@ class Analytics(object):
                 return user_id
         except LockError:
             msg = "Failed to acquire '{}'"
-            logger.debug(msg.format(self.user_id_file_lock.lock_file))
+            logger.debug(msg.format(self.user_id_file_lock.lockfile))
 
     def _collect_windows(self):
         import sys
@@ -213,29 +214,35 @@ class Analytics(object):
         return core.get(Config.SECTION_CORE_ANALYTICS, True)
 
     @staticmethod
-    def _is_enabled(cmd=None):
+    def _get_current_config():
         from dvc.config import Config
         from dvc.repo import Repo
         from dvc.exceptions import NotDvcRepoError
+
+        try:
+            dvc_dir = Repo.find_dvc_dir()
+            config = Config(dvc_dir)
+        except NotDvcRepoError:
+            config = Config(validate=False)
+        return config
+
+    @staticmethod
+    def is_enabled(cmd=None):
         from dvc.command.daemon import CmdDaemonBase
 
-        if os.getenv("DVC_TEST"):
+        if env2bool("DVC_TEST"):
             return False
 
         if isinstance(cmd, CmdDaemonBase):
             return False
 
-        if cmd is None or not hasattr(cmd, "config"):
-            try:
-                dvc_dir = Repo.find_dvc_dir()
-                config = Config(dvc_dir)
-                assert config is not None
-            except NotDvcRepoError:
-                config = Config(validate=False)
-                assert config is not None
-        else:
-            config = cmd.config
-            assert config is not None
+        config = (
+            Analytics._get_current_config()
+            if cmd is None or not hasattr(cmd, "config")
+            else cmd.config
+        )
+
+        assert config is not None
 
         enabled = Analytics._is_enabled_config(config)
         logger.debug(
@@ -253,7 +260,7 @@ class Analytics(object):
         """
         from dvc.daemon import daemon
 
-        if not Analytics._is_enabled(cmd):
+        if not Analytics.is_enabled(cmd):
             return
 
         analytics = Analytics()
@@ -264,7 +271,7 @@ class Analytics(object):
         """Collect and send analytics."""
         import requests
 
-        if not self._is_enabled():
+        if not self.is_enabled():
             return
 
         self.collect()

dvc/api.py

@@ -1,15 +1,71 @@
+from contextlib import contextmanager
+import os
+
+try:
+    from contextlib import _GeneratorContextManager as GCM
+except ImportError:
+    from contextlib import GeneratorContextManager as GCM
+
+from dvc.utils.compat import urlparse
 from dvc.repo import Repo
+from dvc.external_repo import external_repo
 
 
-def get_url(path, repo_dir=None, remote=None):
-    """Returns an url of `path` in default or specified remote"""
-    repo = Repo(repo_dir)
-    out, = repo.find_outs_by_path(path)
-    remote_obj = repo.cloud.get_remote(remote)
-    return str(remote_obj.checksum_to_path_info(out.checksum))
+def get_url(path, repo=None, rev=None, remote=None):
+    """Returns an url of a resource specified by path in repo"""
+    with _make_repo(repo, rev=rev) as _repo:
+        abspath = os.path.join(_repo.root_dir, path)
+        out, = _repo.find_outs_by_path(abspath)
+        remote_obj = _repo.cloud.get_remote(remote)
+        return str(remote_obj.checksum_to_path_info(out.checksum))
 
 
-def open(path, repo_dir=None, remote=None, mode="r", encoding=None):
+def open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
     """Opens a specified resource as a file descriptor"""
-    repo = Repo(repo_dir)
-    return repo.open(path, remote=remote, mode=mode, encoding=encoding)
+    args = (path,)
+    kwargs = {
+        "repo": repo,
+        "remote": remote,
+        "rev": rev,
+        "mode": mode,
+        "encoding": encoding,
+    }
+    return _OpenContextManager(_open, args, kwargs)
+
+
+class _OpenContextManager(GCM):
+    def __init__(self, func, args, kwds):
+        self.gen = func(*args, **kwds)
+        self.func, self.args, self.kwds = func, args, kwds
+
+    def __getattr__(self, name):
+        raise AttributeError(
+            "dvc.api.open() should be used in a with statement"
+        )
+
+
+def _open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
+    with _make_repo(repo, rev=rev) as _repo:
+        abspath = os.path.join(_repo.root_dir, path)
+        with _repo.open(
+            abspath, remote=remote, mode=mode, encoding=encoding
+        ) as fd:
+            yield fd
+
+
+def read(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
+    """Read a specified resource into string"""
+    with open(
+        path, repo=repo, rev=rev, remote=remote, mode=mode, encoding=encoding
+    ) as fd:
+        return fd.read()
+
+
+@contextmanager
+def _make_repo(repo_url, rev=None):
+    if not repo_url or urlparse(repo_url).scheme == "":
+        assert rev is None, "Custom revision is not supported for local repo"
+        yield Repo(repo_url)
+    else:
+        with external_repo(url=repo_url, rev=rev) as repo:
+            yield repo

dvc/cache.py

@@ -3,10 +3,63 @@
 from __future__ import unicode_literals
 
 import os
+from collections import defaultdict
+from funcy import cached_property
 
+from dvc.utils.compat import builtin_str
 from dvc.config import Config
 
 
+class CacheConfig(object):
+    def __init__(self, config):
+        self.config = config
+
+    def set_dir(self, dname, level=None):
+        from dvc.remote.config import RemoteConfig
+
+        configobj = self.config.get_configobj(level)
+        path = RemoteConfig.resolve_path(dname, configobj.filename)
+        self.config.set(
+            Config.SECTION_CACHE, Config.SECTION_CACHE_DIR, path, level=level
+        )
+
+
+def _make_remote_property(name):
+    """
+    The config file is stored in a way that allows you to have a
+    cache for each remote.
+
+    This is needed when specifying external outputs
+    (as they require you to have an external cache location).
+
+    Imagine a config file like the following:
+
+            ['remote "dvc-storage"']
+            url = ssh://localhost/tmp
+            ask_password = true
+
+            [cache]
+            ssh = dvc-storage
+
+    This method creates a cached property, containing cache named `name`:
+
+        self.config == {'ssh': 'dvc-storage'}
+        self.ssh  # a RemoteSSH instance
+    """
+
+    def getter(self):
+        from dvc.remote import Remote
+
+        remote = self.config.get(name)
+        if not remote:
+            return None
+
+        return Remote(self.repo, name=remote)
+
+    getter.__name__ = builtin_str(name)
+    return cached_property(getter)
+
+
 class Cache(object):
     """Class that manages cache locations of a dvc repo.
 
@@ -21,7 +74,7 @@ class Cache(object):
 
         self.repo = repo
 
-        config = repo.config.config[Config.SECTION_CACHE]
+        self.config = config = repo.config.config[Config.SECTION_CACHE]
         local = config.get(Config.SECTION_CACHE_LOCAL)
 
         if local:
@@ -32,6 +85,7 @@ class Cache(object):
             cache_dir = config.get(Config.SECTION_CACHE_DIR, default_cache_dir)
             cache_type = config.get(Config.SECTION_CACHE_TYPE)
             protected = config.get(Config.SECTION_CACHE_PROTECTED)
+            shared = config.get(Config.SECTION_CACHE_SHARED)
 
             settings = {
                 Config.PRIVATE_CWD: config.get(
@@ -40,52 +94,46 @@ class Cache(object):
                 Config.SECTION_REMOTE_URL: cache_dir,
                 Config.SECTION_CACHE_TYPE: cache_type,
                 Config.SECTION_CACHE_PROTECTED: protected,
+                Config.SECTION_CACHE_SHARED: shared,
             }
 
-        self.local = Remote(repo, settings)
-        self.s3 = self._get_remote(config, Config.SECTION_CACHE_S3)
-        self.gs = self._get_remote(config, Config.SECTION_CACHE_GS)
-        self.ssh = self._get_remote(config, Config.SECTION_CACHE_SSH)
-        self.hdfs = self._get_remote(config, Config.SECTION_CACHE_HDFS)
-        self.azure = self._get_remote(config, Config.SECTION_CACHE_AZURE)
+        self.local = Remote(repo, **settings)
 
-    def _get_remote(self, config, name):
-        """
-        The config file is stored in a way that allows you to have a
-        cache for each remote.
+    s3 = _make_remote_property(Config.SECTION_CACHE_S3)
+    gs = _make_remote_property(Config.SECTION_CACHE_GS)
+    ssh = _make_remote_property(Config.SECTION_CACHE_SSH)
+    hdfs = _make_remote_property(Config.SECTION_CACHE_HDFS)
+    azure = _make_remote_property(Config.SECTION_CACHE_AZURE)
 
-        This is needed when specifying external outputs
-        (as they require you to have an external cache location).
 
-        Imagine a config file like the following:
+class NamedCache(object):
+    def __init__(self):
+        self._items = defaultdict(lambda: defaultdict(set))
+        self.external = defaultdict(set)
 
-                ['remote "dvc-storage"']
-                url = ssh://localhost/tmp
-                ask_password = true
+    @classmethod
+    def make(cls, scheme, checksum, name):
+        cache = cls()
+        cache.add(scheme, checksum, name)
+        return cache
 
-                [cache]
-                ssh = dvc-storage
+    def __getitem__(self, key):
+        return self._items[key]
 
-        This method resolves the name under the cache section into the
-        correct Remote instance.
+    def add(self, scheme, checksum, name):
+        self._items[scheme][checksum].add(name)
 
-        Args:
-            config (dict): The cache section on the config file
-            name (str): Name of the section we are interested in to retrieve
+    def add_external(self, url, rev, path):
+        self.external[url, rev].add(path)
 
-        Returns:
-            remote (dvc.Remote): Remote instance that the section is referring.
-                None when there's no remote with that name.
+    def update(self, cache, suffix=""):
+        for scheme, src in cache._items.items():
+            dst = self._items[scheme]
+            for checksum, names in src.items():
+                if suffix:
+                    dst[checksum].update(n + suffix for n in names)
+                else:
+                    dst[checksum].update(names)
 
-        Example:
-            >>> _get_remote(config={'ssh': 'dvc-storage'}, name='ssh')
-        """
-        from dvc.remote import Remote
-
-        remote = config.get(name)
-
-        if not remote:
-            return None
-
-        settings = self.repo.config.get_remote_settings(remote)
-        return Remote(self.repo, settings)
+        for repo_pair, files in cache.external.items():
+            self.external[repo_pair].update(files)

dvc/cli.py

@@ -9,7 +9,8 @@ import logging
 
 from dvc.command.base import fix_subparsers
 import dvc.command.init as init
-import dvc.command.pkg as pkg
+import dvc.command.get as get
+import dvc.command.get_url as get_url
 import dvc.command.destroy as destroy
 import dvc.command.remove as remove
 import dvc.command.move as move
@@ -20,6 +21,7 @@ import dvc.command.data_sync as data_sync
 import dvc.command.gc as gc
 import dvc.command.add as add
 import dvc.command.imp as imp
+import dvc.command.imp_url as imp_url
 import dvc.command.config as config
 import dvc.command.checkout as checkout
 import dvc.command.remote as remote
@@ -34,6 +36,7 @@ import dvc.command.commit as commit
 import dvc.command.tag as tag
 import dvc.command.diff as diff
 import dvc.command.version as version
+import dvc.command.update as update
 from dvc.exceptions import DvcParserError
 
 
@@ -41,7 +44,8 @@ logger = logging.getLogger(__name__)
 
 COMMANDS = [
     init,
-    pkg,
+    get,
+    get_url,
     destroy,
     add,
     remove,
@@ -52,6 +56,7 @@ COMMANDS = [
     data_sync,
     gc,
     imp,
+    imp_url,
     config,
     checkout,
     remote,
@@ -66,6 +71,7 @@ COMMANDS = [
     tag,
     diff,
     version,
+    update,
 ]
 
 
@@ -138,6 +144,19 @@ def parse_args(argv=None):
         description=desc,
         parents=[parent_parser],
         formatter_class=argparse.RawTextHelpFormatter,
+        add_help=False,
+    )
+
+    # NOTE: We are doing this to capitalize help message.
+    # Unfortunately, there is no easier and clearer way to do it,
+    # as adding this argument in get_parent_parser() either in
+    # log_level_group or on parent_parser itself will cause unexpected error.
+    parser.add_argument(
+        "-h",
+        "--help",
+        action="help",
+        default=argparse.SUPPRESS,
+        help="Show this help message and exit.",
     )
 
     # NOTE: On some python versions action='version' prints to stderr

dvc/command/add.py

@@ -54,9 +54,9 @@ def add_parser(subparsers, parent_parser):
         help="Don't put files/directories into cache.",
     )
     add_parser.add_argument(
-        "-f", "--file", help="Specify name of the DVC file it generates."
+        "-f", "--file", help="Specify name of the DVC-file it generates."
     )
     add_parser.add_argument(
-        "targets", nargs="+", help="Input files/directories."
+        "targets", nargs="+", help="Input files/directories to add."
     )
     add_parser.set_defaults(func=CmdAdd)

dvc/command/base.py

@@ -38,10 +38,13 @@ def append_doc_link(help_message, path):
 class CmdBase(object):
     def __init__(self, args):
         from dvc.repo import Repo
+        from dvc.updater import Updater
 
         self.repo = Repo()
         self.config = self.repo.config
         self.args = args
+        updater = Updater(self.repo.dvc_dir)
+        updater.check()
 
     @property
     def default_targets(self):
@@ -52,16 +55,6 @@ class CmdBase(object):
         logger.warning(msg)
         return [Stage.STAGE_FILE]
 
-    def run_cmd(self):
-        from dvc.lock import LockError
-
-        try:
-            with self.repo.lock:
-                return self.run()
-        except LockError:
-            logger.exception("failed to lock before running a command")
-            return 1
-
     # Abstract methods that have to be implemented by any inheritance class
     def run(self):
         pass
@@ -70,6 +63,3 @@ class CmdBase(object):
 class CmdBaseNoRepo(CmdBase):
     def __init__(self, args):
         self.args = args
-
-    def run_cmd(self):
-        return self.run()

dvc/command/cache.py

@@ -3,18 +3,18 @@ from __future__ import unicode_literals
 import argparse
 
 from dvc.command.base import fix_subparsers, append_doc_link
-from dvc.command.remote import CmdRemoteAdd
 from dvc.command.config import CmdConfig
+from dvc.cache import CacheConfig
 
 
 class CmdCacheDir(CmdConfig):
-    def run(self):
-        self.args.name = "cache.dir"
-        self.args.value = CmdRemoteAdd.resolve_path(
-            self.args.value, self.configobj.filename
-        )
+    def __init__(self, args):
+        super(CmdCacheDir, self).__init__(args)
+        self.cache_config = CacheConfig(self.config)
 
-        return super(CmdCacheDir, self).run()
+    def run(self):
+        self.cache_config.set_dir(self.args.value, level=self.args.level)
+        return 0
 
 
 def add_parser(subparsers, parent_parser):
@@ -43,8 +43,8 @@ def add_parser(subparsers, parent_parser):
 
     cache_dir_parser = cache_subparsers.add_parser(
         "dir",
-        parents=[parent_cache_config_parser],
-        description=append_doc_link(CACHE_HELP, "cache-dir"),
+        parents=[parent_parser, parent_cache_config_parser],
+        description=append_doc_link(CACHE_HELP, "cache/dir"),
         help=CACHE_DIR_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
@@ -57,7 +57,6 @@ def add_parser(subparsers, parent_parser):
     )
     cache_dir_parser.add_argument(
         "value",
-        default=None,
         help="Path to cache directory. Relative paths are resolved relative "
         "to the current directory and saved to config relative to the "
         "config file location.",

dvc/command/checkout.py

@@ -7,16 +7,12 @@ from dvc.command.base import CmdBase, append_doc_link
 
 class CmdCheckout(CmdBase):
     def run(self):
-        if not self.args.targets:
-            self.repo.checkout(force=self.args.force)
-        else:
-            for target in self.args.targets:
-                self.repo.checkout(
-                    target=target,
-                    with_deps=self.args.with_deps,
-                    force=self.args.force,
-                    recursive=self.args.recursive,
-                )
+        self.repo.checkout(
+            targets=self.args.targets,
+            with_deps=self.args.with_deps,
+            force=self.args.force,
+            recursive=self.args.recursive,
+        )
         return 0
 
 
@@ -30,13 +26,6 @@ def add_parser(subparsers, parent_parser):
         help=CHECKOUT_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    checkout_parser.add_argument(
-        "-d",
-        "--with-deps",
-        action="store_true",
-        default=False,
-        help="Checkout all dependencies of the specified target.",
-    )
     checkout_parser.add_argument(
         "-f",
         "--force",
@@ -44,6 +33,13 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Do not prompt when removing working directory files.",
     )
+    checkout_parser.add_argument(
+        "-d",
+        "--with-deps",
+        action="store_true",
+        default=False,
+        help="Checkout all dependencies of the specified target.",
+    )
     checkout_parser.add_argument(
         "-R",
         "--recursive",
@@ -51,5 +47,10 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Checkout all subdirectories of the specified directory.",
     )
-    checkout_parser.add_argument("targets", nargs="*", help="DVC files.")
+    checkout_parser.add_argument(
+        "targets",
+        nargs="*",
+        help="DVC-files to checkout. Optional. "
+        "(Finds all DVC-files in the workspace by default.)",
+    )
     checkout_parser.set_defaults(func=CmdCheckout)

dvc/command/commit.py

@@ -34,7 +34,7 @@ class CmdCommit(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    COMMIT_HELP = "Save changed data to cache and update DVC files."
+    COMMIT_HELP = "Save changed data to cache and update DVC-files."
 
     commit_parser = subparsers.add_parser(
         "commit",
@@ -65,6 +65,9 @@ def add_parser(subparsers, parent_parser):
         help="Commit cache for subdirectories of the specified directory.",
     )
     commit_parser.add_argument(
-        "targets", nargs="*", default=None, help="DVC files."
+        "targets",
+        nargs="*",
+        help="DVC-files to commit. Optional. "
+        "(Finds all DVC-files in the workspace by default.)",
     )
     commit_parser.set_defaults(func=CmdCommit)

dvc/command/config.py

@@ -1,12 +1,10 @@
 from __future__ import unicode_literals
 
-import os
 import argparse
 import logging
 
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.config import Config
-from dvc.exceptions import DvcException
 
 
 logger = logging.getLogger(__name__)
@@ -14,87 +12,48 @@ logger = logging.getLogger(__name__)
 
 class CmdConfig(CmdBaseNoRepo):
     def __init__(self, args):
-        from dvc.repo import Repo, NotDvcRepoError
-
         super(CmdConfig, self).__init__(args)
 
-        try:
-            dvc_dir = os.path.join(Repo.find_root(), Repo.DVC_DIR)
-            saved_exc = None
-        except NotDvcRepoError as exc:
-            dvc_dir = None
-            saved_exc = exc
-
-        self.config = Config(dvc_dir, validate=False)
-        if self.args.system:
-            self.configobj = self.config._system_config
-        elif self.args.glob:
-            self.configobj = self.config._global_config
-        elif self.args.local:
-            if dvc_dir is None:
-                raise saved_exc
-            self.configobj = self.config._local_config
-        else:
-            if dvc_dir is None:
-                raise saved_exc
-            self.configobj = self.config._repo_config
-
-    def _unset(self, section, opt=None, configobj=None):
-        if configobj is None:
-            configobj = self.configobj
-
-        try:
-            self.config.unset(configobj, section, opt)
-            self.config.save(configobj)
-        except DvcException:
-            logger.exception("failed to unset '{}'".format(self.args.name))
-            return 1
-        return 0
-
-    def _show(self, section, opt):
-        try:
-            self.config.show(self.configobj, section, opt)
-        except DvcException:
-            logger.exception("failed to show '{}'".format(self.args.name))
-            return 1
-        return 0
-
-    def _set(self, section, opt, value):
-        try:
-            self.config.set(self.configobj, section, opt, value)
-            self.config.save(self.configobj)
-        except DvcException:
-            logger.exception(
-                "failed to set '{}.{}' to '{}'".format(section, opt, value)
-            )
-            return 1
-        return 0
+        self.config = Config(validate=False)
 
     def run(self):
         section, opt = self.args.name.lower().strip().split(".", 1)
 
         if self.args.unset:
-            return self._unset(section, opt)
+            self.config.unset(section, opt, level=self.args.level)
         elif self.args.value is None:
-            return self._show(section, opt)
+            logger.info(self.config.get(section, opt, level=self.args.level))
         else:
-            return self._set(section, opt, self.args.value)
+            self.config.set(
+                section, opt, self.args.value, level=self.args.level
+            )
+
+        return 0
 
 
 parent_config_parser = argparse.ArgumentParser(add_help=False)
 parent_config_parser.add_argument(
     "--global",
-    dest="glob",
-    action="store_true",
-    default=False,
+    dest="level",
+    action="store_const",
+    const=Config.LEVEL_GLOBAL,
     help="Use global config.",
 )
 parent_config_parser.add_argument(
-    "--system", action="store_true", default=False, help="Use system config."
+    "--system",
+    dest="level",
+    action="store_const",
+    const=Config.LEVEL_SYSTEM,
+    help="Use system config.",
 )
 parent_config_parser.add_argument(
-    "--local", action="store_true", default=False, help="Use local config."
+    "--local",
+    dest="level",
+    action="store_const",
+    const=Config.LEVEL_LOCAL,
+    help="Use local config.",
 )
+parent_config_parser.set_defaults(level=Config.LEVEL_REPO)
 
 
 def add_parser(subparsers, parent_parser):
@@ -115,7 +74,5 @@ def add_parser(subparsers, parent_parser):
         help="Unset option.",
     )
     config_parser.add_argument("name", help="Option name.")
-    config_parser.add_argument(
-        "value", nargs="?", default=None, help="Option value."
-    )
+    config_parser.add_argument("value", nargs="?", help="Option value.")
     config_parser.set_defaults(func=CmdConfig)

dvc/command/data_sync.py

@@ -4,48 +4,33 @@ import argparse
 import logging
 
 from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException
 
 
 logger = logging.getLogger(__name__)
 
 
 class CmdDataBase(CmdBase):
-    UP_TO_DATE_MSG = "Everything is up to date."
-
-    def do_run(self, target):
-        pass
-
-    def run(self):
-        if not self.args.targets:
-            return self.do_run()
-
-        ret = 0
-        for target in self.args.targets:
-            if self.do_run(target):
-                ret = 1
-        return ret
-
     @classmethod
     def check_up_to_date(cls, processed_files_count):
         if processed_files_count == 0:
-            logger.info(cls.UP_TO_DATE_MSG)
+            logger.info("Everything is up to date.")
 
 
 class CmdDataPull(CmdDataBase):
-    def do_run(self, target=None):
+    def run(self):
         try:
             processed_files_count = self.repo.pull(
-                target=target,
+                targets=self.args.targets,
                 jobs=self.args.jobs,
                 remote=self.args.remote,
-                show_checksums=self.args.show_checksums,
                 all_branches=self.args.all_branches,
                 all_tags=self.args.all_tags,
                 with_deps=self.args.with_deps,
                 force=self.args.force,
                 recursive=self.args.recursive,
             )
-        except Exception:
+        except DvcException:
             logger.exception("failed to pull data from the cloud")
             return 1
         self.check_up_to_date(processed_files_count)
@@ -53,19 +38,18 @@ class CmdDataPull(CmdDataBase):
 
 
 class CmdDataPush(CmdDataBase):
-    def do_run(self, target=None):
+    def run(self):
         try:
             processed_files_count = self.repo.push(
-                target=target,
+                targets=self.args.targets,
                 jobs=self.args.jobs,
                 remote=self.args.remote,
-                show_checksums=self.args.show_checksums,
                 all_branches=self.args.all_branches,
                 all_tags=self.args.all_tags,
                 with_deps=self.args.with_deps,
                 recursive=self.args.recursive,
             )
-        except Exception:
+        except DvcException:
             logger.exception("failed to push data to the cloud")
             return 1
         self.check_up_to_date(processed_files_count)
@@ -73,19 +57,18 @@ class CmdDataPush(CmdDataBase):
 
 
 class CmdDataFetch(CmdDataBase):
-    def do_run(self, target=None):
+    def run(self):
         try:
             processed_files_count = self.repo.fetch(
-                target=target,
+                targets=self.args.targets,
                 jobs=self.args.jobs,
                 remote=self.args.remote,
-                show_checksums=self.args.show_checksums,
                 all_branches=self.args.all_branches,
                 all_tags=self.args.all_tags,
                 with_deps=self.args.with_deps,
                 recursive=self.args.recursive,
             )
-        except Exception:
+        except DvcException:
             logger.exception("failed to fetch data from the cloud")
             return 1
         self.check_up_to_date(processed_files_count)
@@ -100,20 +83,13 @@ def shared_parent_parser():
         add_help=False, parents=[get_parent_parser()]
     )
     shared_parent_parser.add_argument(
-        "-j",
-        "--jobs",
-        type=int,
-        default=None,
-        help="Number of jobs to run simultaneously.",
+        "-j", "--jobs", type=int, help="Number of jobs to run simultaneously."
     )
     shared_parent_parser.add_argument(
-        "--show-checksums",
-        action="store_true",
-        default=False,
-        help="Show checksums instead of file names.",
-    )
-    shared_parent_parser.add_argument(
-        "targets", nargs="*", default=None, help="DVC files."
+        "targets",
+        nargs="*",
+        help="Limit command scope to these DVC-files. "
+        "Using -R, directories to search DVC-files in can also be given.",
     )
 
     return shared_parent_parser
@@ -149,13 +125,6 @@ def add_parser(subparsers, _parent_parser):
         default=False,
         help="Fetch cache for all tags.",
     )
-    pull_parser.add_argument(
-        "-d",
-        "--with-deps",
-        action="store_true",
-        default=False,
-        help="Fetch cache for all dependencies of the specified target.",
-    )
     pull_parser.add_argument(
         "-f",
         "--force",
@@ -163,6 +132,13 @@ def add_parser(subparsers, _parent_parser):
         default=False,
         help="Do not prompt when removing working directory files.",
     )
+    pull_parser.add_argument(
+        "-d",
+        "--with-deps",
+        action="store_true",
+        default=False,
+        help="Fetch cache for all dependencies of the specified target.",
+    )
     pull_parser.add_argument(
         "-R",
         "--recursive",
@@ -278,7 +254,7 @@ def add_parser(subparsers, _parent_parser):
         default=False,
         help=(
             "Suppresses all output."
-            " Exit with 0 if pipeline is up to date, otherwise 1."
+            " Exit with 0 if pipelines are up to date, otherwise 1."
         ),
     )
     status_parser.add_argument(

dvc/command/destroy.py

@@ -12,7 +12,7 @@ logger = logging.getLogger(__name__)
 
 
 class CmdDestroy(CmdBase):
-    def run_cmd(self):
+    def run(self):
         try:
             statement = (
                 "This will destroy all information about your pipelines,"
@@ -35,7 +35,7 @@ class CmdDestroy(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    DESTROY_HELP = "Remove DVC files, local DVC config and data cache."
+    DESTROY_HELP = "Remove DVC-files, local DVC config and data cache."
 
     destroy_parser = subparsers.add_parser(
         "destroy",

dvc/command/diff.py

@@ -5,18 +5,18 @@ import argparse
 import humanize
 import inflect
 import logging
+from funcy import compact
 
 from dvc.exceptions import DvcException
 from dvc.command.base import CmdBase, append_doc_link
-from dvc.utils.collections import compact
-import dvc.repo.diff as diff
 
 
 logger = logging.getLogger(__name__)
 
 
 class CmdDiff(CmdBase):
-    def _print_size(self, size):
+    @staticmethod
+    def _print_size(size):
         if size < 0:
             change = "decreased by {}"
         elif size > 0:
@@ -26,17 +26,21 @@ class CmdDiff(CmdBase):
         natur_size = humanize.naturalsize(abs(size))
         return change.format(natur_size)
 
-    def _get_md5_string(self, sign, file_name, checksum):
+    @staticmethod
+    def _get_md5_string(sign, file_name, checksum):
         sample_msg = ""
         if file_name:
             sample_msg = "{}{} with md5 {}\n"
             sample_msg = sample_msg.format(sign, file_name, checksum)
         return sample_msg
 
-    def _get_dir_changes(self, dct):
+    @classmethod
+    def _get_dir_changes(cls, dct):
+        import dvc.repo.diff as diff
+
         engine = inflect.engine()
         changes_msg = (
-            "{} {} not changed, {} {} modified, {} {} added, "
+            "{} {} untouched, {} {} modified, {} {} added, "
             "{} {} deleted, size was {}"
         )
         changes_msg = changes_msg.format(
@@ -48,11 +52,14 @@ class CmdDiff(CmdBase):
             engine.plural("file", dct[diff.DIFF_NEW]),
             dct[diff.DIFF_DEL],
             engine.plural("file", dct[diff.DIFF_DEL]),
-            self._print_size(dct[diff.DIFF_SIZE]),
+            cls._print_size(dct[diff.DIFF_SIZE]),
         )
         return changes_msg
 
-    def _get_file_changes(self, dct):
+    @classmethod
+    def _get_file_changes(cls, dct):
+        import dvc.repo.diff as diff
+
         if (
             dct.get(diff.DIFF_OLD_FILE)
             and dct.get(diff.DIFF_NEW_FILE)
@@ -69,19 +76,25 @@ class CmdDiff(CmdBase):
             )
         else:
             msg = "file was modified, file size {}".format(
-                self._print_size(dct[diff.DIFF_SIZE])
+                cls._print_size(dct[diff.DIFF_SIZE])
             )
         return msg
 
-    def _get_royal_changes(self, dct):
+    @classmethod
+    def _get_royal_changes(cls, dct):
+        import dvc.repo.diff as diff
+
         if dct[diff.DIFF_SIZE] != diff.DIFF_SIZE_UNKNOWN:
             if dct.get("is_dir"):
-                return self._get_dir_changes(dct)
+                return cls._get_dir_changes(dct)
             else:
-                return self._get_file_changes(dct)
+                return cls._get_file_changes(dct)
         return "size is ?"
 
-    def _show(self, diff_dct):
+    @classmethod
+    def _show(cls, diff_dct):
+        import dvc.repo.diff as diff
+
         msg = "dvc diff from {} to {}".format(
             diff_dct[diff.DIFF_A_REF], diff_dct[diff.DIFF_B_REF]
         )
@@ -90,18 +103,18 @@ class CmdDiff(CmdBase):
             return
         for dct in diff_dct[diff.DIFF_LIST]:
             msg += "\n\ndiff for '{}'\n".format(dct[diff.DIFF_TARGET])
-            msg += self._get_md5_string(
+            msg += cls._get_md5_string(
                 "-",
                 dct.get(diff.DIFF_OLD_FILE),
                 dct.get(diff.DIFF_OLD_CHECKSUM),
             )
-            msg += self._get_md5_string(
+            msg += cls._get_md5_string(
                 "+",
                 dct.get(diff.DIFF_NEW_FILE),
                 dct.get(diff.DIFF_NEW_CHECKSUM),
             )
             msg += "\n"
-            msg += self._get_royal_changes(dct)
+            msg += cls._get_royal_changes(dct)
         logger.info(msg)
         return msg
 
@@ -138,7 +151,6 @@ def add_parser(subparsers, parent_parser):
     diff_parser.add_argument(
         "-t",
         "--target",
-        default=None,
         help=(
             "Source path to a data file or directory. Default None. "
             "If not specified, compares all files and directories "
@@ -151,10 +163,9 @@ def add_parser(subparsers, parent_parser):
     diff_parser.add_argument(
         "b_ref",
         help=(
-            "Git reference untill which diff calculates, if omitted "
+            "Git reference until which diff calculates, if omitted "
             "diff shows the difference between current HEAD and a_ref"
         ),
         nargs="?",
-        default=None,
     )
     diff_parser.set_defaults(func=CmdDiff)

dvc/command/gc.py

@@ -13,17 +13,19 @@ logger = logging.getLogger(__name__)
 
 class CmdGC(CmdBase):
     def run(self):
-        msg = "this will remove all cache except the cache that is used in "
-        if not self.args.all_branches and not self.args.all_tags:
-            msg += "the current git branch"
-        elif self.args.all_branches and not self.args.all_tags:
-            msg += "all git branches"
-        elif not self.args.all_branches and self.args.all_tags:
-            msg += "all git tags"
-        else:
-            msg += "all git branches and all git tags"
+        msg = "This will remove all cache except items used in "
 
-        if self.args.repos is not None and len(self.args.repos) > 0:
+        msg += "the working tree"
+        if self.args.all_commits:
+            msg += " and all git commits"
+        elif self.args.all_branches and self.args.all_tags:
+            msg += " and all git branches and tags"
+        elif self.args.all_branches:
+            msg += " and all git branches"
+        elif self.args.all_tags:
+            msg += " and all git tags"
+
+        if self.args.repos:
             msg += " of the current and the following repos:"
 
             for repo_path in self.args.repos:
@@ -40,6 +42,7 @@ class CmdGC(CmdBase):
         self.repo.gc(
             all_branches=self.args.all_branches,
             all_tags=self.args.all_tags,
+            all_commits=self.args.all_commits,
             cloud=self.args.cloud,
             remote=self.args.remote,
             force=self.args.force,
@@ -76,6 +79,12 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Keep data files for all git tags.",
     )
+    gc_parser.add_argument(
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help="Keep data files for all commits.",
+    )
     gc_parser.add_argument(
         "-c",
         "--cloud",
@@ -94,11 +103,7 @@ def add_parser(subparsers, parent_parser):
         help="Force garbage collection - automatically agree to all prompts.",
     )
     gc_parser.add_argument(
-        "-j",
-        "--jobs",
-        type=int,
-        default=None,
-        help="Number of jobs to run simultaneously.",
+        "-j", "--jobs", type=int, help="Number of jobs to run simultaneously."
     )
     gc_parser.add_argument(
         "-p",
@@ -106,7 +111,6 @@ def add_parser(subparsers, parent_parser):
         dest="repos",
         type=str,
         nargs="*",
-        default=None,
         help="Keep data files required by these projects "
         "in addition to the current one. "
         "Useful if you share a single cache across repos.",

dvc/command/get.py

@@ -0,0 +1,53 @@
+from __future__ import unicode_literals
+
+import argparse
+import logging
+
+from dvc.exceptions import DvcException
+from .base import CmdBaseNoRepo, append_doc_link
+
+
+logger = logging.getLogger(__name__)
+
+
+class CmdGet(CmdBaseNoRepo):
+    def run(self):
+        from dvc.repo import Repo
+
+        try:
+            Repo.get(
+                self.args.url,
+                path=self.args.path,
+                out=self.args.out,
+                rev=self.args.rev,
+            )
+            return 0
+        except DvcException:
+            logger.exception(
+                "failed to get '{}' from '{}'".format(
+                    self.args.path, self.args.url
+                )
+            )
+            return 1
+
+
+def add_parser(subparsers, parent_parser):
+    GET_HELP = "Download data from DVC repository."
+    get_parser = subparsers.add_parser(
+        "get",
+        parents=[parent_parser],
+        description=append_doc_link(GET_HELP, "get"),
+        help=GET_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    get_parser.add_argument(
+        "url", help="URL of Git repository with DVC project to download from."
+    )
+    get_parser.add_argument("path", help="Path to data within DVC repository.")
+    get_parser.add_argument(
+        "-o", "--out", nargs="?", help="Destination path to put data to."
+    )
+    get_parser.add_argument(
+        "--rev", nargs="?", help="DVC repository git revision."
+    )
+    get_parser.set_defaults(func=CmdGet)

dvc/command/get_url.py

@@ -0,0 +1,40 @@
+from __future__ import unicode_literals
+
+import argparse
+import logging
+
+from dvc.exceptions import DvcException
+from .base import CmdBaseNoRepo, append_doc_link
+
+
+logger = logging.getLogger(__name__)
+
+
+class CmdGetUrl(CmdBaseNoRepo):
+    def run(self):
+        from dvc.repo import Repo
+
+        try:
+            Repo.get_url(self.args.url, out=self.args.out)
+            return 0
+        except DvcException:
+            logger.exception("failed to get '{}'".format(self.args.url))
+            return 1
+
+
+def add_parser(subparsers, parent_parser):
+    GET_HELP = "Download or copy files from URL."
+    get_parser = subparsers.add_parser(
+        "get-url",
+        parents=[parent_parser],
+        description=append_doc_link(GET_HELP, "get-url"),
+        help=GET_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    get_parser.add_argument(
+        "url", help="See `dvc import-url -h` for full list of supported URLs."
+    )
+    get_parser.add_argument(
+        "out", nargs="?", help="Destination path to put data to."
+    )
+    get_parser.set_defaults(func=CmdGetUrl)

dvc/command/imp.py

@@ -1,10 +1,8 @@
 from __future__ import unicode_literals
 
 import argparse
-import os
 import logging
 
-from dvc.utils.compat import urlparse
 from dvc.exceptions import DvcException
 from dvc.command.base import CmdBase, append_doc_link
 
@@ -15,18 +13,16 @@ logger = logging.getLogger(__name__)
 class CmdImport(CmdBase):
     def run(self):
         try:
-            default_out = os.path.basename(urlparse(self.args.url).path)
-
-            out = self.args.out or default_out
-
             self.repo.imp(
-                self.args.url, out, self.args.resume, fname=self.args.file
+                self.args.url,
+                self.args.path,
+                out=self.args.out,
+                rev=self.args.rev,
             )
         except DvcException:
             logger.exception(
-                "failed to import {}. You could also try downloading "
-                "it manually and adding it with `dvc add` command.".format(
-                    self.args.url
+                "failed to import '{}' from '{}'.".format(
+                    self.args.path, self.args.url
                 )
             )
             return 1
@@ -34,7 +30,9 @@ class CmdImport(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    IMPORT_HELP = "Download or copy files from URL and take under DVC control."
+    IMPORT_HELP = (
+        "Download data from DVC repository and take it under DVC control."
+    )
 
     import_parser = subparsers.add_parser(
         "import",
@@ -44,27 +42,15 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawTextHelpFormatter,
     )
     import_parser.add_argument(
-        "url",
-        help="Supported urls:\n"
-        "/path/to/file\n"
-        "C:\\\\path\\to\\file\n"
-        "https://example.com/path/to/file\n"
-        "s3://bucket/path/to/file\n"
-        "gs://bucket/path/to/file\n"
-        "hdfs://example.com/path/to/file\n"
-        "ssh://example.com:/path/to/file\n"
-        "remote://myremote/path/to/file (see `dvc remote`)",
+        "url", help="URL of Git repository with DVC project to download from."
     )
     import_parser.add_argument(
-        "--resume",
-        action="store_true",
-        default=False,
-        help="Resume previously started download.",
+        "path", help="Path to data within DVC repository."
     )
     import_parser.add_argument(
-        "out", nargs="?", help="Destination path to put files to."
+        "-o", "--out", nargs="?", help="Destination path to put data to."
     )
     import_parser.add_argument(
-        "-f", "--file", help="Specify name of the DVC file it generates."
+        "--rev", nargs="?", help="DVC repository git revision."
     )
     import_parser.set_defaults(func=CmdImport)

dvc/command/imp_url.py

@@ -0,0 +1,60 @@
+from __future__ import unicode_literals
+
+import argparse
+import logging
+
+from dvc.exceptions import DvcException
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
+
+
+class CmdImportUrl(CmdBase):
+    def run(self):
+        try:
+            self.repo.imp_url(
+                self.args.url, out=self.args.out, fname=self.args.file
+            )
+        except DvcException:
+            logger.exception(
+                "failed to import {}. You could also try downloading "
+                "it manually and adding it with `dvc add` command.".format(
+                    self.args.url
+                )
+            )
+            return 1
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    IMPORT_HELP = (
+        "Download or copy file from URL and take it under DVC control."
+    )
+
+    import_parser = subparsers.add_parser(
+        "import-url",
+        parents=[parent_parser],
+        description=append_doc_link(IMPORT_HELP, "import-url"),
+        help=IMPORT_HELP,
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    import_parser.add_argument(
+        "url",
+        help="Supported urls:\n"
+        "/path/to/file\n"
+        "C:\\\\path\\to\\file\n"
+        "https://example.com/path/to/file\n"
+        "s3://bucket/path/to/file\n"
+        "gs://bucket/path/to/file\n"
+        "hdfs://example.com/path/to/file\n"
+        "ssh://example.com:/path/to/file\n"
+        "remote://myremote/path/to/file (see `dvc remote`)",
+    )
+    import_parser.add_argument(
+        "out", nargs="?", help="Destination path to put files to."
+    )
+    import_parser.add_argument(
+        "-f", "--file", help="Specify name of the DVC-file it generates."
+    )
+    import_parser.set_defaults(func=CmdImportUrl)

dvc/command/install.py

@@ -10,7 +10,7 @@ logger = logging.getLogger(__name__)
 
 
 class CmdInstall(CmdBase):
-    def run_cmd(self):
+    def run(self):
         try:
             self.repo.install()
         except Exception:

dvc/command/lock.py

@@ -37,7 +37,7 @@ class CmdUnlock(CmdLockBase):
 
 
 def add_parser(subparsers, parent_parser):
-    LOCK_HELP = "Lock DVC file."
+    LOCK_HELP = "Lock DVC-files."
     lock_parser = subparsers.add_parser(
         "lock",
         parents=[parent_parser],
@@ -45,10 +45,10 @@ def add_parser(subparsers, parent_parser):
         help=LOCK_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    lock_parser.add_argument("targets", nargs="+", help="DVC files.")
+    lock_parser.add_argument("targets", nargs="+", help="DVC-files to lock.")
     lock_parser.set_defaults(func=CmdLock)
 
-    UNLOCK_HELP = "Unlock DVC file."
+    UNLOCK_HELP = "Unlock DVC-files."
     unlock_parser = subparsers.add_parser(
         "unlock",
         parents=[parent_parser],
@@ -56,5 +56,7 @@ def add_parser(subparsers, parent_parser):
         help=UNLOCK_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    unlock_parser.add_argument("targets", nargs="+", help="DVC files.")
+    unlock_parser.add_argument(
+        "targets", nargs="+", help="DVC-files to unlock."
+    )
     unlock_parser.set_defaults(func=CmdUnlock)

dvc/command/metrics.py

@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.exceptions import DvcException
+from dvc.exceptions import DvcException, BadMetricError
 from dvc.command.base import CmdBase, fix_subparsers, append_doc_link
 
 
@@ -16,6 +16,10 @@ def show_metrics(metrics, all_branches=False, all_tags=False):
         metrics (list): Where each element is either a `list`
             if an xpath was specified, otherwise a `str`
     """
+    # When `metrics` contains a `None` key, it means that some files
+    # specified as `targets` in `repo.metrics.show` didn't contain any metrics.
+    missing = metrics.pop(None, None)
+
     for branch, val in metrics.items():
         if all_branches or all_tags:
             logger.info("{branch}:".format(branch=branch))
@@ -32,16 +36,17 @@ def show_metrics(metrics, all_branches=False, all_tags=False):
             else:
                 logger.info("\t{}: {}".format(fname, metric))
 
+    if missing:
+        raise BadMetricError(missing)
+
 
 class CmdMetricsShow(CmdBase):
     def run(self):
-        typ = self.args.type
-        xpath = self.args.xpath
         try:
             metrics = self.repo.metrics.show(
-                self.args.path,
-                typ=typ,
-                xpath=xpath,
+                self.args.targets,
+                typ=self.args.type,
+                xpath=self.args.xpath,
                 all_branches=self.args.all_branches,
                 all_tags=self.args.all_tags,
                 recursive=self.args.recursive,
@@ -116,12 +121,15 @@ def add_parser(subparsers, parent_parser):
     metrics_show_parser = metrics_subparsers.add_parser(
         "show",
         parents=[parent_parser],
-        description=append_doc_link(METRICS_SHOW_HELP, "metrics-show"),
+        description=append_doc_link(METRICS_SHOW_HELP, "metrics/show"),
         help=METRICS_SHOW_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     metrics_show_parser.add_argument(
-        "path", nargs="?", help="Path to a metric file or a directory."
+        "targets",
+        nargs="*",
+        help="Metric files or directories (see -R) to show "
+        "(leave empty to display all)",
     )
     metrics_show_parser.add_argument(
         "-t",
@@ -155,8 +163,8 @@ def add_parser(subparsers, parent_parser):
         action="store_true",
         default=False,
         help=(
-            "If path is a directory, recursively search and process metric "
-            "files in path."
+            "If any target is a directory, recursively search and process "
+            "metric files."
         ),
     )
     metrics_show_parser.set_defaults(func=CmdMetricsShow)
@@ -165,7 +173,7 @@ def add_parser(subparsers, parent_parser):
     metrics_add_parser = metrics_subparsers.add_parser(
         "add",
         parents=[parent_parser],
-        description=append_doc_link(METRICS_ADD_HELP, "metrics-add"),
+        description=append_doc_link(METRICS_ADD_HELP, "metrics/add"),
         help=METRICS_ADD_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
@@ -182,7 +190,7 @@ def add_parser(subparsers, parent_parser):
     metrics_modify_parser = metrics_subparsers.add_parser(
         "modify",
         parents=[parent_parser],
-        description=append_doc_link(METRICS_MODIFY_HELP, "metrics-modify"),
+        description=append_doc_link(METRICS_MODIFY_HELP, "metrics/modify"),
         help=METRICS_MODIFY_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
@@ -199,7 +207,7 @@ def add_parser(subparsers, parent_parser):
     metrics_remove_parser = metrics_subparsers.add_parser(
         "remove",
         parents=[parent_parser],
-        description=append_doc_link(METRICS_REMOVE_HELP, "metrics-remove"),
+        description=append_doc_link(METRICS_REMOVE_HELP, "metrics/remove"),
         help=METRICS_REMOVE_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )

dvc/command/move.py

@@ -27,7 +27,7 @@ def add_parser(subparsers, parent_parser):
     MOVE_HELP = "Rename or move a DVC controlled data file or a directory."
     MOVE_DESCRIPTION = (
         "Rename or move a DVC controlled data file or a directory.\n"
-        "It renames and modifies the corresponding DVC file to reflect the"
+        "It renames and modifies the corresponding DVC-file to reflect the"
         " changes."
     )
 

dvc/command/pipeline.py

@@ -1,9 +1,9 @@
 from __future__ import unicode_literals
 
+from dvc.utils import relpath
 from dvc.utils.compat import str
 
 import argparse
-import os
 import logging
 
 from dvc.exceptions import DvcException
@@ -19,9 +19,9 @@ class CmdPipelineShow(CmdBase):
         from dvc.stage import Stage
 
         stage = Stage.load(self.repo, target)
-        G = self.repo.graph()[0]
+        G = self.repo.graph
         stages = networkx.get_node_attributes(G, "stage")
-        node = os.path.relpath(stage.path, self.repo.root_dir)
+        node = relpath(stage.path, self.repo.root_dir)
         nodes = networkx.dfs_postorder_nodes(G, node)
 
         if locked:
@@ -29,6 +29,8 @@ class CmdPipelineShow(CmdBase):
 
         for n in nodes:
             if commands:
+                if stages[n].cmd is None:
+                    continue
                 logger.info(stages[n].cmd)
             elif outs:
                 for out in stages[n].outs:
@@ -39,20 +41,16 @@ class CmdPipelineShow(CmdBase):
     def __build_graph(self, target, commands, outs):
         import networkx
         from dvc.stage import Stage
+        from dvc.repo.graph import get_pipeline
 
         stage = Stage.load(self.repo, target)
-        node = os.path.relpath(stage.path, self.repo.root_dir)
+        node = relpath(stage.path, self.repo.root_dir)
 
-        pipelines = list(
-            filter(lambda g: node in g.nodes(), self.repo.pipelines())
-        )
-
-        assert len(pipelines) == 1
-        G = pipelines[0]
+        G = get_pipeline(self.repo.pipelines, node)
         stages = networkx.get_node_attributes(G, "stage")
 
         nodes = []
-        for n in G.nodes():
+        for n in G:
             stage = stages[n]
             if commands:
                 if stage.cmd is None:
@@ -166,14 +164,14 @@ class CmdPipelineList(CmdBase):
     def run(self):
         import networkx
 
-        pipelines = self.repo.pipelines()
+        pipelines = self.repo.pipelines
         for p in pipelines:
             stages = networkx.get_node_attributes(p, "stage")
             for stage in stages:
                 logger.info(stage)
             if len(stages) != 0:
                 logger.info("=" * 80)
-        logger.info("{} pipeline(s) total".format(len(pipelines)))
+        logger.info("{} pipelines total".format(len(pipelines)))
 
         return 0
 
@@ -195,11 +193,11 @@ def add_parser(subparsers, parent_parser):
 
     fix_subparsers(pipeline_subparsers)
 
-    PIPELINE_SHOW_HELP = "Show pipeline."
+    PIPELINE_SHOW_HELP = "Show pipelines."
     pipeline_show_parser = pipeline_subparsers.add_parser(
         "show",
         parents=[parent_parser],
-        description=append_doc_link(PIPELINE_SHOW_HELP, "pipeline-show"),
+        description=append_doc_link(PIPELINE_SHOW_HELP, "pipeline/show"),
         help=PIPELINE_SHOW_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
@@ -209,14 +207,14 @@ def add_parser(subparsers, parent_parser):
         "--commands",
         action="store_true",
         default=False,
-        help="Print commands instead of paths to DVC files.",
+        help="Print commands instead of paths to DVC-files.",
     )
     pipeline_show_group.add_argument(
         "-o",
         "--outs",
         action="store_true",
         default=False,
-        help="Print output files instead of paths to DVC files.",
+        help="Print output files instead of paths to DVC-files.",
     )
     pipeline_show_parser.add_argument(
         "-l",
@@ -244,7 +242,10 @@ def add_parser(subparsers, parent_parser):
         help="Output DAG as Dependencies Tree.",
     )
     pipeline_show_parser.add_argument(
-        "targets", nargs="*", help="DVC files. 'Dvcfile' by default."
+        "targets",
+        nargs="*",
+        help="DVC-files to show pipeline for. Optional. "
+        "(Finds all DVC-files in the workspace by default.)",
     )
     pipeline_show_parser.set_defaults(func=CmdPipelineShow)
 
@@ -252,7 +253,7 @@ def add_parser(subparsers, parent_parser):
     pipeline_list_parser = pipeline_subparsers.add_parser(
         "list",
         parents=[parent_parser],
-        description=append_doc_link(PIPELINE_LIST_HELP, "pipeline-list"),
+        description=append_doc_link(PIPELINE_LIST_HELP, "pipeline/list"),
         help=PIPELINE_LIST_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )

dvc/command/pkg.py

@@ -1,89 +0,0 @@
-from __future__ import unicode_literals
-
-import argparse
-import logging
-
-from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, fix_subparsers, append_doc_link
-
-
-logger = logging.getLogger(__name__)
-
-
-class CmdPkgInstall(CmdBase):
-    def run(self):
-        try:
-            self.repo.pkg.install(
-                self.args.address,
-                self.args.target_dir,
-                self.args.select,
-                self.args.file,
-            )
-            return 0
-        except DvcException:
-            logger.exception(
-                "failed to install package '{}'".format(self.args.address)
-            )
-            return 1
-
-
-def add_parser(subparsers, parent_parser):
-    from dvc.command.config import parent_config_parser
-
-    PKG_HELP = "Manage DVC packages."
-    pkg_parser = subparsers.add_parser(
-        "pkg",
-        parents=[parent_parser],
-        description=append_doc_link(PKG_HELP, "pkg"),
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-        add_help=False,
-    )
-
-    pkg_subparsers = pkg_parser.add_subparsers(
-        dest="cmd", help="Use dvc pkg CMD --help for command-specific help."
-    )
-
-    fix_subparsers(pkg_subparsers)
-
-    PKG_INSTALL_HELP = "Install package."
-    pkg_install_parser = pkg_subparsers.add_parser(
-        "install",
-        parents=[parent_config_parser, parent_parser],
-        description=append_doc_link(PKG_INSTALL_HELP, "pkg-install"),
-        help=PKG_INSTALL_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    pkg_install_parser.add_argument(
-        "address",
-        nargs="?",
-        default="",
-        help="Package address: git://<url> or https://github.com/...",
-    )
-    pkg_install_parser.add_argument(
-        "target_dir",
-        metavar="target",
-        nargs="?",
-        default=".",
-        help="Target directory to deploy package outputs. "
-        "Default value is the current dir.",
-    )
-    pkg_install_parser.add_argument(
-        "-s",
-        "--select",
-        metavar="OUT",
-        action="append",
-        default=[],
-        help="Select and persist only specified outputs from a package. "
-        "The parameter can be used multiple times. "
-        "All outputs will be selected by default.",
-    )
-    pkg_install_parser.add_argument(
-        "-f",
-        "--file",
-        help="Specify name of the stage file. It should be "
-        "either 'Dvcfile' or have a '.dvc' suffix (e.g. "
-        "'prepare.dvc', 'clean.dvc', etc). "
-        "By default the file has 'mod_' prefix and imported package name "
-        "followed by .dvc",
-    )
-    pkg_install_parser.set_defaults(func=CmdPkgInstall)

dvc/command/remote.py

@@ -1,130 +1,64 @@
 from __future__ import unicode_literals
 
 import argparse
-import os
-import re
 import logging
 
 from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig
-from dvc.config import Config
-
+from dvc.remote.config import RemoteConfig
 
 logger = logging.getLogger(__name__)
 
 
-class CmdRemoteAdd(CmdConfig):
-    @staticmethod
-    def resolve_path(path, config_file):
-        """Resolve path relative to config file location.
+class CmdRemoteConfig(CmdConfig):
+    def __init__(self, args):
+        super(CmdRemoteConfig, self).__init__(args)
+        self.remote_config = RemoteConfig(self.config)
 
-        Args:
-            path: Path to be resolved.
-            config_file: Path to config file, which `path` is specified
-                relative to.
-
-        Returns:
-            Path relative to the `config_file` location. If `path` is an
-            absolute path then it will be returned without change.
-
-        """
-        if os.path.isabs(path):
-            return path
-        return os.path.relpath(path, os.path.dirname(config_file))
 
+class CmdRemoteAdd(CmdRemoteConfig):
     def run(self):
-        from dvc.remote import _get, RemoteLOCAL
-
-        remote = _get({Config.SECTION_REMOTE_URL: self.args.url})
-        if remote == RemoteLOCAL and not self.args.url.startswith("remote://"):
-            self.args.url = self.resolve_path(
-                self.args.url, self.configobj.filename
-            )
-
-        section = Config.SECTION_REMOTE_FMT.format(self.args.name)
-        if (section in self.configobj.keys()) and not self.args.force:
-            logger.error(
-                "Remote with name {} already exists. "
-                "Use -f (--force) to overwrite remote "
-                "with new value".format(self.args.name)
-            )
-            return 1
-
-        ret = self._set(section, Config.SECTION_REMOTE_URL, self.args.url)
-        if ret != 0:
-            return ret
-
-        if self.args.default:
-            msg = "Setting '{}' as a default remote.".format(self.args.name)
-            logger.info(msg)
-            ret = self._set(
-                Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, self.args.name
-            )
-
-        return ret
-
-
-class CmdRemoteRemove(CmdConfig):
-    def _remove_default(self, config):
-        core = config.get(Config.SECTION_CORE, None)
-        if core is None:
-            return 0
-
-        default = core.get(Config.SECTION_CORE_REMOTE, None)
-        if default is None:
-            return 0
-
-        if default == self.args.name:
-            return self._unset(
-                Config.SECTION_CORE,
-                opt=Config.SECTION_CORE_REMOTE,
-                configobj=config,
-            )
-
-    def run(self):
-        section = Config.SECTION_REMOTE_FMT.format(self.args.name)
-        ret = self._unset(section)
-        if ret != 0:
-            return ret
-
-        for configobj in [
-            self.config._local_config,
-            self.config._repo_config,
-            self.config._global_config,
-            self.config._system_config,
-        ]:
-            self._remove_default(configobj)
-            self.config.save(configobj)
-            if configobj == self.configobj:
-                break
-
+        self.remote_config.add(
+            self.args.name,
+            self.args.url,
+            force=self.args.force,
+            default=self.args.default,
+            level=self.args.level,
+        )
         return 0
 
 
-class CmdRemoteModify(CmdConfig):
+class CmdRemoteRemove(CmdRemoteConfig):
     def run(self):
-        section = Config.SECTION_REMOTE_FMT.format(self.args.name)
-        self.args.name = "{}.{}".format(section, self.args.option)
-        return super(CmdRemoteModify, self).run()
+        self.remote_config.remove(self.args.name, level=self.args.level)
+        return 0
 
 
-class CmdRemoteDefault(CmdConfig):
+class CmdRemoteModify(CmdRemoteConfig):
     def run(self):
-        self.args.value = self.args.name
-        self.args.name = "core.remote"
-        return super(CmdRemoteDefault, self).run()
+        self.remote_config.modify(
+            self.args.name,
+            self.args.option,
+            self.args.value,
+            level=self.args.level,
+        )
+        return 0
 
 
-class CmdRemoteList(CmdConfig):
+class CmdRemoteDefault(CmdRemoteConfig):
     def run(self):
-        for section in self.configobj.keys():
-            r = re.match(Config.SECTION_REMOTE_REGEX, section)
-            if r:
-                name = r.group("name")
-                url = self.configobj[section].get(
-                    Config.SECTION_REMOTE_URL, ""
-                )
-                logger.info("{}\t{}".format(name, url))
+        self.remote_config.set_default(
+            self.args.name, unset=self.args.unset, level=self.args.level
+        )
+        return 0
+
+
+class CmdRemoteList(CmdRemoteConfig):
+    def run(self):
+        for name, url in self.remote_config.list(
+            level=self.args.level
+        ).items():
+            logger.info("{}\t{}".format(name, url))
         return 0
 
 
@@ -151,11 +85,11 @@ def add_parser(subparsers, parent_parser):
     remote_add_parser = remote_subparsers.add_parser(
         "add",
         parents=[parent_config_parser, parent_parser],
-        description=append_doc_link(REMOTE_ADD_HELP, "remote-add"),
+        description=append_doc_link(REMOTE_ADD_HELP, "remote/add"),
         help=REMOTE_ADD_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    remote_add_parser.add_argument("name", help="Name.")
+    remote_add_parser.add_argument("name", help="Name of the remote")
     remote_add_parser.add_argument(
         "url",
         help="URL. See full list of supported urls at " "man.dvc.org/remote",
@@ -180,7 +114,7 @@ def add_parser(subparsers, parent_parser):
     remote_default_parser = remote_subparsers.add_parser(
         "default",
         parents=[parent_config_parser, parent_parser],
-        description=append_doc_link(REMOTE_DEFAULT_HELP, "remote-default"),
+        description=append_doc_link(REMOTE_DEFAULT_HELP, "remote/default"),
         help=REMOTE_DEFAULT_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
@@ -200,24 +134,30 @@ def add_parser(subparsers, parent_parser):
     remote_remove_parser = remote_subparsers.add_parser(
         "remove",
         parents=[parent_config_parser, parent_parser],
-        description=append_doc_link(REMOTE_REMOVE_HELP, "remote-remove"),
+        description=append_doc_link(REMOTE_REMOVE_HELP, "remote/remove"),
         help=REMOTE_REMOVE_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    remote_remove_parser.add_argument("name", help="Name")
+    remote_remove_parser.add_argument(
+        "name", help="Name of the remote to remove."
+    )
     remote_remove_parser.set_defaults(func=CmdRemoteRemove)
 
     REMOTE_MODIFY_HELP = "Modify remote."
     remote_modify_parser = remote_subparsers.add_parser(
         "modify",
         parents=[parent_config_parser, parent_parser],
-        description=append_doc_link(REMOTE_MODIFY_HELP, "remote-modify"),
+        description=append_doc_link(REMOTE_MODIFY_HELP, "remote/modify"),
         help=REMOTE_MODIFY_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    remote_modify_parser.add_argument("name", help="Name.")
-    remote_modify_parser.add_argument("option", help="Option.")
-    remote_modify_parser.add_argument("value", nargs="?", help="Value.")
+    remote_modify_parser.add_argument("name", help="Name of the remote.")
+    remote_modify_parser.add_argument(
+        "option", help="Name of the option to modify."
+    )
+    remote_modify_parser.add_argument(
+        "value", nargs="?", help="(optional) Value of the option."
+    )
     remote_modify_parser.add_argument(
         "-u",
         "--unset",
@@ -231,7 +171,7 @@ def add_parser(subparsers, parent_parser):
     remote_list_parser = remote_subparsers.add_parser(
         "list",
         parents=[parent_config_parser, parent_parser],
-        description=append_doc_link(REMOTE_LIST_HELP, "remote-list"),
+        description=append_doc_link(REMOTE_LIST_HELP, "remote/list"),
         help=REMOTE_LIST_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )

dvc/command/remove.py

@@ -43,7 +43,7 @@ class CmdRemove(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    REMOVE_HELP = "Remove outputs of DVC file."
+    REMOVE_HELP = "Remove DVC-file outputs."
     remove_parser = subparsers.add_parser(
         "remove",
         parents=[parent_parser],
@@ -57,14 +57,14 @@ def add_parser(subparsers, parent_parser):
         "--outs",
         action="store_true",
         default=True,
-        help="Only remove DVC file outputs.(default)",
+        help="Only remove DVC-file outputs. (Default)",
     )
     remove_parser_group.add_argument(
         "-p",
         "--purge",
         action="store_true",
         default=False,
-        help="Remove DVC file and all its outputs.",
+        help="Remove DVC-file and all its outputs.",
     )
     remove_parser.add_argument(
         "-f",
@@ -73,5 +73,7 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Force purge.",
     )
-    remove_parser.add_argument("targets", nargs="+", help="DVC files.")
+    remove_parser.add_argument(
+        "targets", nargs="+", help="DVC-files to remove."
+    )
     remove_parser.set_defaults(func=CmdRemove)

dvc/command/repro.py

@@ -58,7 +58,7 @@ class CmdRepro(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    REPRO_HELP = "Check for changes and reproduce DVC file and dependencies."
+    REPRO_HELP = "Check for changes and reproduce stages and dependencies."
     repro_parser = subparsers.add_parser(
         "repro",
         parents=[parent_parser],
@@ -69,7 +69,7 @@ def add_parser(subparsers, parent_parser):
     repro_parser.add_argument(
         "targets",
         nargs="*",
-        help="DVC file to reproduce (default - 'Dvcfile').",
+        help="DVC-file to reproduce. 'Dvcfile' by default.",
     )
     repro_parser.add_argument(
         "-f",
@@ -152,6 +152,6 @@ def add_parser(subparsers, parent_parser):
         "--downstream",
         action="store_true",
         default=False,
-        help="Reproduce the pipeline starting from the specified stage.",
+        help="Start from the specified stages when reproducing pipelines.",
     )
     repro_parser.set_defaults(func=CmdRepro)

dvc/command/root.py

@@ -1,19 +1,19 @@
 from __future__ import unicode_literals
 
 import argparse
-import os
 import logging
 
-from dvc.repo import Repo
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
-
+from dvc.utils import relpath
 
 logger = logging.getLogger(__name__)
 
 
 class CmdRoot(CmdBaseNoRepo):
     def run(self):
-        logger.info(os.path.relpath(Repo.find_root()))
+        from dvc.repo import Repo
+
+        logger.info(relpath(Repo.find_root()))
         return 0
 
 

dvc/command/run.py

@@ -51,6 +51,7 @@ class CmdRun(CmdBase):
                 no_commit=self.args.no_commit,
                 outs_persist=self.args.outs_persist,
                 outs_persist_no_cache=self.args.outs_persist_no_cache,
+                always_changed=self.args.always_changed,
             )
         except DvcException:
             logger.exception("failed to run command")
@@ -127,15 +128,14 @@ def add_parser(subparsers, parent_parser):
         "(do not put into DVC cache).",
     )
     run_parser.add_argument(
-        "-f", "--file", help="Specify name of the DVC file it generates."
+        "-f", "--file", help="Specify name of the DVC-file it generates."
     )
     run_parser.add_argument(
-        "-c", "--cwd", default=None, help="Deprecated, use -w and -f instead."
+        "-c", "--cwd", help="Deprecated, use -w and -f instead."
     )
     run_parser.add_argument(
         "-w",
         "--wdir",
-        default=None,
         help="Directory within your repo to run your command in.",
     )
     run_parser.add_argument(
@@ -155,7 +155,7 @@ def add_parser(subparsers, parent_parser):
         "--overwrite-dvcfile",
         action="store_true",
         default=False,
-        help="Overwrite existing dvc file without asking for confirmation.",
+        help="Overwrite existing DVC-file without asking for confirmation.",
     )
     run_parser.add_argument(
         "--ignore-build-cache",
@@ -190,6 +190,12 @@ def add_parser(subparsers, parent_parser):
         help="Declare output file or directory that will not be "
         "removed upon repro (do not put into DVC cache).",
     )
+    run_parser.add_argument(
+        "--always-changed",
+        action="store_true",
+        default=False,
+        help="Always consider this DVC-file as changed.",
+    )
     run_parser.add_argument(
         "command", nargs=argparse.REMAINDER, help="Command to execute."
     )

dvc/command/status.py

@@ -12,7 +12,7 @@ logger = logging.getLogger(__name__)
 class CmdDataStatus(CmdDataBase):
     STATUS_LEN = 20
     STATUS_INDENT = "\t"
-    UP_TO_DATE_MSG = "Pipeline is up to date. Nothing to reproduce."
+    UP_TO_DATE_MSG = "Data and pipelines are up to date."
 
     def _normalize(self, s):
         s += ":"
@@ -40,14 +40,13 @@ class CmdDataStatus(CmdDataBase):
                 logger.info("{}{}:".format(ind, key))
                 self._show(value, indent + 1)
 
-    def do_run(self, target=None):
+    def run(self):
         indent = 1 if self.args.cloud else 0
         try:
             st = self.repo.status(
-                target=target,
+                targets=self.args.targets,
                 jobs=self.args.jobs,
                 cloud=self.args.cloud,
-                show_checksums=self.args.show_checksums,
                 remote=self.args.remote,
                 all_branches=self.args.all_branches,
                 all_tags=self.args.all_tags,

dvc/command/tag.py

@@ -82,14 +82,14 @@ def add_parser(subparsers, parent_parser):
     )
     tag_add_parser.add_argument("tag", help="Dvc tag.")
     tag_add_parser.add_argument(
-        "targets", nargs="*", default=[None], help="Dvc files."
+        "targets", nargs="*", default=[None], help="DVC-files."
     )
     tag_add_parser.add_argument(
         "-d",
         "--with-deps",
         action="store_true",
         default=False,
-        help="Add tag for all dependencies of the specified DVC file.",
+        help="Add tag for all dependencies of the specified DVC-file.",
     )
     tag_add_parser.add_argument(
         "-R",
@@ -109,14 +109,14 @@ def add_parser(subparsers, parent_parser):
     )
     tag_remove_parser.add_argument("tag", help="Dvc tag.")
     tag_remove_parser.add_argument(
-        "targets", nargs="*", default=[None], help="Dvc files."
+        "targets", nargs="*", default=[None], help="DVC-files."
     )
     tag_remove_parser.add_argument(
         "-d",
         "--with-deps",
         action="store_true",
         default=False,
-        help="Remove tag for all dependencies of the specified DVC file.",
+        help="Remove tag for all dependencies of the specified DVC-file.",
     )
     tag_remove_parser.add_argument(
         "-R",
@@ -135,14 +135,14 @@ def add_parser(subparsers, parent_parser):
         help=TAG_LIST_HELP,
     )
     tag_list_parser.add_argument(
-        "targets", nargs="*", default=[None], help="Dvc files."
+        "targets", nargs="*", default=[None], help="DVC-files."
     )
     tag_list_parser.add_argument(
         "-d",
         "--with-deps",
         action="store_true",
         default=False,
-        help="List tags for all dependencies of the specified DVC file.",
+        help="List tags for all dependencies of the specified DVC-file.",
     )
     tag_list_parser.add_argument(
         "-R",

dvc/command/unprotect.py

@@ -23,7 +23,7 @@ class CmdUnprotect(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    UNPROTECT_HELP = "Unprotect a data file/directory."
+    UNPROTECT_HELP = "Unprotect data files or directories."
     unprotect_parser = subparsers.add_parser(
         "unprotect",
         parents=[parent_parser],
@@ -32,6 +32,6 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     unprotect_parser.add_argument(
-        "targets", nargs="+", help="Data files/directory."
+        "targets", nargs="+", help="Data files/directories to unprotect."
     )
     unprotect_parser.set_defaults(func=CmdUnprotect)

dvc/command/update.py

@@ -0,0 +1,37 @@
+from __future__ import unicode_literals
+
+import argparse
+import logging
+
+from dvc.exceptions import DvcException
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
+
+
+class CmdUpdate(CmdBase):
+    def run(self):
+        ret = 0
+        for target in self.args.targets:
+            try:
+                self.repo.update(target)
+            except DvcException:
+                logger.exception("failed to update '{}'.".format(target))
+                ret = 1
+        return ret
+
+
+def add_parser(subparsers, parent_parser):
+    UPDATE_HELP = "Update data artifacts imported from other DVC repositories."
+    update_parser = subparsers.add_parser(
+        "update",
+        parents=[parent_parser],
+        description=append_doc_link(UPDATE_HELP, "update"),
+        help=UPDATE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    update_parser.add_argument(
+        "targets", nargs="+", help="DVC-files to update."
+    )
+    update_parser.set_defaults(func=CmdUpdate)

dvc/command/version.py

@@ -1,11 +1,23 @@
 from __future__ import unicode_literals
 
+import os
 import platform
 import argparse
 import logging
+import uuid
+import itertools
 
+try:
+    import psutil
+except ImportError:
+    psutil = None
+
+from dvc.utils import is_binary, relpath
+from dvc.utils.compat import pathlib
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.version import __version__
+from dvc.exceptions import DvcException, NotDvcRepoError
+from dvc.system import System
 
 
 logger = logging.getLogger(__name__)
@@ -13,21 +25,109 @@ logger = logging.getLogger(__name__)
 
 class CmdVersion(CmdBaseNoRepo):
     def run(self):
+        from dvc.repo import Repo
+
+        dvc_version = __version__
+        python_version = platform.python_version()
+        platform_type = platform.platform()
+        binary = is_binary()
+
         info = (
             "DVC version: {dvc_version}\n"
             "Python version: {python_version}\n"
-            "Platform: {platform_type}"
+            "Platform: {platform_type}\n"
+            "Binary: {binary}\n"
         ).format(
-            dvc_version=__version__,
-            python_version=platform.python_version(),
-            platform_type=platform.platform(),
+            dvc_version=dvc_version,
+            python_version=python_version,
+            platform_type=platform_type,
+            binary=binary,
         )
+
+        try:
+            repo = Repo()
+            root_directory = repo.root_dir
+
+            # cache_dir might not exist yet (e.g. after `dvc init`), and we
+            # can't auto-create it, as it might cause issues if the user
+            # later decides to enable shared cache mode with
+            # `dvc config cache.shared group`.
+            if os.path.exists(repo.cache.local.cache_dir):
+                info += "Cache: {cache}\n".format(
+                    cache=self.get_linktype_support_info(repo)
+                )
+            else:
+                logger.warning(
+                    "Unable to detect supported link types, as cache "
+                    "directory '{}' doesn't exist. It is usually auto-created "
+                    "by commands such as `dvc add/fetch/pull/run/import`, "
+                    "but you could create it manually to enable this "
+                    "check.".format(relpath(repo.cache.local.cache_dir))
+                )
+
+            if psutil:
+                info += (
+                    "Filesystem type (cache directory): {fs_cache}\n"
+                ).format(fs_cache=self.get_fs_type(repo.cache.local.cache_dir))
+        except NotDvcRepoError:
+            root_directory = os.getcwd()
+
+        if psutil:
+            info += ("Filesystem type (workspace): {fs_root}").format(
+                fs_root=self.get_fs_type(os.path.abspath(root_directory))
+            )
+
         logger.info(info)
         return 0
 
+    @staticmethod
+    def get_fs_type(path):
+        partition = {
+            pathlib.Path(part.mountpoint): (part.fstype, part.device)
+            for part in psutil.disk_partitions(all=True)
+        }
+
+        path = pathlib.Path(path)
+
+        for parent in itertools.chain([path], path.parents):
+            if parent in partition:
+                return partition[parent]
+        return ("unkown", "none")
+
+    @staticmethod
+    def get_linktype_support_info(repo):
+        links = {
+            "reflink": System.reflink,
+            "hardlink": System.hardlink,
+            "symlink": System.symlink,
+        }
+
+        fname = "." + str(uuid.uuid4())
+        src = os.path.join(repo.cache.local.cache_dir, fname)
+        open(src, "w").close()
+        dst = os.path.join(repo.root_dir, fname)
+
+        cache = []
+
+        for name, link in links.items():
+            try:
+                link(src, dst)
+                os.unlink(dst)
+                supported = True
+            except DvcException:
+                supported = False
+            cache.append(
+                "{name} - {supported}".format(
+                    name=name, supported=True if supported else False
+                )
+            )
+        os.remove(src)
+
+        return ", ".join(cache)
+
 
 def add_parser(subparsers, parent_parser):
-    VERSION_HELP = "Show DVC version and system/environment informaion."
+    VERSION_HELP = "Show DVC version and system/environment information."
 
     version_parser = subparsers.add_parser(
         "version",

dvc/config.py

@@ -2,15 +2,17 @@
 
 from __future__ import unicode_literals
 
-from dvc.utils.compat import str, open, urlparse
+from dvc.utils.compat import str, open
 
 import os
+import re
+import copy
 import errno
 import configobj
 import logging
 
-from schema import Schema, Optional, And, Use, Regex
-from dvc.exceptions import DvcException
+from schema import Schema, Optional, And, Use, Regex, SchemaError
+from dvc.exceptions import DvcException, NotDvcRepoError
 
 logger = logging.getLogger(__name__)
 
@@ -20,15 +22,27 @@ class ConfigError(DvcException):
 
     Args:
         msg (str): error message.
-        ex (Exception): optional exception that has caused this error.
+        cause (Exception): optional exception that has caused this error.
     """
 
-    def __init__(self, msg, ex=None):
+    def __init__(self, msg, cause=None):
         super(ConfigError, self).__init__(
-            "config file error: {}".format(msg), ex
+            "config file error: {}".format(msg), cause=cause
         )
 
 
+class NoRemoteError(ConfigError):
+    def __init__(self, command, cause=None):
+        msg = (
+            "no remote specified. Setup default remote with\n"
+            "    dvc config core.remote <name>\n"
+            "or use:\n"
+            "    dvc {} -r <name>\n".format(command)
+        )
+
+        super(NoRemoteError, self).__init__(msg, cause=cause)
+
+
 def supported_cache_type(types):
     """Checks if link type config option has a valid value.
 
@@ -43,24 +57,6 @@ def supported_cache_type(types):
     return True
 
 
-def supported_loglevel(level):
-    """Checks if log level config option has a valid value.
-
-    Args:
-        level (str): log level name.
-    """
-    return level in ["info", "debug", "warning", "error"]
-
-
-def supported_cloud(cloud):
-    """Checks if obsoleted cloud option has a valid value.
-
-    Args:
-        cloud (str): cloud type name.
-    """
-    return cloud in ["aws", "gcp", "local", ""]
-
-
 def is_bool(val):
     """Checks that value is a boolean.
 
@@ -109,6 +105,23 @@ def is_percent(val):
     return int(val) >= 0 and int(val) <= 100
 
 
+class Choices(object):
+    """Checks that value belongs to the specified set of values
+
+    Args:
+        *choices: pass allowed values as arguments, or pass a list or
+            tuple as a single argument
+    """
+
+    def __init__(self, *choices):
+        if len(choices) == 1 and isinstance(choices[0], (list, tuple)):
+            choices = choices[0]
+        self.choices = choices
+
+    def __call__(self, value):
+        return value in self.choices
+
+
 class Config(object):  # pylint: disable=too-many-instance-attributes
     """Class that manages configuration files for a dvc repo.
 
@@ -134,22 +147,33 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     CONFIG = "config"
     CONFIG_LOCAL = "config.local"
 
+    LEVEL_LOCAL = 0
+    LEVEL_REPO = 1
+    LEVEL_GLOBAL = 2
+    LEVEL_SYSTEM = 3
+
     BOOL_SCHEMA = And(str, is_bool, Use(to_bool))
 
     SECTION_CORE = "core"
     SECTION_CORE_LOGLEVEL = "loglevel"
-    SECTION_CORE_LOGLEVEL_SCHEMA = And(Use(str.lower), supported_loglevel)
+    SECTION_CORE_LOGLEVEL_SCHEMA = And(
+        Use(str.lower), Choices("info", "debug", "warning", "error")
+    )
     SECTION_CORE_REMOTE = "remote"
     SECTION_CORE_INTERACTIVE_SCHEMA = BOOL_SCHEMA
     SECTION_CORE_INTERACTIVE = "interactive"
     SECTION_CORE_ANALYTICS = "analytics"
     SECTION_CORE_ANALYTICS_SCHEMA = BOOL_SCHEMA
+    SECTION_CORE_CHECKSUM_JOBS = "checksum_jobs"
+    SECTION_CORE_CHECKSUM_JOBS_SCHEMA = And(Use(int), lambda x: x > 0)
 
     SECTION_CACHE = "cache"
     SECTION_CACHE_DIR = "dir"
     SECTION_CACHE_TYPE = "type"
     SECTION_CACHE_TYPE_SCHEMA = supported_cache_type
     SECTION_CACHE_PROTECTED = "protected"
+    SECTION_CACHE_SHARED = "shared"
+    SECTION_CACHE_SHARED_SCHEMA = And(Use(str.lower), Choices("group"))
     SECTION_CACHE_LOCAL = "local"
     SECTION_CACHE_S3 = "s3"
     SECTION_CACHE_GS = "gs"
@@ -167,15 +191,11 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         Optional(SECTION_CACHE_DIR): str,
         Optional(SECTION_CACHE_TYPE, default=None): SECTION_CACHE_TYPE_SCHEMA,
         Optional(SECTION_CACHE_PROTECTED, default=False): BOOL_SCHEMA,
+        Optional(SECTION_CACHE_SHARED): SECTION_CACHE_SHARED_SCHEMA,
         Optional(PRIVATE_CWD): str,
         Optional(SECTION_CACHE_SLOW_LINK_WARNING, default=True): BOOL_SCHEMA,
     }
 
-    # backward compatibility
-    SECTION_CORE_CLOUD = "cloud"
-    SECTION_CORE_CLOUD_SCHEMA = And(Use(str.lower), supported_cloud)
-    SECTION_CORE_STORAGEPATH = "storagepath"
-
     SECTION_CORE_SCHEMA = {
         Optional(SECTION_CORE_LOGLEVEL): And(
             str, Use(str.lower), SECTION_CORE_LOGLEVEL_SCHEMA
@@ -187,9 +207,9 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         Optional(
             SECTION_CORE_ANALYTICS, default=True
         ): SECTION_CORE_ANALYTICS_SCHEMA,
-        # backward compatibility
-        Optional(SECTION_CORE_CLOUD, default=""): SECTION_CORE_CLOUD_SCHEMA,
-        Optional(SECTION_CORE_STORAGEPATH, default=""): str,
+        Optional(
+            SECTION_CORE_CHECKSUM_JOBS, default=None
+        ): SECTION_CORE_CHECKSUM_JOBS_SCHEMA,
     }
 
     # backward compatibility
@@ -201,6 +221,8 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     SECTION_AWS_REGION = "region"
     SECTION_AWS_PROFILE = "profile"
     SECTION_AWS_USE_SSL = "use_ssl"
+    SECTION_AWS_SSE = "sse"
+    SECTION_AWS_ACL = "acl"
     SECTION_AWS_SCHEMA = {
         SECTION_AWS_STORAGEPATH: str,
         Optional(SECTION_AWS_REGION): str,
@@ -209,6 +231,8 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         Optional(SECTION_AWS_ENDPOINT_URL): str,
         Optional(SECTION_AWS_LIST_OBJECTS, default=False): BOOL_SCHEMA,
         Optional(SECTION_AWS_USE_SSL, default=True): BOOL_SCHEMA,
+        Optional(SECTION_AWS_SSE): str,
+        Optional(SECTION_AWS_ACL): str,
     }
 
     # backward compatibility
@@ -241,6 +265,8 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     SECTION_REMOTE_TIMEOUT = "timeout"
     SECTION_REMOTE_PASSWORD = "password"
     SECTION_REMOTE_ASK_PASSWORD = "ask_password"
+    SECTION_REMOTE_GSS_AUTH = "gss_auth"
+    SECTION_REMOTE_NO_TRAVERSE = "no_traverse"
     SECTION_REMOTE_SCHEMA = {
         SECTION_REMOTE_URL: str,
         Optional(SECTION_AWS_REGION): str,
@@ -249,6 +275,8 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         Optional(SECTION_AWS_ENDPOINT_URL): str,
         Optional(SECTION_AWS_LIST_OBJECTS, default=False): BOOL_SCHEMA,
         Optional(SECTION_AWS_USE_SSL, default=True): BOOL_SCHEMA,
+        Optional(SECTION_AWS_SSE): str,
+        Optional(SECTION_AWS_ACL): str,
         Optional(SECTION_GCP_PROJECTNAME): str,
         Optional(SECTION_CACHE_TYPE): SECTION_CACHE_TYPE_SCHEMA,
         Optional(SECTION_CACHE_PROTECTED, default=False): BOOL_SCHEMA,
@@ -258,11 +286,13 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         Optional(SECTION_REMOTE_TIMEOUT): Use(int),
         Optional(SECTION_REMOTE_PASSWORD): str,
         Optional(SECTION_REMOTE_ASK_PASSWORD): BOOL_SCHEMA,
+        Optional(SECTION_REMOTE_GSS_AUTH): BOOL_SCHEMA,
         Optional(SECTION_AZURE_CONNECTION_STRING): str,
         Optional(SECTION_OSS_ACCESS_KEY_ID): str,
         Optional(SECTION_OSS_ACCESS_KEY_SECRET): str,
         Optional(SECTION_OSS_ENDPOINT): str,
         Optional(PRIVATE_CWD): str,
+        Optional(SECTION_REMOTE_NO_TRAVERSE, default=True): BOOL_SCHEMA,
     }
 
     SECTION_STATE = "state"
@@ -285,30 +315,20 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     }
 
     def __init__(self, dvc_dir=None, validate=True):
-        self.system_config_file = os.path.join(
-            self.get_system_config_dir(), self.CONFIG
-        )
-        self.global_config_file = os.path.join(
-            self.get_global_config_dir(), self.CONFIG
-        )
+        self.dvc_dir = dvc_dir
+        self.validate = validate
 
-        if dvc_dir is not None:
-            self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
-            self.config_file = os.path.join(dvc_dir, self.CONFIG)
-            self.config_local_file = os.path.join(dvc_dir, self.CONFIG_LOCAL)
+        if not dvc_dir:
+            try:
+                from dvc.repo import Repo
+
+                self.dvc_dir = os.path.join(Repo.find_dvc_dir())
+            except NotDvcRepoError:
+                self.dvc_dir = None
         else:
-            self.dvc_dir = None
-            self.config_file = None
-            self.config_local_file = None
+            self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
 
-        self._system_config = None
-        self._global_config = None
-        self._repo_config = None
-        self._local_config = None
-
-        self.config = None
-
-        self.load(validate=validate)
+        self.load()
 
     @staticmethod
     def get_global_config_dir():
@@ -350,35 +370,7 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         open(config_file, "w+").close()
         return Config(dvc_dir)
 
-    def _load(self):
-        self._system_config = configobj.ConfigObj(self.system_config_file)
-        self._global_config = configobj.ConfigObj(self.global_config_file)
-
-        if self.config_file is not None:
-            self._repo_config = configobj.ConfigObj(self.config_file)
-        else:
-            self._repo_config = configobj.ConfigObj()
-
-        if self.config_local_file is not None:
-            self._local_config = configobj.ConfigObj(self.config_local_file)
-        else:
-            self._local_config = configobj.ConfigObj()
-
-        self.config = None
-
-    def _load_config(self, path):
-        config = configobj.ConfigObj(path)
-        config = self._lower(config)
-        self._resolve_paths(config, path)
-        return config
-
-    @staticmethod
-    def _resolve_path(path, config_file):
-        assert os.path.isabs(config_file)
-        config_dir = os.path.dirname(config_file)
-        return os.path.abspath(os.path.join(config_dir, path))
-
-    def _resolve_cache_path(self, config, fname):
+    def _resolve_cache_path(self, config):
         cache = config.get(self.SECTION_CACHE)
         if cache is None:
             return
@@ -387,72 +379,86 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         if cache_dir is None:
             return
 
-        cache[self.PRIVATE_CWD] = os.path.dirname(fname)
+        cache[self.PRIVATE_CWD] = os.path.dirname(config.filename)
 
-    def _resolve_paths(self, config, fname):
-        if fname is None:
-            return
+    def _resolve_paths(self, config):
+        if config.filename is None:
+            return config
 
-        self._resolve_cache_path(config, fname)
-        for section in config.values():
+        ret = copy.deepcopy(config)
+        self._resolve_cache_path(ret)
+
+        for section in ret.values():
             if self.SECTION_REMOTE_URL not in section.keys():
                 continue
 
-            section[self.PRIVATE_CWD] = os.path.dirname(fname)
+            section[self.PRIVATE_CWD] = os.path.dirname(ret.filename)
 
-    def load(self, validate=True):
+        return ret
+
+    def _load_configs(self):
+        system_config_file = os.path.join(
+            self.get_system_config_dir(), self.CONFIG
+        )
+
+        global_config_file = os.path.join(
+            self.get_global_config_dir(), self.CONFIG
+        )
+
+        self._system_config = configobj.ConfigObj(system_config_file)
+        self._global_config = configobj.ConfigObj(global_config_file)
+        self._repo_config = configobj.ConfigObj()
+        self._local_config = configobj.ConfigObj()
+
+        if not self.dvc_dir:
+            return
+
+        config_file = os.path.join(self.dvc_dir, self.CONFIG)
+        config_local_file = os.path.join(self.dvc_dir, self.CONFIG_LOCAL)
+
+        self._repo_config = configobj.ConfigObj(config_file)
+        self._local_config = configobj.ConfigObj(config_local_file)
+
+    @property
+    def config_local_file(self):
+        return self._local_config.filename
+
+    @property
+    def config_file(self):
+        return self._repo_config.filename
+
+    def load(self):
         """Loads config from all the config files.
 
-        Args:
-            validate (bool): optional flag to tell dvc if it should validate
-                the config or just load it as is. 'True' by default.
-
-
         Raises:
             dvc.config.ConfigError: thrown if config has invalid format.
         """
-        self._load()
+        self._load_configs()
+
+        self.config = configobj.ConfigObj()
+        for c in [
+            self._system_config,
+            self._global_config,
+            self._repo_config,
+            self._local_config,
+        ]:
+            c = self._resolve_paths(c)
+            c = self._lower(c)
+            self.config.merge(c)
+
+        if not self.validate:
+            return
+
+        d = self.config.dict()
         try:
-            self.config = self._load_config(self.system_config_file)
-            user = self._load_config(self.global_config_file)
-            config = self._load_config(self.config_file)
-            local = self._load_config(self.config_local_file)
-
-            # NOTE: schema doesn't support ConfigObj.Section validation, so we
-            # need to convert our config to dict before passing it to
-            for conf in [user, config, local]:
-                self.config = self._merge(self.config, conf)
-
-            if validate:
-                self.config = Schema(self.SCHEMA).validate(self.config)
-
-            # NOTE: now converting back to ConfigObj
-            self.config = configobj.ConfigObj(
-                self.config, write_empty_values=True
-            )
-            self.config.filename = self.config_file
-            self._resolve_paths(self.config, self.config_file)
-        except Exception as ex:
-            raise ConfigError(ex)
-
-    @staticmethod
-    def _get_key(conf, name, add=False):
-        for k in conf.keys():
-            if k.lower() == name.lower():
-                return k
-
-        if add:
-            conf[name] = {}
-            return name
-
-        return None
+            d = Schema(self.SCHEMA).validate(d)
+        except SchemaError as exc:
+            raise ConfigError("config format error", cause=exc)
+        self.config = configobj.ConfigObj(d, write_empty_values=True)
 
     def save(self, config=None):
         """Saves config to config files.
 
-        Args:
-            config (configobj.ConfigObj): optional config object to save.
-
         Raises:
             dvc.config.ConfigError: thrown if failed to write config file.
         """
@@ -467,129 +473,103 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
             ]
 
         for conf in clist:
-            if conf.filename is None:
-                continue
+            self._save(conf)
 
-            try:
-                logger.debug("Writing '{}'.".format(conf.filename))
-                dname = os.path.dirname(os.path.abspath(conf.filename))
-                try:
-                    os.makedirs(dname)
-                except OSError as exc:
-                    if exc.errno != errno.EEXIST:
-                        raise
-                conf.write()
-            except Exception as exc:
-                msg = "failed to write config '{}'".format(conf.filename)
-                raise ConfigError(msg, exc)
-
-    def get_remote_settings(self, name):
-        import posixpath
-
-        """
-        Args:
-            name (str): The name of the remote that we want to retrieve
-
-        Returns:
-            dict: The content beneath the given remote name.
-
-        Example:
-            >>> config = {'remote "server"': {'url': 'ssh://localhost/'}}
-            >>> get_remote_settings("server")
-            {'url': 'ssh://localhost/'}
-        """
-        settings = self.config.get(
-            self.SECTION_REMOTE_FMT.format(name.lower())
-        )
-
-        if settings is None:
-            raise ConfigError(
-                "unable to find remote section '{}'".format(name)
-            )
-
-        parsed = urlparse(settings["url"])
-
-        # Support for cross referenced remotes.
-        # This will merge the settings, giving priority to the outer reference.
-        # For example, having:
-        #
-        #       dvc remote add server ssh://localhost
-        #       dvc remote modify server user root
-        #       dvc remote modify server ask_password true
-        #
-        #       dvc remote add images remote://server/tmp/pictures
-        #       dvc remote modify images user alice
-        #       dvc remote modify images ask_password false
-        #       dvc remote modify images password asdf1234
-        #
-        # Results on a config dictionary like:
-        #
-        #       {
-        #           "url": "ssh://localhost/tmp/pictures",
-        #           "user": "alice",
-        #           "password": "asdf1234",
-        #           "ask_password": False,
-        #       }
-        #
-        if parsed.scheme == "remote":
-            reference = self.get_remote_settings(parsed.netloc)
-            url = posixpath.join(reference["url"], parsed.path.lstrip("/"))
-            merged = reference.copy()
-            merged.update(settings)
-            merged["url"] = url
-            return merged
-
-        return settings
+        self.load()
 
     @staticmethod
-    def unset(config, section, opt=None):
+    def _save(config):
+        if config.filename is None:
+            return
+
+        logger.debug("Writing '{}'.".format(config.filename))
+        dname = os.path.dirname(os.path.abspath(config.filename))
+        try:
+            os.makedirs(dname)
+        except OSError as exc:
+            if exc.errno != errno.EEXIST:
+                raise
+        config.write()
+
+    def unset(self, section, opt=None, level=None, force=False):
         """Unsets specified option and/or section in the config.
 
         Args:
-            config (configobj.ConfigObj): config to work on.
             section (str): section name.
             opt (str): optional option name.
+            level (int): config level to use.
+            force (bool): don't error-out even if section doesn't exist. False
+                by default.
+
+        Raises:
+            dvc.config.ConfigError: thrown if section doesn't exist and
+                `force != True`.
         """
+        config = self.get_configobj(level)
+
         if section not in config.keys():
+            if force:
+                return
             raise ConfigError("section '{}' doesn't exist".format(section))
 
-        if opt is None:
-            del config[section]
-            return
+        if opt:
+            if opt not in config[section].keys():
+                if force:
+                    return
+                raise ConfigError(
+                    "option '{}.{}' doesn't exist".format(section, opt)
+                )
+            del config[section][opt]
 
-        if opt not in config[section].keys():
-            raise ConfigError(
-                "option '{}.{}' doesn't exist".format(section, opt)
-            )
-        del config[section][opt]
-
-        if not config[section]:
+            if not config[section]:
+                del config[section]
+        else:
             del config[section]
 
-    @staticmethod
-    def set(config, section, opt, value):
+        self.save(config)
+
+    def set(self, section, opt, value, level=None, force=True):
         """Sets specified option in the config.
 
         Args:
-            config (configobj.ConfigObj): config to work on.
             section (str): section name.
             opt (str): option name.
             value: value to set option to.
+            level (int): config level to use.
+            force (bool): set option even if section already exists. True by
+                default.
+
+        Raises:
+            dvc.config.ConfigError: thrown if section already exists and
+                `force != True`.
+
         """
+        config = self.get_configobj(level)
+
         if section not in config.keys():
             config[section] = {}
+        elif not force:
+            raise ConfigError(
+                "Section '{}' already exists. Use `-f|--force` to overwrite "
+                "section with new value.".format(section)
+            )
 
         config[section][opt] = value
+        self.save(config)
 
-    @staticmethod
-    def show(config, section, opt):
-        """Prints option value from the config.
+    def get(self, section, opt=None, level=None):
+        """Return option value from the config.
 
         Args:
-            config (configobj.ConfigObj): config to work on.
             section (str): section name.
             opt (str): option name.
+            level (int): config level to use.
+
+        Returns:
+            value (str, int): option value.
         """
+        config = self.get_configobj(level)
+
         if section not in config.keys():
             raise ConfigError("section '{}' doesn't exist".format(section))
 
@@ -598,25 +578,35 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
                 "option '{}.{}' doesn't exist".format(section, opt)
             )
 
-        logger.info(config[section][opt])
-
-    @staticmethod
-    def _merge(first, second):
-        res = {}
-        sections = list(first.keys()) + list(second.keys())
-        for section in sections:
-            first_copy = first.get(section, {}).copy()
-            second_copy = second.get(section, {}).copy()
-            first_copy.update(second_copy)
-            res[section] = first_copy
-        return res
+        return config[section][opt]
 
     @staticmethod
     def _lower(config):
-        new_config = {}
+        new_config = configobj.ConfigObj()
         for s_key, s_value in config.items():
             new_s = {}
             for key, value in s_value.items():
                 new_s[key.lower()] = str(value)
             new_config[s_key.lower()] = new_s
         return new_config
+
+    def get_configobj(self, level):
+        configs = {
+            self.LEVEL_LOCAL: self._local_config,
+            self.LEVEL_REPO: self._repo_config,
+            self.LEVEL_GLOBAL: self._global_config,
+            self.LEVEL_SYSTEM: self._system_config,
+        }
+
+        return configs.get(level, self._repo_config)
+
+    def list_options(self, section_regex, option, level=None):
+        ret = {}
+        config = self.get_configobj(level)
+        for section in config.keys():
+            r = re.match(section_regex, section)
+            if r:
+                name = r.group("name")
+                value = config[section].get(option, "")
+                ret[name] = value
+        return ret

dvc/daemon.py

@@ -94,7 +94,7 @@ def daemon(args):
 
     cmd = [sys.executable]
     if not is_binary():
-        cmd += ["-m", "dvc"]
+        cmd += [sys.argv[0]]
     cmd += ["daemon", "-q"] + args
 
     env = fix_env()

dvc/data_cloud.py

@@ -4,16 +4,8 @@ from __future__ import unicode_literals
 
 import logging
 
-from dvc.config import Config, ConfigError
+from dvc.config import Config, NoRemoteError
 from dvc.remote import Remote
-from dvc.remote.s3 import RemoteS3
-from dvc.remote.gs import RemoteGS
-from dvc.remote.azure import RemoteAZURE
-from dvc.remote.oss import RemoteOSS
-from dvc.remote.ssh import RemoteSSH
-from dvc.remote.hdfs import RemoteHDFS
-from dvc.remote.local import RemoteLOCAL
-from dvc.remote.http import RemoteHTTP
 
 
 logger = logging.getLogger(__name__)
@@ -25,24 +17,11 @@ class DataCloud(object):
     Args:
         repo (dvc.repo.Repo): repo instance that belongs to the repo that
             we are working on.
-        config (dict): config of the repo.
 
     Raises:
         config.ConfigError: thrown when config has invalid format.
     """
 
-    CLOUD_MAP = {
-        "aws": RemoteS3,
-        "gcp": RemoteGS,
-        "azure": RemoteAZURE,
-        "oss": RemoteOSS,
-        "ssh": RemoteSSH,
-        "hdfs": RemoteHDFS,
-        "local": RemoteLOCAL,
-        "http": RemoteHTTP,
-        "https": RemoteHTTP,
-    }
-
     def __init__(self, repo):
         self.repo = repo
 
@@ -61,58 +40,16 @@ class DataCloud(object):
         if remote:
             return self._init_remote(remote)
 
-        # Old config format support for backward compatibility
-        if Config.SECTION_CORE_CLOUD in self._core:
-            msg = "using obsoleted config format. Consider updating."
-            logger.warning(msg)
-            return self._init_compat()
-
-        raise ConfigError(
-            "No remote repository specified. Setup default repository with\n"
-            "    dvc config core.remote <name>\n"
-            "or use:\n"
-            "    dvc {} -r <name>\n".format(command)
-        )
+        raise NoRemoteError(command)
 
     def _init_remote(self, remote):
-        config = self.repo.config.get_remote_settings(remote)
-        return Remote(self.repo, config)
+        return Remote(self.repo, name=remote)
 
-    def _init_compat(self):
-        name = self._core.get(Config.SECTION_CORE_CLOUD, "").strip().lower()
-        if name == "":
-            return None
-
-        cloud_type = self.CLOUD_MAP.get(name, None)
-        if not cloud_type:
-            msg = "wrong cloud type '{}' specified".format(name)
-            raise ConfigError(msg)
-
-        cloud_config = self._config.get(name, None)
-        if not cloud_config:
-            msg = "can't find cloud section '{}' in config".format(name)
-            raise ConfigError(msg)
-
-        # NOTE: check if the class itself has everything needed for operation.
-        # E.g. all the imported packages.
-        if not cloud_type.supported(cloud_type.compat_config(cloud_config)):
-            raise ConfigError("unsupported cloud '{}'".format(name))
-
-        return self._init_cloud(cloud_config, cloud_type)
-
-    def _init_cloud(self, cloud_config, cloud_type):
-        global_storage_path = self._core.get(Config.SECTION_CORE_STORAGEPATH)
-        if global_storage_path:
-            logger.warning("using obsoleted config format. Consider updating.")
-
-        cloud = cloud_type(self.repo, cloud_config)
-        return cloud
-
-    def push(self, targets, jobs=None, remote=None, show_checksums=False):
+    def push(self, cache, jobs=None, remote=None, show_checksums=False):
         """Push data items in a cloud-agnostic way.
 
         Args:
-            targets (list): list of targets to push to the cloud.
+            cache (NamedCache): named checksums to push to the cloud.
             jobs (int): number of jobs that can be running simultaneously.
             remote (dvc.remote.base.RemoteBASE): optional remote to push to.
                 By default remote from core.remote config option is used.
@@ -120,17 +57,17 @@ class DataCloud(object):
                 information messages.
         """
         return self.repo.cache.local.push(
-            targets,
+            cache,
             jobs=jobs,
             remote=self.get_remote(remote, "push"),
             show_checksums=show_checksums,
         )
 
-    def pull(self, targets, jobs=None, remote=None, show_checksums=False):
+    def pull(self, cache, jobs=None, remote=None, show_checksums=False):
         """Pull data items in a cloud-agnostic way.
 
         Args:
-            targets (list): list of targets to pull from the cloud.
+            cache (NamedCache): named checksums to pull from the cloud.
             jobs (int): number of jobs that can be running simultaneously.
             remote (dvc.remote.base.RemoteBASE): optional remote to pull from.
                 By default remote from core.remote config option is used.
@@ -138,25 +75,25 @@ class DataCloud(object):
                 information messages.
         """
         return self.repo.cache.local.pull(
-            targets,
+            cache,
             jobs=jobs,
             remote=self.get_remote(remote, "pull"),
             show_checksums=show_checksums,
         )
 
-    def status(self, targets, jobs=None, remote=None, show_checksums=False):
+    def status(self, cache, jobs=None, remote=None, show_checksums=False):
         """Check status of data items in a cloud-agnostic way.
 
         Args:
-            targets (list): list of targets to check status for.
+            cache (NamedCache): named checksums to check status for.
             jobs (int): number of jobs that can be running simultaneously.
             remote (dvc.remote.base.RemoteBASE): optional remote to compare
-                targets to. By default remote from core.remote config option
+                cache to. By default remote from core.remote config option
                 is used.
             show_checksums (bool): show checksums instead of file names in
                 information messages.
         """
         remote = self.get_remote(remote, "status")
         return self.repo.cache.local.status(
-            targets, jobs=jobs, remote=remote, show_checksums=show_checksums
+            cache, jobs=jobs, remote=remote, show_checksums=show_checksums
         )

dvc/dependency/__init__.py

@@ -13,9 +13,11 @@ from dvc.dependency.hdfs import DependencyHDFS
 from dvc.dependency.ssh import DependencySSH
 from dvc.dependency.http import DependencyHTTP
 from dvc.dependency.https import DependencyHTTPS
+from .repo import DependencyREPO
 
 from dvc.remote import Remote
 
+
 DEPS = [
     DependencyGS,
     DependencyHDFS,
@@ -44,6 +46,7 @@ DEP_MAP = {
 SCHEMA = output.SCHEMA.copy()
 del SCHEMA[schema.Optional(OutputBase.PARAM_CACHE)]
 del SCHEMA[schema.Optional(OutputBase.PARAM_METRIC)]
+SCHEMA[schema.Optional(DependencyREPO.PARAM_REPO)] = DependencyREPO.REPO_SCHEMA
 
 
 def _get(stage, p, info):
@@ -52,10 +55,13 @@ def _get(stage, p, info):
     parsed = urlparse(p)
 
     if parsed.scheme == "remote":
-        settings = stage.repo.config.get_remote_settings(parsed.netloc)
-        remote = Remote(stage.repo, settings)
+        remote = Remote(stage.repo, name=parsed.netloc)
         return DEP_MAP[remote.scheme](stage, p, info, remote=remote)
 
+    if info and info.get(DependencyREPO.PARAM_REPO):
+        repo = info.pop(DependencyREPO.PARAM_REPO)
+        return DependencyREPO(repo, stage, p, info)
+
     for d in DEPS:
         if d.supported(p):
             return d(stage, p, info)
@@ -70,8 +76,9 @@ def loadd_from(stage, d_list):
     return ret
 
 
-def loads_from(stage, s_list):
+def loads_from(stage, s_list, erepo=None):
     ret = []
     for s in s_list:
-        ret.append(_get(stage, s, {}))
+        info = {DependencyREPO.PARAM_REPO: erepo} if erepo else {}
+        ret.append(_get(stage, s, info))
     return ret

dvc/dependency/base.py

@@ -15,8 +15,19 @@ class DependencyIsNotFileOrDirError(DvcException):
         super(DependencyIsNotFileOrDirError, self).__init__(msg)
 
 
+class DependencyIsStageFileError(DvcException):
+    def __init__(self, path):
+        super(DependencyIsStageFileError, self).__init__(
+            "Stage file '{}' cannot be a dependency.".format(path)
+        )
+
+
 class DependencyBase(object):
     IS_DEPENDENCY = True
 
     DoesNotExistError = DependencyDoesNotExistError
     IsNotFileOrDirError = DependencyIsNotFileOrDirError
+    IsStageFileError = DependencyIsStageFileError
+
+    def update(self):
+        pass

dvc/dependency/http.py

@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-from dvc.path import Path
-from dvc.utils.compat import urlparse, urljoin
 from dvc.output.base import OutputBase
 from dvc.remote.http import RemoteHTTP
 from dvc.dependency.base import DependencyBase
@@ -9,12 +7,3 @@ from dvc.dependency.base import DependencyBase
 
 class DependencyHTTP(DependencyBase, OutputBase):
     REMOTE = RemoteHTTP
-
-    def __init__(self, stage, path, info=None, remote=None):
-        super(DependencyHTTP, self).__init__(
-            stage, path, info=info, remote=remote
-        )
-        if path.startswith("remote"):
-            path = urljoin(self.remote.cache_dir, urlparse(path).path)
-
-        self.path_info = Path(self.scheme, url=self.url, path=path)

dvc/dependency/repo.py

@@ -0,0 +1,88 @@
+from __future__ import unicode_literals
+
+import copy
+
+from funcy import merge
+from schema import Optional
+from contextlib import contextmanager
+
+from dvc.external_repo import external_repo
+from dvc.utils.compat import str
+
+from .local import DependencyLOCAL
+
+
+class DependencyREPO(DependencyLOCAL):
+    PARAM_REPO = "repo"
+    PARAM_URL = "url"
+    PARAM_REV = "rev"
+    PARAM_REV_LOCK = "rev_lock"
+
+    REPO_SCHEMA = {
+        Optional(PARAM_URL): str,
+        Optional(PARAM_REV): str,
+        Optional(PARAM_REV_LOCK): str,
+    }
+
+    def __init__(self, def_repo, stage, *args, **kwargs):
+        self.def_repo = def_repo
+        super(DependencyREPO, self).__init__(stage, *args, **kwargs)
+
+    def _parse_path(self, remote, path):
+        return None
+
+    @property
+    def is_in_repo(self):
+        return False
+
+    @property
+    def repo_pair(self):
+        d = self.def_repo
+        return d[self.PARAM_URL], d[self.PARAM_REV_LOCK] or d[self.PARAM_REV]
+
+    def __str__(self):
+        return "{} ({})".format(self.def_path, self.def_repo[self.PARAM_URL])
+
+    @contextmanager
+    def _make_repo(self, **overrides):
+        with external_repo(**merge(self.def_repo, overrides)) as repo:
+            yield repo
+
+    def status(self):
+        with self._make_repo() as repo:
+            current = repo.find_out_by_relpath(self.def_path).info
+
+        with self._make_repo(rev_lock=None) as repo:
+            updated = repo.find_out_by_relpath(self.def_path).info
+
+        if current != updated:
+            return {str(self): "update available"}
+
+        return {}
+
+    def save(self):
+        pass
+
+    def dumpd(self):
+        return {self.PARAM_PATH: self.def_path, self.PARAM_REPO: self.def_repo}
+
+    def fetch(self):
+        with self._make_repo(
+            cache_dir=self.repo.cache.local.cache_dir
+        ) as repo:
+            self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()
+
+            out = repo.find_out_by_relpath(self.def_path)
+            with repo.state:
+                repo.cloud.pull(out.get_used_cache())
+
+        return out
+
+    def download(self, to):
+        out = self.fetch()
+        to.info = copy.copy(out.info)
+        to.checkout()
+
+    def update(self):
+        with self._make_repo(rev_lock=None) as repo:
+            self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()

dvc/exceptions.py

@@ -4,9 +4,10 @@ from __future__ import unicode_literals
 
 from dvc.utils.compat import str, builtin_str
 
-import os
 import traceback
 
+from dvc.utils import relpath
+
 
 class DvcException(Exception):
     """Base class for all dvc exceptions.
@@ -46,7 +47,7 @@ class OutputDuplicationError(DvcException):
             for stage in stages
         )
         msg = (
-            "file/directory '{}' is specified as an output in more than one"
+            "file/directory '{}' is specified as an output in more than one "
             "stage: {}"
         ).format(output, "\n    ".join(stages))
         super(OutputDuplicationError, self).__init__(msg)
@@ -61,8 +62,8 @@ class OutputNotFoundError(DvcException):
 
     def __init__(self, output):
         super(OutputNotFoundError, self).__init__(
-            "unable to find stage file with output '{path}'".format(
-                path=os.path.relpath(output)
+            "unable to find DVC-file with output '{path}'".format(
+                path=relpath(output)
             )
         )
 
@@ -73,7 +74,7 @@ class StagePathAsOutputError(DvcException):
 
     Args:
         cwd (str): path to the directory.
-        fname (str): path to the stage file that has cwd specified as an
+        fname (str): path to the DVC-file that has cwd specified as an
             output.
     """
 
@@ -81,8 +82,8 @@ class StagePathAsOutputError(DvcException):
         assert isinstance(wdir, str) or isinstance(wdir, builtin_str)
         assert isinstance(fname, str) or isinstance(fname, builtin_str)
         msg = (
-            "current working directory '{cwd}' is specified as an output in"
-            " '{fname}'. Use another CWD to prevent any data removal.".format(
+            "current working directory '{cwd}' is specified as an output in "
+            "'{fname}'. Use another CWD to prevent any data removal.".format(
                 cwd=wdir, fname=fname
             )
         )
@@ -98,9 +99,7 @@ class CircularDependencyError(DvcException):
     """
 
     def __init__(self, dependency):
-        assert isinstance(dependency, str) or isinstance(
-            dependency, builtin_str
-        )
+        assert isinstance(dependency, (str, builtin_str))
 
         msg = (
             "file/directory '{}' is specified as an output and as a "
@@ -118,7 +117,7 @@ class ArgumentDuplicationError(DvcException):
     """
 
     def __init__(self, path):
-        assert isinstance(path, str) or isinstance(path, builtin_str)
+        assert isinstance(path, (str, builtin_str))
         msg = "file '{}' is specified more than once."
         super(ArgumentDuplicationError, self).__init__(msg.format(path))
 
@@ -168,8 +167,8 @@ class CyclicGraphError(DvcException):
         assert isinstance(stages, list)
         stages = "\n".join("\t- {}".format(stage) for stage in stages)
         msg = (
-            "you've introduced a cycle in your pipeline that involves"
-            " the following stages:"
+            "you've introduced a cycle in your pipeline that involves "
+            "the following stages:"
             "\n"
             "{stages}".format(stages=stages)
         )
@@ -197,10 +196,11 @@ class ReproductionError(DvcException):
 
 
 class BadMetricError(DvcException):
-    def __init__(self, path):
+    def __init__(self, paths):
         super(BadMetricError, self).__init__(
-            "'{}' does not exist, not a metric or is malformed".format(
-                os.path.relpath(path)
+            "the following metrics do not exists, "
+            "are not metric files or are malformed: {paths}".format(
+                paths=", ".join("'{}'".format(path) for path in paths)
             )
         )
 
@@ -215,9 +215,9 @@ class NoMetricsError(DvcException):
 
 class StageFileCorruptedError(DvcException):
     def __init__(self, path, cause=None):
-        path = os.path.relpath(path)
+        path = relpath(path)
         super(StageFileCorruptedError, self).__init__(
-            "unable to read stage file: {} "
+            "unable to read DVC-file: {} "
             "YAML file structure is corrupted".format(path),
             cause=cause,
         )
@@ -244,13 +244,6 @@ class OverlappingOutputPathsError(DvcException):
         )
 
 
-class TargetNotDirectoryError(DvcException):
-    def __init__(self, path):
-        super(TargetNotDirectoryError, self).__init__(
-            "Target: {} is not a directory".format(path)
-        )
-
-
 class CheckoutErrorSuggestGit(DvcException):
     def __init__(self, target, cause):
         super(CheckoutErrorSuggestGit, self).__init__(
@@ -264,3 +257,75 @@ class ETagMismatchError(DvcException):
             "ETag mismatch detected when copying file to cache! "
             "(expected: '{}', actual: '{}')".format(etag, cached_etag)
         )
+
+
+class FileMissingError(DvcException):
+    def __init__(self, path, cause=None):
+        super(FileMissingError, self).__init__(
+            "Can't find '{}' neither locally nor on remote".format(path),
+            cause=cause,
+        )
+
+
+class DvcIgnoreInCollectedDirError(DvcException):
+    def __init__(self, ignore_dirname):
+        super(DvcIgnoreInCollectedDirError, self).__init__(
+            ".dvcignore file should not be in collected dir path: "
+            "'{}'".format(ignore_dirname)
+        )
+
+
+class UrlNotDvcRepoError(DvcException):
+    def __init__(self, url):
+        super(UrlNotDvcRepoError, self).__init__(
+            "URL '{}' is not a dvc repository.".format(url)
+        )
+
+
+class GetDVCFileError(DvcException):
+    def __init__(self):
+        super(GetDVCFileError, self).__init__(
+            "the given path is a DVC-file, you must specify a data file "
+            "or a directory"
+        )
+
+
+class GitHookAlreadyExistsError(DvcException):
+    def __init__(self, hook_name):
+        super(GitHookAlreadyExistsError, self).__init__(
+            "Hook '{}' already exists. Please refer to "
+            "https://man.dvc.org/install "
+            "for more info.".format(hook_name)
+        )
+
+
+class DownloadError(DvcException):
+    def __init__(self, amount):
+        self.amount = amount
+
+        super(DownloadError, self).__init__(
+            "{amount} files failed to download".format(amount=amount)
+        )
+
+
+class UploadError(DvcException):
+    def __init__(self, amount):
+        self.amount = amount
+
+        super(UploadError, self).__init__(
+            "{amount} files failed to upload".format(amount=amount)
+        )
+
+
+class CheckoutError(DvcException):
+    def __init__(self, target_infos):
+        targets = [str(t) for t in target_infos]
+        m = (
+            "Checkout failed for following targets:\n {}\nDid you "
+            "forget to fetch?".format("\n".join(targets))
+        )
+        super(CheckoutError, self).__init__(m)
+
+
+class CollectCacheError(DvcException):
+    pass

dvc/external_repo.py

@@ -0,0 +1,91 @@
+from __future__ import unicode_literals
+
+import os
+import tempfile
+from distutils.dir_util import copy_tree
+
+from contextlib import contextmanager
+from funcy import retry
+
+from dvc.utils import remove
+
+
+REPO_CACHE = {}
+
+
+@contextmanager
+def external_repo(url=None, rev=None, rev_lock=None, cache_dir=None):
+    from dvc.repo import Repo
+
+    path = _external_repo(url=url, rev=rev_lock or rev, cache_dir=cache_dir)
+    repo = Repo(path)
+    yield repo
+    repo.close()
+
+
+def _external_repo(url=None, rev=None, cache_dir=None):
+    from dvc.config import Config
+    from dvc.cache import CacheConfig
+    from dvc.repo import Repo
+
+    key = (url, rev, cache_dir)
+    if key in REPO_CACHE:
+        return REPO_CACHE[key]
+
+    new_path = tempfile.mkdtemp("dvc-erepo")
+
+    # Copy and adjust existing clone
+    if (url, None, None) in REPO_CACHE:
+        old_path = REPO_CACHE[url, None, None]
+
+        # This one unlike shutil.copytree() works with an existing dir
+        copy_tree(old_path, new_path)
+    else:
+        # Create a new clone
+        _clone_repo(url, new_path)
+
+        # Save clean clone dir so that we will have access to a default branch
+        clean_clone_path = tempfile.mkdtemp("dvc-erepo")
+        copy_tree(new_path, clean_clone_path)
+        REPO_CACHE[url, None, None] = clean_clone_path
+
+    # Adjust new clone/copy to fit rev and cache_dir
+    repo = Repo(new_path)
+    try:
+        if rev is not None:
+            repo.scm.checkout(rev)
+
+        if cache_dir is not None:
+            cache_config = CacheConfig(repo.config)
+            cache_config.set_dir(cache_dir, level=Config.LEVEL_LOCAL)
+    finally:
+        # Need to close/reopen repo to force config reread
+        repo.close()
+
+    REPO_CACHE[key] = new_path
+    return new_path
+
+
+def clean_repos():
+    # Outside code should not see cache while we are removing
+    repo_paths = list(REPO_CACHE.values())
+    REPO_CACHE.clear()
+
+    for path in repo_paths:
+        _remove(path)
+
+
+def _remove(path):
+    if os.name == "nt":
+        # git.exe may hang for a while not permitting to remove temp dir
+        os_retry = retry(5, errors=OSError, timeout=0.1)
+        os_retry(remove)(path)
+    else:
+        remove(path)
+
+
+def _clone_repo(url, path):
+    from dvc.scm.git import Git
+
+    git = Git.clone(url, path)
+    git.close()

dvc/ignore.py

@@ -1,31 +1,15 @@
+from __future__ import unicode_literals
+
+import logging
 import os
+from pathspec import PathSpec
+from pathspec.patterns import GitWildMatchPattern
 
-from dulwich.ignore import match_pattern, read_ignore_patterns
-from dvc.utils.compat import cast_bytes
-from dvc.utils.fs import get_parent_dirs_up_to
+from dvc.utils import relpath
 
+from dvc.utils.compat import open
 
-class DvcIgnoreFileHandler(object):
-    def __init__(self, tree):
-        self.tree = tree
-
-    def read_patterns(self, path):
-        with self.tree.open(path, binary=True) as stream:
-            return self._read_patterns(stream)
-
-    def get_repo_root(self):
-        return self.tree.tree_root
-
-    def _read_patterns(self, binary_stream):
-        negate_patterns = []
-        patterns = []
-        for pattern in read_ignore_patterns(binary_stream):
-            if pattern.lstrip().startswith(b"!"):
-                negate_patterns.append(pattern)
-            else:
-                patterns.append(pattern)
-
-        return negate_patterns, patterns
+logger = logging.getLogger(__name__)
 
 
 class DvcIgnore(object):
@@ -35,16 +19,15 @@ class DvcIgnore(object):
         raise NotImplementedError
 
 
-class DvcIgnoreFromFile(DvcIgnore):
-    def __init__(self, ignore_file_path, ignore_handler):
+class DvcIgnorePatterns(DvcIgnore):
+    def __init__(self, ignore_file_path):
+        assert os.path.isabs(ignore_file_path)
+
         self.ignore_file_path = ignore_file_path
         self.dirname = os.path.normpath(os.path.dirname(ignore_file_path))
-        self.patterns = []
-        self.negate_patterns = []
 
-        self.negate_patterns, self.patterns = ignore_handler.read_patterns(
-            ignore_file_path
-        )
+        with open(ignore_file_path, encoding="utf-8") as fobj:
+            self.ignore_spec = PathSpec.from_lines(GitWildMatchPattern, fobj)
 
     def __call__(self, root, dirs, files):
         files = [f for f in files if not self.matches(root, f)]
@@ -52,82 +35,45 @@ class DvcIgnoreFromFile(DvcIgnore):
 
         return dirs, files
 
-    def get_match(self, abs_path):
-        rel_path = os.path.relpath(abs_path, self.dirname)
-        if os.name == "nt":
-            rel_path = rel_path.replace("\\", "/")
-        rel_path = cast_bytes(rel_path, "utf-8")
-
-        for pattern in self.patterns:
-            if match_pattern(
-                rel_path, pattern
-            ) and self._no_negate_pattern_matches(rel_path):
-                return (abs_path, pattern, self.ignore_file_path)
-        return None
-
     def matches(self, dirname, basename):
-        if self.get_match(os.path.join(dirname, basename)):
-            return True
-        return False
+        abs_path = os.path.join(dirname, basename)
+        rel_path = relpath(abs_path, self.dirname)
 
-    def _no_negate_pattern_matches(self, path):
-        return all([not match_pattern(path, p) for p in self.negate_patterns])
+        if os.pardir + os.sep in rel_path:
+            return False
+        return self.ignore_spec.match_file(rel_path)
 
     def __hash__(self):
         return hash(self.ignore_file_path)
 
-
-class DvcIgnoreConstant(DvcIgnore):
-    def __init__(self, basename):
-        self.basename = basename
+    def __eq__(self, other):
+        return self.ignore_file_path == other.ignore_file_path
 
 
-class DvcIgnoreDir(DvcIgnoreConstant):
+class DvcIgnoreDirs(DvcIgnore):
+    def __init__(self, basenames):
+        self.basenames = set(basenames)
+
     def __call__(self, root, dirs, files):
-        dirs = [d for d in dirs if not d == self.basename]
-
-        return dirs, files
-
-
-class DvcIgnoreFile(DvcIgnoreConstant):
-    def __call__(self, root, dirs, files):
-        files = [f for f in files if not f == self.basename]
+        dirs = [d for d in dirs if d not in self.basenames]
 
         return dirs, files
 
 
 class DvcIgnoreFilter(object):
-    def __init__(self, wdir, ignore_file_handler=None):
-        self.ignores = [
-            DvcIgnoreDir(".git"),
-            DvcIgnoreDir(".hg"),
-            DvcIgnoreDir(".dvc"),
-            DvcIgnoreFile(".dvcignore"),
-        ]
+    def __init__(self, root_dir):
+        self.ignores = {DvcIgnoreDirs([".git", ".hg", ".dvc"])}
+        self._update(root_dir)
+        for root, dirs, _ in os.walk(root_dir):
+            for d in dirs:
+                self._update(os.path.join(root, d))
 
-        self.ignore_file_handler = ignore_file_handler
-        self._process_ignores_in_parent_dirs(wdir)
-
-    def _process_ignores_in_parent_dirs(self, wdir):
-        if self.ignore_file_handler:
-            wdir = os.path.normpath(os.path.abspath(wdir))
-            ignore_search_end_dir = self.ignore_file_handler.get_repo_root()
-            parent_dirs = get_parent_dirs_up_to(wdir, ignore_search_end_dir)
-            for d in parent_dirs:
-                self.update(d)
-
-    def update(self, wdir):
-        ignore_file_path = os.path.join(wdir, DvcIgnore.DVCIGNORE_FILE)
+    def _update(self, dirname):
+        ignore_file_path = os.path.join(dirname, DvcIgnore.DVCIGNORE_FILE)
         if os.path.exists(ignore_file_path):
-            file_ignore = DvcIgnoreFromFile(
-                ignore_file_path, ignore_handler=self.ignore_file_handler
-            )
-            self.ignores.append(file_ignore)
+            self.ignores.add(DvcIgnorePatterns(ignore_file_path))
 
     def __call__(self, root, dirs, files):
-        if self.ignore_file_handler:
-            self.update(root)
-
         for ignore in self.ignores:
             dirs, files = ignore(root, dirs, files)
 

dvc/lock.py

@@ -3,58 +3,132 @@
 from __future__ import unicode_literals
 
 import os
+import hashlib
 import time
-import zc.lockfile
+from datetime import timedelta
+
+from funcy.py3 import lkeep
 
 from dvc.exceptions import DvcException
+from dvc.utils import makedirs
+from dvc.utils.compat import is_py3
+
+
+DEFAULT_TIMEOUT = 5
 
 
 class LockError(DvcException):
     """Thrown when unable to acquire the lock for dvc repo."""
 
 
-class Lock(object):
-    """Class for dvc repo lock.
+if is_py3:
+    import flufl.lock
 
-    Args:
-        dvc_dir (str): path to the directory that the lock should be created
-            in.
-        name (str): name of the lock file.
-    """
+    class Lock(flufl.lock.Lock):
+        """Class for dvc repo lock.
 
-    LOCK_FILE = "lock"
-    TIMEOUT = 5
+        Args:
+            lockfile (str): the lock filename
+                in.
+            tmp_dir (str): a directory to store claim files.
+        """
 
-    def __init__(self, dvc_dir, name=LOCK_FILE):
-        self.lock_file = os.path.join(dvc_dir, name)
-        self._lock = None
+        def __init__(self, lockfile, tmp_dir=None):
+            import socket
 
-    def _do_lock(self):
-        try:
-            self._lock = zc.lockfile.LockFile(self.lock_file)
-        except zc.lockfile.LockError:
-            raise LockError(
-                "cannot perform the cmd since DVC is busy and "
-                "locked. Please retry the cmd later."
-            )
+            self._tmp_dir = tmp_dir
+            if self._tmp_dir is not None:
+                makedirs(self._tmp_dir, exist_ok=True)
+
+            # NOTE: this is basically Lock.__init__ copy-paste, except that
+            # instead of using `socket.getfqdn()` we use `socket.gethostname()`
+            # to speed this up. We've seen [1] `getfqdn()` take ~5sec to return
+            # anything, which is way too slow. `gethostname()` is actually a
+            # fallback for `getfqdn()` when it is not able to resolve a
+            # canonical hostname through network. The claimfile that uses
+            # `self._hostname` is still usable, as it uses `pid` and random
+            # number to generate the resulting lock file name, which is unique
+            # enough for our application.
+            #
+            # [1] https://github.com/iterative/dvc/issues/2582
+            self._hostname = socket.gethostname()
+
+            self._lockfile = lockfile
+            self._lifetime = timedelta(days=365)  # Lock for good by default
+            self._separator = flufl.lock.SEP
+            self._set_claimfile()
+            self._owned = True
+            self._retry_errnos = []
+
+        @property
+        def lockfile(self):
+            return self._lockfile
+
+        @property
+        def files(self):
+            return lkeep([self._lockfile, self._tmp_dir])
+
+        def lock(self):
+            try:
+                super(Lock, self).lock(timedelta(seconds=DEFAULT_TIMEOUT))
+            except flufl.lock.TimeOutError:
+                raise LockError(
+                    "cannot perform the cmd since DVC is busy and "
+                    "locked. Please retry the cmd later."
+                )
+
+        def _set_claimfile(self, pid=None):
+            super(Lock, self)._set_claimfile(pid)
+
+            if self._tmp_dir is not None:
+                # Under Windows file path length is limited so we hash it
+                filename = hashlib.md5(self._claimfile.encode()).hexdigest()
+                self._claimfile = os.path.join(
+                    self._tmp_dir, filename + ".lock"
+                )
+
+
+else:
+    import zc.lockfile
+
+    class Lock(object):
+        """Class for dvc repo lock.
+
+        Uses zc.lockfile as backend.
+        """
+
+        def __init__(self, lockfile, tmp_dir=None):
+            self.lockfile = lockfile
+            self._lock = None
+
+        @property
+        def files(self):
+            return [self.lockfile]
+
+        def _do_lock(self):
+            try:
+                self._lock = zc.lockfile.LockFile(self.lockfile)
+            except zc.lockfile.LockError:
+                raise LockError(
+                    "cannot perform the cmd since DVC is busy and "
+                    "locked. Please retry the cmd later."
+                )
+
+        def lock(self):
+            try:
+                self._do_lock()
+                return
+            except LockError:
+                time.sleep(DEFAULT_TIMEOUT)
 
-    def lock(self):
-        """Acquire lock for dvc repo."""
-        try:
             self._do_lock()
-            return
-        except LockError:
-            time.sleep(self.TIMEOUT)
 
-        self._do_lock()
+        def unlock(self):
+            self._lock.close()
+            self._lock = None
 
-    def unlock(self):
-        """Release lock for dvc repo."""
-        self._lock.close()
-        self._lock = None
+        def __enter__(self):
+            self.lock()
 
-    def __enter__(self):
-        self.lock()
-
-    def __exit__(self, typ, value, tbck):
-        self.unlock()
+        def __exit__(self, typ, value, tbck):
+            self.unlock()

dvc/logger.py

@@ -2,7 +2,8 @@
 
 from __future__ import unicode_literals
 
-from dvc.utils.compat import str, StringIO
+from dvc.utils.compat import str, StringIO, RecursionError
+from dvc.progress import Tqdm
 
 import logging
 import logging.handlers
@@ -10,9 +11,15 @@ import logging.config
 import colorama
 
 
+class LoggingException(Exception):
+    def __init__(self, record):
+        msg = "failed to log {}".format(str(record))
+        super(LoggingException, self).__init__(msg)
+
+
 class ExcludeErrorsFilter(logging.Filter):
     def filter(self, record):
-        return record.levelno < logging.ERROR
+        return record.levelno < logging.WARNING
 
 
 class ColorFormatter(logging.Formatter):
@@ -37,7 +44,7 @@ class ColorFormatter(logging.Formatter):
     }
 
     footer = (
-        "{yellow}Having any troubles?{nc}."
+        "{yellow}Having any troubles?{nc}"
         " Hit us up at {blue}https://dvc.org/support{nc},"
         " we are always happy to help!"
     ).format(
@@ -47,9 +54,6 @@ class ColorFormatter(logging.Formatter):
     )
 
     def format(self, record):
-        if self._is_visible(record):
-            self._progress_aware()
-
         if record.levelname == "INFO":
             return record.msg
 
@@ -140,13 +144,24 @@ class ColorFormatter(logging.Formatter):
 
         return (exception, stack_trace)
 
-    def _progress_aware(self):
-        """Add a new line if progress bar hasn't finished"""
-        from dvc.progress import progress
 
-        if not progress.is_finished:
-            progress._print()
-        progress.clearln()
+class LoggerHandler(logging.StreamHandler):
+    def handleError(self, record):
+        super(LoggerHandler, self).handleError(record)
+        raise LoggingException(record)
+
+    def emit(self, record):
+        """Write to Tqdm's stream so as to not break progressbars"""
+        try:
+            msg = self.format(record)
+            Tqdm.write(
+                msg, file=self.stream, end=getattr(self, "terminator", "\n")
+            )
+            self.flush()
+        except RecursionError:
+            raise
+        except Exception:
+            self.handleError(record)
 
 
 def setup(level=logging.INFO):
@@ -159,15 +174,15 @@ def setup(level=logging.INFO):
             "formatters": {"color": {"()": ColorFormatter}},
             "handlers": {
                 "console": {
-                    "class": "logging.StreamHandler",
+                    "class": "dvc.logger.LoggerHandler",
                     "level": "DEBUG",
                     "formatter": "color",
                     "stream": "ext://sys.stdout",
                     "filters": ["exclude_errors"],
                 },
                 "console_errors": {
-                    "class": "logging.StreamHandler",
-                    "level": "ERROR",
+                    "class": "dvc.logger.LoggerHandler",
+                    "level": "WARNING",
                     "formatter": "color",
                     "stream": "ext://sys.stderr",
                 },
@@ -176,7 +191,15 @@ def setup(level=logging.INFO):
                 "dvc": {
                     "level": level,
                     "handlers": ["console", "console_errors"],
-                }
+                },
+                "paramiko": {
+                    "level": logging.CRITICAL,
+                    "handlers": ["console", "console_errors"],
+                },
+                "flufl.lock": {
+                    "level": logging.CRITICAL,
+                    "handlers": ["console", "console_errors"],
+                },
             },
         }
     )

dvc/main.py

@@ -5,8 +5,13 @@ from __future__ import unicode_literals
 import logging
 
 from dvc.cli import parse_args
+from dvc.lock import LockError
+from dvc.config import ConfigError
 from dvc.analytics import Analytics
 from dvc.exceptions import NotDvcRepoError, DvcParserError
+from dvc.remote.pool import close_pools
+from dvc.external_repo import clean_repos
+from dvc.utils.compat import is_py2
 
 
 logger = logging.getLogger("dvc")
@@ -35,7 +40,13 @@ def main(argv=None):
             logger.setLevel(logging.DEBUG)
 
         cmd = args.func(args)
-        ret = cmd.run_cmd()
+        ret = cmd.run()
+    except LockError:
+        logger.exception("failed to lock before running a command")
+        ret = 250
+    except ConfigError:
+        logger.exception("configuration error")
+        ret = 251
     except KeyboardInterrupt:
         logger.exception("interrupted by the user")
         ret = 252
@@ -44,12 +55,26 @@ def main(argv=None):
         ret = 253
     except DvcParserError:
         ret = 254
-    except Exception:  # pylint: disable=broad-except
-        logger.exception("unexpected error")
+    except Exception as exc:  # pylint: disable=broad-except
+        if isinstance(exc, UnicodeError) and is_py2:
+            logger.exception(
+                "unicode is not supported in DVC for Python 2 "
+                "(end-of-life January 1, 2020), please upgrade to Python 3"
+            )
+        else:
+            logger.exception("unexpected error")
         ret = 255
     finally:
         logger.setLevel(outerLogLevel)
 
+        # Python 2 fails to close these clean occasionally and users see
+        # weird error messages, so we do it manually
+        close_pools()
+
+        # Remove cached repos in the end of the call, these are anonymous
+        # so won't be reused by any other subsequent run anyway.
+        clean_repos()
+
     Analytics().send_cmd(cmd, args, ret)
 
     return ret

dvc/output/__init__.py

@@ -61,8 +61,7 @@ def _get(stage, p, info, cache, metric, persist=False, tags=None):
     parsed = urlparse(p)
 
     if parsed.scheme == "remote":
-        settings = stage.repo.config.get_remote_settings(parsed.netloc)
-        remote = Remote(stage.repo, settings)
+        remote = Remote(stage.repo, name=parsed.netloc)
         return OUTS_MAP[remote.scheme](
             stage,
             p,

dvc/output/base.py

@@ -1,13 +1,14 @@
 from __future__ import unicode_literals
 
-import re
 import logging
 from copy import copy
 
 from schema import Or, Optional
 
-from dvc.exceptions import DvcException
-from dvc.utils.compat import str
+from dvc.cache import NamedCache
+import dvc.prompt as prompt
+from dvc.exceptions import DvcException, CollectCacheError
+from dvc.utils.compat import str, urlparse
 from dvc.remote.base import RemoteBASE
 
 
@@ -32,6 +33,13 @@ class OutputAlreadyTrackedError(DvcException):
         super(OutputAlreadyTrackedError, self).__init__(msg)
 
 
+class OutputIsStageFileError(DvcException):
+    def __init__(self, path):
+        super(OutputIsStageFileError, self).__init__(
+            "Stage file '{}' cannot be an output.".format(path)
+        )
+
+
 class OutputBase(object):
     IS_DEPENDENCY = False
 
@@ -57,6 +65,9 @@ class OutputBase(object):
 
     DoesNotExistError = OutputDoesNotExistError
     IsNotFileOrDirError = OutputIsNotFileOrDirError
+    IsStageFileError = OutputIsStageFileError
+
+    sep = "/"
 
     def __init__(
         self,
@@ -69,9 +80,20 @@ class OutputBase(object):
         persist=False,
         tags=None,
     ):
+        self._validate_output_path(path)
+        # This output (and dependency) objects have too many paths/urls
+        # here is a list and comments:
+        #
+        #   .def_path - path from definition in stage file
+        #   .path_info - PathInfo/URLInfo structured resolved path
+        #   .fspath - local only, resolved
+        #   .__str__ - for presentation purposes, def_path/relpath
+        #
+        # By resolved path, which contains actual location,
+        # should be absolute and don't contain remote:// refs.
         self.stage = stage
-        self.repo = stage.repo
-        self.url = path
+        self.repo = stage.repo if stage else None
+        self.def_path = path
         self.info = info
         self.remote = remote or self.REMOTE(self.repo, {})
         self.use_cache = False if self.IS_DEPENDENCY else cache
@@ -86,18 +108,37 @@ class OutputBase(object):
                 )
             )
 
+        self.path_info = self._parse_path(remote, path)
+
+    def _parse_path(self, remote, path):
+        if remote:
+            parsed = urlparse(path)
+            return remote.path_info / parsed.path.lstrip("/")
+        return self.REMOTE.path_cls(path)
+
     def __repr__(self):
-        return "{class_name}: '{url}'".format(
-            class_name=type(self).__name__, url=(self.url or "No url")
+        return "{class_name}: '{def_path}'".format(
+            class_name=type(self).__name__, def_path=self.def_path
         )
 
     def __str__(self):
-        return self.url
+        return self.def_path
+
+    @property
+    def scheme(self):
+        return self.REMOTE.scheme
 
     @property
     def is_in_repo(self):
         return False
 
+    @property
+    def use_scm_ignore(self):
+        if not self.is_in_repo:
+            return False
+
+        return self.use_cache or self.stage.is_repo_import
+
     @property
     def cache(self):
         return getattr(self.repo.cache, self.scheme)
@@ -106,40 +147,13 @@ class OutputBase(object):
     def dir_cache(self):
         return self.cache.get_dir_cache(self.checksum)
 
-    def assign_to_stage_file(self, target_repo):
-        raise DvcException(
-            "change repo is not supported for {}".format(self.scheme)
-        )
-
-    @classmethod
-    def match(cls, url):
-        return re.match(cls.REMOTE.REGEX, url)
-
-    def group(self, name):
-        match = self.match(self.url)
-        if not match:
-            return None
-        return match.group(name)
-
     @classmethod
     def supported(cls, url):
-        return cls.match(url) is not None
-
-    @property
-    def scheme(self):
-        return self.REMOTE.scheme
-
-    @property
-    def path(self):
-        return self.path_info.path
+        return cls.REMOTE.supported(url)
 
     @property
     def cache_path(self):
-        return self.cache.checksum_to_path(self.checksum)
-
-    @property
-    def sep(self):
-        return "/"
+        return self.cache.checksum_to_path_info(self.checksum).url
 
     @property
     def checksum(self):
@@ -207,6 +221,12 @@ class OutputBase(object):
         if self.is_empty:
             logger.warning("'{}' is empty.".format(self))
 
+        if self.use_scm_ignore:
+            if self.repo.scm.is_tracked(self.fspath):
+                raise OutputAlreadyTrackedError(self)
+
+            self.repo.scm.ignore(self.fspath)
+
         if not self.use_cache:
             self.info = self.remote.save_info(self.path_info)
             if self.metric:
@@ -227,13 +247,6 @@ class OutputBase(object):
             )
             return
 
-        if self.is_in_repo:
-            if self.repo.scm.is_tracked(self.path):
-                raise OutputAlreadyTrackedError(self)
-
-            if self.use_cache:
-                self.repo.scm.ignore(self.path)
-
         self.info = self.remote.save_info(self.path_info)
 
     def commit(self):
@@ -242,7 +255,7 @@ class OutputBase(object):
 
     def dumpd(self):
         ret = copy(self.info)
-        ret[self.PARAM_PATH] = self.url
+        ret[self.PARAM_PATH] = self.def_path
 
         if self.IS_DEPENDENCY:
             return ret
@@ -269,19 +282,20 @@ class OutputBase(object):
             "verify metric is not supported for {}".format(self.scheme)
         )
 
-    def download(self, to_info, resume=False):
-        self.remote.download([self.path_info], [to_info], resume=resume)
+    def download(self, to):
+        self.remote.download(self.path_info, to.path_info)
 
     def checkout(self, force=False, progress_callback=None, tag=None):
         if not self.use_cache:
-            return
+            progress_callback(str(self.path_info), self.get_files_number())
+            return None
 
         if tag:
             info = self.tags[tag]
         else:
             info = self.info
 
-        self.cache.checkout(
+        return self.cache.checkout(
             self.path_info,
             info,
             force=force,
@@ -293,31 +307,118 @@ class OutputBase(object):
         if self.scheme != "local":
             return
 
-        if ignore_remove and self.use_cache and self.is_in_repo:
-            self.repo.scm.ignore_remove(self.path)
+        if ignore_remove and self.use_scm_ignore:
+            self.repo.scm.ignore_remove(self.fspath)
 
     def move(self, out):
-        if self.scheme == "local" and self.use_cache and self.is_in_repo:
-            self.repo.scm.ignore_remove(self.path)
+        if self.scheme == "local" and self.use_scm_ignore:
+            self.repo.scm.ignore_remove(self.fspath)
 
         self.remote.move(self.path_info, out.path_info)
-        self.url = out.url
-        self.path_info = copy(out.path_info)
+        self.def_path = out.def_path
+        self.path_info = out.path_info
         self.save()
         self.commit()
 
-        if self.scheme == "local" and self.use_cache and self.is_in_repo:
-            self.repo.scm.ignore(self.path)
+        if self.scheme == "local" and self.use_scm_ignore:
+            self.repo.scm.ignore(self.fspath)
 
     def get_files_number(self):
-        if not self.use_cache or not self.checksum:
+        if not self.use_cache:
             return 0
 
-        if self.is_dir_checksum:
-            return len(self.dir_cache)
-
-        return 1
+        return self.cache.get_files_number(self.checksum)
 
     def unprotect(self):
         if self.exists:
             self.remote.unprotect(self.path_info)
+
+    def _collect_used_dir_cache(self, remote=None, force=False, jobs=None):
+        """Get a list of `info`s retaled to the given directory.
+
+        - Pull the directory entry from the remote cache if it was changed.
+
+        Example:
+
+            Given the following commands:
+
+            $ echo "foo" > directory/foo
+            $ echo "bar" > directory/bar
+            $ dvc add directory
+
+            It will return a NamedCache like:
+
+            nc = NamedCache()
+            nc.add(self.scheme, 'c157a79031e1', 'directory/foo')
+            nc.add(self.scheme, 'd3b07384d113', 'directory/bar')
+        """
+
+        cache = NamedCache()
+
+        if self.cache.changed_cache_file(self.checksum):
+            try:
+                self.repo.cloud.pull(
+                    NamedCache.make("local", self.checksum, str(self)),
+                    jobs=jobs,
+                    remote=remote,
+                    show_checksums=False,
+                )
+            except DvcException:
+                logger.debug("failed to pull cache for '{}'".format(self))
+
+        if self.cache.changed_cache_file(self.checksum):
+            msg = (
+                "Missing cache for directory '{}'. "
+                "Cache for files inside will be lost. "
+                "Would you like to continue? Use '-f' to force."
+            )
+            if not force and not prompt.confirm(msg.format(self.path_info)):
+                raise CollectCacheError(
+                    "unable to fully collect used cache"
+                    " without cache for directory '{}'".format(self)
+                )
+            else:
+                return cache
+
+        for entry in self.dir_cache:
+            checksum = entry[self.remote.PARAM_CHECKSUM]
+            path_info = self.path_info / entry[self.remote.PARAM_RELPATH]
+            cache.add(self.scheme, checksum, str(path_info))
+
+        return cache
+
+    def get_used_cache(self, **kwargs):
+        """Get a dumpd of the given `out`, with an entry including the branch.
+
+        The `used_cache` of an output is no more than its `info`.
+
+        In case that the given output is a directory, it will also
+        include the `info` of its files.
+        """
+
+        if not self.use_cache:
+            return NamedCache()
+
+        if not self.info:
+            logger.warning(
+                "Output '{}'({}) is missing version info. Cache for it will "
+                "not be collected. Use dvc repro to get your pipeline up to "
+                "date.".format(self, self.stage)
+            )
+            return NamedCache()
+
+        ret = NamedCache.make(self.scheme, self.checksum, str(self))
+
+        if not self.is_dir_checksum:
+            return ret
+
+        ret.update(self._collect_used_dir_cache(**kwargs))
+
+        return ret
+
+    @classmethod
+    def _validate_output_path(cls, path):
+        from dvc.stage import Stage
+
+        if Stage.is_valid_filename(path):
+            raise cls.IsStageFileError(path)

dvc/output/hdfs.py

@@ -1,38 +1,8 @@
 from __future__ import unicode_literals
 
-import posixpath
-
-from dvc.path.hdfs import PathHDFS
-from dvc.utils.compat import urlparse
 from dvc.output.base import OutputBase
 from dvc.remote.hdfs import RemoteHDFS
 
 
 class OutputHDFS(OutputBase):
     REMOTE = RemoteHDFS
-
-    def __init__(
-        self,
-        stage,
-        path,
-        info=None,
-        remote=None,
-        cache=True,
-        metric=False,
-        persist=False,
-        tags=None,
-    ):
-        super(OutputHDFS, self).__init__(
-            stage,
-            path,
-            info=info,
-            remote=remote,
-            cache=cache,
-            metric=metric,
-            persist=persist,
-            tags=tags,
-        )
-        if remote:
-            path = posixpath.join(remote.url, urlparse(path).path.lstrip("/"))
-        user = remote.user if remote else self.group("user")
-        self.path_info = PathHDFS(user=user, url=self.url, path=path)

dvc/output/local.py

@@ -3,8 +3,7 @@ from __future__ import unicode_literals
 import os
 import logging
 
-from dvc.path.local import PathLOCAL
-from dvc.utils.compat import urlparse
+from dvc.utils.compat import urlparse, fspath_py35, str
 from dvc.istextfile import istextfile
 from dvc.exceptions import DvcException
 from dvc.remote.local import RemoteLOCAL
@@ -16,82 +15,44 @@ logger = logging.getLogger(__name__)
 
 class OutputLOCAL(OutputBase):
     REMOTE = RemoteLOCAL
+    sep = os.sep
 
-    def __init__(
-        self,
-        stage,
-        path,
-        info=None,
-        remote=None,
-        cache=True,
-        metric=False,
-        persist=False,
-        tags=None,
-    ):
-        super(OutputLOCAL, self).__init__(
-            stage,
-            path,
-            info,
-            remote=remote,
-            cache=cache,
-            metric=metric,
-            persist=persist,
-            tags=tags,
-        )
-        if remote:
-            p = os.path.join(
-                remote.prefix, urlparse(self.url).path.lstrip("/")
-            )
+    def _parse_path(self, remote, path):
+        parsed = urlparse(path)
+        if parsed.scheme == "remote":
+            p = remote.path_info / parsed.path.lstrip("/")
         else:
-            p = path
+            # NOTE: we can path either from command line or .dvc file,
+            # so we should expect both posix and windows style paths.
+            # PathInfo accepts both, i.e. / works everywhere, \ only on win.
+            #
+            # FIXME: if we have Windows path containig / or posix one with \
+            # then we have #2059 bug and can't really handle that.
+            p = self.REMOTE.path_cls(path)
+            if not p.is_absolute():
+                p = self.stage.wdir / p
 
-        if not os.path.isabs(p):
-            p = self.remote.to_ospath(p)
-            p = os.path.join(stage.wdir, p)
-        p = os.path.abspath(os.path.normpath(p))
-
-        self.path_info = PathLOCAL(url=self.url, path=p)
+        abs_p = os.path.abspath(os.path.normpath(fspath_py35(p)))
+        return self.REMOTE.path_cls(abs_p)
 
     def __str__(self):
-        return self.rel_path
+        return str(self.path_info)
+
+    @property
+    def fspath(self):
+        return self.path_info.fspath
 
     @property
     def is_in_repo(self):
-        return urlparse(self.url).scheme != "remote" and not os.path.isabs(
-            self.url
-        )
-
-    def assign_to_stage_file(self, stage):
-        from dvc.repo import Repo
-
-        fullpath = os.path.abspath(stage.wdir)
-        self.path_info.path = os.path.join(fullpath, self.stage_path)
-
-        self.repo = Repo(self.path)
-
-        self.stage = stage
-        return self
-
-    @property
-    def sep(self):
-        return os.sep
-
-    @property
-    def rel_path(self):
-        return os.path.relpath(self.path)
-
-    @property
-    def stage_path(self):
-        return os.path.relpath(self.path, self.stage.wdir)
+        def_scheme = urlparse(self.def_path).scheme
+        return def_scheme != "remote" and not os.path.isabs(self.def_path)
 
     def dumpd(self):
         ret = super(OutputLOCAL, self).dumpd()
         if self.is_in_repo:
-            path = self.remote.to_posixpath(
-                os.path.relpath(self.path, self.stage.wdir)
-            )
+            path = self.path_info.relpath(self.stage.wdir).as_posix()
         else:
-            path = self.url
+            path = self.def_path
 
         ret[self.PARAM_PATH] = path
 
@@ -101,13 +62,14 @@ class OutputLOCAL(OutputBase):
         if not self.metric:
             return
 
-        if not os.path.exists(self.path):
+        path = fspath_py35(self.path_info)
+        if not os.path.exists(path):
             return
 
-        if os.path.isdir(self.path):
+        if os.path.isdir(path):
             msg = "directory '{}' cannot be used as metrics."
-            raise DvcException(msg.format(self.rel_path))
+            raise DvcException(msg.format(self.path_info))
 
-        if not istextfile(self.path):
+        if not istextfile(path):
             msg = "binary file '{}' cannot be used as metrics."
-            raise DvcException(msg.format(self.rel_path))
+            raise DvcException(msg.format(self.path_info))

dvc/output/s3.py

@@ -1,42 +1,8 @@
 from __future__ import unicode_literals
 
-import posixpath
-
-from dvc.path import Path
 from dvc.remote.s3 import RemoteS3
-from dvc.utils.compat import urlparse
 from dvc.output.base import OutputBase
 
 
 class OutputS3(OutputBase):
     REMOTE = RemoteS3
-
-    def __init__(
-        self,
-        stage,
-        path,
-        info=None,
-        remote=None,
-        cache=True,
-        metric=False,
-        persist=False,
-        tags=None,
-    ):
-        super(OutputS3, self).__init__(
-            stage,
-            path,
-            info=info,
-            remote=remote,
-            cache=cache,
-            metric=metric,
-            persist=persist,
-            tags=tags,
-        )
-        bucket = remote.bucket if remote else urlparse(path).netloc
-        path = urlparse(path).path.lstrip("/")
-        if remote:
-            path = posixpath.join(remote.prefix, path)
-
-        self.path_info = Path(
-            self.scheme, bucket=bucket, path=path, url=self.url
-        )

dvc/output/ssh.py

@@ -1,54 +1,8 @@
 from __future__ import unicode_literals
 
-import getpass
-import posixpath
-
-from dvc.path.ssh import PathSSH
-from dvc.utils.compat import urlparse
 from dvc.output.base import OutputBase
 from dvc.remote.ssh import RemoteSSH
 
 
 class OutputSSH(OutputBase):
     REMOTE = RemoteSSH
-
-    def __init__(
-        self,
-        stage,
-        path,
-        info=None,
-        remote=None,
-        cache=True,
-        metric=False,
-        persist=False,
-        tags=None,
-    ):
-        super(OutputSSH, self).__init__(
-            stage,
-            path,
-            info=info,
-            remote=remote,
-            cache=cache,
-            metric=metric,
-            persist=persist,
-            tags=tags,
-        )
-        parsed = urlparse(path)
-        host = remote.host if remote else parsed.hostname
-        port = (
-            remote.port if remote else (parsed.port or RemoteSSH.DEFAULT_PORT)
-        )
-        user = (
-            remote.user if remote else (parsed.username or getpass.getuser())
-        )
-
-        if remote:
-            path = posixpath.join(
-                remote.prefix, urlparse(path).path.lstrip("/")
-            )
-        else:
-            path = parsed.path
-
-        self.path_info = PathSSH(
-            host=host, user=user, port=port, url=self.url, path=path
-        )

dvc/path/__init__.py

@@ -1,29 +0,0 @@
-from dvc.scheme import Schemes
-
-from dvc.path.azure import PathAZURE
-from dvc.path.gs import PathGS
-from dvc.path.hdfs import PathHDFS
-from dvc.path.http import PathHTTP
-from dvc.path.https import PathHTTPS
-from dvc.path.local import PathLOCAL
-from dvc.path.oss import PathOSS
-from dvc.path.s3 import PathS3
-from dvc.path.ssh import PathSSH
-
-
-PATH_MAP = {
-    Schemes.SSH: PathSSH,
-    Schemes.HDFS: PathHDFS,
-    Schemes.S3: PathS3,
-    Schemes.AZURE: PathAZURE,
-    Schemes.HTTP: PathHTTP,
-    Schemes.HTTPS: PathHTTPS,
-    Schemes.GS: PathGS,
-    Schemes.LOCAL: PathLOCAL,
-    Schemes.OSS: PathOSS,
-}
-
-
-def Path(scheme, *args, **kwargs):
-    cls = PATH_MAP[scheme]
-    return cls(*args, **kwargs)

dvc/path/azure.py

@@ -1,6 +0,0 @@
-from dvc.scheme import Schemes
-from .base import PathCloudBASE
-
-
-class PathAZURE(PathCloudBASE):
-    scheme = Schemes.AZURE

dvc/path/base.py

@@ -1,23 +0,0 @@
-from dvc.utils.compat import urlunsplit
-
-
-class PathBASE(object):
-    scheme = None
-
-    def __init__(self, url=None, path=None):
-        self.url = url
-        self.path = path
-
-    def __str__(self):
-        return self.url
-
-
-class PathCloudBASE(PathBASE):
-    def __init__(self, bucket, url=None, path=None):
-        super(PathCloudBASE, self).__init__(url, path)
-        self.bucket = bucket
-
-    def __str__(self):
-        if not self.url:
-            return urlunsplit((self.scheme, self.bucket, self.path, "", ""))
-        return self.url

dvc/path/gs.py

@@ -1,6 +0,0 @@
-from dvc.scheme import Schemes
-from .base import PathCloudBASE
-
-
-class PathGS(PathCloudBASE):
-    scheme = Schemes.GS

dvc/path/hdfs.py

@@ -1,13 +0,0 @@
-from dvc.scheme import Schemes
-from .base import PathBASE
-
-
-class PathHDFS(PathBASE):
-    scheme = Schemes.HDFS
-
-    def __init__(self, user, url=None, path=None):
-        super(PathHDFS, self).__init__(url, path)
-        self.user = user
-
-    def __str__(self):
-        return self.path

dvc/path/http.py

@@ -1,13 +0,0 @@
-from dvc.scheme import Schemes
-from dvc.utils.compat import urlunsplit
-
-from .base import PathBASE
-
-
-class PathHTTP(PathBASE):
-    scheme = Schemes.HTTP
-
-    def __str__(self):
-        if not self.url:
-            return urlunsplit((self.scheme, self.path, "", "", ""))
-        return self.url

dvc/path/https.py

@@ -1,6 +0,0 @@
-from dvc.scheme import Schemes
-from .http import PathHTTP
-
-
-class PathHTTPS(PathHTTP):
-    scheme = Schemes.HTTPS

dvc/path/local.py

@@ -1,17 +0,0 @@
-import os
-
-from dvc.scheme import Schemes
-from .base import PathBASE
-
-
-class PathLOCAL(PathBASE):
-    scheme = Schemes.LOCAL
-
-    def __str__(self):
-        if os.name == "nt" and not os.path.commonprefix(
-            [os.getcwd(), self.path]
-        ):
-            # In case of windows, when cache is on different drive than
-            # workspace, we will get ValueError when trying to get relpath
-            return self.path
-        return os.path.relpath(self.path)

dvc/path/oss.py

@@ -1,6 +0,0 @@
-from dvc.scheme import Schemes
-from .base import PathCloudBASE
-
-
-class PathOSS(PathCloudBASE):
-    scheme = Schemes.OSS

dvc/path/s3.py

@@ -1,6 +0,0 @@
-from dvc.scheme import Schemes
-from .base import PathCloudBASE
-
-
-class PathS3(PathCloudBASE):
-    scheme = Schemes.S3

dvc/path/ssh.py

@@ -1,26 +0,0 @@
-from dvc.utils.compat import urlunsplit
-from dvc.scheme import Schemes
-from .base import PathBASE
-
-
-class PathSSH(PathBASE):
-    scheme = Schemes.SSH
-
-    def __init__(self, host, user, port, url=None, path=None):
-        super(PathSSH, self).__init__(url, path)
-        self.host = host
-        self.user = user
-        self.port = port
-
-    def __str__(self):
-        if not self.url:
-            return urlunsplit(
-                (
-                    self.scheme,
-                    "{}@{}:{}".format(self.user, self.host, self.port),
-                    self.path,
-                    "",
-                    "",
-                )
-            )
-        return self.url

dvc/path_info.py

@@ -0,0 +1,276 @@
+from __future__ import unicode_literals, absolute_import
+
+import sys
+import os
+import posixpath
+
+from funcy import cached_property
+
+from dvc.utils.compat import str, builtin_str, basestring, is_py2
+from dvc.utils.compat import pathlib, urlparse
+
+
+# On Python 2.7/Windows sys.getfilesystemencoding() is set to mbcs,
+# which is lossy, thus we can't use that,
+# see https://github.com/mcmtroffaes/pathlib2/issues/56.
+from dvc.utils import relpath
+
+if is_py2:
+    fs_encoding = "utf-8"
+
+
+class PathInfo(pathlib.PurePath):
+    # Use __slots__ in PathInfo objects following PurePath implementation.
+    # This makes objects smaller and speeds up attribute access.
+    # We don't add any fields so it's empty.
+    __slots__ = ()
+    scheme = "local"
+
+    def __new__(cls, *args):
+        # Construct a proper subclass depending on current os
+        if cls is PathInfo:
+            cls = WindowsPathInfo if os.name == "nt" else PosixPathInfo
+        return cls._from_parts(args)
+
+    @classmethod
+    def from_posix(cls, s):
+        return cls(PosixPathInfo(s))
+
+    def as_posix(self):
+        f = self._flavour
+        # Unlike original implementation [1] that uses `str()` we actually need
+        # to use `fspath`, because we've overriden `__str__` method to return
+        # relative paths, which will break original `as_posix`.
+        #
+        # [1] https://github.com/python/cpython/blob/v3.7.0/Lib/pathlib.py#L692
+        return self.fspath.replace(f.sep, "/")
+
+    def __str__(self):
+        path = self.__fspath__()
+        return relpath(path)
+
+    def __repr__(self):
+        return builtin_str("{}: '{}'").format(type(self).__name__, self)
+
+    # This permits passing it to file utils directly in Python 3.6+
+    # With Python 2.7, Python 3.5+ we are stuck with path_info.fspath for now
+    def __fspath__(self):
+        return pathlib.PurePath.__str__(self)
+
+    @property
+    def fspath(self):
+        return self.__fspath__()
+
+    url = fspath
+
+    def relpath(self, other):
+        return self.__class__(relpath(self, other))
+
+    def isin(self, other):
+        if isinstance(other, basestring):
+            other = self.__class__(other)
+        elif self.__class__ != other.__class__:
+            return False
+        # Use cached casefolded parts to compare paths
+        n = len(other._cparts)
+        return len(self._cparts) > n and self._cparts[:n] == other._cparts
+
+    # pathlib2 uses bytes internally in Python 2, and we use unicode everywhere
+    # for paths in both pythons, thus we need this glue.
+    if is_py2:
+        __unicode__ = __str__
+
+        def __str__(self):
+            return self.__unicode__().encode(sys.getfilesystemencoding())
+
+        @classmethod
+        def _parse_args(cls, args):
+            args = [
+                a.encode(fs_encoding)
+                if isinstance(a, unicode)  # noqa: F821
+                else a
+                for a in args
+            ]
+            return super(PathInfo, cls)._parse_args(args)
+
+        @property
+        def name(self):
+            return super(PathInfo, self).name.decode(fs_encoding)
+
+        def __fspath__(self):  # noqa: F811
+            return pathlib.PurePath.__str__(self).decode(fs_encoding)
+
+        def with_name(self, name):
+            return pathlib.PurePath.with_name(self, name.encode(fs_encoding))
+
+
+class WindowsPathInfo(PathInfo, pathlib.PureWindowsPath):
+    pass
+
+
+class PosixPathInfo(PathInfo, pathlib.PurePosixPath):
+    pass
+
+
+class _URLPathParents(object):
+    def __init__(self, pathcls, scheme, netloc, path):
+        self._scheme = scheme
+        self._netloc = netloc
+        self._parents = path.parents
+        self._pathcls = pathcls
+
+    def __len__(self):
+        return len(self._parents)
+
+    def __getitem__(self, idx):
+        return self._pathcls.from_parts(
+            scheme=self._scheme,
+            netloc=self._netloc,
+            path=self._parents[idx].fspath,
+        )
+
+    def __repr__(self):
+        return "<{}.parents>".format(self._pathcls.__name__)
+
+
+class URLInfo(object):
+    DEFAULT_PORTS = {"http": 80, "https": 443, "ssh": 22, "hdfs": 0}
+
+    def __init__(self, url):
+        self.parsed = urlparse(url)
+        assert self.parsed.scheme != "remote"
+
+    @classmethod
+    def from_parts(
+        cls, scheme=None, netloc=None, host=None, user=None, port=None, path=""
+    ):
+        assert scheme and (bool(host) ^ bool(netloc))
+
+        if netloc is None:
+            netloc = host
+            if user:
+                netloc = user + "@" + host
+            if port:
+                netloc += ":" + str(port)
+        return cls("{}://{}{}".format(scheme, netloc, path))
+
+    @cached_property
+    def url(self):
+        p = self.parsed
+        return "{}://{}{}".format(p.scheme, self.netloc, p.path)
+
+    def __str__(self):
+        return self.url
+
+    def __repr__(self):
+        return "{}: '{}'".format(type(self).__name__, self)
+
+    def __eq__(self, other):
+        if isinstance(other, basestring):
+            other = self.__class__(other)
+        return (
+            self.__class__ == other.__class__
+            and self.scheme == other.scheme
+            and self.netloc == other.netloc
+            and self._path == other._path
+        )
+
+    def __hash__(self):
+        return hash(self.url)
+
+    def __div__(self, other):
+        p = self.parsed
+        new_path = posixpath.join(p.path, str(other))
+        if not new_path.startswith("/"):
+            new_path = "/" + new_path
+        new_url = "{}://{}{}".format(p.scheme, p.netloc, new_path)
+        return self.__class__(new_url)
+
+    __truediv__ = __div__
+
+    def __getattr__(self, name):
+        # When deepcopy is called, it creates and object without __init__,
+        # self.parsed is not initialized and it causes infinite recursion.
+        # More on this special casing here:
+        # https://stackoverflow.com/a/47300262/298182
+        if name.startswith("__"):
+            raise AttributeError(name)
+        return getattr(self.parsed, name)
+
+    @cached_property
+    def netloc(self):
+        p = self.parsed
+        netloc = p.hostname
+        if p.username:
+            netloc = p.username + "@" + netloc
+        if p.port and int(p.port) != self.DEFAULT_PORTS.get(p.scheme):
+            netloc += ":" + str(p.port)
+        return netloc
+
+    @property
+    def port(self):
+        return self.parsed.port or self.DEFAULT_PORTS.get(self.parsed.scheme)
+
+    @property
+    def host(self):
+        return self.parsed.hostname
+
+    @property
+    def user(self):
+        return self.parsed.username
+
+    @cached_property
+    def _path(self):
+        return PosixPathInfo(self.parsed.path)
+
+    @property
+    def name(self):
+        return self._path.name
+
+    @property
+    def parts(self):
+        return (self.scheme, self.netloc) + self._path.parts
+
+    @property
+    def bucket(self):
+        return self.parsed.netloc
+
+    @property
+    def parent(self):
+        return self.from_parts(
+            scheme=self.scheme,
+            netloc=self.parsed.netloc,
+            path=self._path.parent.fspath,
+        )
+
+    @property
+    def parents(self):
+        return _URLPathParents(
+            type(self), self.scheme, self.parsed.netloc, self._path
+        )
+
+    def relative_to(self, other):
+        if isinstance(other, str):
+            other = URLInfo(other)
+        if self.scheme != other.scheme or self.netloc != other.netloc:
+            raise ValueError(
+                "'{}' does not start with '{}'".format(self, other)
+            )
+        return self._path.relative_to(other._path)
+
+    def isin(self, other):
+        if isinstance(other, basestring):
+            other = self.__class__(other)
+        elif self.__class__ != other.__class__:
+            return False
+        return (
+            self.scheme == other.scheme
+            and self.netloc == other.netloc
+            and self._path.isin(other._path)
+        )
+
+
+class CloudURLInfo(URLInfo):
+    @property
+    def path(self):
+        return self.parsed.path.lstrip("/")

dvc/progress.py

@@ -1,154 +1,140 @@
 """Manages progress bars for dvc repo."""
-
 from __future__ import print_function
-from __future__ import unicode_literals
-
-from dvc.utils.compat import str
-
+import logging
 import sys
-import threading
+from tqdm import tqdm
+from concurrent.futures import ThreadPoolExecutor
+from funcy import merge
+from dvc.utils import env2bool
 
-CLEARLINE_PATTERN = "\r\x1b[K"
+logger = logging.getLogger(__name__)
 
 
-class Progress(object):
+class TqdmThreadPoolExecutor(ThreadPoolExecutor):
     """
-    Simple multi-target progress bar.
+    Ensure worker progressbars are cleared away properly.
     """
 
-    def __init__(self):
-        self._n_total = 0
-        self._n_finished = 0
-        self._lock = threading.Lock()
-        self._line = None
-
-    def set_n_total(self, total):
-        """Sets total number of targets."""
-        self._n_total = total
-        self._n_finished = 0
-
-    @property
-    def is_finished(self):
-        """Returns if all targets have finished."""
-        return self._n_total == self._n_finished
-
-    def clearln(self):
-        self._print(CLEARLINE_PATTERN, end="")
-
-    def _writeln(self, line):
-        self.clearln()
-        self._print(line, end="")
-        sys.stdout.flush()
-
-    def reset(self):
-        with self._lock:
-            self._n_total = 0
-            self._n_finished = 0
-            self._line = None
-
-    def refresh(self, line=None):
-        """Refreshes progress bar."""
-        # Just go away if it is locked. Will update next time
-        if not self._lock.acquire(False):
-            return
-
-        if line is None:
-            line = self._line
-
-        if sys.stdout.isatty() and line is not None:
-            self._writeln(line)
-            self._line = line
-
-        self._lock.release()
-
-    def update_target(self, name, current, total):
-        """Updates progress bar for a specified target."""
-        self.refresh(self._bar(name, current, total))
-
-    def finish_target(self, name):
-        """Finishes progress bar for a specified target."""
-        # We have to write a msg about finished target
-        with self._lock:
-            pbar = self._bar(name, 100, 100)
-
-            if sys.stdout.isatty():
-                self.clearln()
-
-            self._print(pbar)
-
-            self._n_finished += 1
-            self._line = None
-
-    def _bar(self, target_name, current, total):
-        """
-        Make a progress bar out of info, which looks like:
-        (1/2): [########################################] 100% master.zip
-        """
-        bar_len = 30
-
-        if total is None:
-            state = 0
-            percent = "?% "
-        else:
-            total = int(total)
-            state = int((100 * current) / total) if current < total else 100
-            percent = str(state) + "% "
-
-        if self._n_total > 1:
-            num = "({}/{}): ".format(self._n_finished + 1, self._n_total)
-        else:
-            num = ""
-
-        n_sh = int((state * bar_len) / 100)
-        n_sp = bar_len - n_sh
-        pbar = "[" + "#" * n_sh + " " * n_sp + "] "
-
-        return num + pbar + percent + target_name
-
-    @staticmethod
-    def _print(*args, **kwargs):
-        import logging
-
-        logger = logging.getLogger(__name__)
-
-        if logger.getEffectiveLevel() == "CRITICAL":
-            return
-
-        print(*args, **kwargs)
-
     def __enter__(self):
-        self._lock.acquire(True)
-        if self._line is not None:
-            self.clearln()
+        """
+        Creates a blank initial dummy progress bar if needed so that workers
+        are forced to create "nested" bars.
+        """
+        blank_bar = Tqdm(bar_format="Multi-Threaded:", leave=False)
+        if blank_bar.pos > 0:
+            # already nested - don't need a placeholder bar
+            blank_bar.close()
+        self.bar = blank_bar
+        return super(TqdmThreadPoolExecutor, self).__enter__()
 
-    def __exit__(self, typ, value, tbck):
-        if self._line is not None:
-            self.refresh()
-        self._lock.release()
-
-    def __call__(self, seq, name="", total=None):
-        if total is None:
-            total = len(seq)
-
-        self.update_target(name, 0, total)
-        for done, item in enumerate(seq, start=1):
-            yield item
-            self.update_target(name, done, total)
-        self.finish_target(name)
+    def __exit__(self, *a, **k):
+        super(TqdmThreadPoolExecutor, self).__exit__(*a, **k)
+        self.bar.close()
 
 
-class ProgressCallback(object):
-    def __init__(self, total):
-        self.total = total
-        self.current = 0
-        progress.reset()
+class Tqdm(tqdm):
+    """
+    maximum-compatibility tqdm-based progressbars
+    """
 
-    def update(self, name, progress_to_add=1):
-        self.current += progress_to_add
-        progress.update_target(name, self.current, self.total)
+    BAR_FMT_DEFAULT = (
+        "{percentage:3.0f}%|{bar:10}|"
+        "{desc:{ncols_desc}.{ncols_desc}}{n}/{total}"
+        " [{elapsed}<{remaining}, {rate_fmt:>11}{postfix}]"
+    )
+    BAR_FMT_NOTOTAL = (
+        "{desc:{ncols_desc}.{ncols_desc}}{n}"
+        " [{elapsed}<??:??, {rate_fmt:>11}{postfix}]"
+    )
 
-    def finish(self, name):
-        progress.finish_target(name)
+    def __init__(
+        self,
+        iterable=None,
+        disable=None,
+        level=logging.ERROR,
+        desc=None,
+        leave=False,
+        bar_format=None,
+        bytes=False,  # pylint: disable=W0622
+        file=None,
+        **kwargs
+    ):
+        """
+        bytes   : shortcut for
+            `unit='B', unit_scale=True, unit_divisor=1024, miniters=1`
+        desc  : persists after `close()`
+        level  : effective logging level for determining `disable`;
+            used only if `disable` is unspecified
+        disable  : If (default: None), will be determined by logging level.
+            May be overridden to `True` due to non-TTY status.
+            Skip override by specifying env var `DVC_IGNORE_ISATTY`.
+        kwargs  : anything accepted by `tqdm.tqdm()`
+        """
+        kwargs = kwargs.copy()
+        kwargs.setdefault("unit_scale", True)
+        if bytes:
+            bytes_defaults = dict(
+                unit="B", unit_scale=True, unit_divisor=1024, miniters=1
+            )
+            kwargs = merge(bytes_defaults, kwargs)
+        if file is None:
+            file = sys.stderr
+        self.desc_persist = desc
+        # auto-disable based on `logger.level`
+        if disable is None:
+            disable = logger.getEffectiveLevel() > level
+        # auto-disable based on TTY
+        if (
+            not disable
+            and not env2bool("DVC_IGNORE_ISATTY")
+            and hasattr(file, "isatty")
+        ):
+            disable = not file.isatty()
+        super(Tqdm, self).__init__(
+            iterable=iterable,
+            disable=disable,
+            leave=leave,
+            desc=desc,
+            bar_format="!",
+            **kwargs
+        )
+        if bar_format is None:
+            if self.__len__():
+                self.bar_format = self.BAR_FMT_DEFAULT
+            else:
+                self.bar_format = self.BAR_FMT_NOTOTAL
+        else:
+            self.bar_format = bar_format
+        self.refresh()
 
+    def update_desc(self, desc, n=1):
+        """
+        Calls `set_description_str(desc)` and `update(n)`
+        """
+        self.set_description_str(desc, refresh=False)
+        self.update(n)
 
-progress = Progress()  # pylint: disable=invalid-name
+    def update_to(self, current, total=None):
+        if total:
+            self.total = total  # pylint: disable=W0613,W0201
+        self.update(current - self.n)
+
+    def close(self):
+        if self.desc_persist is not None:
+            self.set_description_str(self.desc_persist, refresh=False)
+        super(Tqdm, self).close()
+
+    @property
+    def format_dict(self):
+        """inject `ncols_desc` to fill the display width (`ncols`)"""
+        d = super(Tqdm, self).format_dict
+        ncols = d["ncols"] or 80
+        ncols_desc = ncols - len(self.format_meter(ncols_desc=1, **d)) + 1
+        ncols_desc = max(ncols_desc, 0)
+        if ncols_desc:
+            d["ncols_desc"] = ncols_desc
+        else:
+            # work-around for zero-width description
+            d["ncols_desc"] = 1
+            d["prefix"] = ""
+        return d

dvc/remote/__init__.py

@@ -10,6 +10,8 @@ from dvc.remote.http import RemoteHTTP
 from dvc.remote.https import RemoteHTTPS
 from dvc.remote.oss import RemoteOSS
 
+from .config import RemoteConfig
+
 
 REMOTES = [
     RemoteAZURE,
@@ -31,5 +33,11 @@ def _get(config):
     return RemoteLOCAL
 
 
-def Remote(repo, config):
-    return _get(config)(repo, config)
+def Remote(repo, **kwargs):
+    name = kwargs.get("name")
+    if name:
+        remote_config = RemoteConfig(repo.config)
+        settings = remote_config.get_settings(name)
+    else:
+        settings = kwargs
+    return _get(settings)(repo, settings)

dvc/remote/azure.py

@@ -4,101 +4,90 @@ from __future__ import unicode_literals
 import os
 import re
 import logging
+from datetime import datetime, timedelta
+
+from funcy import cached_property
 
 from dvc.scheme import Schemes
-from dvc.path.azure import PathAZURE
 
-try:
-    from azure.storage.blob import BlockBlobService
-    from azure.common import AzureMissingResourceHttpError
-except ImportError:
-    BlockBlobService = None
-
-from dvc.utils import tmp_fname, move
-from dvc.utils.compat import urlparse, makedirs
-from dvc.progress import progress
+from dvc.utils.compat import urlparse
+from dvc.progress import Tqdm
 from dvc.config import Config
 from dvc.remote.base import RemoteBASE
+from dvc.path_info import CloudURLInfo
 
 
 logger = logging.getLogger(__name__)
 
 
-class Callback(object):
-    def __init__(self, name):
-        self.name = name
-
-    def __call__(self, current, total):
-        progress.update_target(self.name, current, total)
-
-
 class RemoteAZURE(RemoteBASE):
     scheme = Schemes.AZURE
+    path_cls = CloudURLInfo
     REGEX = (
         r"azure://((?P<path>[^=;]*)?|("
         # backward compatibility
         r"(ContainerName=(?P<container_name>[^;]+);?)?"
         r"(?P<connection_string>.+)?)?)$"
     )
-    REQUIRES = {"azure-storage-blob": BlockBlobService}
+    REQUIRES = {"azure-storage-blob": "azure.storage.blob"}
     PARAM_CHECKSUM = "etag"
     COPY_POLL_SECONDS = 5
 
     def __init__(self, repo, config):
         super(RemoteAZURE, self).__init__(repo, config)
 
-        self.url = config.get(Config.SECTION_REMOTE_URL, "azure://")
-        match = re.match(self.REGEX, self.url)  # backward compatibility
+        url = config.get(Config.SECTION_REMOTE_URL, "azure://")
 
+        match = re.match(self.REGEX, url)  # backward compatibility
         path = match.group("path")
-        self.bucket = (
-            urlparse(self.url if path else "").netloc
+        bucket = (
+            urlparse(url if path else "").netloc
             or match.group("container_name")  # backward compatibility
             or os.getenv("AZURE_STORAGE_CONTAINER_NAME")
         )
 
-        self.prefix = urlparse(self.url).path.lstrip("/") if path else ""
-
         self.connection_string = (
             config.get(Config.SECTION_AZURE_CONNECTION_STRING)
             or match.group("connection_string")  # backward compatibility
             or os.getenv("AZURE_STORAGE_CONNECTION_STRING")
         )
 
-        if not self.bucket:
+        if not bucket:
             raise ValueError("azure storage container name missing")
 
         if not self.connection_string:
             raise ValueError("azure storage connection string missing")
 
-        self.__blob_service = None
-        self.path_info = PathAZURE(bucket=self.bucket)
+        self.path_info = (
+            self.path_cls(url)
+            if path
+            else self.path_cls.from_parts(scheme=self.scheme, netloc=bucket)
+        )
 
-    @property
+    @cached_property
     def blob_service(self):
-        if self.__blob_service is None:
-            logger.debug("URL {}".format(self.url))
-            logger.debug("Connection string {}".format(self.connection_string))
-            self.__blob_service = BlockBlobService(
-                connection_string=self.connection_string
+        from azure.storage.blob import BlockBlobService
+        from azure.common import AzureMissingResourceHttpError
+
+        logger.debug("URL {}".format(self.path_info))
+        logger.debug("Connection string {}".format(self.connection_string))
+        blob_service = BlockBlobService(
+            connection_string=self.connection_string
+        )
+        logger.debug("Container name {}".format(self.path_info.bucket))
+        try:  # verify that container exists
+            blob_service.list_blobs(
+                self.path_info.bucket, delimiter="/", num_results=1
             )
-            logger.debug("Container name {}".format(self.bucket))
-            try:  # verify that container exists
-                self.__blob_service.list_blobs(
-                    self.bucket, delimiter="/", num_results=1
-                )
-            except AzureMissingResourceHttpError:
-                self.__blob_service.create_container(self.bucket)
-        return self.__blob_service
+        except AzureMissingResourceHttpError:
+            blob_service.create_container(self.path_info.bucket)
+        return blob_service
 
     def remove(self, path_info):
         if path_info.scheme != self.scheme:
             raise NotImplementedError
 
-        logger.debug(
-            "Removing azure://{}/{}".format(path_info.bucket, path_info.path)
-        )
-
+        logger.debug("Removing {}".format(path_info))
         self.blob_service.delete_blob(path_info.bucket, path_info.path)
 
     def _list_paths(self, bucket, prefix):
@@ -118,85 +107,46 @@ class RemoteAZURE(RemoteBASE):
             next_marker = blobs.next_marker
 
     def list_cache_paths(self):
-        return self._list_paths(self.bucket, self.prefix)
+        return self._list_paths(self.path_info.bucket, self.path_info.path)
 
-    def upload(self, from_infos, to_infos, names=None, no_progress_bar=False):
-        names = self._verify_path_args(to_infos, from_infos, names)
-
-        for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info.scheme != self.scheme:
-                raise NotImplementedError
-
-            if from_info.scheme != "local":
-                raise NotImplementedError
-
-            bucket = to_info.bucket
-            path = to_info.path
-
-            logger.debug(
-                "Uploading '{}' to '{}/{}'".format(
-                    from_info.path, bucket, path
-                )
-            )
-
-            if not name:
-                name = os.path.basename(from_info.path)
-
-            cb = None if no_progress_bar else Callback(name)
-
-            try:
-                self.blob_service.create_blob_from_path(
-                    bucket, path, from_info.path, progress_callback=cb
-                )
-            except Exception:
-                msg = "failed to upload '{}'".format(from_info.path)
-                logger.warning(msg)
-            else:
-                progress.finish_target(name)
-
-    def download(
-        self,
-        from_infos,
-        to_infos,
-        no_progress_bar=False,
-        names=None,
-        resume=False,
+    def _upload(
+        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
     ):
-        names = self._verify_path_args(from_infos, to_infos, names)
-
-        for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info.scheme != self.scheme:
-                raise NotImplementedError
-
-            if to_info.scheme != "local":
-                raise NotImplementedError
-
-            bucket = from_info.bucket
-            path = from_info.path
-
-            logger.debug(
-                "Downloading '{}/{}' to '{}'".format(
-                    bucket, path, to_info.path
-                )
+        with Tqdm(desc=name, disable=no_progress_bar, bytes=True) as pbar:
+            self.blob_service.create_blob_from_path(
+                to_info.bucket,
+                to_info.path,
+                from_file,
+                progress_callback=pbar.update_to,
             )
 
-            tmp_file = tmp_fname(to_info.path)
-            if not name:
-                name = os.path.basename(to_info.path)
+    def _download(
+        self, from_info, to_file, name=None, no_progress_bar=False, **_kwargs
+    ):
+        with Tqdm(desc=name, disable=no_progress_bar, bytes=True) as pbar:
+            self.blob_service.get_blob_to_path(
+                from_info.bucket,
+                from_info.path,
+                to_file,
+                progress_callback=pbar.update_to,
+            )
 
-            cb = None if no_progress_bar else Callback(name)
+    def exists(self, path_info):
+        paths = self._list_paths(path_info.bucket, path_info.path)
+        return any(path_info.path == path for path in paths)
 
-            makedirs(os.path.dirname(to_info.path), exist_ok=True)
+    def _generate_download_url(self, path_info, expires=3600):
+        from azure.storage.blob import BlobPermissions
 
-            try:
-                self.blob_service.get_blob_to_path(
-                    bucket, path, tmp_file, progress_callback=cb
-                )
-            except Exception:
-                msg = "failed to download '{}/{}'".format(bucket, path)
-                logger.warning(msg)
-            else:
-                move(tmp_file, to_info.path)
+        expires_at = datetime.utcnow() + timedelta(seconds=expires)
 
-                if not no_progress_bar:
-                    progress.finish_target(name)
+        sas_token = self.blob_service.generate_blob_shared_access_signature(
+            path_info.bucket,
+            path_info.path,
+            permission=BlobPermissions.READ,
+            expiry=expires_at,
+        )
+        download_url = self.blob_service.make_blob_url(
+            path_info.bucket, path_info.path, sas_token=sas_token
+        )
+        return download_url

dvc/remote/base.py

@@ -1,37 +1,39 @@
 from __future__ import unicode_literals
 
-from copy import copy
-
-from dvc.path.local import PathLOCAL
-from dvc.utils.compat import str
-
-import os
-import re
-import json
-import ntpath
-import logging
-import tempfile
-import posixpath
 from operator import itemgetter
 from multiprocessing import cpu_count
+import json
+import logging
+import tempfile
+import itertools
+from functools import partial
+from concurrent.futures import ThreadPoolExecutor
+from copy import copy
+
+from dvc.ignore import DvcIgnore
+from dvc.utils.compat import str, basestring, urlparse, FileNotFoundError
+from dvc.remote.slow_link_detection import slow_link_guard
 
 import dvc.prompt as prompt
 from dvc.config import Config
-from dvc.exceptions import DvcException, ConfirmRemoveError
-from dvc.progress import progress
-from dvc.utils import LARGE_DIR_SIZE, tmp_fname
-from dvc.state import StateBase
-
+from dvc.exceptions import (
+    DvcException,
+    ConfirmRemoveError,
+    DvcIgnoreInCollectedDirError,
+)
+from dvc.progress import Tqdm
+from dvc.utils import LARGE_DIR_SIZE, tmp_fname, move, relpath, makedirs
+from dvc.state import StateNoop
+from dvc.path_info import PathInfo, URLInfo
+from dvc.utils.http import open_url
 
 logger = logging.getLogger(__name__)
 
-
 STATUS_OK = 1
 STATUS_MISSING = 2
 STATUS_NEW = 3
 STATUS_DELETED = 4
 
-
 STATUS_MAP = {
     # (local_exists, remote_exists)
     (True, True): STATUS_OK,
@@ -66,91 +68,103 @@ class RemoteMissingDepsError(DvcException):
     pass
 
 
+class DirCacheError(DvcException):
+    def __init__(self, checksum, cause=None):
+        super(DirCacheError, self).__init__(
+            "Failed to load dir cache for checksum: '{}'.".format(checksum),
+            cause=cause,
+        )
+
+
 class RemoteBASE(object):
-    scheme = None
-    REGEX = None
+    scheme = "base"
+    path_cls = URLInfo
     REQUIRES = {}
     JOBS = 4 * cpu_count()
 
     PARAM_RELPATH = "relpath"
     CHECKSUM_DIR_SUFFIX = ".dir"
+    CHECKSUM_JOBS = max(1, min(4, cpu_count() // 2))
+    DEFAULT_CACHE_TYPES = ["copy"]
+
+    state = StateNoop()
 
     def __init__(self, repo, config):
         self.repo = repo
-        deps_ok = all(self.REQUIRES.values())
-        if not deps_ok:
-            missing = [k for k, v in self.REQUIRES.items() if v is None]
-            url = config.get(
-                Config.SECTION_REMOTE_URL, "{}://".format(self.scheme)
-            )
-            msg = (
-                "URL '{}' is supported but requires these missing "
-                "dependencies: {}. If you have installed dvc using pip, "
-                "choose one of these options to proceed: \n"
-                "\n"
-                "    1) Install specific missing dependencies:\n"
-                "        pip install {}\n"
-                "    2) Install dvc package that includes those missing "
-                "dependencies: \n"
-                "        pip install dvc[{}]\n"
-                "    3) Install dvc package with all possible "
-                "dependencies included: \n"
-                "        pip install dvc[all]\n"
-                "\n"
-                "If you have installed dvc from a binary package and you "
-                "are still seeing this message, please report it to us "
-                "using https://github.com/iterative/dvc/issues. Thank you!"
-            ).format(url, missing, " ".join(missing), self.scheme)
-            raise RemoteMissingDepsError(msg)
 
-        self.protected = False
-        self.state = StateBase()
+        self._check_requires(config)
 
-        self._dir_info = {}
-
-    def __repr__(self):
-        return "{class_name}: '{url}'".format(
-            class_name=type(self).__name__, url=(self.url or "No url")
+        core = config.get(Config.SECTION_CORE, {})
+        self.checksum_jobs = core.get(
+            Config.SECTION_CORE_CHECKSUM_JOBS, self.CHECKSUM_JOBS
         )
 
-    def compat_config(config):
-        return config.copy()
+        self.protected = False
+        self.no_traverse = config.get(Config.SECTION_REMOTE_NO_TRAVERSE, True)
+        self._dir_info = {}
+
+        types = config.get(Config.SECTION_CACHE_TYPE, None)
+        if types:
+            if isinstance(types, str):
+                types = [t.strip() for t in types.split(",")]
+            self.cache_types = types
+        else:
+            self.cache_types = copy(self.DEFAULT_CACHE_TYPES)
+        self.cache_type_confirmed = False
+
+    def _check_requires(self, config):
+        import importlib
+
+        missing = []
+
+        for package, module in self.REQUIRES.items():
+            try:
+                importlib.import_module(module)
+            except ImportError:
+                missing.append(package)
+
+        if not missing:
+            return
+
+        url = config.get(
+            Config.SECTION_REMOTE_URL, "{}://".format(self.scheme)
+        )
+        msg = (
+            "URL '{}' is supported but requires these missing "
+            "dependencies: {}. If you have installed dvc using pip, "
+            "choose one of these options to proceed: \n"
+            "\n"
+            "    1) Install specific missing dependencies:\n"
+            "        pip install {}\n"
+            "    2) Install dvc package that includes those missing "
+            "dependencies: \n"
+            "        pip install 'dvc[{}]'\n"
+            "    3) Install dvc package with all possible "
+            "dependencies included: \n"
+            "        pip install 'dvc[all]'\n"
+            "\n"
+            "If you have installed dvc from a binary package and you "
+            "are still seeing this message, please report it to us "
+            "using https://github.com/iterative/dvc/issues. Thank you!"
+        ).format(url, missing, " ".join(missing), self.scheme)
+        raise RemoteMissingDepsError(msg)
+
+    def __repr__(self):
+        return "{class_name}: '{path_info}'".format(
+            class_name=type(self).__name__,
+            path_info=self.path_info or "No path",
+        )
 
     @classmethod
     def supported(cls, config):
-        url = config[Config.SECTION_REMOTE_URL]
-        return cls.match(url) is not None
+        if isinstance(config, basestring):
+            url = config
+        else:
+            url = config[Config.SECTION_REMOTE_URL]
 
-    @staticmethod
-    def _replace_sep(path, from_sep, to_sep):
-        assert not ntpath.isabs(path)
-        assert not posixpath.isabs(path)
-        return path.replace(from_sep, to_sep)
-
-    @classmethod
-    def to_posixpath(cls, path):
-        return cls._replace_sep(path, "\\", "/")
-
-    @classmethod
-    def to_ntpath(cls, path):
-        return cls._replace_sep(path, "/", "\\")
-
-    def to_ospath(self, path):
-        if self.ospath == ntpath:
-            return self.to_ntpath(path)
-        elif self.ospath == posixpath:
-            return self.to_posixpath(path)
-        assert False
-
-    @classmethod
-    def match(cls, url):
-        return re.match(cls.REGEX, url)
-
-    def group(self, name):
-        m = self.match(self.url)
-        if not m:
-            return None
-        return m.group(name)
+        # NOTE: silently skipping remote, calling code should handle that
+        parsed = urlparse(url)
+        return parsed.scheme == cls.scheme
 
     @property
     def cache(self):
@@ -159,65 +173,84 @@ class RemoteBASE(object):
     def get_file_checksum(self, path_info):
         raise NotImplementedError
 
+    def _calculate_checksums(self, file_infos):
+        file_infos = list(file_infos)
+        with ThreadPoolExecutor(max_workers=self.checksum_jobs) as executor:
+            tasks = executor.map(self.get_file_checksum, file_infos)
+
+            if len(file_infos) > LARGE_DIR_SIZE:
+                logger.info(
+                    (
+                        "Computing md5 for a large number of files. "
+                        "This is only done once."
+                    )
+                )
+                tasks = Tqdm(tasks, total=len(file_infos), unit="md5")
+            checksums = dict(zip(file_infos, tasks))
+        return checksums
+
     def _collect_dir(self, path_info):
-        dir_info = []
 
-        p_info = copy(path_info)
-        dpath = p_info.path
-        for root, dirs, files in self.walk(path_info):
-            if len(files) > LARGE_DIR_SIZE:
-                msg = (
-                    "Computing md5 for a large directory {}. "
-                    "This is only done once."
-                )
-                relpath = self.ospath.relpath(root)
-                logger.info(msg.format(relpath))
-                files = progress(files, name=relpath)
+        file_infos = set()
+        for root, _dirs, files in self.walk(path_info):
 
-            for fname in files:
-                path = self.ospath.join(root, fname)
-                p_info.path = path
-                relpath = self.to_posixpath(self.ospath.relpath(path, dpath))
+            if DvcIgnore.DVCIGNORE_FILE in files:
+                raise DvcIgnoreInCollectedDirError(root)
 
-                checksum = self.get_file_checksum(p_info)
-                dir_info.append(
-                    {
-                        self.PARAM_RELPATH: relpath,
-                        self.PARAM_CHECKSUM: checksum,
-                    }
-                )
+            file_infos.update(path_info / root / fname for fname in files)
 
-        # NOTE: sorting the list by path to ensure reproducibility
-        return sorted(dir_info, key=itemgetter(self.PARAM_RELPATH))
+        checksums = {fi: self.state.get(fi) for fi in file_infos}
+        not_in_state = {
+            fi for fi, checksum in checksums.items() if checksum is None
+        }
+
+        new_checksums = self._calculate_checksums(not_in_state)
+
+        checksums.update(new_checksums)
+
+        result = [
+            {
+                self.PARAM_CHECKSUM: checksums[fi],
+                # NOTE: this is lossy transformation:
+                #   "hey\there" -> "hey/there"
+                #   "hey/there" -> "hey/there"
+                # The latter is fine filename on Windows, which
+                # will transform to dir/file on back transform.
+                #
+                # Yes, this is a BUG, as long as we permit "/" in
+                # filenames on Windows and "\" on Unix
+                self.PARAM_RELPATH: fi.relative_to(path_info).as_posix(),
+            }
+            for fi in file_infos
+        ]
+
+        # Sorting the list by path to ensure reproducibility
+        return sorted(result, key=itemgetter(self.PARAM_RELPATH))
 
     def get_dir_checksum(self, path_info):
         dir_info = self._collect_dir(path_info)
-        checksum, from_info, to_info = self._get_dir_info_checksum(
-            dir_info, path_info
-        )
+        checksum, tmp_info = self._get_dir_info_checksum(dir_info)
+        new_info = self.cache.checksum_to_path_info(checksum)
         if self.cache.changed_cache_file(checksum):
-            self.cache.move(from_info, to_info)
+            self.cache.makedirs(new_info.parent)
+            self.cache.move(tmp_info, new_info)
 
         self.state.save(path_info, checksum)
-        self.state.save(to_info, checksum)
+        self.state.save(new_info, checksum)
 
         return checksum
 
-    def _get_dir_info_checksum(self, dir_info, path_info):
-        to_info = copy(path_info)
-        to_info.path = self.cache.ospath.join(self.cache.prefix, tmp_fname(""))
-
+    def _get_dir_info_checksum(self, dir_info):
         tmp = tempfile.NamedTemporaryFile(delete=False).name
         with open(tmp, "w+") as fobj:
             json.dump(dir_info, fobj, sort_keys=True)
 
-        from_info = PathLOCAL(path=tmp)
-        self.cache.upload([from_info], [to_info], no_progress_bar=True)
+        from_info = PathInfo(tmp)
+        to_info = self.cache.path_info / tmp_fname("")
+        self.cache.upload(from_info, to_info, no_progress_bar=True)
 
         checksum = self.get_file_checksum(to_info) + self.CHECKSUM_DIR_SUFFIX
-        from_info = copy(to_info)
-        to_info.path = self.cache.checksum_to_path(checksum)
-        return checksum, from_info, to_info
+        return checksum, to_info
 
     def get_dir_cache(self, checksum):
         assert checksum
@@ -226,34 +259,32 @@ class RemoteBASE(object):
         if dir_info:
             return dir_info
 
-        dir_info = self.load_dir_cache(checksum)
+        try:
+            dir_info = self.load_dir_cache(checksum)
+        except DirCacheError:
+            dir_info = []
+
         self._dir_info[checksum] = dir_info
         return dir_info
 
     def load_dir_cache(self, checksum):
         path_info = self.checksum_to_path_info(checksum)
 
-        fobj = tempfile.NamedTemporaryFile(delete=False)
-        path = fobj.name
-        to_info = PathLOCAL(path=path)
-        self.cache.download([path_info], [to_info], no_progress_bar=True)
-
         try:
-            with open(path, "r") as fobj:
+            with self.cache.open(path_info, "r") as fobj:
                 d = json.load(fobj)
-        except ValueError:
-            logger.exception("Failed to load dir cache '{}'".format(path_info))
-            return []
-        finally:
-            os.unlink(path)
+        except (ValueError, FileNotFoundError) as exc:
+            raise DirCacheError(checksum, cause=exc)
 
         if not isinstance(d, list):
             msg = "dir cache file format error '{}' [skipping the file]"
-            logger.error(msg.format(os.path.relpath(path)))
+            logger.error(msg.format(relpath(path_info)))
             return []
 
         for info in d:
-            info[self.PARAM_RELPATH] = self.to_ospath(info[self.PARAM_RELPATH])
+            # NOTE: here is a BUG, see comment to .as_posix() below
+            relative_path = PathInfo.from_posix(info[self.PARAM_RELPATH])
+            info[self.PARAM_RELPATH] = relative_path.fspath
 
         return d
 
@@ -266,6 +297,17 @@ class RemoteBASE(object):
             return None
 
         checksum = self.state.get(path_info)
+
+        # If we have dir checksum in state db, but dir cache file is lost,
+        # then we need to recollect the dir via .get_dir_checksum() call below,
+        # see https://github.com/iterative/dvc/issues/2219 for context
+        if (
+            checksum
+            and self.is_dir_checksum(checksum)
+            and not self.exists(self.cache.checksum_to_path_info(checksum))
+        ):
+            checksum = None
+
         if checksum:
             return checksum
 
@@ -334,7 +376,47 @@ class RemoteBASE(object):
         return False
 
     def link(self, from_info, to_info):
-        self.copy(from_info, to_info)
+        self._link(from_info, to_info, self.cache_types)
+
+    def _link(self, from_info, to_info, link_types):
+        assert self.isfile(from_info)
+
+        self.makedirs(to_info.parent)
+
+        self._try_links(from_info, to_info, link_types)
+
+    @slow_link_guard
+    def _try_links(self, from_info, to_info, link_types):
+        while link_types:
+            link_method = getattr(self, link_types[0])
+            try:
+                self._do_link(from_info, to_info, link_method)
+                self.cache_type_confirmed = True
+                return
+
+            except DvcException as exc:
+                msg = "Cache type '{}' is not supported: {}"
+                logger.debug(msg.format(link_types[0], str(exc)))
+                del link_types[0]
+
+        raise DvcException("no possible cache types left to try out.")
+
+    def _do_link(self, from_info, to_info, link_method):
+        if self.exists(to_info):
+            raise DvcException("Link '{}' already exists!".format(to_info))
+
+        link_method(from_info, to_info)
+
+        if self.protected:
+            self.protect(to_info)
+
+        msg = "Created {}'{}': {} -> {}".format(
+            "protected " if self.protected else "",
+            self.cache_types[0],
+            from_info,
+            to_info,
+        )
+        logger.debug(msg)
 
     def _save_file(self, path_info, checksum, save_link=True):
         assert checksum
@@ -360,13 +442,9 @@ class RemoteBASE(object):
         cache_info = self.checksum_to_path_info(checksum)
         dir_info = self.get_dir_cache(checksum)
 
-        entry_info = copy(path_info)
         for entry in dir_info:
+            entry_info = path_info / entry[self.PARAM_RELPATH]
             entry_checksum = entry[self.PARAM_CHECKSUM]
-            entry_info.path = self.ospath.join(
-                path_info.path, entry[self.PARAM_RELPATH]
-            )
-
             self._save_file(entry_info, entry_checksum, save_link=False)
 
         self.state.save_link(path_info)
@@ -377,15 +455,22 @@ class RemoteBASE(object):
         return False
 
     def isfile(self, path_info):
-        raise NotImplementedError
+        """Optional: Overwrite only if the remote has a way to distinguish
+        between a directory and a file.
+        """
+        return True
 
     def isdir(self, path_info):
+        """Optional: Overwrite only if the remote has a way to distinguish
+        between a directory and a file.
+        """
         return False
 
     def walk(self, path_info):
         raise NotImplementedError
 
-    def protect(self, path_info):
+    @staticmethod
+    def protect(path_info):
         pass
 
     def save(self, path_info, checksum_info):
@@ -404,24 +489,88 @@ class RemoteBASE(object):
 
     def _save(self, path_info, checksum):
         to_info = self.checksum_to_path_info(checksum)
-        logger.info("Saving '{}' to '{}'.".format(path_info, to_info))
+        logger.debug("Saving '{}' to '{}'.".format(path_info, to_info))
         if self.isdir(path_info):
             self._save_dir(path_info, checksum)
             return
         self._save_file(path_info, checksum)
 
+    def upload(self, from_info, to_info, name=None, no_progress_bar=False):
+        if not hasattr(self, "_upload"):
+            raise RemoteActionNotImplemented("upload", self.scheme)
+
+        if to_info.scheme != self.scheme:
+            raise NotImplementedError
+
+        if from_info.scheme != "local":
+            raise NotImplementedError
+
+        logger.debug("Uploading '{}' to '{}'".format(from_info, to_info))
+
+        name = name or from_info.name
+
+        try:
+            self._upload(
+                from_info.fspath,
+                to_info,
+                name=name,
+                no_progress_bar=no_progress_bar,
+            )
+        except Exception:
+            msg = "failed to upload '{}' to '{}'"
+            logger.exception(msg.format(from_info, to_info))
+            return 1  # 1 fail
+
+        return 0
+
     def download(
         self,
-        from_infos,
-        to_infos,
-        no_progress_bar=False,
+        from_info,
+        to_info,
         name=None,
-        resume=False,
+        no_progress_bar=False,
+        file_mode=None,
+        dir_mode=None,
     ):
-        raise RemoteActionNotImplemented("download", self.scheme)
+        if not hasattr(self, "_download"):
+            raise RemoteActionNotImplemented("download", self.scheme)
 
-    def upload(self, from_infos, to_infos, names=None, no_progress_bar=False):
-        raise RemoteActionNotImplemented("upload", self.scheme)
+        if from_info.scheme != self.scheme:
+            raise NotImplementedError
+
+        if to_info.scheme == self.scheme != "local":
+            self.copy(from_info, to_info)
+            return 0
+
+        if to_info.scheme != "local":
+            raise NotImplementedError
+
+        logger.debug("Downloading '{}' to '{}'".format(from_info, to_info))
+
+        name = name or to_info.name
+
+        makedirs(to_info.parent, exist_ok=True, mode=dir_mode)
+        tmp_file = tmp_fname(to_info)
+
+        try:
+            self._download(
+                from_info, tmp_file, name=name, no_progress_bar=no_progress_bar
+            )
+        except Exception:
+            msg = "failed to download '{}' to '{}'"
+            logger.exception(msg.format(from_info, to_info))
+            return 1  # 1 fail
+
+        move(tmp_file, to_info, mode=file_mode)
+
+        return 0
+
+    def open(self, path_info, mode="r", encoding=None):
+        if hasattr(self, "_generate_download_url"):
+            get_url = partial(self._generate_download_url, path_info)
+            return open_url(get_url, mode=mode, encoding=encoding)
+
+        raise RemoteActionNotImplemented("open", self.scheme)
 
     def remove(self, path_info):
         raise RemoteActionNotImplemented("remove", self.scheme)
@@ -433,42 +582,28 @@ class RemoteBASE(object):
     def copy(self, from_info, to_info):
         raise RemoteActionNotImplemented("copy", self.scheme)
 
-    def exists(self, path_infos):
+    def symlink(self, from_info, to_info):
+        raise RemoteActionNotImplemented("symlink", self.scheme)
+
+    def hardlink(self, from_info, to_info):
+        raise RemoteActionNotImplemented("hardlink", self.scheme)
+
+    def reflink(self, from_info, to_info):
+        raise RemoteActionNotImplemented("reflink", self.scheme)
+
+    def exists(self, path_info):
         raise NotImplementedError
 
-    @classmethod
-    def _verify_path_args(cls, from_infos, to_infos, names=None):
-        assert isinstance(from_infos, list)
-        assert isinstance(to_infos, list)
-        assert len(from_infos) == len(to_infos)
-
-        if not names:
-            names = len(to_infos) * [None]
-        else:
-            assert isinstance(names, list)
-            assert len(names) == len(to_infos)
-
-        return names
-
-    @property
-    def ospath(self):
-        return posixpath
-
-    def checksum_to_path(self, checksum):
-        assert checksum
-        return self.ospath.join(self.prefix, checksum[0:2], checksum[2:])
-
     def path_to_checksum(self, path):
-        relpath = self.ospath.relpath(path, self.prefix)
-        return self.ospath.dirname(relpath) + self.ospath.basename(relpath)
+        parts = self.path_cls(path).parts[-2:]
+
+        if not (len(parts) == 2 and parts[0] and len(parts[0]) == 2):
+            raise ValueError("Bad cache file path")
+
+        return "".join(parts)
 
     def checksum_to_path_info(self, checksum):
-        path_info = copy(self.path_info)
-        path_info.path = self.checksum_to_path(checksum)
-        return path_info
-
-    def md5s_to_path_infos(self, md5s):
-        return [self.checksum_to_path_info(md5) for md5 in md5s]
+        return self.path_info / checksum[0:2] / checksum[2:]
 
     def list_cache_paths(self):
         raise NotImplementedError
@@ -478,20 +613,18 @@ class RemoteBASE(object):
         # is 32 bytes, so ~3200Mb list) and we don't really need all of it at
         # the same time, so it makes sense to use a generator to gradually
         # iterate over it, without keeping all of it in memory.
-        return (
-            self.path_to_checksum(path) for path in self.list_cache_paths()
-        )
+        for path in self.list_cache_paths():
+            try:
+                yield self.path_to_checksum(path)
+            except ValueError:
+                # We ignore all the non-cache looking files
+                pass
 
-    def gc(self, cinfos):
-        from dvc.remote.local import RemoteLOCAL
-
-        used = {info[RemoteLOCAL.PARAM_CHECKSUM] for info in cinfos["local"]}
+    def gc(self, named_cache):
+        used = self.extract_used_local_checksums(named_cache)
 
         if self.scheme != "":
-            used |= {
-                info[self.PARAM_CHECKSUM]
-                for info in cinfos.get(self.scheme, [])
-            }
+            used.update(named_cache[self.scheme])
 
         removed = False
         for checksum in self.all():
@@ -539,11 +672,15 @@ class RemoteBASE(object):
         if self.changed_cache_file(checksum):
             return True
 
+        if not self._changed_unpacked_dir(checksum):
+            return False
+
         for entry in self.get_dir_cache(checksum):
-            checksum = entry[self.PARAM_CHECKSUM]
-            if self.changed_cache_file(checksum):
+            entry_checksum = entry[self.PARAM_CHECKSUM]
+            if self.changed_cache_file(entry_checksum):
                 return True
 
+        self._update_unpacked_dir(checksum)
         return False
 
     def changed_cache(self, checksum):
@@ -551,16 +688,49 @@ class RemoteBASE(object):
             return self._changed_dir_cache(checksum)
         return self.changed_cache_file(checksum)
 
-    def cache_exists(self, checksums):
-        # NOTE: The reason for such an odd logic is that most of the remotes
-        # take much shorter time to just retrieve everything they have under
-        # a certain prefix(e.g. s3, gs, ssh, hdfs). Other remotes that can
-        # check if particular file exists much quicker, use their own
-        # implementation of cache_exists(see http, local).
-        #
-        # Result of all() might be way too big, so we should walk through
-        # it in one pass.
-        return list(filter(lambda checksum: checksum in checksums, self.all()))
+    def cache_exists(self, checksums, jobs=None, name=None):
+        """Check if the given checksums are stored in the remote.
+
+        There are two ways of performing this check:
+
+        - Traverse: Get a list of all the files in the remote
+            (traversing the cache directory) and compare it with
+            the given checksums.
+
+        - No traverse: For each given checksum, run the `exists`
+            method and filter the checksums that aren't on the remote.
+            This is done in parallel threads.
+            It also shows a progress bar when performing the check.
+
+        The reason for such an odd logic is that most of the remotes
+        take much shorter time to just retrieve everything they have under
+        a certain prefix (e.g. s3, gs, ssh, hdfs). Other remotes that can
+        check if particular file exists much quicker, use their own
+        implementation of cache_exists (see ssh, local).
+
+        Returns:
+            A list with checksums that were found in the remote
+        """
+        if not self.no_traverse:
+            return list(set(checksums) & set(self.all()))
+
+        with Tqdm(
+            desc="Querying "
+            + ("cache in " + name if name else "remote cache"),
+            total=len(checksums),
+            unit="file",
+        ) as pbar:
+
+            def exists_with_progress(path_info):
+                ret = self.exists(path_info)
+                pbar.update_desc(str(path_info))
+                return ret
+
+            with ThreadPoolExecutor(max_workers=jobs or self.JOBS) as executor:
+                path_infos = map(self.checksum_to_path_info, checksums)
+                in_remote = executor.map(exists_with_progress, path_infos)
+                ret = list(itertools.compress(checksums, in_remote))
+                return ret
 
     def already_cached(self, path_info):
         current = self.get_checksum(path_info)
@@ -586,22 +756,50 @@ class RemoteBASE(object):
         self.remove(path_info)
 
     def _checkout_file(
-        self, path_info, checksum, force, progress_callback=None
+        self,
+        path_info,
+        checksum,
+        force,
+        progress_callback=None,
+        save_link=True,
     ):
-        cache_info = self.checksum_to_path_info(checksum)
-        if self.exists(path_info):
-            msg = "data '{}' exists. Removing before checkout."
-            logger.warning(msg.format(str(path_info)))
+        # NOTE: In case if path_info is already cached and path_info's
+        # link type matches cache link type, we would like to avoid
+        # relinking.
+        if self.changed(
+            path_info, {self.PARAM_CHECKSUM: checksum}
+        ) or not self._link_matches(path_info):
             self.safe_remove(path_info, force=force)
 
-        self.link(cache_info, path_info)
-        self.state.save_link(path_info)
-        self.state.save(path_info, checksum)
+            cache_info = self.checksum_to_path_info(checksum)
+            self.link(cache_info, path_info)
+
+            if save_link:
+                self.state.save_link(path_info)
+
+            self.state.save(path_info, checksum)
+        else:
+            # NOTE: performing (un)protection costs us +/- the same as checking
+            # if path_info is protected. Instead of implementing logic,
+            # just (un)protect according to self.protected.
+            if self.protected:
+                self.protect(path_info)
+            else:
+                # NOTE dont allow copy, because we checked before that link
+                # type matches cache, and we don't want data duplication
+                self.unprotect(path_info, allow_copy=False)
+
         if progress_callback:
-            progress_callback.update(path_info.url)
+            progress_callback(str(path_info))
+
+    def _link_matches(self, path_info):
+        return True
 
     def makedirs(self, path_info):
-        raise NotImplementedError
+        """Optional: Implement only if the remote needs to create
+        directories before copying/linking/moving data
+        """
+        pass
 
     def _checkout_dir(
         self, path_info, checksum, force, progress_callback=None
@@ -615,22 +813,17 @@ class RemoteBASE(object):
 
         logger.debug("Linking directory '{}'.".format(path_info))
 
-        entry_info = copy(path_info)
         for entry in dir_info:
-            relpath = entry[self.PARAM_RELPATH]
+            relative_path = entry[self.PARAM_RELPATH]
             entry_checksum = entry[self.PARAM_CHECKSUM]
-            entry_cache_info = self.checksum_to_path_info(entry_checksum)
-            entry_info.url = self.ospath.join(path_info.url, relpath)
-            entry_info.path = self.ospath.join(path_info.path, relpath)
-
-            entry_checksum_info = {self.PARAM_CHECKSUM: entry_checksum}
-            if self.changed(entry_info, entry_checksum_info):
-                if self.exists(entry_info):
-                    self.safe_remove(entry_info, force=force)
-                self.link(entry_cache_info, entry_info)
-                self.state.save(entry_info, entry_checksum)
-            if progress_callback:
-                progress_callback.update(entry_info.url)
+            entry_info = path_info / relative_path
+            self._checkout_file(
+                entry_info,
+                entry_checksum,
+                force,
+                progress_callback,
+                save_link=False,
+            )
 
         self._remove_redundant_files(path_info, dir_info, force)
 
@@ -639,53 +832,52 @@ class RemoteBASE(object):
 
     def _remove_redundant_files(self, path_info, dir_info, force):
         existing_files = set(
-            self.ospath.join(root, fname)
+            path_info / root / fname
             for root, _, files in self.walk(path_info)
             for fname in files
         )
 
-        needed_files = set(
-            self.ospath.join(path_info.path, entry[self.PARAM_RELPATH])
-            for entry in dir_info
-        )
+        needed_files = {
+            path_info / entry[self.PARAM_RELPATH] for entry in dir_info
+        }
 
-        delta = existing_files - needed_files
-
-        d_info = copy(path_info)
-        for path in delta:
-            d_info.path = path
-            self.safe_remove(d_info, force)
+        for path in existing_files - needed_files:
+            self.safe_remove(path, force)
 
     def checkout(
         self, path_info, checksum_info, force=False, progress_callback=None
     ):
-        if (
-            path_info.scheme not in ["local"]
-            and path_info.scheme != self.scheme
-        ):
+        if path_info.scheme not in ["local", self.scheme]:
             raise NotImplementedError
 
         checksum = checksum_info.get(self.PARAM_CHECKSUM)
+        failed = None
         if not checksum:
-            msg = "No checksum info for '{}'."
-            logger.debug(msg.format(str(path_info)))
-            return
+            logger.warning(
+                "No checksum info found for '{}'. "
+                "It won't be created.".format(str(path_info))
+            )
+            self.safe_remove(path_info, force=force)
+            failed = path_info
 
-        if not self.changed(path_info, checksum_info):
-            msg = "Data '{}' didn't change."
-            logger.debug(msg.format(str(path_info)))
-            return
-
-        if self.changed_cache(checksum):
+        elif self.changed_cache(checksum):
             msg = "Cache '{}' not found. File '{}' won't be created."
             logger.warning(msg.format(checksum, str(path_info)))
             self.safe_remove(path_info, force=force)
-            return
+            failed = path_info
+
+        if failed:
+            if progress_callback:
+                progress_callback(
+                    str(path_info), self.get_files_number(checksum)
+                )
+            return failed
 
         msg = "Checking out '{}' with cache '{}'."
         logger.debug(msg.format(str(path_info), checksum))
 
-        return self._checkout(path_info, checksum, force, progress_callback)
+        self._checkout(path_info, checksum, force, progress_callback)
+        return None
 
     def _checkout(
         self, path_info, checksum, force=False, progress_callback=None
@@ -698,6 +890,29 @@ class RemoteBASE(object):
             path_info, checksum, force, progress_callback=progress_callback
         )
 
+    def get_files_number(self, checksum):
+        if not checksum:
+            return 0
+
+        if self.is_dir_checksum(checksum):
+            return len(self.get_dir_cache(checksum))
+
+        return 1
+
     @staticmethod
-    def unprotect(path_info):
+    def unprotect(path_info, allow_copy=True):
+        pass
+
+    def _get_unpacked_dir_names(self, checksums):
+        return set()
+
+    def extract_used_local_checksums(self, named_cache):
+        used = set(named_cache["local"])
+        unpacked = self._get_unpacked_dir_names(used)
+        return used | unpacked
+
+    def _changed_unpacked_dir(self, checksum):
+        return True
+
+    def _update_unpacked_dir(self, checksum):
         pass

dvc/remote/config.py

@@ -0,0 +1,152 @@
+import os
+import logging
+import posixpath
+
+from dvc.utils import relpath
+from dvc.utils.compat import urlparse
+from dvc.config import Config, ConfigError
+
+
+logger = logging.getLogger(__name__)
+
+
+class RemoteConfig(object):
+    def __init__(self, config):
+        self.config = config
+
+    def get_settings(self, name):
+        """
+        Args:
+            name (str): The name of the remote that we want to retrieve
+
+        Returns:
+            dict: The content beneath the given remote name.
+
+        Example:
+            >>> config = {'remote "server"': {'url': 'ssh://localhost/'}}
+            >>> get_settings("server")
+            {'url': 'ssh://localhost/'}
+        """
+        settings = self.config.config.get(
+            Config.SECTION_REMOTE_FMT.format(name.lower())
+        )
+
+        if settings is None:
+            raise ConfigError(
+                "unable to find remote section '{}'".format(name)
+            )
+
+        parsed = urlparse(settings["url"])
+
+        # Support for cross referenced remotes.
+        # This will merge the settings, giving priority to the outer reference.
+        # For example, having:
+        #
+        #       dvc remote add server ssh://localhost
+        #       dvc remote modify server user root
+        #       dvc remote modify server ask_password true
+        #
+        #       dvc remote add images remote://server/tmp/pictures
+        #       dvc remote modify images user alice
+        #       dvc remote modify images ask_password false
+        #       dvc remote modify images password asdf1234
+        #
+        # Results on a config dictionary like:
+        #
+        #       {
+        #           "url": "ssh://localhost/tmp/pictures",
+        #           "user": "alice",
+        #           "password": "asdf1234",
+        #           "ask_password": False,
+        #       }
+        #
+        if parsed.scheme == "remote":
+            reference = self.get_settings(parsed.netloc)
+            url = posixpath.join(reference["url"], parsed.path.lstrip("/"))
+            merged = reference.copy()
+            merged.update(settings)
+            merged["url"] = url
+            return merged
+
+        return settings
+
+    @staticmethod
+    def resolve_path(path, config_file):
+        """Resolve path relative to config file location.
+
+        Args:
+            path: Path to be resolved.
+            config_file: Path to config file, which `path` is specified
+                relative to.
+
+        Returns:
+            Path relative to the `config_file` location. If `path` is an
+            absolute path then it will be returned without change.
+
+        """
+        if os.path.isabs(path):
+            return path
+        return relpath(path, os.path.dirname(config_file))
+
+    def add(self, name, url, default=False, force=False, level=None):
+        from dvc.remote import _get, RemoteLOCAL
+
+        configobj = self.config.get_configobj(level)
+        remote = _get({Config.SECTION_REMOTE_URL: url})
+        if remote == RemoteLOCAL and not url.startswith("remote://"):
+            url = self.resolve_path(url, configobj.filename)
+
+        self.config.set(
+            Config.SECTION_REMOTE_FMT.format(name),
+            Config.SECTION_REMOTE_URL,
+            url,
+            force=force,
+            level=level,
+        )
+        if default:
+            logger.info("Setting '{}' as a default remote.".format(name))
+            self.config.set(
+                Config.SECTION_CORE,
+                Config.SECTION_CORE_REMOTE,
+                name,
+                level=level,
+            )
+
+    def remove(self, name, level=None):
+        self.config.unset(Config.SECTION_REMOTE_FMT.format(name), level=level)
+
+        if level is None:
+            level = Config.LEVEL_REPO
+
+        for lev in [
+            Config.LEVEL_LOCAL,
+            Config.LEVEL_REPO,
+            Config.LEVEL_GLOBAL,
+            Config.LEVEL_SYSTEM,
+        ]:
+            self.config.unset(
+                Config.SECTION_CORE,
+                Config.SECTION_CORE_REMOTE,
+                level=lev,
+                force=True,
+            )
+            if lev == level:
+                break
+
+    def modify(self, name, option, value, level=None):
+        self.config.set(
+            Config.SECTION_REMOTE_FMT.format(name), option, value, level=level
+        )
+
+    def list(self, level=None):
+        return self.config.list_options(
+            Config.SECTION_REMOTE_REGEX, Config.SECTION_REMOTE_URL, level=level
+        )
+
+    def set_default(self, name, unset=False, level=None):
+        if unset:
+            self.config.unset(Config.SECTION_CORE, Config.SECTION_CORE_REMOTE)
+            return
+        self.config.set(
+            Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, name, level=level
+        )

dvc/remote/gs.py

@@ -1,61 +1,43 @@
 from __future__ import unicode_literals
 
-import os
 import logging
+from datetime import timedelta
+from funcy import cached_property
+from dvc.utils.compat import FileNotFoundError
 
-from dvc.scheme import Schemes
-from dvc.path.gs import PathGS
-
-try:
-    from google.cloud import storage
-except ImportError:
-    storage = None
-
-from dvc.utils import tmp_fname, move
-from dvc.utils.compat import urlparse, makedirs
 from dvc.remote.base import RemoteBASE
 from dvc.config import Config
-from dvc.progress import progress
 from dvc.exceptions import DvcException
-
+from dvc.path_info import CloudURLInfo
+from dvc.scheme import Schemes
 
 logger = logging.getLogger(__name__)
 
 
 class RemoteGS(RemoteBASE):
     scheme = Schemes.GS
-    REGEX = r"^gs://(?P<path>.*)$"
-    REQUIRES = {"google.cloud.storage": storage}
+    path_cls = CloudURLInfo
+    REQUIRES = {"google-cloud-storage": "google.cloud.storage"}
     PARAM_CHECKSUM = "md5"
 
     def __init__(self, repo, config):
         super(RemoteGS, self).__init__(repo, config)
-        storagepath = "gs://"
-        storagepath += config.get(Config.SECTION_AWS_STORAGEPATH, "/")
-        storagepath.lstrip("/")
-        self.url = config.get(Config.SECTION_REMOTE_URL, storagepath)
+
+        storagepath = "gs://" + config.get(Config.SECTION_GCP_STORAGEPATH, "/")
+        url = config.get(Config.SECTION_REMOTE_URL, storagepath)
+        self.path_info = self.path_cls(url)
+
         self.projectname = config.get(Config.SECTION_GCP_PROJECTNAME, None)
         self.credentialpath = config.get(Config.SECTION_GCP_CREDENTIALPATH)
 
-        parsed = urlparse(self.url)
-        self.bucket = parsed.netloc
-        self.prefix = parsed.path.lstrip("/")
-
-        self.path_info = PathGS(bucket=self.bucket)
-
-    @staticmethod
-    def compat_config(config):
-        ret = config.copy()
-        url = "gs://" + ret.pop(Config.SECTION_AWS_STORAGEPATH, "").lstrip("/")
-        ret[Config.SECTION_REMOTE_URL] = url
-        return ret
-
-    @property
+    @cached_property
     def gs(self):
+        from google.cloud.storage import Client
+
         return (
-            storage.Client.from_service_account_json(self.credentialpath)
+            Client.from_service_account_json(self.credentialpath)
             if self.credentialpath
-            else storage.Client(self.projectname)
+            else Client(self.projectname)
         )
 
     def get_file_checksum(self, path_info):
@@ -72,27 +54,21 @@ class RemoteGS(RemoteBASE):
         md5 = base64.b64decode(b64_md5)
         return codecs.getencoder("hex")(md5)[0].decode("utf-8")
 
-    def copy(self, from_info, to_info, gs=None):
-        gs = gs if gs else self.gs
-
-        blob = gs.bucket(from_info.bucket).get_blob(from_info.path)
+    def copy(self, from_info, to_info):
+        from_bucket = self.gs.bucket(from_info.bucket)
+        blob = from_bucket.get_blob(from_info.path)
         if not blob:
             msg = "'{}' doesn't exist in the cloud".format(from_info.path)
             raise DvcException(msg)
 
-        bucket = self.gs.bucket(to_info.bucket)
-        bucket.copy_blob(
-            blob, self.gs.bucket(to_info.bucket), new_name=to_info.path
-        )
+        to_bucket = self.gs.bucket(to_info.bucket)
+        from_bucket.copy_blob(blob, to_bucket, new_name=to_info.path)
 
     def remove(self, path_info):
         if path_info.scheme != "gs":
             raise NotImplementedError
 
-        logger.debug(
-            "Removing gs://{}/{}".format(path_info.bucket, path_info.path)
-        )
-
+        logger.debug("Removing gs://{}".format(path_info))
         blob = self.gs.bucket(path_info.bucket).get_blob(path_info.path)
         if not blob:
             return
@@ -104,103 +80,27 @@ class RemoteGS(RemoteBASE):
             yield blob.name
 
     def list_cache_paths(self):
-        return self._list_paths(self.bucket, self.prefix)
+        return self._list_paths(self.path_info.bucket, self.path_info.path)
 
     def exists(self, path_info):
-        assert not isinstance(path_info, list)
-        assert path_info.scheme == "gs"
-
-        paths = self._list_paths(path_info.bucket, path_info.path)
+        paths = set(self._list_paths(path_info.bucket, path_info.path))
         return any(path_info.path == path for path in paths)
 
-    def upload(self, from_infos, to_infos, names=None, no_progress_bar=False):
-        names = self._verify_path_args(to_infos, from_infos, names)
+    def _upload(self, from_file, to_info, **_kwargs):
+        bucket = self.gs.bucket(to_info.bucket)
+        blob = bucket.blob(to_info.path)
+        blob.upload_from_filename(from_file)
 
-        gs = self.gs
+    def _download(self, from_info, to_file, **_kwargs):
+        bucket = self.gs.bucket(from_info.bucket)
+        blob = bucket.get_blob(from_info.path)
+        blob.download_to_filename(to_file)
 
-        for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info.scheme != "gs":
-                raise NotImplementedError
+    def _generate_download_url(self, path_info, expires=3600):
+        expiration = timedelta(seconds=int(expires))
 
-            if from_info.scheme != "local":
-                raise NotImplementedError
-
-            logger.debug(
-                "Uploading '{}' to '{}/{}'".format(
-                    from_info.path, to_info.bucket, to_info.path
-                )
-            )
-
-            if not name:
-                name = os.path.basename(from_info.path)
-
-            if not no_progress_bar:
-                progress.update_target(name, 0, None)
-
-            try:
-                bucket = gs.bucket(to_info.bucket)
-                blob = bucket.blob(to_info.path)
-                blob.upload_from_filename(from_info.path)
-            except Exception:
-                msg = "failed to upload '{}' to '{}/{}'"
-                logger.exception(
-                    msg.format(from_info.path, to_info.bucket, to_info.path)
-                )
-                continue
-
-            progress.finish_target(name)
-
-    def download(
-        self,
-        from_infos,
-        to_infos,
-        no_progress_bar=False,
-        names=None,
-        resume=False,
-    ):
-        names = self._verify_path_args(from_infos, to_infos, names)
-
-        gs = self.gs
-
-        for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info.scheme != "gs":
-                raise NotImplementedError
-
-            if to_info.scheme == "gs":
-                self.copy(from_info, to_info, gs=gs)
-                continue
-
-            if to_info.scheme != "local":
-                raise NotImplementedError
-
-            msg = "Downloading '{}/{}' to '{}'".format(
-                from_info.bucket, from_info.path, to_info.path
-            )
-            logger.debug(msg)
-
-            tmp_file = tmp_fname(to_info.path)
-            if not name:
-                name = os.path.basename(to_info.path)
-
-            if not no_progress_bar:
-                # percent_cb is not available for download_to_filename, so
-                # lets at least update progress at pathpoints(start, finish)
-                progress.update_target(name, 0, None)
-
-            makedirs(os.path.dirname(to_info.path), exist_ok=True)
-
-            try:
-                bucket = gs.bucket(from_info.bucket)
-                blob = bucket.get_blob(from_info.path)
-                blob.download_to_filename(tmp_file)
-            except Exception:
-                msg = "failed to download '{}/{}' to '{}'"
-                logger.exception(
-                    msg.format(from_info.bucket, from_info.path, to_info.path)
-                )
-                continue
-
-            move(tmp_file, to_info.path)
-
-            if not no_progress_bar:
-                progress.finish_target(name)
+        bucket = self.gs.bucket(path_info.bucket)
+        blob = bucket.get_blob(path_info.path)
+        if blob is None:
+            raise FileNotFoundError
+        return blob.generate_signed_url(expiration=expiration)

dvc/remote/hdfs.py

@@ -2,17 +2,20 @@ from __future__ import unicode_literals
 
 import os
 import re
-import getpass
+import io
 import posixpath
 import logging
 from subprocess import Popen, PIPE
+from contextlib import contextmanager, closing
 
 from dvc.config import Config
 from dvc.scheme import Schemes
-from dvc.path.hdfs import PathHDFS
-from dvc.remote.base import RemoteBASE, RemoteCmdError
+
+from dvc.utils.compat import urlparse, FileNotFoundError
 from dvc.utils import fix_env, tmp_fname
 
+from .pool import get_connection
+from .base import RemoteBASE, RemoteCmdError
 
 logger = logging.getLogger(__name__)
 
@@ -21,18 +24,36 @@ class RemoteHDFS(RemoteBASE):
     scheme = Schemes.HDFS
     REGEX = r"^hdfs://((?P<user>.*)@)?.*$"
     PARAM_CHECKSUM = "checksum"
+    REQUIRES = {"pyarrow": "pyarrow"}
 
     def __init__(self, repo, config):
         super(RemoteHDFS, self).__init__(repo, config)
-        self.url = config.get(Config.SECTION_REMOTE_URL, "/")
-        self.prefix = self.url
-        self.user = self.group("user")
-        if not self.user:
-            self.user = config.get(
-                Config.SECTION_REMOTE_USER, getpass.getuser()
-            )
+        self.path_info = None
+        url = config.get(Config.SECTION_REMOTE_URL)
+        if not url:
+            return
 
-        self.path_info = PathHDFS(user=self.user)
+        parsed = urlparse(url)
+
+        user = parsed.username or config.get(Config.SECTION_REMOTE_USER)
+
+        self.path_info = self.path_cls.from_parts(
+            scheme=self.scheme,
+            host=parsed.hostname,
+            user=user,
+            port=parsed.port,
+            path=parsed.path,
+        )
+
+    def hdfs(self, path_info):
+        import pyarrow
+
+        return get_connection(
+            pyarrow.hdfs.connect,
+            path_info.host,
+            path_info.port,
+            user=path_info.user,
+        )
 
     def hadoop_fs(self, cmd, user=None):
         cmd = "hadoop fs -" + cmd
@@ -66,114 +87,83 @@ class RemoteHDFS(RemoteBASE):
         return match.group(gname)
 
     def get_file_checksum(self, path_info):
+        # NOTE: pyarrow doesn't support checksum, so we need to use hadoop
         regex = r".*\t.*\t(?P<checksum>.*)"
         stdout = self.hadoop_fs(
             "checksum {}".format(path_info.path), user=path_info.user
         )
         return self._group(regex, stdout, "checksum")
 
-    def copy(self, from_info, to_info):
+    def copy(self, from_info, to_info, **_kwargs):
         dname = posixpath.dirname(to_info.path)
-        self.hadoop_fs("mkdir -p {}".format(dname), user=to_info.user)
-        self.hadoop_fs(
-            "cp -f {} {}".format(from_info.path, to_info.path),
-            user=to_info.user,
-        )
-
-    def rm(self, path_info):
-        self.hadoop_fs("rm -f {}".format(path_info.path), user=path_info.user)
+        with self.hdfs(to_info) as hdfs:
+            hdfs.mkdir(dname)
+            # NOTE: this is how `hadoop fs -cp` works too: it copies through
+            # your local machine.
+            with hdfs.open(from_info.path, "rb") as from_fobj:
+                with hdfs.open(to_info.path, "wb") as to_fobj:
+                    to_fobj.upload(from_fobj)
 
     def remove(self, path_info):
         if path_info.scheme != "hdfs":
             raise NotImplementedError
 
-        assert path_info.path
-
-        logger.debug("Removing {}".format(path_info.path))
-
-        self.rm(path_info)
+        if self.exists(path_info):
+            logger.debug("Removing {}".format(path_info.path))
+            with self.hdfs(path_info) as hdfs:
+                hdfs.rm(path_info.path)
 
     def exists(self, path_info):
         assert not isinstance(path_info, list)
         assert path_info.scheme == "hdfs"
+        with self.hdfs(path_info) as hdfs:
+            return hdfs.exists(path_info.path)
+
+    def _upload(self, from_file, to_info, **_kwargs):
+        with self.hdfs(to_info) as hdfs:
+            hdfs.mkdir(posixpath.dirname(to_info.path))
+            tmp_file = tmp_fname(to_info.path)
+            with open(from_file, "rb") as fobj:
+                hdfs.upload(tmp_file, fobj)
+            hdfs.rename(tmp_file, to_info.path)
+
+    def _download(self, from_info, to_file, **_kwargs):
+        with self.hdfs(from_info) as hdfs:
+            with open(to_file, "wb+") as fobj:
+                hdfs.download(from_info.path, fobj)
+
+    @contextmanager
+    def open(self, path_info, mode="r", encoding=None):
+        assert mode in {"r", "rt", "rb"}
 
         try:
-            self.hadoop_fs("test -e {}".format(path_info.path))
-            return True
-        except RemoteCmdError:
-            return False
-
-    def upload(self, from_infos, to_infos, names=None, no_progress_bar=False):
-        names = self._verify_path_args(to_infos, from_infos, names)
-
-        for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info.scheme != "hdfs":
-                raise NotImplementedError
-
-            if from_info.scheme != "local":
-                raise NotImplementedError
-
-            self.hadoop_fs(
-                "mkdir -p {}".format(posixpath.dirname(to_info.path)),
-                user=to_info.user,
-            )
-
-            tmp_file = tmp_fname(to_info.path)
-
-            self.hadoop_fs(
-                "copyFromLocal {} {}".format(from_info.path, tmp_file),
-                user=to_info.user,
-            )
-
-            self.hadoop_fs(
-                "mv {} {}".format(tmp_file, to_info.path), user=to_info.user
-            )
-
-    def download(
-        self,
-        from_infos,
-        to_infos,
-        no_progress_bar=False,
-        names=None,
-        resume=False,
-    ):
-        names = self._verify_path_args(from_infos, to_infos, names)
-
-        for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info.scheme != "hdfs":
-                raise NotImplementedError
-
-            if to_info.scheme == "hdfs":
-                self.copy(from_info, to_info)
-                continue
-
-            if to_info.scheme != "local":
-                raise NotImplementedError
-
-            dname = os.path.dirname(to_info.path)
-            if not os.path.exists(dname):
-                os.makedirs(dname)
-
-            tmp_file = tmp_fname(to_info.path)
-
-            self.hadoop_fs(
-                "copyToLocal {} {}".format(from_info.path, tmp_file),
-                user=from_info.user,
-            )
-
-            os.rename(tmp_file, to_info.path)
+            with self.hdfs(path_info) as hdfs, closing(
+                hdfs.open(path_info.path, mode="rb")
+            ) as fd:
+                if mode == "rb":
+                    yield fd
+                else:
+                    yield io.TextIOWrapper(fd, encoding=encoding)
+        except IOError as e:
+            # Empty .errno and not specific enough error class in pyarrow,
+            # see https://issues.apache.org/jira/browse/ARROW-6248
+            if "file does not exist" in str(e):
+                raise FileNotFoundError(*e.args)
+            raise
 
     def list_cache_paths(self):
-        try:
-            self.hadoop_fs("test -e {}".format(self.prefix))
-        except RemoteCmdError:
+        if not self.exists(self.path_info):
             return []
 
-        stdout = self.hadoop_fs("ls -R {}".format(self.prefix))
-        lines = stdout.split("\n")
-        flist = []
-        for line in lines:
-            if not line.startswith("-"):
-                continue
-            flist.append(line.split()[-1])
-        return flist
+        files = []
+        dirs = [self.path_info.path]
+
+        with self.hdfs(self.path_info) as hdfs:
+            while dirs:
+                for entry in hdfs.ls(dirs.pop(), detail=True):
+                    if entry["kind"] == "directory":
+                        dirs.append(urlparse(entry["name"]).path)
+                    elif entry["kind"] == "file":
+                        files.append(urlparse(entry["name"]).path)
+
+        return files

dvc/remote/http.py

@@ -1,123 +1,70 @@
 from __future__ import unicode_literals
 
-from dvc.scheme import Schemes
-from dvc.path import Path
-from dvc.utils.compat import open, makedirs
-
-import os
-import threading
-import requests
 import logging
 
-from dvc.progress import progress
-from dvc.exceptions import DvcException
-from dvc.config import Config
-from dvc.remote.base import RemoteBASE
-from dvc.utils import move
+from funcy import cached_property
 
+from dvc.scheme import Schemes
+from dvc.utils.compat import open
+from dvc.progress import Tqdm
+from dvc.exceptions import DvcException
+from dvc.config import Config, ConfigError
+from dvc.remote.base import RemoteBASE
 
 logger = logging.getLogger(__name__)
 
 
-class ProgressBarCallback(object):
-    def __init__(self, name, total):
-        self.name = name
-        self.total = total
-        self.current = 0
-        self.lock = threading.Lock()
-
-    def __call__(self, byts):
-        with self.lock:
-            self.current += byts
-            progress.update_target(self.name, self.current, self.total)
-
-
 class RemoteHTTP(RemoteBASE):
     scheme = Schemes.HTTP
-    REGEX = r"^http://.*$"
+    SESSION_RETRIES = 5
+    SESSION_BACKOFF_FACTOR = 0.1
     REQUEST_TIMEOUT = 10
     CHUNK_SIZE = 2 ** 16
     PARAM_CHECKSUM = "etag"
 
     def __init__(self, repo, config):
         super(RemoteHTTP, self).__init__(repo, config)
-        self.cache_dir = config.get(Config.SECTION_REMOTE_URL)
-        self.url = self.cache_dir
 
-        self.path_info = Path(self.scheme)
+        url = config.get(Config.SECTION_REMOTE_URL)
+        self.path_info = self.path_cls(url) if url else None
 
-    @property
-    def prefix(self):
-        return self.cache_dir
-
-    def download(
-        self,
-        from_infos,
-        to_infos,
-        no_progress_bar=False,
-        names=None,
-        resume=False,
-    ):
-        names = self._verify_path_args(to_infos, from_infos, names)
-
-        for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info.scheme != self.scheme:
-                raise NotImplementedError
-
-            if to_info.scheme != "local":
-                raise NotImplementedError
-
-            msg = "Downloading '{}' to '{}'".format(
-                from_info.path, to_info.path
+        if not self.no_traverse:
+            raise ConfigError(
+                "HTTP doesn't support traversing the remote to list existing "
+                "files. Use: `dvc remote modify <name> no_traverse true`"
             )
-            logger.debug(msg)
 
-            if not name:
-                name = os.path.basename(to_info.path)
-
-            makedirs(os.path.dirname(to_info.path), exist_ok=True)
-
-            total = self._content_length(from_info.path)
-
-            if no_progress_bar or not total:
-                cb = None
-            else:
-                cb = ProgressBarCallback(name, total)
-
-            try:
-                self._download_to(
-                    from_info.path, to_info.path, callback=cb, resume=resume
-                )
-
-            except Exception:
-                msg = "failed to download '{}'".format(from_info.path)
-                logger.exception(msg)
-                continue
-
-            if not no_progress_bar:
-                progress.finish_target(name)
+    def _download(self, from_info, to_file, name=None, no_progress_bar=False):
+        request = self._request("GET", from_info.url, stream=True)
+        with Tqdm(
+            total=None if no_progress_bar else self._content_length(from_info),
+            leave=False,
+            bytes=True,
+            desc=from_info.url if name is None else name,
+            disable=no_progress_bar,
+        ) as pbar:
+            with open(to_file, "wb") as fd:
+                for chunk in request.iter_content(chunk_size=self.CHUNK_SIZE):
+                    fd.write(chunk)
+                    fd.flush()
+                    pbar.update(len(chunk))
 
     def exists(self, path_info):
-        assert not isinstance(path_info, list)
-        assert path_info.scheme == self.scheme
-        return bool(self._request("HEAD", path_info.path))
+        return bool(self._request("HEAD", path_info.url))
 
-    def cache_exists(self, md5s):
-        assert isinstance(md5s, list)
-
-        def func(md5):
-            return bool(self._request("HEAD", self.checksum_to_path(md5)))
-
-        return list(filter(func, md5s))
-
-    def _content_length(self, url):
-        return self._request("HEAD", url).headers.get("Content-Length")
+    def _content_length(self, url_or_request):
+        headers = getattr(
+            url_or_request,
+            "headers",
+            self._request("HEAD", url_or_request).headers,
+        )
+        res = headers.get("Content-Length")
+        return int(res) if res else None
 
     def get_file_checksum(self, path_info):
-        url = path_info.path
-        etag = self._request("HEAD", url).headers.get("ETag") or self._request(
-            "HEAD", url
-        ).headers.get("Content-MD5")
+        url = path_info.url
+        headers = self._request("HEAD", url).headers
+        etag = headers.get("ETag") or headers.get("Content-MD5")
 
         if not etag:
             raise DvcException(
@@ -133,61 +80,32 @@ class RemoteHTTP(RemoteBASE):
 
         return etag
 
-    def _download_to(self, url, target_file, callback=None, resume=False):
-        request = self._request("GET", url, stream=True)
-        partial_file = target_file + ".part"
+    @cached_property
+    def _session(self):
+        import requests
+        from requests.adapters import HTTPAdapter
+        from urllib3.util.retry import Retry
 
-        mode, transferred_bytes = self._determine_mode_get_transferred_bytes(
-            partial_file, resume
+        session = requests.Session()
+
+        retries = Retry(
+            total=self.SESSION_RETRIES,
+            backoff_factor=self.SESSION_BACKOFF_FACTOR,
         )
 
-        self._validate_existing_file_size(transferred_bytes, partial_file)
+        session.mount("http://", HTTPAdapter(max_retries=retries))
+        session.mount("https://", HTTPAdapter(max_retries=retries))
 
-        self._write_request_content(
-            mode, partial_file, request, transferred_bytes, callback
-        )
-
-        move(partial_file, target_file)
-
-    def _write_request_content(
-        self, mode, partial_file, request, transferred_bytes, callback=None
-    ):
-        with open(partial_file, mode) as fd:
-
-            for index, chunk in enumerate(
-                request.iter_content(chunk_size=self.CHUNK_SIZE)
-            ):
-                chunk_number = index + 1
-                if chunk_number * self.CHUNK_SIZE > transferred_bytes:
-                    fd.write(chunk)
-                    fd.flush()
-                    transferred_bytes += len(chunk)
-
-                if callback:
-                    callback(transferred_bytes)
-
-    def _validate_existing_file_size(self, bytes_transferred, partial_file):
-        if bytes_transferred % self.CHUNK_SIZE != 0:
-            raise DvcException(
-                "File {}, might be corrupted, please remove "
-                "it and retry importing".format(partial_file)
-            )
-
-    def _determine_mode_get_transferred_bytes(self, partial_file, resume):
-        if os.path.exists(partial_file) and resume:
-            mode = "ab"
-            bytes_transfered = os.path.getsize(partial_file)
-        else:
-            mode = "wb"
-            bytes_transfered = 0
-        return mode, bytes_transfered
+        return session
 
     def _request(self, method, url, **kwargs):
+        import requests
+
         kwargs.setdefault("allow_redirects", True)
         kwargs.setdefault("timeout", self.REQUEST_TIMEOUT)
 
         try:
-            return requests.request(method, url, **kwargs)
+            return self._session.request(method, url, **kwargs)
         except requests.exceptions.RequestException:
             raise DvcException("could not perform a {} request".format(method))
 

dvc/remote/https.py

@@ -7,4 +7,3 @@ from .http import RemoteHTTP
 
 class RemoteHTTPS(RemoteHTTP):
     scheme = Schemes.HTTPS
-    REGEX = r"^https://.*$"

dvc/remote/local.py

@@ -0,0 +1,556 @@
+from __future__ import unicode_literals
+import os
+import stat
+import errno
+import logging
+from functools import partial
+
+from shortuuid import uuid
+
+from dvc.scheme import Schemes
+from dvc.utils.compat import str, fspath_py35, open
+from dvc.system import System
+from dvc.remote.base import (
+    RemoteBASE,
+    STATUS_MAP,
+    STATUS_NEW,
+    STATUS_DELETED,
+    STATUS_MISSING,
+)
+from dvc.utils import (
+    remove,
+    move,
+    copyfile,
+    tmp_fname,
+    file_md5,
+    walk_files,
+    relpath,
+    dvc_walk,
+    makedirs,
+)
+from dvc.config import Config
+from dvc.exceptions import DvcException, DownloadError, UploadError
+from dvc.progress import Tqdm, TqdmThreadPoolExecutor
+
+from dvc.path_info import PathInfo
+
+logger = logging.getLogger(__name__)
+
+
+class RemoteLOCAL(RemoteBASE):
+    scheme = Schemes.LOCAL
+    path_cls = PathInfo
+    PARAM_CHECKSUM = "md5"
+    PARAM_PATH = "path"
+
+    UNPACKED_DIR_SUFFIX = ".unpacked"
+
+    DEFAULT_CACHE_TYPES = ["reflink", "copy"]
+
+    SHARED_MODE_MAP = {None: (0o644, 0o755), "group": (0o664, 0o775)}
+
+    def __init__(self, repo, config):
+        super(RemoteLOCAL, self).__init__(repo, config)
+        self.protected = config.get(Config.SECTION_CACHE_PROTECTED, False)
+
+        shared = config.get(Config.SECTION_CACHE_SHARED)
+        self._file_mode, self._dir_mode = self.SHARED_MODE_MAP[shared]
+
+        if self.protected:
+            # cache files are set to be read-only for everyone
+            self._file_mode = stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH
+
+        # A clunky way to detect cache dir
+        storagepath = config.get(Config.SECTION_LOCAL_STORAGEPATH, None)
+        cache_dir = config.get(Config.SECTION_REMOTE_URL, storagepath)
+
+        if cache_dir is not None and not os.path.isabs(cache_dir):
+            cwd = config[Config.PRIVATE_CWD]
+            cache_dir = os.path.abspath(os.path.join(cwd, cache_dir))
+
+        self.path_info = PathInfo(cache_dir) if cache_dir else None
+        self._dir_info = {}
+
+    @property
+    def state(self):
+        return self.repo.state
+
+    @property
+    def cache_dir(self):
+        return self.path_info.fspath if self.path_info else None
+
+    @classmethod
+    def supported(cls, config):
+        return True
+
+    def list_cache_paths(self):
+        assert self.path_info is not None
+
+        clist = []
+        for entry in os.listdir(fspath_py35(self.path_info)):
+            subdir = self.path_info / entry
+            if not os.path.isdir(fspath_py35(subdir)):
+                continue
+            clist.extend(
+                subdir / cache for cache in os.listdir(fspath_py35(subdir))
+            )
+
+        return clist
+
+    def get(self, md5):
+        if not md5:
+            return None
+
+        return self.checksum_to_path_info(md5).url
+
+    @staticmethod
+    def exists(path_info):
+        assert path_info.scheme == "local"
+        return os.path.lexists(fspath_py35(path_info))
+
+    def makedirs(self, path_info):
+        makedirs(path_info, exist_ok=True, mode=self._dir_mode)
+
+    def already_cached(self, path_info):
+        assert path_info.scheme in ["", "local"]
+
+        current_md5 = self.get_checksum(path_info)
+
+        if not current_md5:
+            return False
+
+        return not self.changed_cache(current_md5)
+
+    def is_empty(self, path_info):
+        path = path_info.fspath
+
+        if self.isfile(path_info) and os.path.getsize(path) == 0:
+            return True
+
+        if self.isdir(path_info) and len(os.listdir(path)) == 0:
+            return True
+
+        return False
+
+    @staticmethod
+    def isfile(path_info):
+        return os.path.isfile(fspath_py35(path_info))
+
+    @staticmethod
+    def isdir(path_info):
+        return os.path.isdir(fspath_py35(path_info))
+
+    @staticmethod
+    def getsize(path_info):
+        return os.path.getsize(fspath_py35(path_info))
+
+    def walk(self, path_info):
+        return dvc_walk(path_info, self.repo.dvcignore)
+
+    def get_file_checksum(self, path_info):
+        return file_md5(fspath_py35(path_info))[0]
+
+    def remove(self, path_info):
+        if path_info.scheme != "local":
+            raise NotImplementedError
+
+        if self.exists(path_info):
+            remove(path_info.fspath)
+
+    def move(self, from_info, to_info):
+        if from_info.scheme != "local" or to_info.scheme != "local":
+            raise NotImplementedError
+
+        self.makedirs(to_info.parent)
+
+        if self.isfile(from_info):
+            mode = self._file_mode
+        else:
+            mode = self._dir_mode
+
+        move(from_info, to_info, mode=mode)
+
+    @staticmethod
+    def copy(from_info, to_info):
+        System.copy(from_info, to_info)
+
+    @staticmethod
+    def symlink(from_info, to_info):
+        System.symlink(from_info, to_info)
+
+    def hardlink(self, from_info, to_info):
+        # If there are a lot of empty files (which happens a lot in datasets),
+        # and the cache type is `hardlink`, we might reach link limits and
+        # will get something like: `too many links error`
+        #
+        # This is because all those empty files will have the same checksum
+        # (i.e. 68b329da9893e34099c7d8ad5cb9c940), therfore, they will be
+        # linked to the same file in the cache.
+        #
+        # From https://en.wikipedia.org/wiki/Hard_link
+        #   * ext4 limits the number of hard links on a file to 65,000
+        #   * Windows with NTFS has a limit of 1024 hard links on a file
+        #
+        # That's why we simply create an empty file rather than a link.
+        if self.getsize(from_info) == 0:
+            self.open(to_info, "w").close()
+
+            logger.debug(
+                "Created empty file: {src} -> {dest}".format(
+                    src=str(from_info), dest=str(to_info)
+                )
+            )
+            return
+
+        System.hardlink(from_info, to_info)
+
+    @staticmethod
+    def reflink(from_info, to_info):
+        System.reflink(from_info, to_info)
+
+    def cache_exists(self, checksums, jobs=None, name=None):
+        return [
+            checksum
+            for checksum in Tqdm(
+                checksums,
+                unit="file",
+                desc="Querying "
+                + ("cache in " + name if name else "local cache"),
+            )
+            if not self.changed_cache_file(checksum)
+        ]
+
+    def _upload(
+        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
+    ):
+        makedirs(fspath_py35(to_info.parent), exist_ok=True)
+
+        tmp_file = tmp_fname(to_info)
+        copyfile(
+            from_file, tmp_file, name=name, no_progress_bar=no_progress_bar
+        )
+        os.rename(tmp_file, fspath_py35(to_info))
+
+    def _download(
+        self, from_info, to_file, name=None, no_progress_bar=False, **_kwargs
+    ):
+        copyfile(
+            fspath_py35(from_info),
+            to_file,
+            no_progress_bar=no_progress_bar,
+            name=name,
+        )
+
+    @staticmethod
+    def open(path_info, mode="r", encoding=None):
+        return open(fspath_py35(path_info), mode=mode, encoding=encoding)
+
+    def status(
+        self,
+        named_cache,
+        remote,
+        jobs=None,
+        show_checksums=False,
+        download=False,
+    ):
+        logger.debug(
+            "Preparing to collect status from {}".format(remote.path_info)
+        )
+        md5s = list(named_cache[self.scheme])
+
+        logger.debug("Collecting information from local cache...")
+        local_exists = self.cache_exists(md5s, jobs=jobs, name=self.cache_dir)
+
+        # This is a performance optimization. We can safely assume that,
+        # if the resources that we want to fetch are already cached,
+        # there's no need to check the remote storage for the existance of
+        # those files.
+        if download and sorted(local_exists) == sorted(md5s):
+            remote_exists = local_exists
+        else:
+            logger.debug("Collecting information from remote cache...")
+            remote_exists = list(
+                remote.cache_exists(
+                    md5s, jobs=jobs, name=str(remote.path_info)
+                )
+            )
+
+        ret = {
+            checksum: {"name": checksum if show_checksums else " ".join(names)}
+            for checksum, names in named_cache[self.scheme].items()
+        }
+        self._fill_statuses(ret, local_exists, remote_exists)
+
+        self._log_missing_caches(ret)
+
+        return ret
+
+    @staticmethod
+    def _fill_statuses(checksum_info_dir, local_exists, remote_exists):
+        # Using sets because they are way faster for lookups
+        local = set(local_exists)
+        remote = set(remote_exists)
+
+        for md5, info in checksum_info_dir.items():
+            status = STATUS_MAP[(md5 in local, md5 in remote)]
+            info["status"] = status
+
+    def _get_plans(self, download, remote, status_info, status):
+        cache = []
+        path_infos = []
+        names = []
+        for md5, info in Tqdm(
+            status_info.items(), desc="Analysing status", unit="file"
+        ):
+            if info["status"] == status:
+                cache.append(self.checksum_to_path_info(md5))
+                path_infos.append(remote.checksum_to_path_info(md5))
+                names.append(info["name"])
+
+        if download:
+            to_infos = cache
+            from_infos = path_infos
+        else:
+            to_infos = path_infos
+            from_infos = cache
+
+        return from_infos, to_infos, names
+
+    def _process(
+        self,
+        named_cache,
+        remote,
+        jobs=None,
+        show_checksums=False,
+        download=False,
+    ):
+        logger.debug(
+            "Preparing to {} '{}'".format(
+                "download data from" if download else "upload data to",
+                remote.path_info,
+            )
+        )
+
+        if download:
+            func = partial(
+                remote.download,
+                dir_mode=self._dir_mode,
+                file_mode=self._file_mode,
+            )
+            status = STATUS_DELETED
+        else:
+            func = remote.upload
+            status = STATUS_NEW
+
+        if jobs is None:
+            jobs = remote.JOBS
+
+        status_info = self.status(
+            named_cache,
+            remote,
+            jobs=jobs,
+            show_checksums=show_checksums,
+            download=download,
+        )
+
+        plans = self._get_plans(download, remote, status_info, status)
+
+        if len(plans[0]) == 0:
+            return 0
+
+        if jobs > 1:
+            with TqdmThreadPoolExecutor(max_workers=jobs) as executor:
+                fails = sum(executor.map(func, *plans))
+        else:
+            fails = sum(map(func, *plans))
+
+        if fails:
+            if download:
+                raise DownloadError(fails)
+            raise UploadError(fails)
+
+        return len(plans[0])
+
+    def push(self, named_cache, remote, jobs=None, show_checksums=False):
+        return self._process(
+            named_cache,
+            remote,
+            jobs=jobs,
+            show_checksums=show_checksums,
+            download=False,
+        )
+
+    def pull(self, named_cache, remote, jobs=None, show_checksums=False):
+        return self._process(
+            named_cache,
+            remote,
+            jobs=jobs,
+            show_checksums=show_checksums,
+            download=True,
+        )
+
+    @staticmethod
+    def _log_missing_caches(checksum_info_dict):
+        missing_caches = [
+            (md5, info)
+            for md5, info in checksum_info_dict.items()
+            if info["status"] == STATUS_MISSING
+        ]
+        if missing_caches:
+            missing_desc = "".join(
+                "\nname: {}, md5: {}".format(info["name"], md5)
+                for md5, info in missing_caches
+            )
+            msg = (
+                "Some of the cache files do not exist neither locally "
+                "nor on remote. Missing cache files: {}".format(missing_desc)
+            )
+            logger.warning(msg)
+
+    @staticmethod
+    def _unprotect_file(path, allow_copy=True):
+        if System.is_symlink(path) or System.is_hardlink(path):
+            if allow_copy:
+                logger.debug("Unprotecting '{}'".format(path))
+                tmp = os.path.join(os.path.dirname(path), "." + str(uuid()))
+
+                # The operations order is important here - if some application
+                # would access the file during the process of copyfile then it
+                # would get only the part of file. So, at first, the file
+                # should be copied with the temporary name, and then
+                # original file should be replaced by new.
+                copyfile(
+                    path, tmp, name="Unprotecting '{}'".format(relpath(path))
+                )
+                remove(path)
+                os.rename(tmp, path)
+
+        else:
+            logger.debug(
+                "Skipping copying for '{}', since it is not "
+                "a symlink or a hardlink.".format(path)
+            )
+
+        os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
+
+    def _unprotect_dir(self, path, allow_copy=True):
+        for fname in walk_files(path, self.repo.dvcignore):
+            self._unprotect_file(fname, allow_copy)
+
+    def unprotect(self, path_info, allow_copy=True):
+        path = path_info.fspath
+        if not os.path.exists(path):
+            raise DvcException(
+                "can't unprotect non-existing data '{}'".format(path)
+            )
+
+        if os.path.isdir(path):
+            self._unprotect_dir(path, allow_copy)
+        else:
+            self._unprotect_file(path, allow_copy)
+
+    @staticmethod
+    def protect(path_info):
+        path = fspath_py35(path_info)
+        mode = stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH
+
+        try:
+            os.chmod(path, mode)
+        except OSError as exc:
+            # In share cache scenario, we might not own the cache file, so we
+            # need to check if cache file is already protected.
+            if exc.errno not in [errno.EPERM, errno.EACCES]:
+                raise
+
+            actual = os.stat(path).st_mode
+            if actual & mode != mode:
+                raise
+
+    def _get_unpacked_dir_path_info(self, checksum):
+        info = self.checksum_to_path_info(checksum)
+        return info.with_name(info.name + self.UNPACKED_DIR_SUFFIX)
+
+    def _path_info_changed(self, path_info):
+        if self.exists(path_info) and self.state.get(path_info):
+            return False
+        return True
+
+    def _update_unpacked_dir(self, checksum):
+        unpacked_dir_info = self._get_unpacked_dir_path_info(checksum)
+
+        if not self._path_info_changed(unpacked_dir_info):
+            return
+
+        self.remove(unpacked_dir_info)
+
+        try:
+            dir_info = self.get_dir_cache(checksum)
+            self._create_unpacked_dir(checksum, dir_info, unpacked_dir_info)
+        except DvcException:
+            logger.warning("Could not create '{}'".format(unpacked_dir_info))
+
+            self.remove(unpacked_dir_info)
+
+    def _create_unpacked_dir(self, checksum, dir_info, unpacked_dir_info):
+        self.makedirs(unpacked_dir_info)
+
+        for entry in Tqdm(dir_info, desc="Creating unpacked dir", unit="file"):
+            entry_cache_info = self.checksum_to_path_info(
+                entry[self.PARAM_CHECKSUM]
+            )
+            relative_path = entry[self.PARAM_RELPATH]
+            # In shared cache mode some cache files might not be owned by the
+            # user, so we need to use symlinks because, unless
+            # /proc/sys/fs/protected_hardlinks is disabled, the user is not
+            # allowed to create hardlinks to files that he doesn't own.
+            link_types = ["hardlink", "symlink"]
+            self._link(
+                entry_cache_info, unpacked_dir_info / relative_path, link_types
+            )
+
+        self.state.save(unpacked_dir_info, checksum)
+
+    def _changed_unpacked_dir(self, checksum):
+        status_unpacked_dir_info = self._get_unpacked_dir_path_info(checksum)
+
+        return not self.state.get(status_unpacked_dir_info)
+
+    def _get_unpacked_dir_names(self, checksums):
+        unpacked = set()
+        for c in checksums:
+            if self.is_dir_checksum(c):
+                unpacked.add(c + self.UNPACKED_DIR_SUFFIX)
+        return unpacked
+
+    def _get_cache_type(self, path_info):
+        if self.cache_type_confirmed:
+            return self.cache_types[0]
+
+        workspace_file = path_info.with_name("." + uuid())
+        test_cache_file = self.path_info / ".cache_type_test_file"
+        if not self.exists(test_cache_file):
+            with open(fspath_py35(test_cache_file), "wb") as fobj:
+                fobj.write(bytes(1))
+        try:
+            self.link(test_cache_file, workspace_file)
+        finally:
+            self.remove(workspace_file)
+            self.remove(test_cache_file)
+
+        self.cache_type_confirmed = True
+        return self.cache_types[0]
+
+    def _link_matches(self, path_info):
+        is_hardlink = System.is_hardlink(path_info)
+        is_symlink = System.is_symlink(path_info)
+        is_copy_or_reflink = not is_hardlink and not is_symlink
+
+        cache_type = self._get_cache_type(path_info)
+
+        if cache_type == "symlink":
+            return is_symlink
+
+        if cache_type == "hardlink":
+            return is_hardlink
+
+        return is_copy_or_reflink

dvc/remote/local/__init__.py

@@ -1,562 +0,0 @@
-from __future__ import unicode_literals
-
-from copy import copy
-
-from dvc.scheme import Schemes
-from dvc.path.base import PathBASE
-from dvc.path.local import PathLOCAL
-from dvc.remote.local.slow_link_detection import slow_link_guard
-from dvc.utils.compat import str, makedirs
-
-import os
-import stat
-import uuid
-import ntpath
-import shutil
-import posixpath
-import logging
-
-from dvc.system import System
-from dvc.remote.base import (
-    RemoteBASE,
-    STATUS_MAP,
-    STATUS_NEW,
-    STATUS_DELETED,
-    STATUS_MISSING,
-)
-from dvc.utils import (
-    remove,
-    move,
-    copyfile,
-    to_chunks,
-    tmp_fname,
-    file_md5,
-    walk_files,
-)
-from dvc.config import Config
-from dvc.exceptions import DvcException
-from dvc.progress import progress
-from concurrent.futures import ThreadPoolExecutor
-
-
-logger = logging.getLogger(__name__)
-
-
-class RemoteLOCAL(RemoteBASE):
-    scheme = Schemes.LOCAL
-    REGEX = r"^(?P<path>.*)$"
-    PARAM_CHECKSUM = "md5"
-    PARAM_PATH = "path"
-
-    DEFAULT_CACHE_TYPES = ["reflink", "copy"]
-    CACHE_TYPE_MAP = {
-        "copy": shutil.copyfile,
-        "symlink": System.symlink,
-        "hardlink": System.hardlink,
-        "reflink": System.reflink,
-    }
-
-    def __init__(self, repo, config):
-        super(RemoteLOCAL, self).__init__(repo, config)
-        self.state = self.repo.state if self.repo else None
-        self.protected = config.get(Config.SECTION_CACHE_PROTECTED, False)
-        storagepath = config.get(Config.SECTION_AWS_STORAGEPATH, None)
-        self.cache_dir = config.get(Config.SECTION_REMOTE_URL, storagepath)
-
-        if self.cache_dir is not None and not os.path.isabs(self.cache_dir):
-            cwd = config[Config.PRIVATE_CWD]
-            self.cache_dir = os.path.abspath(os.path.join(cwd, self.cache_dir))
-
-        types = config.get(Config.SECTION_CACHE_TYPE, None)
-        if types:
-            if isinstance(types, str):
-                types = [t.strip() for t in types.split(",")]
-            self.cache_types = types
-        else:
-            self.cache_types = copy(self.DEFAULT_CACHE_TYPES)
-
-        if self.cache_dir is not None and not os.path.exists(self.cache_dir):
-            os.mkdir(self.cache_dir)
-
-        self._dir_info = {}
-        self.path_info = PathLOCAL()
-
-    @staticmethod
-    def compat_config(config):
-        ret = config.copy()
-        url = ret.pop(Config.SECTION_AWS_STORAGEPATH, "")
-        ret[Config.SECTION_REMOTE_URL] = url
-        return ret
-
-    @property
-    def url(self):
-        return self.cache_dir
-
-    @property
-    def prefix(self):
-        return self.cache_dir
-
-    def list_cache_paths(self):
-        clist = []
-        for entry in os.listdir(self.cache_dir):
-            subdir = os.path.join(self.cache_dir, entry)
-            if not os.path.isdir(subdir):
-                continue
-
-            for cache in os.listdir(subdir):
-                clist.append(os.path.join(subdir, cache))
-
-        return clist
-
-    def get(self, md5):
-        if not md5:
-            return None
-
-        return self.checksum_to_path(md5)
-
-    def exists(self, path_info):
-        assert isinstance(path_info, PathBASE)
-        assert path_info.scheme == "local"
-        return os.path.lexists(path_info.path)
-
-    def makedirs(self, path_info):
-        if not self.exists(path_info):
-            os.makedirs(path_info.path)
-
-    @slow_link_guard
-    def link(self, cache_info, path_info):
-        cache = cache_info.path
-        path = path_info.path
-
-        assert os.path.isfile(cache)
-
-        dname = os.path.dirname(path)
-        if not os.path.exists(dname):
-            os.makedirs(dname)
-
-        # NOTE: just create an empty file for an empty cache
-        if os.path.getsize(cache) == 0:
-            open(path, "w+").close()
-
-            msg = "Created empty file: {} -> {}".format(cache, path)
-            logger.debug(msg)
-            return
-
-        i = len(self.cache_types)
-        while i > 0:
-            try:
-                self.CACHE_TYPE_MAP[self.cache_types[0]](cache, path)
-
-                if self.protected:
-                    self.protect(path_info)
-
-                msg = "Created {}'{}': {} -> {}".format(
-                    "protected " if self.protected else "",
-                    self.cache_types[0],
-                    cache,
-                    path,
-                )
-
-                logger.debug(msg)
-                return
-
-            except DvcException as exc:
-                msg = "Cache type '{}' is not supported: {}"
-                logger.debug(msg.format(self.cache_types[0], str(exc)))
-                del self.cache_types[0]
-                i -= 1
-
-        raise DvcException("no possible cache types left to try out.")
-
-    @property
-    def ospath(self):
-        if os.name == "nt":
-            return ntpath
-        return posixpath
-
-    def already_cached(self, path_info):
-        assert path_info.scheme in ["", "local"]
-
-        current_md5 = self.get_checksum(path_info)
-
-        if not current_md5:
-            return False
-
-        return not self.changed_cache(current_md5)
-
-    def is_empty(self, path_info):
-        path = path_info.path
-
-        if self.isfile(path_info) and os.path.getsize(path) == 0:
-            return True
-
-        if self.isdir(path_info) and len(os.listdir(path)) == 0:
-            return True
-
-        return False
-
-    def isfile(self, path_info):
-        return os.path.isfile(path_info.path)
-
-    def isdir(self, path_info):
-        return os.path.isdir(path_info.path)
-
-    def walk(self, path_info):
-        return os.walk(path_info.path)
-
-    def get_file_checksum(self, path_info):
-        return file_md5(path_info.path)[0]
-
-    def remove(self, path_info):
-        if path_info.scheme != "local":
-            raise NotImplementedError
-
-        remove(path_info.path)
-
-    def move(self, from_info, to_info):
-        if from_info.scheme != "local" or to_info.scheme != "local":
-            raise NotImplementedError
-
-        inp = from_info.path
-        outp = to_info.path
-
-        # moving in two stages to make the whole operation atomic in
-        # case inp and outp are in different filesystems and actual
-        # physical copying of data is happening
-        tmp = "{}.{}".format(outp, str(uuid.uuid4()))
-        move(inp, tmp)
-        move(tmp, outp)
-
-    def cache_exists(self, md5s):
-        assert isinstance(md5s, list)
-        return list(filter(lambda md5: not self.changed_cache_file(md5), md5s))
-
-    def upload(self, from_infos, to_infos, names=None, no_progress_bar=False):
-        names = self._verify_path_args(to_infos, from_infos, names)
-
-        for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info.scheme != "local":
-                raise NotImplementedError
-
-            if from_info.scheme != "local":
-                raise NotImplementedError
-
-            logger.debug(
-                "Uploading '{}' to '{}'".format(from_info.path, to_info.path)
-            )
-
-            if not name:
-                name = os.path.basename(from_info.path)
-
-            makedirs(os.path.dirname(to_info.path), exist_ok=True)
-            tmp_file = tmp_fname(to_info.path)
-
-            try:
-                copyfile(
-                    from_info.path,
-                    tmp_file,
-                    name=name,
-                    no_progress_bar=no_progress_bar,
-                )
-                os.rename(tmp_file, to_info.path)
-            except Exception:
-                logger.exception(
-                    "failed to upload '{}' to '{}'".format(
-                        from_info.path, to_info.path
-                    )
-                )
-
-    def download(
-        self,
-        from_infos,
-        to_infos,
-        no_progress_bar=False,
-        names=None,
-        resume=False,
-    ):
-        names = self._verify_path_args(from_infos, to_infos, names)
-
-        for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info.scheme != "local":
-                raise NotImplementedError
-
-            if to_info.scheme != "local":
-                raise NotImplementedError
-
-            logger.debug(
-                "Downloading '{}' to '{}'".format(from_info.path, to_info.path)
-            )
-
-            if not name:
-                name = os.path.basename(to_info.path)
-
-            makedirs(os.path.dirname(to_info.path), exist_ok=True)
-            tmp_file = tmp_fname(to_info.path)
-            try:
-                copyfile(
-                    from_info.path,
-                    tmp_file,
-                    no_progress_bar=no_progress_bar,
-                    name=name,
-                )
-
-                move(tmp_file, to_info.path)
-            except Exception:
-                logger.exception(
-                    "failed to download '{}' to '{}'".format(
-                        from_info.path, to_info.path
-                    )
-                )
-
-                continue
-
-    def _group(self, checksum_infos, show_checksums=False):
-        by_md5 = {}
-
-        for info in checksum_infos:
-            md5 = info[self.PARAM_CHECKSUM]
-
-            if show_checksums:
-                by_md5[md5] = {"name": md5}
-                continue
-
-            name = info[self.PARAM_PATH]
-            branch = info.get("branch")
-            if branch:
-                name += "({})".format(branch)
-
-            if md5 not in by_md5.keys():
-                by_md5[md5] = {"name": name}
-            else:
-                by_md5[md5]["name"] += " " + name
-
-        return by_md5
-
-    def status(
-        self,
-        checksum_infos,
-        remote,
-        jobs=None,
-        show_checksums=False,
-        download=False,
-    ):
-        logger.info("Preparing to collect status from {}".format(remote.url))
-        title = "Collecting information"
-
-        ret = {}
-
-        progress.set_n_total(1)
-        progress.update_target(title, 0, 100)
-
-        progress.update_target(title, 10, 100)
-
-        ret = self._group(checksum_infos, show_checksums=show_checksums)
-        md5s = list(ret.keys())
-
-        progress.update_target(title, 30, 100)
-
-        local_exists = self.cache_exists(md5s)
-
-        progress.update_target(title, 40, 100)
-
-        # This is a performance optimization. We can safely assume that,
-        # if the resources that we want to fetch are already cached,
-        # there's no need to check the remote storage for the existance of
-        # those files.
-        if download and sorted(local_exists) == sorted(md5s):
-            remote_exists = local_exists
-        else:
-            remote_exists = list(remote.cache_exists(md5s))
-
-        progress.update_target(title, 90, 100)
-
-        progress.finish_target(title)
-
-        self._fill_statuses(ret, local_exists, remote_exists)
-
-        self._log_missing_caches(ret)
-
-        return ret
-
-    def _fill_statuses(self, checksum_info_dir, local_exists, remote_exists):
-        # Using sets because they are way faster for lookups
-        local = set(local_exists)
-        remote = set(remote_exists)
-
-        for md5, info in checksum_info_dir.items():
-            status = STATUS_MAP[(md5 in local, md5 in remote)]
-            info["status"] = status
-
-    def _get_chunks(self, download, remote, status_info, status, jobs):
-        title = "Analysing status."
-
-        progress.set_n_total(1)
-        total = len(status_info)
-        current = 0
-
-        cache = []
-        path_infos = []
-        names = []
-        for md5, info in status_info.items():
-            if info["status"] == status:
-                cache.append(self.checksum_to_path_info(md5))
-                path_infos.append(remote.checksum_to_path_info(md5))
-                names.append(info["name"])
-            current += 1
-            progress.update_target(title, current, total)
-
-        progress.finish_target(title)
-
-        progress.set_n_total(len(names))
-
-        if download:
-            to_infos = cache
-            from_infos = path_infos
-        else:
-            to_infos = path_infos
-            from_infos = cache
-
-        return list(
-            zip(
-                to_chunks(from_infos, jobs),
-                to_chunks(to_infos, jobs),
-                to_chunks(names, jobs),
-            )
-        )
-
-    def _process(
-        self,
-        checksum_infos,
-        remote,
-        jobs=None,
-        show_checksums=False,
-        download=False,
-    ):
-        msg = "Preparing to {} data {} '{}'"
-        logger.info(
-            msg.format(
-                "download" if download else "upload",
-                "from" if download else "to",
-                remote.url,
-            )
-        )
-
-        if download:
-            func = remote.download
-            status = STATUS_DELETED
-        else:
-            func = remote.upload
-            status = STATUS_NEW
-
-        if jobs is None:
-            jobs = remote.JOBS
-
-        status_info = self.status(
-            checksum_infos,
-            remote,
-            jobs=jobs,
-            show_checksums=show_checksums,
-            download=download,
-        )
-
-        chunks = self._get_chunks(download, remote, status_info, status, jobs)
-
-        if len(chunks) == 0:
-            return 0
-
-        futures = []
-        with ThreadPoolExecutor(max_workers=jobs) as executor:
-            for from_infos, to_infos, names in chunks:
-                res = executor.submit(func, from_infos, to_infos, names=names)
-                futures.append(res)
-
-        for f in futures:
-            f.result()
-
-        return len(chunks)
-
-    def push(self, checksum_infos, remote, jobs=None, show_checksums=False):
-        return self._process(
-            checksum_infos,
-            remote,
-            jobs=jobs,
-            show_checksums=show_checksums,
-            download=False,
-        )
-
-    def pull(self, checksum_infos, remote, jobs=None, show_checksums=False):
-        return self._process(
-            checksum_infos,
-            remote,
-            jobs=jobs,
-            show_checksums=show_checksums,
-            download=True,
-        )
-
-    def _log_missing_caches(self, checksum_info_dict):
-        missing_caches = [
-            (md5, info)
-            for md5, info in checksum_info_dict.items()
-            if info["status"] == STATUS_MISSING
-        ]
-        if missing_caches:
-            missing_desc = "".join(
-                [
-                    "\nname: {}, md5: {}".format(info["name"], md5)
-                    for md5, info in missing_caches
-                ]
-            )
-            msg = (
-                "Some of the cache files do not exist neither locally "
-                "nor on remote. Missing cache files: {}".format(missing_desc)
-            )
-            logger.warning(msg)
-
-    @staticmethod
-    def _unprotect_file(path):
-        if System.is_symlink(path) or System.is_hardlink(path):
-            logger.debug("Unprotecting '{}'".format(path))
-            tmp = os.path.join(os.path.dirname(path), "." + str(uuid.uuid4()))
-
-            # The operations order is important here - if some application
-            # would access the file during the process of copyfile then it
-            # would get only the part of file. So, at first, the file should be
-            # copied with the temporary name, and then original file should be
-            # replaced by new.
-            copyfile(
-                path,
-                tmp,
-                name="Unprotecting '{}'".format(os.path.relpath(path)),
-            )
-            remove(path)
-            os.rename(tmp, path)
-
-        else:
-            logger.debug(
-                "Skipping copying for '{}', since it is not "
-                "a symlink or a hardlink.".format(path)
-            )
-
-        os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
-
-    @staticmethod
-    def _unprotect_dir(path):
-        for path in walk_files(path):
-            RemoteLOCAL._unprotect_file(path)
-
-    @staticmethod
-    def unprotect(path_info):
-        path = path_info.path
-        if not os.path.exists(path):
-            raise DvcException(
-                "can't unprotect non-existing data '{}'".format(path)
-            )
-
-        if os.path.isdir(path):
-            RemoteLOCAL._unprotect_dir(path)
-        else:
-            RemoteLOCAL._unprotect_file(path)
-
-    @staticmethod
-    def protect(path_info):
-        os.chmod(path_info.path, stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH)

dvc/remote/local/slow_link_detection.py

@@ -1,62 +0,0 @@
-import logging
-import time
-
-import colorama
-from dvc.config import Config
-
-logger = logging.getLogger(__name__)
-
-
-class SlowLinkDetectorDecorator(object):
-    LINKING_TIMEOUT_SECONDS = 10.0
-    was_displayed = False
-
-    @classmethod
-    def should_display(cls):
-        if not cls.was_displayed:
-            cls.was_displayed = True
-            return True
-        return False
-
-    def __init__(self, method):
-        self.method = method
-
-    def __call__(self, *args, **kwargs):
-        start = time.time()
-        result = self.method(*args, **kwargs)
-        execution_time_seconds = time.time() - start
-
-        if (
-            execution_time_seconds >= self.LINKING_TIMEOUT_SECONDS
-            and self.should_display()
-        ):
-            msg = (
-                "You can cut execution time considerably by using a different"
-                " `cache.type` configuration.\nSee "
-                "{blue}https://dvc.org/doc/commands-reference/config#cache{"
-                "reset} for more information.\n"
-                "To disable this message, run:\n"
-                "'dvc config cache.slow_link_warning false'".format(
-                    blue=colorama.Fore.BLUE, reset=colorama.Fore.RESET
-                )
-            )
-            logger.warning(msg)
-
-        return result
-
-
-def slow_link_guard(method):
-    def call(remote_local, *args, **kwargs):
-        cache_config = remote_local.repo.config.config.get(
-            Config.SECTION_CACHE, {}
-        )
-        should_warn = cache_config.get(
-            Config.SECTION_CACHE_SLOW_LINK_WARNING, True
-        ) and not cache_config.get(Config.SECTION_CACHE_TYPE, None)
-
-        if should_warn:
-            decorated = SlowLinkDetectorDecorator(method)
-            return decorated(remote_local, *args, **kwargs)
-        return method(remote_local, *args, **kwargs)
-
-    return call

dvc/remote/oss.py

@@ -5,19 +5,11 @@ import os
 import logging
 
 from dvc.scheme import Schemes
-from dvc.path.oss import PathOSS
 
-try:
-    import oss2
-except ImportError:
-    oss2 = None
-
-from dvc.utils import tmp_fname, move
-from dvc.utils.compat import urlparse, makedirs
-from dvc.progress import progress
 from dvc.config import Config
 from dvc.remote.base import RemoteBASE
-from dvc.remote.azure import Callback
+from dvc.progress import Tqdm
+from dvc.path_info import CloudURLInfo
 
 
 logger = logging.getLogger(__name__)
@@ -43,18 +35,16 @@ class RemoteOSS(RemoteBASE):
     """
 
     scheme = Schemes.OSS
-    REGEX = r"^oss://(?P<path>.*)?$"
-    REQUIRES = {"oss2": oss2}
+    path_cls = CloudURLInfo
+    REQUIRES = {"oss2": "oss2"}
     PARAM_CHECKSUM = "etag"
     COPY_POLL_SECONDS = 5
 
     def __init__(self, repo, config):
         super(RemoteOSS, self).__init__(repo, config)
 
-        self.url = config.get(Config.SECTION_REMOTE_URL)
-        parsed = urlparse(self.url)
-        self.bucket = parsed.netloc
-        self.prefix = parsed.path.lstrip("/")
+        url = config.get(Config.SECTION_REMOTE_URL)
+        self.path_info = self.path_cls(url) if url else None
 
         self.endpoint = config.get(Config.SECTION_OSS_ENDPOINT) or os.getenv(
             "OSS_ENDPOINT"
@@ -73,17 +63,19 @@ class RemoteOSS(RemoteBASE):
         )
 
         self._bucket = None
-        self.path_info = PathOSS(bucket=self.bucket)
 
     @property
     def oss_service(self):
+        import oss2
+
         if self._bucket is None:
-            logger.debug("URL {}".format(self.url))
+            logger.debug("URL {}".format(self.path_info))
             logger.debug("key id {}".format(self.key_id))
             logger.debug("key secret {}".format(self.key_secret))
             auth = oss2.Auth(self.key_id, self.key_secret)
-            logger.debug("bucket name {}".format(self.bucket))
-            self._bucket = oss2.Bucket(auth, self.endpoint, self.bucket)
+            self._bucket = oss2.Bucket(
+                auth, self.endpoint, self.path_info.bucket
+            )
             try:  # verify that bucket exists
                 self._bucket.get_bucket_info()
             except oss2.exceptions.NoSuchBucket:
@@ -99,94 +91,39 @@ class RemoteOSS(RemoteBASE):
         if path_info.scheme != self.scheme:
             raise NotImplementedError
 
-        logger.debug(
-            "Removing oss://{}/{}".format(path_info.bucket, path_info.path)
-        )
-
+        logger.debug("Removing oss://{}".format(path_info))
         self.oss_service.delete_object(path_info.path)
 
     def _list_paths(self, prefix):
+        import oss2
+
         for blob in oss2.ObjectIterator(self.oss_service, prefix=prefix):
             yield blob.key
 
     def list_cache_paths(self):
-        return self._list_paths(self.prefix)
+        return self._list_paths(self.path_info.path)
 
-    def upload(self, from_infos, to_infos, names=None, no_progress_bar=False):
-        names = self._verify_path_args(to_infos, from_infos, names)
-
-        for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info.scheme != self.scheme:
-                raise NotImplementedError
-
-            if from_info.scheme != "local":
-                raise NotImplementedError
-
-            bucket = to_info.bucket
-            path = to_info.path
-
-            logger.debug(
-                "Uploading '{}' to 'oss://{}/{}'".format(
-                    from_info.path, bucket, path
-                )
-            )
-
-            if not name:
-                name = os.path.basename(from_info.path)
-
-            cb = None if no_progress_bar else Callback(name)
-
-            try:
-                self.oss_service.put_object_from_file(
-                    path, from_info.path, progress_callback=cb
-                )
-            except Exception:
-                msg = "failed to upload '{}'".format(from_info.path)
-                logger.warning(msg)
-            else:
-                progress.finish_target(name)
-
-    def download(
-        self,
-        from_infos,
-        to_infos,
-        names=None,
-        no_progress_bar=False,
-        resume=False,
+    def _upload(
+        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
     ):
-        names = self._verify_path_args(from_infos, to_infos, names)
-        for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info.scheme != self.scheme:
-                raise NotImplementedError
-            if to_info.scheme != "local":
-                raise NotImplementedError
-
-            bucket = from_info.bucket
-            path = from_info.path
-
-            logger.debug(
-                "Downloading 'oss://{}/{}' to '{}'".format(
-                    bucket, path, to_info.path
-                )
+        with Tqdm(desc=name, disable=no_progress_bar, bytes=True) as pbar:
+            self.oss_service.put_object_from_file(
+                to_info.path, from_file, progress_callback=pbar.update_to
             )
 
-            tmp_file = tmp_fname(to_info.path)
-            if not name:
-                name = os.path.basename(to_info.path)
+    def _download(
+        self, from_info, to_file, name=None, no_progress_bar=False, **_kwargs
+    ):
+        with Tqdm(desc=name, disable=no_progress_bar, bytes=True) as pbar:
+            self.oss_service.get_object_to_file(
+                from_info.path, to_file, progress_callback=pbar.update_to
+            )
 
-            cb = None if no_progress_bar else Callback(name)
+    def _generate_download_url(self, path_info, expires=3600):
+        assert path_info.bucket == self.path_info.bucket
 
-            makedirs(os.path.dirname(to_info.path), exist_ok=True)
+        return self.oss_service.sign_url("GET", path_info.path, expires)
 
-            try:
-                self.oss_service.get_object_to_file(
-                    path, tmp_file, progress_callback=cb
-                )
-            except Exception:
-                msg = "failed to download 'oss://{}/{}'".format(bucket, path)
-                logger.warning(msg)
-            else:
-                move(tmp_file, to_info.path)
-
-                if not no_progress_bar:
-                    progress.finish_target(name)
+    def exists(self, path_info):
+        paths = self._list_paths(path_info.path)
+        return any(path_info.path == path for path in paths)

dvc/remote/pool.py

@@ -0,0 +1,56 @@
+from collections import deque
+from contextlib import contextmanager
+from funcy import memoize
+
+
+@contextmanager
+def get_connection(conn_func, *args, **kwargs):
+    pool = get_pool(conn_func, *args, **kwargs)
+    conn = pool.get_connection()
+    try:
+        yield conn
+    except BaseException:
+        conn.close()
+        raise
+    else:
+        pool.release(conn)
+
+
+@memoize
+def get_pool(conn_func, *args, **kwargs):
+    return Pool(conn_func, *args, **kwargs)
+
+
+def close_pools():
+    for pool in get_pool.memory.values():
+        pool.close()
+    get_pool.memory.clear()
+
+
+class Pool(object):
+    def __init__(self, conn_func, *conn_args, **conn_kwargs):
+        self._conn_func = conn_func
+        self._conn_args = conn_args
+        self._conn_kwargs = conn_kwargs
+        self._conns = deque()
+        self._closed = False
+
+    def __del__(self):
+        self.close()
+
+    def close(self):
+        while self._conns:
+            self._conns.pop().close()
+        self._closed = True
+
+    def get_connection(self):
+        try:
+            return self._conns.popleft()
+        except IndexError:
+            return self._conn_func(*self._conn_args, **self._conn_kwargs)
+
+    def release(self, conn):
+        if self._closed:
+            conn.close()
+        else:
+            self._conns.append(conn)

dvc/remote/s3.py

@@ -1,44 +1,23 @@
 from __future__ import unicode_literals
 
 import os
-import threading
 import logging
+from funcy import cached_property
 
-from dvc.scheme import Schemes
-from dvc.path.s3 import PathS3
-
-try:
-    import boto3
-except ImportError:
-    boto3 = None
-
-from dvc.utils import tmp_fname, move
-from dvc.utils.compat import urlparse, makedirs
-from dvc.progress import progress
+from dvc.progress import Tqdm
 from dvc.config import Config
 from dvc.remote.base import RemoteBASE
 from dvc.exceptions import DvcException, ETagMismatchError
+from dvc.path_info import CloudURLInfo
+from dvc.scheme import Schemes
 
 logger = logging.getLogger(__name__)
 
 
-class Callback(object):
-    def __init__(self, name, total):
-        self.name = name
-        self.total = total
-        self.current = 0
-        self.lock = threading.Lock()
-
-    def __call__(self, byts):
-        with self.lock:
-            self.current += byts
-            progress.update_target(self.name, self.current, self.total)
-
-
 class RemoteS3(RemoteBASE):
     scheme = Schemes.S3
-    REGEX = r"^s3://(?P<path>.*)$"
-    REQUIRES = {"boto3": boto3}
+    path_cls = CloudURLInfo
+    REQUIRES = {"boto3": "boto3"}
     PARAM_CHECKSUM = "etag"
 
     def __init__(self, repo, config):
@@ -48,15 +27,12 @@ class RemoteS3(RemoteBASE):
             config.get(Config.SECTION_AWS_STORAGEPATH, "").lstrip("/")
         )
 
-        self.url = config.get(Config.SECTION_REMOTE_URL, storagepath)
+        url = config.get(Config.SECTION_REMOTE_URL, storagepath)
+        self.path_info = self.path_cls(url)
 
-        self.region = os.environ.get("AWS_DEFAULT_REGION") or config.get(
-            Config.SECTION_AWS_REGION
-        )
+        self.region = config.get(Config.SECTION_AWS_REGION)
 
-        self.profile = os.environ.get("AWS_PROFILE") or config.get(
-            Config.SECTION_AWS_PROFILE
-        )
+        self.profile = config.get(Config.SECTION_AWS_PROFILE)
 
         self.endpoint_url = config.get(Config.SECTION_AWS_ENDPOINT_URL)
 
@@ -64,25 +40,24 @@ class RemoteS3(RemoteBASE):
 
         self.use_ssl = config.get(Config.SECTION_AWS_USE_SSL, True)
 
+        self.extra_args = {}
+
+        self.sse = config.get(Config.SECTION_AWS_SSE, "")
+        if self.sse:
+            self.extra_args["ServerSideEncryption"] = self.sse
+
+        self.acl = config.get(Config.SECTION_AWS_ACL, "")
+        if self.acl:
+            self.extra_args["ACL"] = self.acl
+
         shared_creds = config.get(Config.SECTION_AWS_CREDENTIALPATH)
         if shared_creds:
             os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", shared_creds)
 
-        parsed = urlparse(self.url)
-        self.bucket = parsed.netloc
-        self.prefix = parsed.path.lstrip("/")
-
-        self.path_info = PathS3(bucket=self.bucket)
-
-    @staticmethod
-    def compat_config(config):
-        ret = config.copy()
-        url = "s3://" + ret.pop(Config.SECTION_AWS_STORAGEPATH, "").lstrip("/")
-        ret[Config.SECTION_REMOTE_URL] = url
-        return ret
-
-    @property
+    @cached_property
     def s3(self):
+        import boto3
+
         session = boto3.session.Session(
             profile_name=self.profile, region_name=self.region
         )
@@ -112,9 +87,11 @@ class RemoteS3(RemoteBASE):
         return obj
 
     @classmethod
-    def _copy_multipart(cls, s3, from_info, to_info, size, n_parts):
+    def _copy_multipart(
+        cls, s3, from_info, to_info, size, n_parts, extra_args
+    ):
         mpu = s3.create_multipart_upload(
-            Bucket=to_info.bucket, Key=to_info.path
+            Bucket=to_info.bucket, Key=to_info.path, **extra_args
         )
         mpu_id = mpu["UploadId"]
 
@@ -154,7 +131,7 @@ class RemoteS3(RemoteBASE):
         )
 
     @classmethod
-    def _copy(cls, s3, from_info, to_info):
+    def _copy(cls, s3, from_info, to_info, extra_args):
         # NOTE: object's etag depends on the way it was uploaded to s3 or the
         # way it was copied within the s3. More specifically, it depends on
         # the chunk size that was used to transfer it, which would affect
@@ -182,38 +159,35 @@ class RemoteS3(RemoteBASE):
         _, _, parts_suffix = etag.partition("-")
         if parts_suffix:
             n_parts = int(parts_suffix)
-            cls._copy_multipart(s3, from_info, to_info, size, n_parts)
+            cls._copy_multipart(
+                s3, from_info, to_info, size, n_parts, extra_args=extra_args
+            )
         else:
             source = {"Bucket": from_info.bucket, "Key": from_info.path}
-            s3.copy(source, to_info.bucket, to_info.path)
+            s3.copy(source, to_info.bucket, to_info.path, ExtraArgs=extra_args)
 
         cached_etag = cls.get_etag(s3, to_info.bucket, to_info.path)
         if etag != cached_etag:
             raise ETagMismatchError(etag, cached_etag)
 
-    def copy(self, from_info, to_info, s3=None):
-        s3 = s3 if s3 else self.s3
-        self._copy(s3, from_info, to_info)
+    def copy(self, from_info, to_info):
+        self._copy(self.s3, from_info, to_info, self.extra_args)
 
     def remove(self, path_info):
         if path_info.scheme != "s3":
             raise NotImplementedError
 
-        logger.debug(
-            "Removing s3://{}/{}".format(path_info.bucket, path_info.path)
-        )
-
+        logger.debug("Removing {}".format(path_info))
         self.s3.delete_object(Bucket=path_info.bucket, Key=path_info.path)
 
     def _list_paths(self, bucket, prefix):
         """ Read config for list object api, paginate through list objects."""
-        s3 = self.s3
         kwargs = {"Bucket": bucket, "Prefix": prefix}
         if self.list_objects:
             list_objects_api = "list_objects"
         else:
             list_objects_api = "list_objects_v2"
-        paginator = s3.get_paginator(list_objects_api)
+        paginator = self.s3.get_paginator(list_objects_api)
         for page in paginator.paginate(**kwargs):
             contents = page.get("Contents", None)
             if not contents:
@@ -222,104 +196,41 @@ class RemoteS3(RemoteBASE):
                 yield item["Key"]
 
     def list_cache_paths(self):
-        return self._list_paths(self.bucket, self.prefix)
+        return self._list_paths(self.path_info.bucket, self.path_info.path)
 
     def exists(self, path_info):
-        assert not isinstance(path_info, list)
-        assert path_info.scheme == "s3"
-
         paths = self._list_paths(path_info.bucket, path_info.path)
         return any(path_info.path == path for path in paths)
 
-    def upload(self, from_infos, to_infos, names=None, no_progress_bar=False):
-        names = self._verify_path_args(to_infos, from_infos, names)
-
-        s3 = self.s3
-
-        for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info.scheme != "s3":
-                raise NotImplementedError
-
-            if from_info.scheme != "local":
-                raise NotImplementedError
-
-            logger.debug(
-                "Uploading '{}' to '{}/{}'".format(
-                    from_info.path, to_info.bucket, to_info.path
-                )
+    def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
+        total = os.path.getsize(from_file)
+        with Tqdm(
+            disable=no_progress_bar, total=total, bytes=True, desc=name
+        ) as pbar:
+            self.s3.upload_file(
+                from_file,
+                to_info.bucket,
+                to_info.path,
+                Callback=pbar.update,
+                ExtraArgs=self.extra_args,
             )
 
-            if not name:
-                name = os.path.basename(from_info.path)
-
-            total = os.path.getsize(from_info.path)
-            cb = None if no_progress_bar else Callback(name, total)
-
-            try:
-                s3.upload_file(
-                    from_info.path, to_info.bucket, to_info.path, Callback=cb
-                )
-            except Exception:
-                msg = "failed to upload '{}'".format(from_info.path)
-                logger.exception(msg)
-                continue
-
-            progress.finish_target(name)
-
-    def download(
-        self,
-        from_infos,
-        to_infos,
-        no_progress_bar=False,
-        names=None,
-        resume=False,
-    ):
-        names = self._verify_path_args(from_infos, to_infos, names)
-
-        s3 = self.s3
-
-        for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info.scheme != "s3":
-                raise NotImplementedError
-
-            if to_info.scheme == "s3":
-                self.copy(from_info, to_info, s3=s3)
-                continue
-
-            if to_info.scheme != "local":
-                raise NotImplementedError
-
-            msg = "Downloading '{}/{}' to '{}'".format(
-                from_info.bucket, from_info.path, to_info.path
+    def _download(self, from_info, to_file, name=None, no_progress_bar=False):
+        if no_progress_bar:
+            total = None
+        else:
+            total = self.s3.head_object(
+                Bucket=from_info.bucket, Key=from_info.path
+            )["ContentLength"]
+        with Tqdm(
+            disable=no_progress_bar, total=total, bytes=True, desc=name
+        ) as pbar:
+            self.s3.download_file(
+                from_info.bucket, from_info.path, to_file, Callback=pbar.update
             )
-            logger.debug(msg)
 
-            tmp_file = tmp_fname(to_info.path)
-            if not name:
-                name = os.path.basename(to_info.path)
-
-            makedirs(os.path.dirname(to_info.path), exist_ok=True)
-
-            try:
-                if no_progress_bar:
-                    cb = None
-                else:
-                    total = s3.head_object(
-                        Bucket=from_info.bucket, Key=from_info.path
-                    )["ContentLength"]
-                    cb = Callback(name, total)
-
-                s3.download_file(
-                    from_info.bucket, from_info.path, tmp_file, Callback=cb
-                )
-            except Exception:
-                msg = "failed to download '{}/{}'".format(
-                    from_info.bucket, from_info.path
-                )
-                logger.exception(msg)
-                continue
-
-            move(tmp_file, to_info.path)
-
-            if not no_progress_bar:
-                progress.finish_target(name)
+    def _generate_download_url(self, path_info, expires=3600):
+        params = {"Bucket": path_info.bucket, "Key": path_info.path}
+        return self.s3.generate_presigned_url(
+            ClientMethod="get_object", Params=params, ExpiresIn=int(expires)
+        )

dvc/remote/slow_link_detection.py

@@ -0,0 +1,50 @@
+import logging
+import time
+import sys
+from functools import wraps
+
+import colorama
+from dvc.config import Config
+
+
+logger = logging.getLogger(__name__)
+this = sys.modules[__name__]
+
+this.timeout_seconds = 10.0
+this.already_displayed = False
+this.message = (
+    "You can cut execution time considerably by using a different "
+    "`cache.type` configuration.\n"
+    "See {blue}https://dvc.org/doc/commands-reference/config#cache{reset} "
+    "for more information.\n"
+    "To disable this message, run:\n"
+    "'dvc config cache.slow_link_warning false'".format(
+        blue=colorama.Fore.BLUE, reset=colorama.Fore.RESET
+    )
+)
+
+
+def slow_link_guard(f):
+    @wraps(f)
+    def wrapper(remote, *args, **kwargs):
+        if this.already_displayed:
+            return f(remote, *args, **kwargs)
+
+        config = remote.repo.config.config.get(Config.SECTION_CACHE, {})
+        cache_type = config.get(Config.SECTION_CACHE_TYPE)
+        should_warn = config.get(Config.SECTION_CACHE_SLOW_LINK_WARNING, True)
+
+        if not should_warn or cache_type:
+            return f(remote, *args, **kwargs)
+
+        start = time.time()
+        result = f(remote, *args, **kwargs)
+        delta = time.time() - start
+
+        if delta >= this.timeout_seconds:
+            logger.warning(this.message)
+            this.already_displayed = True
+
+        return result
+
+    return wrapper

dvc/remote/ssh/__init__.py

@@ -1,64 +1,77 @@
 from __future__ import unicode_literals
 
+import errno
+import itertools
+import io
 import os
 import getpass
 import logging
-
-from dvc.path import Schemes
-from dvc.path.ssh import PathSSH
-
-try:
-    import paramiko
-except ImportError:
-    paramiko = None
+import threading
+from concurrent.futures import ThreadPoolExecutor
+from contextlib import contextmanager, closing
 
 import dvc.prompt as prompt
-from dvc.remote.ssh.connection import SSHConnection
 from dvc.config import Config
-from dvc.utils.compat import urlparse
+from dvc.utils import to_chunks
+from dvc.utils.compat import urlparse, StringIO
 from dvc.remote.base import RemoteBASE
-
+from dvc.scheme import Schemes
+from dvc.remote.pool import get_connection
+from dvc.progress import Tqdm
 
 logger = logging.getLogger(__name__)
 
 
+saved_passwords = {}
+saved_passwords_lock = threading.Lock()
+
+
 class RemoteSSH(RemoteBASE):
     scheme = Schemes.SSH
-
-    # NOTE: we support both URL-like (ssh://[user@]host.xz[:port]/path) and
-    # SCP-like (ssh://[user@]host.xz:/absolute/path) urls.
-    REGEX = r"^ssh://.*$"
-
-    REQUIRES = {"paramiko": paramiko}
+    REQUIRES = {"paramiko": "paramiko"}
 
     JOBS = 4
     PARAM_CHECKSUM = "md5"
     DEFAULT_PORT = 22
     TIMEOUT = 1800
+    # At any given time some of the connections will go over network and
+    # paramiko stuff, so we would ideally have it double of server processors.
+    # We use conservative setting of 4 instead to not exhaust max sessions.
+    CHECKSUM_JOBS = 4
+
+    DEFAULT_CACHE_TYPES = ["copy"]
 
     def __init__(self, repo, config):
         super(RemoteSSH, self).__init__(repo, config)
-        self.url = config.get(Config.SECTION_REMOTE_URL, "ssh://")
+        url = config.get(Config.SECTION_REMOTE_URL)
+        if url:
+            parsed = urlparse(url)
+            user_ssh_config = self._load_user_ssh_config(parsed.hostname)
 
-        parsed = urlparse(self.url)
-        self.host = parsed.hostname
+            host = user_ssh_config.get("hostname", parsed.hostname)
+            user = (
+                config.get(Config.SECTION_REMOTE_USER)
+                or parsed.username
+                or user_ssh_config.get("user")
+                or getpass.getuser()
+            )
+            port = (
+                config.get(Config.SECTION_REMOTE_PORT)
+                or parsed.port
+                or self._try_get_ssh_config_port(user_ssh_config)
+                or self.DEFAULT_PORT
+            )
+            self.path_info = self.path_cls.from_parts(
+                scheme=self.scheme,
+                host=host,
+                user=user,
+                port=port,
+                path=parsed.path,
+            )
+        else:
+            self.path_info = None
+            user_ssh_config = {}
 
-        user_ssh_config = self._load_user_ssh_config(self.host)
-
-        self.host = user_ssh_config.get("hostname", self.host)
-        self.user = (
-            config.get(Config.SECTION_REMOTE_USER)
-            or parsed.username
-            or user_ssh_config.get("user")
-            or getpass.getuser()
-        )
-        self.prefix = parsed.path or "/"
-        self.port = (
-            config.get(Config.SECTION_REMOTE_PORT)
-            or parsed.port
-            or self._try_get_ssh_config_port(user_ssh_config)
-            or self.DEFAULT_PORT
-        )
         self.keyfile = config.get(
             Config.SECTION_REMOTE_KEY_FILE
         ) or self._try_get_ssh_config_keyfile(user_ssh_config)
@@ -67,10 +80,7 @@ class RemoteSSH(RemoteBASE):
         self.ask_password = config.get(
             Config.SECTION_REMOTE_ASK_PASSWORD, False
         )
-
-        self.path_info = PathSSH(
-            host=self.host, user=self.user, port=self.port
-        )
+        self.gss_auth = config.get(Config.SECTION_REMOTE_GSS_AUTH, False)
 
     @staticmethod
     def ssh_config_filename():
@@ -78,13 +88,17 @@ class RemoteSSH(RemoteBASE):
 
     @staticmethod
     def _load_user_ssh_config(hostname):
+        import paramiko
+
         user_config_file = RemoteSSH.ssh_config_filename()
-        user_ssh_config = dict()
+        user_ssh_config = {}
         if hostname and os.path.exists(user_config_file):
+            ssh_config = paramiko.SSHConfig()
             with open(user_config_file) as f:
-                ssh_config = paramiko.SSHConfig()
-                ssh_config.parse(f)
-                user_ssh_config = ssh_config.lookup(hostname)
+                # For whatever reason parsing directly from f is unreliable
+                f_copy = StringIO(f.read())
+                ssh_config.parse(f_copy)
+            user_ssh_config = ssh_config.lookup(hostname)
         return user_ssh_config
 
     @staticmethod
@@ -101,37 +115,42 @@ class RemoteSSH(RemoteBASE):
             return identity_file[0]
         return None
 
-    def ssh(self, path_info):
+    def ensure_credentials(self, path_info=None):
+        if path_info is None:
+            path_info = self.path_info
         host, user, port = path_info.host, path_info.user, path_info.port
+        # NOTE: we use the same password regardless of the server :(
+        if self.ask_password and self.password is None:
+            with saved_passwords_lock:
+                server_key = (host, user, port)
+                password = saved_passwords.get(server_key)
 
-        logger.debug(
-            "Establishing ssh connection with '{host}' "
-            "through port '{port}' as user '{user}'".format(
-                host=host, user=user, port=port
-            )
-        )
+                if password is None:
+                    saved_passwords[server_key] = password = prompt.password(
+                        "Enter a private key passphrase or a password for "
+                        "host '{host}' port '{port}' user '{user}'".format(
+                            host=host, port=port, user=user
+                        )
+                    )
+                self.password = password
 
-        if self.ask_password and not self.password:
-            self.password = prompt.password(
-                "Enter a private key passphrase or a password for "
-                "host '{host}' port '{port}' user '{user}'".format(
-                    host=host, port=port, user=user
-                )
-            )
+    def ssh(self, path_info):
+        self.ensure_credentials(path_info)
 
-        return SSHConnection(
-            host,
-            username=user,
-            port=port,
+        from .connection import SSHConnection
+
+        return get_connection(
+            SSHConnection,
+            path_info.host,
+            username=path_info.user,
+            port=path_info.port,
             key_filename=self.keyfile,
             timeout=self.timeout,
             password=self.password,
+            gss_auth=self.gss_auth,
         )
 
     def exists(self, path_info):
-        assert not isinstance(path_info, list)
-        assert path_info.scheme == self.scheme
-
         with self.ssh(path_info) as ssh:
             return ssh.exists(path_info.path)
 
@@ -146,12 +165,54 @@ class RemoteSSH(RemoteBASE):
         with self.ssh(path_info) as ssh:
             return ssh.isdir(path_info.path)
 
+    def isfile(self, path_info):
+        with self.ssh(path_info) as ssh:
+            return ssh.isfile(path_info.path)
+
+    def getsize(self, path_info):
+        with self.ssh(path_info) as ssh:
+            return ssh.getsize(path_info.path)
+
     def copy(self, from_info, to_info):
+        if not from_info.scheme == to_info.scheme == self.scheme:
+            raise NotImplementedError
+
+        with self.ssh(from_info) as ssh:
+            ssh.copy(from_info.path, to_info.path)
+
+    def symlink(self, from_info, to_info):
+        if not from_info.scheme == to_info.scheme == self.scheme:
+            raise NotImplementedError
+
+        with self.ssh(from_info) as ssh:
+            ssh.symlink(from_info.path, to_info.path)
+
+    def hardlink(self, from_info, to_info):
+        if not from_info.scheme == to_info.scheme == self.scheme:
+            raise NotImplementedError
+
+        # See dvc/remote/local/__init__.py - hardlink()
+        if self.getsize(from_info) == 0:
+
+            with self.ssh(to_info) as ssh:
+                ssh.sftp.open(to_info.path, "w").close()
+
+            logger.debug(
+                "Created empty file: {src} -> {dest}".format(
+                    src=str(from_info), dest=str(to_info)
+                )
+            )
+            return
+
+        with self.ssh(from_info) as ssh:
+            ssh.hardlink(from_info.path, to_info.path)
+
+    def reflink(self, from_info, to_info):
         if from_info.scheme != self.scheme or to_info.scheme != self.scheme:
             raise NotImplementedError
 
         with self.ssh(from_info) as ssh:
-            ssh.cp(from_info.path, to_info.path)
+            ssh.reflink(from_info.path, to_info.path)
 
     def remove(self, path_info):
         if path_info.scheme != self.scheme:
@@ -167,81 +228,41 @@ class RemoteSSH(RemoteBASE):
         with self.ssh(from_info) as ssh:
             ssh.move(from_info.path, to_info.path)
 
-    def download(
-        self,
-        from_infos,
-        to_infos,
-        no_progress_bar=False,
-        names=None,
-        resume=False,
-    ):
-        names = self._verify_path_args(from_infos, to_infos, names)
-        ssh = self.ssh(from_infos[0])
-
-        for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info.scheme != self.scheme:
-                raise NotImplementedError
-
-            if to_info.scheme == self.scheme:
-                ssh.cp(from_info.path, to_info.path)
-                continue
-
-            if to_info.scheme != "local":
-                raise NotImplementedError
-
-            logger.debug(
-                "Downloading '{host}/{path}' to '{dest}'".format(
-                    host=from_info.host, path=from_info.path, dest=to_info.path
-                )
+    def _download(self, from_info, to_file, name=None, no_progress_bar=False):
+        assert from_info.isin(self.path_info)
+        with self.ssh(self.path_info) as ssh:
+            ssh.download(
+                from_info.path,
+                to_file,
+                progress_title=name,
+                no_progress_bar=no_progress_bar,
             )
 
-            try:
-                ssh.download(
-                    from_info.path,
-                    to_info.path,
-                    progress_title=name,
-                    no_progress_bar=no_progress_bar,
-                )
-            except Exception:
-                logger.exception(
-                    "failed to download '{host}/{path}' to '{dest}'".format(
-                        host=from_info.host,
-                        path=from_info.path,
-                        dest=to_info.path,
-                    )
-                )
-                continue
+    def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
+        assert to_info.isin(self.path_info)
+        with self.ssh(self.path_info) as ssh:
+            ssh.upload(
+                from_file,
+                to_info.path,
+                progress_title=name,
+                no_progress_bar=no_progress_bar,
+            )
 
-        ssh.close()
+    @contextmanager
+    def open(self, path_info, mode="r", encoding=None):
+        assert mode in {"r", "rt", "rb", "wb"}
 
-    def upload(self, from_infos, to_infos, names=None, no_progress_bar=False):
-        names = self._verify_path_args(to_infos, from_infos, names)
-
-        with self.ssh(to_infos[0]) as ssh:
-            for from_info, to_info, name in zip(from_infos, to_infos, names):
-                if to_info.scheme != self.scheme:
-                    raise NotImplementedError
-
-                if from_info.scheme != "local":
-                    raise NotImplementedError
-
-                try:
-                    ssh.upload(
-                        from_info.path,
-                        to_info.path,
-                        progress_title=name,
-                        no_progress_bar=no_progress_bar,
-                    )
-                except Exception:
-                    logger.exception(
-                        "failed to upload '{}' to '{}'".format(
-                            from_info, to_info
-                        )
-                    )
+        with self.ssh(path_info) as ssh, closing(
+            ssh.sftp.open(path_info.path, mode)
+        ) as fd:
+            if "b" in mode:
+                yield fd
+            else:
+                yield io.TextIOWrapper(fd, encoding=encoding)
 
     def list_cache_paths(self):
         with self.ssh(self.path_info) as ssh:
-            return list(ssh.walk_files(self.prefix))
+            return list(ssh.walk_files(self.path_info.path))
 
     def walk(self, path_info):
         with self.ssh(path_info) as ssh:
@@ -251,3 +272,61 @@ class RemoteSSH(RemoteBASE):
     def makedirs(self, path_info):
         with self.ssh(path_info) as ssh:
             ssh.makedirs(path_info.path)
+
+    def batch_exists(self, path_infos, callback):
+        def _exists(chunk_and_channel):
+            chunk, channel = chunk_and_channel
+            ret = []
+            for path in chunk:
+                try:
+                    channel.stat(path)
+                    ret.append(True)
+                except IOError as exc:
+                    if exc.errno != errno.ENOENT:
+                        raise
+                    ret.append(False)
+                callback(path)
+            return ret
+
+        with self.ssh(path_infos[0]) as ssh:
+            channels = ssh.open_max_sftp_channels()
+            max_workers = len(channels)
+
+            with ThreadPoolExecutor(max_workers=max_workers) as executor:
+                paths = [path_info.path for path_info in path_infos]
+                chunks = to_chunks(paths, num_chunks=max_workers)
+                chunks_and_channels = zip(chunks, channels)
+                outcome = executor.map(_exists, chunks_and_channels)
+                results = list(itertools.chain.from_iterable(outcome))
+
+            return results
+
+    def cache_exists(self, checksums, jobs=None, name=None):
+        """This is older implementation used in remote/base.py
+        We are reusing it in RemoteSSH, because SSH's batch_exists proved to be
+        faster than current approach (relying on exists(path_info)) applied in
+        remote/base.
+        """
+        if not self.no_traverse:
+            return list(set(checksums) & set(self.all()))
+
+        # possibly prompt for credentials before "Querying" progress output
+        self.ensure_credentials()
+
+        with Tqdm(
+            desc="Querying "
+            + ("cache in " + name if name else "remote cache"),
+            total=len(checksums),
+            unit="file",
+        ) as pbar:
+
+            def exists_with_progress(chunks):
+                return self.batch_exists(chunks, callback=pbar.update_desc)
+
+            with ThreadPoolExecutor(max_workers=jobs or self.JOBS) as executor:
+                path_infos = [self.checksum_to_path_info(x) for x in checksums]
+                chunks = to_chunks(path_infos, num_chunks=self.JOBS)
+                results = executor.map(exists_with_progress, chunks)
+                in_remote = itertools.chain.from_iterable(results)
+                ret = list(itertools.compress(checksums, in_remote))
+                return ret

dvc/remote/ssh/connection.py

@@ -1,8 +1,9 @@
-import errno
 import os
 import posixpath
 import logging
-from stat import S_ISDIR
+import errno
+import stat
+from funcy import cached_property
 
 try:
     import paramiko
@@ -10,8 +11,8 @@ except ImportError:
     paramiko = None
 
 from dvc.utils import tmp_fname
-from dvc.utils.compat import makedirs
-from dvc.progress import progress
+from dvc.utils.compat import ignore_file_not_found
+from dvc.progress import Tqdm
 from dvc.exceptions import DvcException
 from dvc.remote.base import RemoteCmdError
 
@@ -28,93 +29,67 @@ def sizeof_fmt(num, suffix="B"):
     return "%.1f%s%s" % (num, "Y", suffix)
 
 
-def percent_cb(name, complete, total):
-    """ Callback for updating target progress """
-    logger.debug(
-        "{}: {} transferred out of {}".format(
-            name, sizeof_fmt(complete), sizeof_fmt(total)
-        )
-    )
-    progress.update_target(name, complete, total)
-
-
-def create_cb(name):
-    """ Create callback function for multipart object """
-    return lambda cur, tot: percent_cb(name, cur, tot)
-
-
 class SSHConnection:
-    def __init__(self, *args, **kwargs):
+    def __init__(self, host, *args, **kwargs):
+        logger.debug(
+            "Establishing ssh connection with '{host}' "
+            "through port '{port}' as user '{username}'".format(
+                host=host, **kwargs
+            )
+        )
         self.timeout = kwargs.get("timeout", 1800)
 
         self._ssh = paramiko.SSHClient()
         self._ssh.load_system_host_keys()
         self._ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
 
-        self._ssh.connect(*args, **kwargs)
-        self._sftp = None
-        self._sftp_alive = False
+        self._ssh.connect(host, *args, **kwargs)
+        self._ssh.get_transport().set_keepalive(10)
+        self._sftp_channels = []
 
-    def __enter__(self):
-        return self
-
-    def __exit__(self, exception_type, exception_value, traceback):
-        self.close()
-
-    def _sftp_connect(self):
-        if not self._sftp or not self._sftp_alive:
-            self._sftp = self._ssh.open_sftp()
-            self._sftp_alive = True
+    @property
+    def sftp(self):
+        if not self._sftp_channels:
+            self._sftp_channels = [self._ssh.open_sftp()]
+        return self._sftp_channels[0]
 
     def close(self):
-        if self._sftp:
-            self._sftp.close()
-            self._sftp_alive = False
-
+        for sftp in self._sftp_channels:
+            sftp.close()
         self._ssh.close()
 
-    def exists(self, path):
-        self._sftp_connect()
-        try:
-            return self._sftp.stat(path)
-        except IOError:
-            return False
-        pass
+    def st_mode(self, path):
+        with ignore_file_not_found():
+            return self.sftp.lstat(path).st_mode
+
+        return 0
+
+    def getsize(self, path):
+        with ignore_file_not_found():
+            return self.sftp.lstat(path).st_size
+
+        return 0
+
+    def exists(self, path, sftp=None):
+        return bool(self.st_mode(path))
 
     def isdir(self, path):
-        from stat import S_ISDIR
-
-        self._sftp_connect()
-        try:
-            return S_ISDIR(self._sftp.stat(path).st_mode)
-        except IOError:
-            return False
+        return stat.S_ISDIR(self.st_mode(path))
 
     def isfile(self, path):
-        from stat import S_ISREG
-
-        self._sftp_connect()
-        try:
-            return S_ISREG(self._sftp.stat(path).st_mode)
-        except IOError:
-            return False
+        return stat.S_ISREG(self.st_mode(path))
 
     def islink(self, path):
-        from stat import S_ISLNK
-
-        self._sftp_connect()
-        try:
-            return S_ISLNK(self._sftp.stat(path).st_mode)
-        except IOError:
-            return False
+        return stat.S_ISLNK(self.st_mode(path))
 
     def makedirs(self, path):
-        self._sftp_connect()
+        # Single stat call will say whether this is a dir, a file or a link
+        st_mode = self.st_mode(path)
 
-        if self.isdir(path):
+        if stat.S_ISDIR(st_mode):
             return
 
-        if self.isfile(path) or self.islink(path):
+        if stat.S_ISREG(st_mode) or stat.S_ISLNK(st_mode):
             raise DvcException(
                 "a file with the same name '{}' already exists".format(path)
             )
@@ -125,18 +100,24 @@ class SSHConnection:
             self.makedirs(head)
 
         if tail:
-            self._sftp.mkdir(path)
+            try:
+                self.sftp.mkdir(path)
+            except IOError as e:
+                # Since paramiko errors are very vague we need to recheck
+                # whether it's because path already exists or something else
+                if e.errno == errno.EACCES or not self.exists(path):
+                    raise DvcException(
+                        "unable to create remote directory '{}'".format(path),
+                        cause=e,
+                    )
 
     def walk(self, directory, topdown=True):
         # NOTE: original os.walk() implementation [1] with default options was
         # used as a template.
         #
         # [1] https://github.com/python/cpython/blob/master/Lib/os.py
-
-        self._sftp_connect()
-
         try:
-            dir_entries = self._sftp.listdir_attr(directory)
+            dir_entries = self.sftp.listdir_attr(directory)
         except IOError as exc:
             raise DvcException(
                 "couldn't get the '{}' remote directory files list".format(
@@ -149,7 +130,7 @@ class SSHConnection:
         nondirs = []
         for entry in dir_entries:
             name = entry.filename
-            if S_ISDIR(entry.st_mode):
+            if stat.S_ISDIR(entry.st_mode):
                 dirs.append(name)
             else:
                 nondirs.append(name)
@@ -171,76 +152,54 @@ class SSHConnection:
                 yield posixpath.join(root, fname)
 
     def _remove_file(self, path):
-        try:
-            self._sftp.remove(path)
-        except IOError as exc:
-            if exc.errno != errno.ENOENT:
-                raise
+        with ignore_file_not_found():
+            self.sftp.remove(path)
 
     def _remove_dir(self, path):
         for root, dirs, files in self.walk(path, topdown=False):
             for fname in files:
                 path = posixpath.join(root, fname)
-                self._remove_file(path)
+                with ignore_file_not_found():
+                    self._remove_file(path)
 
             for dname in dirs:
                 path = posixpath.join(root, dname)
-                self._sftp.rmdir(dname)
-        try:
-            self._sftp.rmdir(path)
-        except IOError as exc:
-            if exc.errno != errno.ENOENT:
-                raise
+                with ignore_file_not_found():
+                    self.sftp.rmdir(dname)
+
+        with ignore_file_not_found():
+            self.sftp.rmdir(path)
 
     def remove(self, path):
-        self._sftp_connect()
-
         if self.isdir(path):
             self._remove_dir(path)
         else:
             self._remove_file(path)
 
     def download(self, src, dest, no_progress_bar=False, progress_title=None):
-        self._sftp_connect()
-
-        makedirs(os.path.dirname(dest), exist_ok=True)
-        tmp_file = tmp_fname(dest)
-
-        if no_progress_bar:
-            self._sftp.get(src, tmp_file)
-        else:
-            if not progress_title:
-                progress_title = os.path.basename(dest)
-
-            self._sftp.get(src, tmp_file, callback=create_cb(progress_title))
-            progress.finish_target(progress_title)
-
-        if os.path.exists(dest):
-            os.remove(dest)
-
-        os.rename(tmp_file, dest)
+        with Tqdm(
+            desc=progress_title or os.path.basename(src),
+            disable=no_progress_bar,
+            bytes=True,
+        ) as pbar:
+            self.sftp.get(src, dest, callback=pbar.update_to)
 
     def move(self, src, dst):
         self.makedirs(posixpath.dirname(dst))
-        self._sftp_connect()
-        self._sftp.rename(src, dst)
+        self.sftp.rename(src, dst)
 
     def upload(self, src, dest, no_progress_bar=False, progress_title=None):
-        self._sftp_connect()
-
         self.makedirs(posixpath.dirname(dest))
         tmp_file = tmp_fname(dest)
+        if not progress_title:
+            progress_title = posixpath.basename(dest)
 
-        if no_progress_bar:
-            self._sftp.put(src, tmp_file)
-        else:
-            if not progress_title:
-                progress_title = posixpath.basename(dest)
+        with Tqdm(
+            desc=progress_title, disable=no_progress_bar, bytes=True
+        ) as pbar:
+            self.sftp.put(src, tmp_file, callback=pbar.update_to)
 
-            self._sftp.put(src, tmp_file, callback=create_cb(progress_title))
-            progress.finish_target(progress_title)
-
-        self._sftp.rename(tmp_file, dest)
+        self.sftp.rename(tmp_file, dest)
 
     def execute(self, cmd):
         stdin, stdout, stderr = self._ssh.exec_command(cmd)
@@ -291,6 +250,10 @@ class SSHConnection:
 
         return b"".join(stdout_chunks).decode("utf-8")
 
+    @cached_property
+    def uname(self):
+        return self.execute("uname").strip()
+
     def md5(self, path):
         """
         Use different md5 commands depending on the OS:
@@ -301,22 +264,49 @@ class SSHConnection:
          Example:
               MD5 (foo.txt) = f3d220a856b52aabbf294351e8a24300
         """
-        uname = self.execute("uname").strip()
-
-        command = {
-            "Darwin": "md5 {}".format(path),
-            "Linux": "md5sum --tag {}".format(path),
-        }.get(uname)
-
-        if not command:
+        if self.uname == "Linux":
+            md5 = self.execute("md5sum " + path).split()[0]
+        elif self.uname == "Darwin":
+            md5 = self.execute("md5 " + path).split()[-1]
+        else:
             raise DvcException(
-                "'{uname}' is not supported as a remote".format(uname=uname)
+                "'{}' is not supported as a SSH remote".format(self.uname)
             )
 
-        md5 = self.execute(command).split()[-1]
         assert len(md5) == 32
         return md5
 
-    def cp(self, src, dest):
-        self.makedirs(posixpath.dirname(dest))
+    def copy(self, src, dest):
         self.execute("cp {} {}".format(src, dest))
+
+    def open_max_sftp_channels(self):
+        # If there are more than 1 it means we've already opened max amount
+        if len(self._sftp_channels) <= 1:
+            while True:
+                try:
+                    self._sftp_channels.append(self._ssh.open_sftp())
+                except paramiko.ssh_exception.ChannelException:
+                    if not self._sftp_channels:
+                        raise
+                    break
+        return self._sftp_channels
+
+    def open(self, *args, **kwargs):
+        return self.sftp.open(*args, **kwargs)
+
+    def symlink(self, src, dest):
+        self.sftp.symlink(src, dest)
+
+    def reflink(self, src, dest):
+        if self.uname == "Linux":
+            return self.execute("cp --reflink {} {}".format(src, dest))
+
+        if self.uname == "Darwin":
+            return self.execute("cp -c {} {}".format(src, dest))
+
+        raise DvcException(
+            "'{}' is not supported as a SSH remote".format(self.uname)
+        )
+
+    def hardlink(self, src, dest):
+        self.execute("ln {} {}".format(src, dest))

dvc/repo/__init__.py

@@ -2,22 +2,42 @@ from __future__ import unicode_literals
 
 import os
 import logging
+from contextlib import contextmanager
+from itertools import chain
 
-import dvc.prompt as prompt
+from functools import wraps
+from funcy import cached_property
 
+from dvc.config import Config
 from dvc.exceptions import (
-    DvcException,
     NotDvcRepoError,
     OutputNotFoundError,
-    TargetNotDirectoryError,
+    FileMissingError,
 )
-from dvc.ignore import DvcIgnoreFileHandler
-from dvc.path.local import PathLOCAL
-from dvc.utils.compat import open as _open
+from dvc.ignore import DvcIgnoreFilter
+from dvc.path_info import PathInfo
+from dvc.remote.base import RemoteActionNotImplemented
+from dvc.utils.compat import open as _open, fspath_py35, FileNotFoundError
+from dvc.utils import relpath
+
+from .graph import check_acyclic, get_pipeline, get_pipelines, get_stages
+
 
 logger = logging.getLogger(__name__)
 
 
+def locked(f):
+    @wraps(f)
+    def wrapper(repo, *args, **kwargs):
+        with repo.lock, repo.state:
+            ret = f(repo, *args, **kwargs)
+            # Our graph cache is no longer valid after we release the repo.lock
+            repo._reset()
+            return ret
+
+    return wrapper
+
+
 class Repo(object):
     DVC_DIR = ".dvc"
 
@@ -29,29 +49,30 @@ class Repo(object):
     from dvc.repo.move import move
     from dvc.repo.run import run
     from dvc.repo.imp import imp
+    from dvc.repo.imp_url import imp_url
     from dvc.repo.reproduce import reproduce
-    from dvc.repo.checkout import checkout
+    from dvc.repo.checkout import _checkout
     from dvc.repo.push import push
-    from dvc.repo.fetch import fetch
+    from dvc.repo.fetch import _fetch
     from dvc.repo.pull import pull
     from dvc.repo.status import status
     from dvc.repo.gc import gc
     from dvc.repo.commit import commit
     from dvc.repo.diff import diff
     from dvc.repo.brancher import brancher
+    from dvc.repo.get import get
+    from dvc.repo.get_url import get_url
+    from dvc.repo.update import update
 
     def __init__(self, root_dir=None):
-        from dvc.config import Config
         from dvc.state import State
         from dvc.lock import Lock
         from dvc.scm import SCM
         from dvc.cache import Cache
         from dvc.data_cloud import DataCloud
-        from dvc.updater import Updater
         from dvc.repo.metrics import Metrics
         from dvc.scm.tree import WorkingTree
         from dvc.repo.tag import Tag
-        from dvc.repo.pkg import Pkg
 
         root_dir = self.find_root(root_dir)
 
@@ -60,10 +81,14 @@ class Repo(object):
 
         self.config = Config(self.dvc_dir)
 
+        self.scm = SCM(self.root_dir)
+
         self.tree = WorkingTree(self.root_dir)
 
-        self.scm = SCM(self.root_dir, repo=self)
-        self.lock = Lock(self.dvc_dir)
+        self.lock = Lock(
+            os.path.join(self.dvc_dir, "lock"),
+            tmp_dir=os.path.join(self.dvc_dir, "tmp"),
+        )
         # NOTE: storing state and link_state in the repository itself to avoid
         # any possible state corruption in 'shared cache dir' scenario.
         self.state = State(self, self.config.config)
@@ -76,28 +101,35 @@ class Repo(object):
 
         self.cache = Cache(self)
         self.cloud = DataCloud(self)
-        self.updater = Updater(self.dvc_dir)
 
         self.metrics = Metrics(self)
         self.tag = Tag(self)
-        self.pkg = Pkg(self)
 
         self._ignore()
 
-        self.updater.check()
+    @property
+    def tree(self):
+        return self._tree
+
+    @tree.setter
+    def tree(self, tree):
+        self._tree = tree
+        # Our graph cache is no longer valid, as it was based on the previous
+        # tree.
+        self._reset()
 
     def __repr__(self):
         return "Repo: '{root_dir}'".format(root_dir=self.root_dir)
 
-    @staticmethod
-    def find_root(root=None):
+    @classmethod
+    def find_root(cls, root=None):
         if root is None:
             root = os.getcwd()
         else:
             root = os.path.abspath(os.path.realpath(root))
 
         while True:
-            dvc_dir = os.path.join(root, Repo.DVC_DIR)
+            dvc_dir = os.path.join(root, cls.DVC_DIR)
             if os.path.isdir(dvc_dir):
                 return root
             if os.path.ismount(root):
@@ -105,10 +137,10 @@ class Repo(object):
             root = os.path.dirname(root)
         raise NotDvcRepoError(root)
 
-    @staticmethod
-    def find_dvc_dir(root=None):
-        root_dir = Repo.find_root(root)
-        return os.path.join(root_dir, Repo.DVC_DIR)
+    @classmethod
+    def find_dvc_dir(cls, root=None):
+        root_dir = cls.find_root(root)
+        return os.path.join(root_dir, cls.DVC_DIR)
 
     @staticmethod
     def init(root_dir=os.curdir, no_scm=False, force=False):
@@ -118,163 +150,70 @@ class Repo(object):
         return Repo(root_dir)
 
     def unprotect(self, target):
-        path_info = PathLOCAL(path=target)
-        return self.cache.local.unprotect(path_info)
+        return self.cache.local.unprotect(PathInfo(target))
 
     def _ignore(self):
-        flist = [
-            self.state.state_file,
-            self.lock.lock_file,
-            self.config.config_local_file,
-            self.updater.updater_file,
-            self.updater.lock.lock_file,
-        ] + self.state.temp_files
+        from dvc.updater import Updater
 
-        if self.cache.local.cache_dir.startswith(self.root_dir):
+        updater = Updater(self.dvc_dir)
+
+        flist = (
+            [self.config.config_local_file, updater.updater_file]
+            + self.state.files
+            + self.lock.files
+            + updater.lock.files
+        )
+
+        if self.cache.local.cache_dir.startswith(self.root_dir + os.sep):
             flist += [self.cache.local.cache_dir]
 
         self.scm.ignore_list(flist)
 
-    def check_dag(self, stages):
+    def check_modified_graph(self, new_stages):
         """Generate graph including the new stage to check for errors"""
-        self.graph(stages=stages)
+        self._collect_graph(self.stages + new_stages)
 
-    @staticmethod
-    def _check_cyclic_graph(graph):
-        import networkx as nx
-        from dvc.exceptions import CyclicGraphError
-
-        cycles = list(nx.simple_cycles(graph))
-
-        if cycles:
-            raise CyclicGraphError(cycles[0])
-
-    def _get_pipeline(self, node):
-        pipelines = [i for i in self.pipelines() if i.has_node(node)]
-        assert len(pipelines) == 1
-        return pipelines[0]
-
-    def collect(self, target, with_deps=False, recursive=False):
+    def collect(self, target, with_deps=False, recursive=False, graph=None):
         import networkx as nx
         from dvc.stage import Stage
 
-        if not target or (recursive and os.path.isdir(target)):
-            return self.active_stages(target)
+        G = graph or self.graph
+
+        if not target:
+            return get_stages(G)
+
+        target = os.path.abspath(target)
+
+        if recursive and os.path.isdir(target):
+            attrs = nx.get_node_attributes(G, "stage")
+            nodes = [node for node in nx.dfs_postorder_nodes(G)]
+
+            ret = []
+            for node in nodes:
+                stage = attrs[node]
+                if stage.path.startswith(target + os.sep):
+                    ret.append(stage)
+            return ret
 
         stage = Stage.load(self, target)
         if not with_deps:
             return [stage]
 
-        node = os.path.relpath(stage.path, self.root_dir)
-        G = self._get_pipeline(node)
+        node = relpath(stage.path, self.root_dir)
+        pipeline = get_pipeline(get_pipelines(G), node)
 
-        ret = []
-        for n in nx.dfs_postorder_nodes(G, node):
-            ret.append(G.node[n]["stage"])
-
-        return ret
-
-    def _collect_dir_cache(
-        self, out, branch=None, remote=None, force=False, jobs=None
-    ):
-        """Get a list of `info`s retaled to the given directory.
-
-        - Pull the directory entry from the remote cache if it was changed.
-
-        Example:
-
-            Given the following commands:
-
-            $ echo "foo" > directory/foo
-            $ echo "bar" > directory/bar
-            $ dvc add directory
-
-            It will return something similar to the following list:
-
-            [
-                { 'path': 'directory',     'md5': '168fd6761b9c.dir', ... },
-                { 'path': 'directory/foo', 'md5': 'c157a79031e1', ... },
-                { 'path': 'directory/bar', 'md5': 'd3b07384d113', ... },
-            ]
-        """
-        info = out.dumpd()
-        ret = [info]
-        r = out.remote
-        md5 = info[r.PARAM_CHECKSUM]
-
-        if self.cache.local.changed_cache_file(md5):
-            try:
-                self.cloud.pull(
-                    ret, jobs=jobs, remote=remote, show_checksums=False
-                )
-            except DvcException as exc:
-                msg = "Failed to pull cache for '{}': {}"
-                logger.debug(msg.format(out, exc))
-
-        if self.cache.local.changed_cache_file(md5):
-            msg = (
-                "Missing cache for directory '{}'. "
-                "Cache for files inside will be lost. "
-                "Would you like to continue? Use '-f' to force. "
-            )
-            if not force and not prompt.confirm(msg):
-                raise DvcException(
-                    "unable to fully collect used cache"
-                    " without cache for directory '{}'".format(out)
-                )
-            else:
-                return ret
-
-        for i in out.dir_cache:
-            i["branch"] = branch
-            i[r.PARAM_PATH] = os.path.join(
-                info[r.PARAM_PATH], i[r.PARAM_RELPATH]
-            )
-            ret.append(i)
-
-        return ret
-
-    def _collect_used_cache(
-        self, out, branch=None, remote=None, force=False, jobs=None
-    ):
-        """Get a dumpd of the given `out`, with an entry including the branch.
-
-        The `used_cache` of an output is no more than its `info`.
-
-        In case that the given output is a directory, it will also
-        include the `info` of its files.
-        """
-        if not out.use_cache or not out.info:
-            if not out.info:
-                logger.warning(
-                    "Output '{}'({}) is missing version "
-                    "info. Cache for it will not be collected. "
-                    "Use dvc repro to get your pipeline up to "
-                    "date.".format(out, out.stage)
-                )
-            return []
-
-        info = out.dumpd()
-        info["branch"] = branch
-        ret = [info]
-
-        if out.scheme != "local":
-            return ret
-
-        if not out.is_dir_checksum:
-            return ret
-
-        return self._collect_dir_cache(
-            out, branch=branch, remote=remote, force=force, jobs=jobs
-        )
+        return [
+            pipeline.node[n]["stage"]
+            for n in nx.dfs_postorder_nodes(pipeline, node)
+        ]
 
     def used_cache(
         self,
-        target=None,
+        targets=None,
         all_branches=False,
-        active=True,
         with_deps=False,
         all_tags=False,
+        all_commits=False,
         remote=None,
         force=False,
         jobs=None,
@@ -293,52 +232,41 @@ class Repo(object):
             A dictionary with Schemes (representing output's location) as keys,
             and a list with the outputs' `dumpd` as values.
         """
+        from dvc.cache import NamedCache
 
-        cache = {}
-        cache["local"] = []
-        cache["s3"] = []
-        cache["gs"] = []
-        cache["hdfs"] = []
-        cache["ssh"] = []
-        cache["azure"] = []
+        cache = NamedCache()
 
         for branch in self.brancher(
-            all_branches=all_branches, all_tags=all_tags
+            all_branches=all_branches,
+            all_tags=all_tags,
+            all_commits=all_commits,
         ):
-            if target:
-                if recursive and os.path.isdir(target):
-                    stages = self.stages(target)
-                else:
-                    stages = self.collect(target, with_deps=with_deps)
-            elif active:
-                stages = self.active_stages()
+            if targets:
+                stages = []
+                for target in targets:
+                    collected = self.collect(
+                        target, recursive=recursive, with_deps=with_deps
+                    )
+                    stages.extend(collected)
             else:
-                stages = self.stages()
+                stages = self.stages
 
             for stage in stages:
-                if active and not target and stage.locked:
-                    logger.warning(
-                        "DVC file '{path}' is locked. Its dependencies are"
-                        " not going to be pushed/pulled/fetched.".format(
-                            path=stage.relpath
-                        )
-                    )
+                if stage.is_repo_import:
+                    dep, = stage.deps
+                    cache.external[dep.repo_pair].add(dep.def_path)
+                    continue
 
                 for out in stage.outs:
-                    scheme = out.path_info.scheme
-                    cache[scheme].extend(
-                        self._collect_used_cache(
-                            out,
-                            branch=branch,
-                            remote=remote,
-                            force=force,
-                            jobs=jobs,
-                        )
+                    used_cache = out.get_used_cache(
+                        remote=remote, force=force, jobs=jobs
                     )
+                    suffix = "({})".format(branch) if branch else ""
+                    cache.update(used_cache, suffix=suffix)
 
         return cache
 
-    def graph(self, stages=None, from_directory=None):
+    def _collect_graph(self, stages=None):
         """Generate a graph by using the given stages on the given directory
 
         The nodes of the graph are the stage's path relative to the root.
@@ -368,11 +296,8 @@ class Repo(object):
               (weakly connected components)
 
         Args:
-            stages (list): used to build a graph, if None given, use the ones
-                on the `from_directory`.
-
-            from_directory (str): directory where to look at for stages, if
-                None is given, use the current working directory
+            stages (list): used to build a graph, if None given, collect stages
+                in the repository.
 
         Raises:
             OutputDuplicationError: two outputs with the same path
@@ -388,72 +313,73 @@ class Repo(object):
         )
 
         G = nx.DiGraph()
-        G_active = nx.DiGraph()
-        stages = stages or self.stages(from_directory, check_dag=False)
+        stages = stages or self.collect_stages()
         stages = [stage for stage in stages if stage]
-        outs = []
+        outs = {}
 
         for stage in stages:
             for out in stage.outs:
-                existing = []
-                for o in outs:
-                    if o.path == out.path:
-                        existing.append(o.stage)
-
-                    in_o_dir = out.path.startswith(o.path + o.sep)
-                    in_out_dir = o.path.startswith(out.path + out.sep)
-                    if in_o_dir or in_out_dir:
-                        raise OverlappingOutputPathsError(o, out)
-
-                if existing:
-                    stages = [stage.relpath, existing[0].relpath]
-                    raise OutputDuplicationError(out.path, stages)
-
-                outs.append(out)
+                if out.path_info in outs:
+                    stages = [stage.relpath, outs[out.path_info].stage.relpath]
+                    raise OutputDuplicationError(str(out), stages)
+                outs[out.path_info] = out
 
         for stage in stages:
-            path_dir = os.path.dirname(stage.path) + os.sep
-            for out in outs:
-                if path_dir.startswith(out.path + os.sep):
+            for out in stage.outs:
+                for p in out.path_info.parents:
+                    if p in outs:
+                        raise OverlappingOutputPathsError(outs[p], out)
+
+        for stage in stages:
+            stage_path_info = PathInfo(stage.path)
+            for p in chain([stage_path_info], stage_path_info.parents):
+                if p in outs:
                     raise StagePathAsOutputError(stage.wdir, stage.relpath)
 
         for stage in stages:
-            node = os.path.relpath(stage.path, self.root_dir)
+            node = relpath(stage.path, self.root_dir)
 
             G.add_node(node, stage=stage)
-            G_active.add_node(node, stage=stage)
 
             for dep in stage.deps:
+                if dep.path_info is None:
+                    continue
+
                 for out in outs:
                     if (
-                        out.path != dep.path
-                        and not dep.path.startswith(out.path + out.sep)
-                        and not out.path.startswith(dep.path + dep.sep)
+                        out == dep.path_info
+                        or dep.path_info.isin(out)
+                        or out.isin(dep.path_info)
                     ):
-                        continue
+                        dep_stage = outs[out].stage
+                        dep_node = relpath(dep_stage.path, self.root_dir)
+                        G.add_node(dep_node, stage=dep_stage)
+                        G.add_edge(node, dep_node)
 
-                    dep_stage = out.stage
-                    dep_node = os.path.relpath(dep_stage.path, self.root_dir)
-                    G.add_node(dep_node, stage=dep_stage)
-                    G.add_edge(node, dep_node)
-                    if not stage.locked:
-                        G_active.add_node(dep_node, stage=dep_stage)
-                        G_active.add_edge(node, dep_node)
+        check_acyclic(G)
 
-        self._check_cyclic_graph(G)
+        return G
 
-        return G, G_active
+    @cached_property
+    def graph(self):
+        return self._collect_graph()
 
-    def pipelines(self, from_directory=None):
-        import networkx as nx
+    @cached_property
+    def pipelines(self):
+        return get_pipelines(self.graph)
 
-        G, G_active = self.graph(from_directory=from_directory)
+    @staticmethod
+    def _filter_out_dirs(dirs, outs, root_dir):
+        def filter_dirs(dname):
+            path = os.path.join(root_dir, dname)
+            for out in outs:
+                if path == os.path.normpath(out):
+                    return False
+            return True
 
-        return [
-            G.subgraph(c).copy() for c in nx.weakly_connected_components(G)
-        ]
+        return list(filter(filter_dirs, dirs))
 
-    def stages(self, from_directory=None, check_dag=True):
+    def collect_stages(self):
         """
         Walks down the root directory looking for Dvcfiles,
         skipping the directories that are related with
@@ -465,17 +391,11 @@ class Repo(object):
         """
         from dvc.stage import Stage
 
-        if not from_directory:
-            from_directory = self.root_dir
-        elif not os.path.isdir(from_directory):
-            raise TargetNotDirectoryError(from_directory)
-
         stages = []
         outs = []
 
-        ignore_file_handler = DvcIgnoreFileHandler(self.tree)
         for root, dirs, files in self.tree.walk(
-            from_directory, ignore_file_handler=ignore_file_handler
+            self.root_dir, dvcignore=self.dvcignore
         ):
             for fname in files:
                 path = os.path.join(root, fname)
@@ -483,49 +403,30 @@ class Repo(object):
                     continue
                 stage = Stage.load(self, path)
                 for out in stage.outs:
-                    outs.append(out.path + out.sep)
+                    if out.scheme == "local":
+                        outs.append(out.fspath + out.sep)
                 stages.append(stage)
 
-            def filter_dirs(dname, root=root):
-                path = os.path.join(root, dname)
-                if path in (self.dvc_dir, self.scm.dir):
-                    return False
-                for out in outs:
-                    if path == os.path.normpath(out) or path.startswith(out):
-                        return False
-                return True
-
-            dirs[:] = list(filter(filter_dirs, dirs))
-
-        if check_dag:
-            self.check_dag(stages)
+            dirs[:] = self._filter_out_dirs(dirs, outs, root)
 
         return stages
 
-    def active_stages(self, from_directory=None):
-        import networkx as nx
-
-        stages = []
-        for G in self.pipelines(from_directory):
-            stages.extend(list(nx.get_node_attributes(G, "stage").values()))
-        return stages
+    @cached_property
+    def stages(self):
+        return get_stages(self.graph)
 
     def find_outs_by_path(self, path, outs=None, recursive=False):
         if not outs:
-            # there is no `from_directory=path` argument because some data
-            # files might be generated to an upper level, and so it is
-            # needed to look at all the files (project root_dir)
-            stages = self.stages()
-            outs = [out for stage in stages for out in stage.outs]
+            outs = [out for stage in self.stages for out in stage.outs]
 
         abs_path = os.path.abspath(path)
         is_dir = self.tree.isdir(abs_path)
 
         def func(out):
-            if out.path == abs_path:
+            if out.scheme == "local" and out.fspath == abs_path:
                 return True
 
-            if is_dir and recursive and out.path.startswith(abs_path + os.sep):
+            if is_dir and recursive and out.path_info.isin(abs_path):
                 return True
 
             return False
@@ -536,19 +437,78 @@ class Repo(object):
 
         return matched
 
+    def find_out_by_relpath(self, relpath):
+        path = os.path.join(self.root_dir, relpath)
+        out, = self.find_outs_by_path(path)
+        return out
+
     def is_dvc_internal(self, path):
         path_parts = os.path.normpath(path).split(os.path.sep)
         return self.DVC_DIR in path_parts
 
+    @contextmanager
     def open(self, path, remote=None, mode="r", encoding=None):
         """Opens a specified resource as a file descriptor"""
-        out, = self.find_outs_by_path(path)
+        cause = None
+        try:
+            out, = self.find_outs_by_path(path)
+        except OutputNotFoundError as e:
+            out = None
+            cause = e
+
+        if out and out.use_cache:
+            try:
+                with self._open_cached(out, remote, mode, encoding) as fd:
+                    yield fd
+                return
+            except FileNotFoundError as e:
+                raise FileMissingError(relpath(path, self.root_dir), cause=e)
+
+        if self.tree.exists(path):
+            with self.tree.open(path, mode, encoding) as fd:
+                yield fd
+            return
+
+        raise FileMissingError(relpath(path, self.root_dir), cause=cause)
+
+    def _open_cached(self, out, remote=None, mode="r", encoding=None):
         if out.isdir():
             raise ValueError("Can't open a dir")
 
-        with self.state:
-            cache_info = self._collect_used_cache(out, remote=remote)
-            self.cloud.pull(cache_info, remote=remote)
+        cache_file = self.cache.local.checksum_to_path_info(out.checksum)
+        cache_file = fspath_py35(cache_file)
 
-        cache_filename = self.cache.local.checksum_to_path(out.checksum)
-        return _open(cache_filename, mode=mode, encoding=encoding)
+        if os.path.exists(cache_file):
+            return _open(cache_file, mode=mode, encoding=encoding)
+
+        try:
+            remote_obj = self.cloud.get_remote(remote)
+            remote_info = remote_obj.checksum_to_path_info(out.checksum)
+            return remote_obj.open(remote_info, mode=mode, encoding=encoding)
+        except RemoteActionNotImplemented:
+            with self.state:
+                cache_info = out.get_used_cache(remote=remote)
+                self.cloud.pull(cache_info, remote=remote)
+
+            return _open(cache_file, mode=mode, encoding=encoding)
+
+    @cached_property
+    def dvcignore(self):
+        return DvcIgnoreFilter(self.root_dir)
+
+    def close(self):
+        self.scm.close()
+
+    @locked
+    def checkout(self, *args, **kwargs):
+        return self._checkout(*args, **kwargs)
+
+    @locked
+    def fetch(self, *args, **kwargs):
+        return self._fetch(*args, **kwargs)
+
+    def _reset(self):
+        self.__dict__.pop("graph", None)
+        self.__dict__.pop("stages", None)
+        self.__dict__.pop("pipelines", None)
+        self.__dict__.pop("dvcignore", None)

dvc/repo/add.py

@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import os
 import logging
 import colorama
@@ -7,10 +9,13 @@ from dvc.stage import Stage
 from dvc.utils import walk_files, LARGE_DIR_SIZE
 from dvc.exceptions import RecursiveAddingWhileUsingFilename
 
+from . import locked
+
 
 logger = logging.getLogger(__name__)
 
 
+@locked
 @scm_context
 def add(repo, target, recursive=False, no_commit=False, fname=None):
     if recursive and fname:
@@ -22,7 +27,7 @@ def add(repo, target, recursive=False, no_commit=False, fname=None):
         logger.warning(
             "You are adding a large directory '{target}' recursively,"
             " consider tracking it as a whole instead.\n"
-            "{purple}HINT:{nc} Remove the generated stage files and then"
+            "{purple}HINT:{nc} Remove the generated DVC-file and then"
             " run {cyan}dvc add {target}{nc}".format(
                 purple=colorama.Fore.MAGENTA,
                 cyan=colorama.Fore.CYAN,
@@ -31,12 +36,18 @@ def add(repo, target, recursive=False, no_commit=False, fname=None):
             )
         )
 
-    stages = _create_stages(repo, targets, fname, no_commit)
+    stages = _create_stages(repo, targets, fname)
 
-    repo.check_dag(repo.stages() + stages)
+    repo.check_modified_graph(stages)
 
     for stage in stages:
+        stage.save()
+
+        if not no_commit:
+            stage.commit()
+
         stage.dump()
+
     return stages
 
 
@@ -44,7 +55,7 @@ def _find_all_targets(repo, target, recursive):
     if os.path.isdir(target) and recursive:
         return [
             fname
-            for fname in walk_files(target)
+            for fname in walk_files(target, repo.dvcignore)
             if not repo.is_dvc_internal(fname)
             if not Stage.is_stage_file(fname)
             if not repo.scm.belongs_to_scm(fname)
@@ -53,21 +64,15 @@ def _find_all_targets(repo, target, recursive):
     return [target]
 
 
-def _create_stages(repo, targets, fname, no_commit):
+def _create_stages(repo, targets, fname):
     stages = []
 
-    with repo.state:
-        for out in targets:
-            stage = Stage.create(repo=repo, outs=[out], add=True, fname=fname)
+    for out in targets:
+        stage = Stage.create(repo, outs=[out], add=True, fname=fname)
 
-            if not stage:
-                continue
+        if not stage:
+            continue
 
-            stage.save()
-
-            if not no_commit:
-                stage.commit()
-
-            stages.append(stage)
+        stages.append(stage)
 
     return stages

dvc/repo/brancher.py

@@ -1,5 +1,14 @@
+from funcy import group_by
+from dvc.scm.tree import WorkingTree
+
+
 def brancher(  # noqa: E302
-    self, branches=None, all_branches=False, tags=None, all_tags=False
+    self,
+    branches=None,
+    all_branches=False,
+    tags=None,
+    all_tags=False,
+    all_commits=False,
 ):
     """Generator that iterates over specified revisions.
 
@@ -15,7 +24,7 @@ def brancher(  # noqa: E302
             - empty string it there is no branches to iterate over
             - "Working Tree" if there are uncommited changes in the SCM repo
     """
-    if not any([branches, all_branches, tags, all_tags]):
+    if not any([branches, all_branches, tags, all_tags, all_commits]):
         yield ""
         return
 
@@ -24,33 +33,31 @@ def brancher(  # noqa: E302
 
     scm = self.scm
 
-    if self.scm.is_dirty():
-        from dvc.scm.tree import WorkingTree
+    self.tree = WorkingTree(self.root_dir)
+    yield "working tree"
 
-        self.tree = WorkingTree(self.root_dir)
-        yield "Working Tree"
-
-    if all_branches:
-        branches = scm.list_branches()
-
-    if all_tags:
-        tags = scm.list_tags()
-
-    if branches is None:
-        revs.extend([scm.active_branch()])
+    if all_commits:
+        revs = scm.list_all_commits()
     else:
-        revs.extend(branches)
+        if all_branches:
+            branches = scm.list_branches()
 
-    if tags is not None:
-        revs.extend(tags)
+        if all_tags:
+            tags = scm.list_tags()
+
+        if branches is not None:
+            revs.extend(branches)
+
+        if tags is not None:
+            revs.extend(tags)
 
     # NOTE: it might be a good idea to wrap this loop in try/finally block
     # to don't leave the tree on some unexpected branch after the
     # `brancher()`, but this could cause problems on exception handling
     # code which might expect the tree on which exception was raised to
     # stay in place. This behavior is a subject to change.
-    for rev in revs:
-        self.tree = scm.get_tree(rev)
-        yield rev
+    for sha, names in group_by(scm.resolve_rev, revs).items():
+        self.tree = scm.get_tree(sha)
+        yield ", ".join(names)
 
     self.tree = saved_tree

dvc/repo/checkout.py

@@ -2,18 +2,20 @@ from __future__ import unicode_literals
 
 import logging
 
-from dvc.exceptions import CheckoutErrorSuggestGit
-from dvc.progress import ProgressCallback
+from dvc.exceptions import CheckoutErrorSuggestGit, CheckoutError
+from dvc.progress import Tqdm
 
 
 logger = logging.getLogger(__name__)
 
 
 def _cleanup_unused_links(self, all_stages):
-    used = []
-    for stage in all_stages:
-        for out in stage.outs:
-            used.append(out.path)
+    used = [
+        out.fspath
+        for stage in all_stages
+        for out in stage.outs
+        if out.scheme == "local"
+    ]
     self.state.remove_unused_links(used)
 
 
@@ -21,36 +23,41 @@ def get_all_files_numbers(stages):
     return sum(stage.get_all_files_number() for stage in stages)
 
 
-def get_progress_callback(stages):
-    total_files_num = get_all_files_numbers(stages)
-    if total_files_num == 0:
-        return None
-    return ProgressCallback(total_files_num)
-
-
-def checkout(self, target=None, with_deps=False, force=False, recursive=False):
+def _checkout(
+    self, targets=None, with_deps=False, force=False, recursive=False
+):
     from dvc.stage import StageFileDoesNotExistError, StageFileBadNameError
 
-    all_stages = self.stages()
-
-    try:
-        stages = self.collect(target, with_deps=with_deps, recursive=recursive)
-    except (StageFileDoesNotExistError, StageFileBadNameError) as exc:
-        if not target:
-            raise
-        raise CheckoutErrorSuggestGit(target, exc)
-
-    with self.state:
-        _cleanup_unused_links(self, all_stages)
-        progress_callback = get_progress_callback(stages)
+    stages = set()
+    targets = targets or [None]
+    for target in targets:
+        try:
+            new = self.collect(
+                target, with_deps=with_deps, recursive=recursive
+            )
+            stages.update(new)
+        except (StageFileDoesNotExistError, StageFileBadNameError) as exc:
+            if not target:
+                raise
+            raise CheckoutErrorSuggestGit(target, exc)
 
+    _cleanup_unused_links(self, self.stages)
+    total = get_all_files_numbers(stages)
+    if total == 0:
+        logger.info("Nothing to do")
+    failed = []
+    with Tqdm(
+        total=total, unit="file", desc="Checkout", disable=total == 0
+    ) as pbar:
         for stage in stages:
             if stage.locked:
                 logger.warning(
-                    "DVC file '{path}' is locked. Its dependencies are"
+                    "DVC-file '{path}' is locked. Its dependencies are"
                     " not going to be checked out.".format(path=stage.relpath)
                 )
 
-            stage.checkout(force=force, progress_callback=progress_callback)
-        if progress_callback:
-            progress_callback.finish("Checkout finished!")
+            failed.extend(
+                stage.checkout(force=force, progress_callback=pbar.update_desc)
+            )
+    if failed:
+        raise CheckoutError(failed)