dvc: bump to 0.40.2

remote: checkout: save checksums without re-computation

.appveyor.yml

@@ -3,6 +3,10 @@ branches:
     - master
     - devel
 
+cache:
+  - C:\ProgramData\chocolatey\bin -> appveyor.yml
+  - C:\ProgramData\chocolatey\lib -> appveyor.yml
+
 platform: x86
 
 environment:
@@ -10,43 +14,35 @@ environment:
     secure: veTfhOrd4wUpG7NlRsvS3tufuaFfST8i7SZa2HVH6Vc=
   aws_secret_access_key:
     secure: nCQcN/Mzce0erwcNOU3yAuWgF6ZuTvPBEqTF3XFp1poGcDbSIg+s67eVxsxWCXQj
-  GOOGLE_APPLICATION_CREDENTIALS: gcp-creds.json
   GCP_CREDS:
     secure: 96fJ3r2i2GohbXHwnSs5N4EplQ7q8YmLpPWM0AC+f4s=
   CODECOV_TOKEN:
     secure: XN4jRtmGE5Bqg8pPZkwNs7kn3UEI73Rkldqc0MGsQISZBm5TNJZOPofDMc1QnUsf
   AZURE_STORAGE_CONTAINER_NAME: appveyor-tests
   AZURE_STORAGE_CONNECTION_STRING: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;
+  OSS_ENDPOINT: localhost:50004
+  OSS_ACCESS_KEY_ID: AccessKeyID
+  OSS_ACCESS_KEY_SECRET: AccessKeySecret
 
   matrix:
 
     - PYTHON: "C:\\Python27"
       PYTHON_VERSION: "2.7.x" # currently 2.7.9
       PYTHON_ARCH: "32"
+      RUBY: C:\Ruby25\bin
 
 # Disabled as unnecessary
 #    - PYTHON: "C:\\Python27-x64"
 #      PYTHON_VERSION: "2.7.x" # currently 2.7.9
 #      PYTHON_ARCH: "64"
-#
-#    - PYTHON: "C:\\Python33"
-#      PYTHON_VERSION: "3.3.x" # currently 3.3.5
-#      PYTHON_ARCH: "32"
-#
-#    - PYTHON: "C:\\Python33-x64"
-#      PYTHON_VERSION: "3.3.x" # currently 3.3.5
-#      PYTHON_ARCH: "64"
-#
-#    - PYTHON: "C:\\Python34"
-#      PYTHON_VERSION: "3.4.x" # currently 3.4.3
-#      PYTHON_ARCH: "32"
-#
-#    - PYTHON: "C:\\Python34-x64"
-#      PYTHON_VERSION: "3.4.x" # currently 3.4.3
-#      PYTHON_ARCH: "64"
 
 install:
-  - ps: Install-Product node
+  - SET PATH=%RUBY%;%PATH%
+  - git clone -q https://github.com/nanaya-tachibana/oss-emulator.git
+  - gem install thor builder
+  - ps: cd oss-emulator
+  - ps: $OSSProcess = Start-Process -FilePath ruby -ArgumentList "bin\emulator -r store -p 50004" -PassThru -NoNewWindow
+  - ps: cd ..
   - npm install -g azurite
   - ps: $AzuriteProcess = Start-Process azurite-blob -PassThru
   - cinst wget
@@ -66,18 +62,20 @@ before_test:
   - aws configure set aws_secret_access_key "%aws_secret_access_key%"
   - aws configure set region us-east-2
   - openssl enc -d -aes-256-cbc -md md5 -k "%GCP_CREDS%" -in scripts\ci\gcp-creds.json.enc -out scripts\ci\gcp-creds.json & exit 0
-  - pip install -r test-requirements.txt
+  - pip install -r tests/requirements.txt
 
 test_script:
   - python -mtests
 
 after_test:
+  - if "%APPVEYOR_PULL_REQUEST_NUMBER%"=="" pip install codecov
   - if "%APPVEYOR_PULL_REQUEST_NUMBER%"=="" codecov
   - if "%APPVEYOR_REPO_TAG%"=="true" python -m dvc pull
   - if "%APPVEYOR_REPO_TAG%"=="true" .\scripts\build_windows.cmd
 
 on_finish:
   - ps: Stop-Process $AzuriteProcess
+  - ps: Stop-Process $OSSProcess
 
 artifacts:
   - path: dvc*.exe

.codecov.yml

@@ -1 +1 @@
-comment: false
+

.dvc/.gitignore

@@ -1,10 +1,8 @@
-link.state
-link.state.lock
-lock
-config.local
-updater
-cache
-state
-state.lock
-state-journal
-state-wal
+/state
+/lock
+/config.local
+/updater
+/updater.lock
+/state-journal
+/state-wal
+/cache

.dvc/config

@@ -1,4 +1,4 @@
 ['remote "dvc"']
-url = s3://dvc-share/dvc/
+url = https://dvc.org/s3/dvc
 [core]
 remote = dvc

.flake8

@@ -0,0 +1,7 @@
+[flake8]
+ignore =
+    E203, # Whitespace before ':'
+    E266, # Too many leading '#' for block comment
+    W503, # Line break occurred before a binary operator
+max-line-length = 79
+select = B,C,E,F,W,T4,B9

.github/CONTRIBUTING.md

@@ -0,0 +1 @@
+### See our contribution guide at [dvc.org](https://dvc.org/doc/user-guide/contributing#contributing).

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,9 @@
+* [ ] Have you followed the guidelines in our
+      [Contributing document](https://dvc.org/doc/user-guide/contributing)?
+
+* [ ] Does your PR affect documented changes or does it add new functionality
+      that should be documented? If yes, have you created a PR for
+      [dvc.org](https://github.com/iterative/dvc.org) documenting it or at
+      least opened an issue for it? If so, please add a link to it.
+
+-----

.gitignore

@@ -3,6 +3,7 @@ neatlynx/__pycache__
 .idea
 cache
 *.pyc
+.env/
 
 cache
 .dvc.conf.lock
@@ -23,6 +24,7 @@ innosetup/config.ini
 
 .coverage
 
-dvc/version.py
-
 *.swp
+
+pip-wheel-metadata/
+.vscode/

.pre-commit-config.yaml

@@ -0,0 +1,11 @@
+repos:
+-   repo: https://github.com/ambv/black
+    rev: 19.3b0
+    hooks:
+    - id: black
+      language_version: python3
+-   repo: https://gitlab.com/pycqa/flake8
+    rev: master
+    hooks:
+    - id: flake8
+      language_version: python3

.pylintrc

@@ -0,0 +1,568 @@
+[MASTER]
+
+# A comma-separated list of package or module names from where C extensions may
+# be loaded. Extensions are loading into the active Python interpreter and may
+# run arbitrary code.
+extension-pkg-whitelist=
+
+# Add files or directories to the blacklist. They should be base names, not
+# paths.
+ignore=CVS
+
+# Add files or directories matching the regex patterns to the blacklist. The
+# regex matches against base names, not paths.
+ignore-patterns=
+
+# Python code to execute, usually for sys.path manipulation such as
+# pygtk.require().
+#init-hook=
+
+# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
+# number of processors available to use.
+jobs=1
+
+# Control the amount of potential inferred values when inferring a single
+# object. This can help the performance when dealing with large functions or
+# complex, nested conditions.
+limit-inference-results=100
+
+# List of plugins (as comma separated values of python modules names) to load,
+# usually to register additional checkers.
+load-plugins=
+
+# Pickle collected data for later comparisons.
+persistent=yes
+
+# Specify a configuration file.
+#rcfile=
+
+# When enabled, pylint would attempt to guess common misconfiguration and emit
+# user-friendly hints instead of false-positive error messages.
+suggestion-mode=yes
+
+# Allow loading of arbitrary C extensions. Extensions are imported into the
+# active Python interpreter and may run arbitrary code.
+unsafe-load-any-extension=no
+
+
+[MESSAGES CONTROL]
+
+# Only show warnings with the listed confidence levels. Leave empty to show
+# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED.
+confidence=
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifiers separated by comma (,) or put this
+# option multiple times (only on the command line, not in the configuration
+# file where it should appear only once). You can also use "--disable=all" to
+# disable everything first and then reenable specific checks. For example, if
+# you want to run only the similarities checker, you can use "--disable=all
+# --enable=similarities". If you want to run only the classes checker, but have
+# no Warning level messages displayed, use "--disable=all --enable=classes
+# --disable=W".
+disable=print-statement,
+        parameter-unpacking,
+        unpacking-in-except,
+        old-raise-syntax,
+        backtick,
+        long-suffix,
+        old-ne-operator,
+        old-octal-literal,
+        import-star-module-level,
+        non-ascii-bytes-literal,
+        raw-checker-failed,
+        bad-inline-option,
+        locally-disabled,
+        file-ignored,
+        suppressed-message,
+        useless-suppression,
+        deprecated-pragma,
+        use-symbolic-message-instead,
+        apply-builtin,
+        basestring-builtin,
+        buffer-builtin,
+        cmp-builtin,
+        coerce-builtin,
+        execfile-builtin,
+        file-builtin,
+        long-builtin,
+        raw_input-builtin,
+        reduce-builtin,
+        standarderror-builtin,
+        unicode-builtin,
+        xrange-builtin,
+        coerce-method,
+        delslice-method,
+        getslice-method,
+        setslice-method,
+        no-absolute-import,
+        old-division,
+        dict-iter-method,
+        dict-view-method,
+        next-method-called,
+        metaclass-assignment,
+        indexing-exception,
+        raising-string,
+        reload-builtin,
+        oct-method,
+        hex-method,
+        nonzero-method,
+        cmp-method,
+        input-builtin,
+        round-builtin,
+        intern-builtin,
+        unichr-builtin,
+        map-builtin-not-iterating,
+        zip-builtin-not-iterating,
+        range-builtin-not-iterating,
+        filter-builtin-not-iterating,
+        using-cmp-argument,
+        eq-without-hash,
+        div-method,
+        idiv-method,
+        rdiv-method,
+        exception-message-attribute,
+        invalid-str-codec,
+        sys-max-int,
+        bad-python3-import,
+        deprecated-string-function,
+        deprecated-str-translate-call,
+        deprecated-itertools-function,
+        deprecated-types-field,
+        next-method-defined,
+        dict-items-not-iterating,
+        dict-keys-not-iterating,
+        dict-values-not-iterating,
+        deprecated-operator-function,
+        deprecated-urllib-function,
+        xreadlines-attribute,
+        deprecated-sys-function,
+        exception-escape,
+        comprehension-escape,
+        useless-object-inheritance
+
+# Enable the message, report, category or checker with the given id(s). You can
+# either give multiple identifier separated by comma (,) or put this option
+# multiple time (only on the command line, not in the configuration file where
+# it should appear only once). See also the "--disable" option for examples.
+enable=c-extension-no-member
+
+
+[REPORTS]
+
+# Python expression which should return a note less than 10 (10 is the highest
+# note). You have access to the variables errors warning, statement which
+# respectively contain the number of errors / warnings messages and the total
+# number of statements analyzed. This is used by the global evaluation report
+# (RP0004).
+evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
+
+# Template used to display messages. This is a python new-style format string
+# used to format the message information. See doc for all details.
+#msg-template=
+
+# Set the output format. Available formats are text, parseable, colorized, json
+# and msvs (visual studio). You can also give a reporter class, e.g.
+# mypackage.mymodule.MyReporterClass.
+output-format=text
+
+# Tells whether to display a full report or only the messages.
+reports=no
+
+# Activate the evaluation score.
+score=yes
+
+
+[REFACTORING]
+
+# Maximum number of nested blocks for function / method body
+max-nested-blocks=5
+
+# Complete name of functions that never returns. When checking for
+# inconsistent-return-statements if a never returning function is called then
+# it will be considered as an explicit return statement and no message will be
+# printed.
+never-returning-functions=sys.exit
+
+
+[LOGGING]
+
+# Format style used to check logging format string. `old` means using %
+# formatting, while `new` is for `{}` formatting.
+logging-format-style=old
+
+# Logging modules to check that the string format arguments are in logging
+# function parameter format.
+logging-modules=logging
+
+
+[SPELLING]
+
+# Limits count of emitted suggestions for spelling mistakes.
+max-spelling-suggestions=4
+
+# Spelling dictionary name. Available dictionaries: none. To make it working
+# install python-enchant package..
+spelling-dict=
+
+# List of comma separated words that should not be checked.
+spelling-ignore-words=
+
+# A path to a file that contains private dictionary; one word per line.
+spelling-private-dict-file=
+
+# Tells whether to store unknown words to indicated private dictionary in
+# --spelling-private-dict-file option instead of raising a message.
+spelling-store-unknown-words=no
+
+
+[TYPECHECK]
+
+# List of decorators that produce context managers, such as
+# contextlib.contextmanager. Add to this list to register other decorators that
+# produce valid context managers.
+contextmanager-decorators=contextlib.contextmanager
+
+# List of members which are set dynamically and missed by pylint inference
+# system, and so shouldn't trigger E1101 when accessed. Python regular
+# expressions are accepted.
+generated-members=
+
+# Tells whether missing members accessed in mixin class should be ignored. A
+# mixin class is detected if its name ends with "mixin" (case insensitive).
+ignore-mixin-members=yes
+
+# Tells whether to warn about missing members when the owner of the attribute
+# is inferred to be None.
+ignore-none=yes
+
+# This flag controls whether pylint should warn about no-member and similar
+# checks whenever an opaque object is returned when inferring. The inference
+# can return multiple potential results while evaluating a Python object, but
+# some branches might not be evaluated, which results in partial inference. In
+# that case, it might be useful to still emit no-member and other checks for
+# the rest of the inferred objects.
+ignore-on-opaque-inference=yes
+
+# List of class names for which member attributes should not be checked (useful
+# for classes with dynamically set attributes). This supports the use of
+# qualified names.
+ignored-classes=optparse.Values,thread._local,_thread._local
+
+# List of module names for which member attributes should not be checked
+# (useful for modules/projects where namespaces are manipulated during runtime
+# and thus existing member attributes cannot be deduced by static analysis. It
+# supports qualified module names, as well as Unix pattern matching.
+ignored-modules=
+
+# Show a hint with possible names when a member name was not found. The aspect
+# of finding the hint is based on edit distance.
+missing-member-hint=yes
+
+# The minimum edit distance a name should have in order to be considered a
+# similar match for a missing member name.
+missing-member-hint-distance=1
+
+# The total number of similar names that should be taken in consideration when
+# showing a hint for a missing member.
+missing-member-max-choices=1
+
+
+[VARIABLES]
+
+# List of additional names supposed to be defined in builtins. Remember that
+# you should avoid defining new builtins when possible.
+additional-builtins=
+
+# Tells whether unused global variables should be treated as a violation.
+allow-global-unused-variables=yes
+
+# List of strings which can identify a callback function by name. A callback
+# name must start or end with one of those strings.
+callbacks=cb_,
+          _cb
+
+# A regular expression matching the name of dummy variables (i.e. expected to
+# not be used).
+dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_
+
+# Argument names that match this expression will be ignored. Default to name
+# with leading underscore.
+ignored-argument-names=_.*|^ignored_|^unused_
+
+# Tells whether we should check for unused import in __init__ files.
+init-import=no
+
+# List of qualified module names which can have objects that can redefine
+# builtins.
+redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io
+
+
+[FORMAT]
+
+# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
+expected-line-ending-format=
+
+# Regexp for a line that is allowed to be longer than the limit.
+ignore-long-lines=^\s*(# )?<?https?://\S+>?$
+
+# Number of spaces of indent required inside a hanging or continued line.
+indent-after-paren=4
+
+# String used as indentation unit. This is usually "    " (4 spaces) or "\t" (1
+# tab).
+indent-string='    '
+
+# Maximum number of characters on a single line.
+max-line-length=100
+
+# Maximum number of lines in a module.
+max-module-lines=1000
+
+# List of optional constructs for which whitespace checking is disabled. `dict-
+# separator` is used to allow tabulation in dicts, etc.: {1  : 1,\n222: 2}.
+# `trailing-comma` allows a space between comma and closing bracket: (a, ).
+# `empty-line` allows space-only lines.
+no-space-check=trailing-comma,
+               dict-separator
+
+# Allow the body of a class to be on the same line as the declaration if body
+# contains single statement.
+single-line-class-stmt=no
+
+# Allow the body of an if to be on the same line as the test if there is no
+# else.
+single-line-if-stmt=no
+
+
+[BASIC]
+
+# Naming style matching correct argument names.
+argument-naming-style=snake_case
+
+# Regular expression matching correct argument names. Overrides argument-
+# naming-style.
+#argument-rgx=
+
+# Naming style matching correct attribute names.
+attr-naming-style=snake_case
+
+# Regular expression matching correct attribute names. Overrides attr-naming-
+# style.
+#attr-rgx=
+
+# Bad variable names which should always be refused, separated by a comma.
+bad-names=foo,
+          bar,
+          baz,
+          toto,
+          tutu,
+          tata
+
+# Naming style matching correct class attribute names.
+class-attribute-naming-style=any
+
+# Regular expression matching correct class attribute names. Overrides class-
+# attribute-naming-style.
+#class-attribute-rgx=
+
+# Naming style matching correct class names.
+class-naming-style=PascalCase
+
+# Regular expression matching correct class names. Overrides class-naming-
+# style.
+#class-rgx=
+
+# Naming style matching correct constant names.
+const-naming-style=UPPER_CASE
+
+# Regular expression matching correct constant names. Overrides const-naming-
+# style.
+#const-rgx=
+
+# Minimum line length for functions/classes that require docstrings, shorter
+# ones are exempt.
+docstring-min-length=-1
+
+# Naming style matching correct function names.
+function-naming-style=snake_case
+
+# Regular expression matching correct function names. Overrides function-
+# naming-style.
+#function-rgx=
+
+# Good variable names which should always be accepted, separated by a comma.
+good-names=i,
+           j,
+           k,
+           ex,
+           Run,
+           _,
+           s3,
+           gs,
+           x, y,
+           x0, y0,
+           x1, y1,
+           dx, dy
+
+# Include a hint for the correct naming format with invalid-name.
+include-naming-hint=no
+
+# Naming style matching correct inline iteration names.
+inlinevar-naming-style=any
+
+# Regular expression matching correct inline iteration names. Overrides
+# inlinevar-naming-style.
+#inlinevar-rgx=
+
+# Naming style matching correct method names.
+method-naming-style=snake_case
+
+# Regular expression matching correct method names. Overrides method-naming-
+# style.
+#method-rgx=
+
+# Naming style matching correct module names.
+module-naming-style=snake_case
+
+# Regular expression matching correct module names. Overrides module-naming-
+# style.
+#module-rgx=
+
+# Colon-delimited sets of names that determine each other's naming style when
+# the name regexes allow several styles.
+name-group=
+
+# Regular expression which should only match function or class names that do
+# not require a docstring.
+no-docstring-rgx=^_
+
+# List of decorators that produce properties, such as abc.abstractproperty. Add
+# to this list to register other decorators that produce valid properties.
+# These decorators are taken in consideration only for invalid-name.
+property-classes=abc.abstractproperty
+
+# Naming style matching correct variable names.
+variable-naming-style=snake_case
+
+# Regular expression matching correct variable names. Overrides variable-
+# naming-style.
+#variable-rgx=
+
+
+[SIMILARITIES]
+
+# Ignore comments when computing similarities.
+ignore-comments=yes
+
+# Ignore docstrings when computing similarities.
+ignore-docstrings=yes
+
+# Ignore imports when computing similarities.
+ignore-imports=no
+
+# Minimum lines number of a similarity.
+min-similarity-lines=4
+
+
+[MISCELLANEOUS]
+
+# List of note tags to take in consideration, separated by a comma.
+notes=FIXME,
+      XXX,
+      TODO
+
+
+[DESIGN]
+
+# Maximum number of arguments for function / method.
+max-args=5
+
+# Maximum number of attributes for a class (see R0902).
+max-attributes=7
+
+# Maximum number of boolean expressions in an if statement.
+max-bool-expr=5
+
+# Maximum number of branch for function / method body.
+max-branches=12
+
+# Maximum number of locals for function / method body.
+max-locals=15
+
+# Maximum number of parents for a class (see R0901).
+max-parents=7
+
+# Maximum number of public methods for a class (see R0904).
+max-public-methods=20
+
+# Maximum number of return / yield for function / method body.
+max-returns=6
+
+# Maximum number of statements in function / method body.
+max-statements=50
+
+# Minimum number of public methods for a class (see R0903).
+min-public-methods=2
+
+
+[CLASSES]
+
+# List of method names used to declare (i.e. assign) instance attributes.
+defining-attr-methods=__init__,
+                      __new__,
+                      setUp
+
+# List of member names, which should be excluded from the protected access
+# warning.
+exclude-protected=_asdict,
+                  _fields,
+                  _replace,
+                  _source,
+                  _make
+
+# List of valid names for the first argument in a class method.
+valid-classmethod-first-arg=cls
+
+# List of valid names for the first argument in a metaclass class method.
+valid-metaclass-classmethod-first-arg=cls
+
+
+[IMPORTS]
+
+# Allow wildcard imports from modules that define __all__.
+allow-wildcard-with-all=no
+
+# Analyse import fallback blocks. This can be used to support both Python 2 and
+# 3 compatible code, which means that the block might have code that exists
+# only in one or another interpreter, leading to false positives when analysed.
+analyse-fallback-blocks=no
+
+# Deprecated modules which should not be used, separated by a comma.
+deprecated-modules=optparse,tkinter.tix
+
+# Create a graph of external dependencies in the given file (report RP0402 must
+# not be disabled).
+ext-import-graph=
+
+# Create a graph of every (i.e. internal and external) dependencies in the
+# given file (report RP0402 must not be disabled).
+import-graph=
+
+# Create a graph of internal dependencies in the given file (report RP0402 must
+# not be disabled).
+int-import-graph=
+
+# Force import order to recognize a module as part of the standard
+# compatibility libraries.
+known-standard-library=
+
+# Force import order to recognize a module as part of a third party library.
+known-third-party=enchant
+
+
+[EXCEPTIONS]
+
+# Exceptions that will emit a warning when being caught. Defaults to
+# "Exception".
+overgeneral-exceptions=Exception

.travis.yml

@@ -1,13 +1,16 @@
 language: python
-cache: pip
-sudo: required
-dist: trusty
-group: deprecated-2017Q4
-python:
-  - "2.7"
-  - "3.4"
-  - "3.5"
-  - "3.6"
+cache:
+  - pip
+  - directories:
+    - /opt/python
+    - $HOME/.pyenv
+sudo: true
+dist: xenial
+python: "3.7"
+env:
+  global:
+    - PYENV_ROOT="$HOME/.pyenv"
+    - PATH="$PYENV_ROOT/bin:$PATH"
 # Travis doesn't support testing python on Mac yet, so we need
 # to workaround it by installing it directly with brew.
 #
@@ -16,10 +19,19 @@ python:
 # version, that causes fpm to crash.
 matrix:
   include:
-  # 3.7 needs Xenial image because of https://github.com/travis-ci/travis-ci/issues/9069
-  - python: "3.7"
-    dist: xenial
-    sudo: true
+  - name: "2.7"
+    env:
+    - PYTHON_VER=2.7.13
+  - name: "3.5"
+    env:
+    - PYTHON_VER=3.5.3
+  - name: "3.6"
+    env:
+    - PYTHON_VER=3.6.2
+  - name: "3.7"
+    env:
+    # NOTE: using default python
+    - PYTHON_VER=
   - if: type = cron OR tag is present
     os: osx
     osx_image: xcode7.3
@@ -27,19 +39,17 @@ matrix:
     rvm: 2.4.0
     env:
     - PYTHON_VER=2.7.10
-    - PYENV_ROOT="$HOME/.pyenv"
-    - PATH="$PYENV_ROOT/bin:$PATH"
     - CPPFLAGS="-I/usr/local/opt/openssl/include $CPPFLAGS"
     - LDFLAGS="-L/usr/local/opt/openssl/lib $LDFLAGS"
+    - PYTHON_CONFIGURE_OPTS="--enable-framework"
   - if: type = cron OR tag is present
     os: osx
     osx_image: xcode7.3
     language: ruby
     rvm: 2.4.0
     env:
-    - PYTHON_VER=3.4.3
-    - PYENV_ROOT="$HOME/.pyenv"
-    - PATH="$PYENV_ROOT/bin:$PATH"
+    - PYTHON_VER=3.6.2
+    - PYTHON_CONFIGURE_OPTS="--enable-framework"
   - if: type = cron OR tag is present
     os: osx
     osx_image: xcode7.3
@@ -47,48 +57,35 @@ matrix:
     rvm: 2.4.0
     env:
     - PYTHON_VER=3.5.0
-    - PYENV_ROOT="$HOME/.pyenv"
-    - PATH="$PYENV_ROOT/bin:$PATH"
+    - PYTHON_CONFIGURE_OPTS="--enable-framework"
+  - if: type = cron OR tag is present
+    os: osx
+    osx_image: xcode10.1
+    language: ruby
+    rvm: 2.4.0
+    env:
+    - PYTHON_VER=2.7.10
+    - CPPFLAGS="-I/usr/local/opt/openssl/include $CPPFLAGS"
+    - LDFLAGS="-L/usr/local/opt/openssl/lib $LDFLAGS"
+    - PYTHON_CONFIGURE_OPTS="--enable-framework"
+  - if: type = cron OR tag is present
+    os: osx
+    osx_image: xcode10.1
+    language: ruby
+    rvm: 2.4.0
+    env:
+    - PYTHON_VER=3.5.0
+    - PYTHON_CONFIGURE_OPTS="--enable-framework"
 before_install:
   - bash ./scripts/ci/before_install.sh
-  - if  [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-      eval "$(pyenv init -)";
-      PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing --keep --verbose "$PYTHON_VER";
-      pyenv shell "$PYTHON_VER";
-      brew install openssl;
-      brew cask install google-cloud-sdk;
-    fi
-  - which python
-  - which pip
+  - source env.sh
   - python --version
-  - if [[ "$TRAVIS_OS_NAME" != "osx" ]]; then
-      bash ./scripts/ci/install_hadoop.sh;
-      source ~/.bashrc;
-      hadoop fs -help;
-      pushd ~;
-      sudo mkdir -p /home/hadoop/hadoopinfra/hdfs/namenode/;
-      sudo chmod 777 /home/hadoop/hadoopinfra/hdfs/namenode/;
-      sudo chmod 777 -R /usr/local/hadoop;
-      hdfs namenode -format;
-      start-dfs.sh;
-      jps;
-      hadoop fs -ls /;
-      hadoop fs -ls hdfs://127.0.0.1/;
-      hadoop fs -mkdir hdfs://127.0.0.1/ololo;
-      popd;
-    fi
 install:
   - bash ./scripts/ci/install.sh
-  - pip install flake8
-before_script:
-  # stop the build if there are any readme formatting errors
-  - python setup.py checkdocs
-  # stop the build if there are any flake8 errors
-  - flake8 dvc
 script:
   - bash ./scripts/ci/script.sh
 deploy:
-  condition: "[[ $(python -c 'import sys; print(sys.version_info[0])') == '2' ]]"
+  condition: "[[ $(./scripts/ci/deploy_condition.sh) == 'true' ]]"
   provider: releases
   api_key:
     secure: nyo17gF4Xl6DFL/KVkMewC805Cv+eK99L7SiYCU+WUAYDIzzFL3YJm7fm8bznPeor7DmpO7uSLSveMnJETBtdurW6JWhCTUss4yD66yFzpk6Z5cvHf/1UA5pdMHkkMM54fRaTRt6L2Yal7RPwGEQTBECLXW1Xmf78Y6+0l8lTvXQ9jXOQSRO/63VDKI5Zww2+XytVicUFlr0hLMGKUuaCjjXcQlAPzBgQz3/5Iz6EtzeTf2o+gTPjocbRbSFPikfiBXUn8XlJ8jBj7i114XJRJffw59KxYGDzxp+enfLBULGNoW0Np2hZqkKkPUIdB08C8XVbRXlFYX8cEYBczjqhv8A05p5F5CnltnqQ0KazcoUP2awPVqn68dWNRF+kqrnMBus8euYKIZ4dy/6bVEKJRa84ESJXZVDcwXTAXRqyovSxOlXqg9ZgAImMKBfKn/fpDX2xIvQPaXcnJxKbmcePfdbC1d0HYJ5jkbNaxKE/T15VvhzIho7mJ2WAv1r8QouQUuzqtpzIGbRn/KXHRUzCtyP0k4+eMKMLFOPzg848WsDarP3v8VYiSy4nFY7ZP7DRd3KttvW5bgVbSeO4jPmTgS6TnnpRwual9BJXSG7bjbfpIMii3bNZ7jWL4zpKT9Jfm8w9RGKLcX9HqkZ3+y60+LtYdhEfBvqsIlWQoIJBXg=

CODE_OF_CONDUCT.md

@@ -0,0 +1,77 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, sex characteristics, gender identity and expression,
+level of experience, education, socio-economic status, nationality, personal
+appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+  advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at info@dvc.org. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq
+

README.rst

@@ -1,4 +1,4 @@
-.. image:: https://dvc.org/static/img/logo-owl-readme.png
+.. image:: https://dvc.org/static/img/logo-github-readme.png
   :target: https://dvc.org
   :alt: DVC logo
 
@@ -6,11 +6,11 @@
 • `Docs <https://dvc.org/doc>`_
 • `Twitter <https://twitter.com/iterativeai>`_
 • `Chat (Community & Support) <https://dvc.org/chat>`_
-• `Tutorial <https://dvc.org/doc/tutorial>`_
+• `Tutorial <https://dvc.org/doc/get-started>`_
 • `Mailing List <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_
 
-.. image:: https://travis-ci.org/iterative/dvc.svg?branch=master
-  :target: https://travis-ci.org/iterative/dvc
+.. image:: https://travis-ci.com/iterative/dvc.svg?branch=master
+  :target: https://travis-ci.com/iterative/dvc
   :alt: Travis
 
 .. image:: https://ci.appveyor.com/api/projects/status/github/iterative/dvc?branch=master&svg=true
@@ -25,20 +25,25 @@
   :target: https://codecov.io/gh/iterative/dvc
   :alt: Codecov
 
+.. image:: https://img.shields.io/badge/patreon-donate-green.svg
+  :target: https://www.patreon.com/DVCorg/overview
+  :alt: Donate
+
 |
 
 **Data Science Version Control** or **DVC** is an **open-source** tool for data science and
-machine learning projects. With a simple and flexible Git-like architecture and interface it
-helps data scientists:
+machine learning projects. Key features:
 
-#. manage **machine learning models** - versioning, including data sets and transformations (scripts) that were
-   used to generate models;
+#. simple **command line** Git-like experience. Does not require installing and maintaining
+   any databases. Does not depend on any proprietary online services;
 
-#. make projects **reproducible**;
+#. it manages and versions **datasets** and **machine learning models**. Data is saved in
+   S3, Google cloud, Azure, Alibaba cloud, SSH server, HDFS or even local HDD RAID;
 
-#. make projects **shareable**;
+#. it makes projects **reproducible** and **shareable**, it helps answering question how
+   the model was build;
 
-#. manage experiments with branching and **metrics** tracking;
+#. it helps manage experiments with Git tags or branches and **metrics** tracking;
 
 It aims to replace tools like Excel and Docs that are being commonly used as a knowledge repo and
 a ledger for the team, ad-hoc scripts to track and move deploy different model versions, ad-hoc
@@ -50,14 +55,30 @@ data file suffixes and prefixes.
 How DVC works
 =============
 
-DVC is compatible with Git for storing code and the dependency graph (DAG), but not data files cache.
-To store and share data files cache DVC supports remotes - any cloud (S3, Azure, Google Cloud, etc) or any on-premise
-network storage (via SSH, for example).
+We encourage you to read our `Get Started <https://dvc.org/doc/get-started>`_ to better understand what DVC
+is and how does it fit your scenarios.
+
+The easiest (but not perfect!) *analogy* to describe it: DVC is Git (or Git-lfs to be precise) + ``makefiles``
+made right and tailored specifically for ML and Data Science scenarios.
+
+#. ``Git/Git-lfs`` part - DVC helps you storing and sharing data artifacts, models. It connects them with your
+   Git repository.
+#. ``Makefiles`` part - DVC describes how one data or model artifact was build from another data.
+
+DVC usually runs along with Git. Git is used as usual to store and version code and DVC meta-files. DVC helps
+to store data and model files seamlessly out of Git while preserving almost the same user experience as if they
+were stored in Git itself. To store and share data files cache DVC supports remotes - any cloud (S3, Azure,
+Google Cloud, etc) or any on-premise network storage (via SSH, for example).
 
 .. image:: https://dvc.org/static/img/flow.gif
    :target: https://dvc.org/static/img/flow.gif
    :alt: how_dvc_works
 
+DVC pipelines (aka computational graph) feature connects code and data together. In a very explicit way you can
+specify, run, and save information that a certain command with certain dependencies needs to be run to produce
+a model. See the quick start section below or check `Get Started <https://dvc.org/doc/get-started>`_ tutorial to
+learn more.
+
 Quick start
 ===========
 
@@ -66,7 +87,7 @@ Please read `Get Started <https://dvc.org/doc/get-started>`_ for the full versio
 +-----------------------------------+-------------------------------------------------------------------+
 | Step                              | Command                                                           |
 +===================================+===================================================================+
-| Track code and data together      | | ``$ git add train.py``                                          |
+| Track data                        | | ``$ git add train.py``                                          |
 |                                   | | ``$ dvc add images.zip``                                        |
 +-----------------------------------+-------------------------------------------------------------------+
 | Connect code and data by commands | | ``$ dvc run -d images.zip -o images/ unzip -q images.zip``      |
@@ -79,27 +100,30 @@ Please read `Get Started <https://dvc.org/doc/get-started>`_ for the full versio
 |                                   | | ``$ git commit -m 'The baseline model'``                        |
 |                                   | | ``$ git push``                                                  |
 +-----------------------------------+-------------------------------------------------------------------+
-| Share data and ML models          | | ``$ dvc remote add myremote s3://mybucket/image_cnn``           |
-|                                   | | ``$ dvc config core.remote myremote``                           |
+| Share data and ML models          | | ``$ dvc remote add myremote -d s3://mybucket/image_cnn``        |
 |                                   | | ``$ dvc push``                                                  |
 +-----------------------------------+-------------------------------------------------------------------+
 
 Installation
 ============
 
-There are three options to install DVC: ``pip``, Homebrew, or an OS-specific package:
+Read this `instruction <https://dvc.org/doc/get-started/install>`_ to get more details. There are three
+options to install DVC: ``pip``, Homebrew, or an OS-specific package:
 
 pip (PyPI)
 ----------
 
-Stable
-^^^^^^
 .. code-block:: bash
 
    pip install dvc
 
-Development
-^^^^^^^^^^^
+Depending on the remote storage type you plan to use to keep and share your data, you might need to specify
+one of the optional dependencies: ``s3``, ``gs``, ``azure``, ``oss``, ``ssh``. Or ``all_remotes`` to include them all.
+The command should look like this: ``pip install dvc[s3]`` - it installs the ``boto3`` library along with
+DVC to support the AWS S3 storage.
+
+To install the development version, run:
+
 .. code-block:: bash
 
    pip install git+git://github.com/iterative/dvc
@@ -123,11 +147,11 @@ Package
 Self-contained packages for Windows, Linux, Mac are available. The latest version of the packages can be found at
 GitHub `releases page <https://github.com/iterative/dvc/releases>`_.
 
-Ubuntu / Debian (apt)
+Ubuntu / Debian (deb)
 ^^^^^^^^^^^^^^^^^^^^^
 .. code-block:: bash
 
-   sudo wget https://dvc.org/deb/dvc.list -O etc/apt/sources.list.d/dvc.list
+   sudo wget https://dvc.org/deb/dvc.list -O /etc/apt/sources.list.d/dvc.list
    sudo apt-get update
    sudo apt-get install dvc
 
@@ -139,7 +163,7 @@ Fedora / CentOS (rpm)
    sudo yum update
    sudo yum install dvc
 
-Arch linux (AUR)
+Arch Linux (AUR)
 ^^^^^^^^^^^^^^^^
 *Unofficial package*, any inquiries regarding the AUR package,
 `refer to the maintainer <https://github.com/mroutis/pkgbuilds>`_.
@@ -164,6 +188,8 @@ Related technologies
 #. `Workflow Management Systems <https://en.wikipedia.org/wiki/Workflow_management_system>`_. DVC is a workflow
    management system designed specifically to manage machine learning experiments. DVC is built on top of Git.
 
+#. `DAGsHub <https://dagshub.com/>`_ Is a Github equivalent for DVC - pushing your Git+DVC based repo to DAGsHub will give you a high level dashboard of your project, including DVC pipeline and metrics visualizations, as well as links to DVC managed files if they are in cloud storage.
+
 Contributing
 ============
 Contributions are welcome! Please see our `Contributing Guide <https://dvc.org/doc/user-guide/contributing/>`_ for more
@@ -204,7 +230,7 @@ details.
 Mailing List
 ============
 
-Want to stay up to date? Want to help improve DVC by participating in our ocassional polls? Subscribe to our `mailing list <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_. No spam, really low traffic.
+Want to stay up to date? Want to help improve DVC by participating in our occasional polls? Subscribe to our `mailing list <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_. No spam, really low traffic.
 
 Copyright
 =========

dvc/__init__.py

@@ -3,53 +3,15 @@ DVC
 ----
 Make your data science projects reproducible and shareable.
 """
-import os
+
+from __future__ import unicode_literals
+
+from dvc.version import __version__  # noqa: F401
 import warnings
+import dvc.logger
 
 
-VERSION_BASE = '0.20.4'
-__version__ = VERSION_BASE
-
-PACKAGEPATH = os.path.abspath(os.path.dirname(__file__))
-HOMEPATH = os.path.dirname(PACKAGEPATH)
-VERSIONPATH = os.path.join(PACKAGEPATH, 'version.py')
-
-if os.path.exists(os.path.join(HOMEPATH, 'setup.py')):
-    # dvc is run directly from source without installation or
-    # __version__ is called from setup.py
-    if os.getenv('APPVEYOR_REPO_TAG', '').lower() != 'true' \
-       and os.getenv('TRAVIS_TAG', '') == '':
-        # Dynamically update version
-        try:
-            import git
-            repo = git.Repo(HOMEPATH)
-            sha = repo.head.object.hexsha
-            short_sha = repo.git.rev_parse(sha, short=6)
-            dirty = '.mod' if repo.is_dirty() else ''
-            __version__ = '{}+{}{}'.format(__version__, short_sha, dirty)
-
-            # Write a helper file, that will be installed with the package
-            # and will provide a true version of the installed dvc
-            with open(VERSIONPATH, 'w+') as fd:
-                fd.write('# AUTOGENERATED by dvc/__init__.py\n')
-                fd.write('version = "{}"\n'.format(__version__))
-        except Exception:  # pragma: no cover
-            pass
-    else:  # pragma: no cover
-        # Remove version.py so that it doesn't get into the release
-        if os.path.exists(VERSIONPATH):
-            os.unlink(VERSIONPATH)
-else:  # pragma: no cover
-    # dvc was installed with pip or something. Hopefully we have our
-    # auto-generated version.py to help us provide a true version
-    try:
-        from dvc.version import version
-        __version__ = version
-    except Exception:
-        pass
-
-VERSION = __version__
-
+dvc.logger.setup()
 
 # Ignore numpy's runtime warnings: https://github.com/numpy/numpy/pull/432.
 # We don't directly import numpy, but our dependency networkx does, causing
@@ -57,3 +19,18 @@ VERSION = __version__
 # we can simply ignore them so that they don't show up when you are using dvc.
 warnings.filterwarnings("ignore", message="numpy.dtype size changed")
 warnings.filterwarnings("ignore", message="numpy.ufunc size changed")
+
+# Ignore paramiko's warning: https://github.com/paramiko/paramiko/issues/1386.
+# This only affects paramiko 2.4.2 and should be fixed in the next version.
+# Cryptography developers decided that it is a brilliant idea not to inherit
+# from DeprecationWarning [1] because it is invisible by default, and decided
+# to spam everyone instead. So it is their fault and not paramiko's.
+#
+# [1] https://github.com/pyca/cryptography/blob/2.6.1/src/cryptography/
+#     utils.py#L14
+try:
+    from cryptography.utils import CryptographyDeprecationWarning
+
+    warnings.simplefilter("ignore", CryptographyDeprecationWarning)
+except ImportError:
+    pass

dvc/__main__.py

@@ -1,6 +1,10 @@
+"""Main entry point for dvc command line tool."""
+
+from __future__ import unicode_literals
+
 import sys
 
 from dvc.main import main
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main(sys.argv[1:])

dvc/analytics.py

@@ -0,0 +1,277 @@
+"""Collect and send usage analytics"""
+
+from __future__ import unicode_literals
+
+from dvc.utils.compat import str
+
+import os
+import json
+import errno
+import logging
+
+from dvc import __version__
+
+
+logger = logging.getLogger(__name__)
+
+
+class Analytics(object):
+    """Class for collecting and sending usage analytics.
+
+    Args:
+        info (dict): optional existing analytics report.
+    """
+
+    URL = "https://analytics.dvc.org"
+    TIMEOUT_POST = 5
+
+    USER_ID_FILE = "user_id"
+
+    PARAM_DVC_VERSION = "dvc_version"
+    PARAM_USER_ID = "user_id"
+    PARAM_SYSTEM_INFO = "system_info"
+
+    PARAM_OS = "os"
+
+    PARAM_WINDOWS_VERSION_MAJOR = "windows_version_major"
+    PARAM_WINDOWS_VERSION_MINOR = "windows_version_minor"
+    PARAM_WINDOWS_VERSION_BUILD = "windows_version_build"
+    PARAM_WINDOWS_VERSION_SERVICE_PACK = "windows_version_service_pack"
+
+    PARAM_MAC_VERSION = "mac_version"
+
+    PARAM_LINUX_DISTRO = "linux_distro"
+    PARAM_LINUX_DISTRO_VERSION = "linux_distro_version"
+    PARAM_LINUX_DISTRO_LIKE = "linux_distro_like"
+
+    PARAM_SCM_CLASS = "scm_class"
+    PARAM_IS_BINARY = "is_binary"
+    PARAM_CMD_CLASS = "cmd_class"
+    PARAM_CMD_RETURN_CODE = "cmd_return_code"
+
+    def __init__(self, info=None):
+        from dvc.config import Config
+        from dvc.lock import Lock
+
+        if info is None:
+            info = {}
+
+        self.info = info
+
+        cdir = Config.get_global_config_dir()
+        try:
+            os.makedirs(cdir)
+        except OSError as exc:
+            if exc.errno != errno.EEXIST:
+                raise
+
+        self.user_id_file = os.path.join(cdir, self.USER_ID_FILE)
+        self.user_id_file_lock = Lock(cdir, self.USER_ID_FILE + ".lock")
+
+    @staticmethod
+    def load(path):
+        """Loads analytics report from json file specified by path.
+
+        Args:
+            path (str): path to json file with analytics report.
+        """
+        with open(path, "r") as fobj:
+            analytics = Analytics(info=json.load(fobj))
+        os.unlink(path)
+        return analytics
+
+    def _write_user_id(self):
+        import uuid
+
+        with open(self.user_id_file, "w+") as fobj:
+            user_id = str(uuid.uuid4())
+            info = {self.PARAM_USER_ID: user_id}
+            json.dump(info, fobj)
+            return user_id
+
+    def _read_user_id(self):
+        if not os.path.exists(self.user_id_file):
+            return None
+
+        with open(self.user_id_file, "r") as fobj:
+            try:
+                info = json.load(fobj)
+            except ValueError as exc:
+                logger.debug("Failed to load user_id: {}".format(exc))
+                return None
+
+            return info[self.PARAM_USER_ID]
+
+    def _get_user_id(self):
+        from dvc.lock import LockError
+
+        try:
+            with self.user_id_file_lock:
+                user_id = self._read_user_id()
+                if user_id is None:
+                    user_id = self._write_user_id()
+                return user_id
+        except LockError:
+            msg = "Failed to acquire '{}'"
+            logger.debug(msg.format(self.user_id_file_lock.lock_file))
+
+    def _collect_windows(self):
+        import sys
+
+        version = sys.getwindowsversion()  # pylint: disable=no-member
+        info = {}
+        info[self.PARAM_OS] = "windows"
+        info[self.PARAM_WINDOWS_VERSION_MAJOR] = version.major
+        info[self.PARAM_WINDOWS_VERSION_MINOR] = version.minor
+        info[self.PARAM_WINDOWS_VERSION_BUILD] = version.build
+        info[self.PARAM_WINDOWS_VERSION_SERVICE_PACK] = version.service_pack
+        return info
+
+    def _collect_darwin(self):
+        import platform
+
+        info = {}
+        info[self.PARAM_OS] = "mac"
+        info[self.PARAM_MAC_VERSION] = platform.mac_ver()[0]
+        return info
+
+    def _collect_linux(self):
+        import distro
+
+        info = {}
+        info[self.PARAM_OS] = "linux"
+        info[self.PARAM_LINUX_DISTRO] = distro.id()
+        info[self.PARAM_LINUX_DISTRO_VERSION] = distro.version()
+        info[self.PARAM_LINUX_DISTRO_LIKE] = distro.like()
+        return info
+
+    def _collect_system_info(self):
+        import platform
+
+        system = platform.system()
+
+        if system == "Windows":
+            return self._collect_windows()
+
+        if system == "Darwin":
+            return self._collect_darwin()
+
+        if system == "Linux":
+            return self._collect_linux()
+
+        raise NotImplementedError
+
+    def collect(self):
+        """Collect analytics report."""
+        from dvc.scm import SCM
+        from dvc.utils import is_binary
+        from dvc.repo import Repo
+        from dvc.exceptions import NotDvcRepoError
+
+        self.info[self.PARAM_DVC_VERSION] = __version__
+        self.info[self.PARAM_IS_BINARY] = is_binary()
+        self.info[self.PARAM_USER_ID] = self._get_user_id()
+
+        self.info[self.PARAM_SYSTEM_INFO] = self._collect_system_info()
+
+        try:
+            scm = SCM(root_dir=Repo.find_root())
+            self.info[self.PARAM_SCM_CLASS] = type(scm).__name__
+        except NotDvcRepoError:
+            pass
+
+    def collect_cmd(self, args, ret):
+        """Collect analytics info from a CLI command."""
+        from dvc.command.daemon import CmdDaemonAnalytics
+
+        assert isinstance(ret, int) or ret is None
+
+        if ret is not None:
+            self.info[self.PARAM_CMD_RETURN_CODE] = ret
+
+        if args is not None and hasattr(args, "func"):
+            assert args.func != CmdDaemonAnalytics
+            self.info[self.PARAM_CMD_CLASS] = args.func.__name__
+
+    def dump(self):
+        """Save analytics report to a temporary file.
+
+        Returns:
+            str: path to the temporary file that contains the analytics report.
+        """
+        import tempfile
+
+        with tempfile.NamedTemporaryFile(delete=False, mode="w") as fobj:
+            json.dump(self.info, fobj)
+            return fobj.name
+
+    @staticmethod
+    def _is_enabled_config(config):
+        from dvc.config import Config
+
+        core = config.config.get(Config.SECTION_CORE, {})
+        return core.get(Config.SECTION_CORE_ANALYTICS, True)
+
+    @staticmethod
+    def _is_enabled(cmd=None):
+        from dvc.config import Config
+        from dvc.repo import Repo
+        from dvc.exceptions import NotDvcRepoError
+        from dvc.command.daemon import CmdDaemonBase
+
+        if os.getenv("DVC_TEST"):
+            return False
+
+        if isinstance(cmd, CmdDaemonBase):
+            return False
+
+        if cmd is None or not hasattr(cmd, "config"):
+            try:
+                dvc_dir = Repo.find_dvc_dir()
+                config = Config(dvc_dir)
+                assert config is not None
+            except NotDvcRepoError:
+                config = Config(validate=False)
+                assert config is not None
+        else:
+            config = cmd.config
+            assert config is not None
+
+        enabled = Analytics._is_enabled_config(config)
+        logger.debug(
+            "Analytics is {}.".format("enabled" if enabled else "disabled")
+        )
+        return enabled
+
+    @staticmethod
+    def send_cmd(cmd, args, ret):
+        """Collect and send analytics for CLI command.
+
+        Args:
+            args (list): parsed args for the CLI command.
+            ret (int): return value of the CLI command.
+        """
+        from dvc.daemon import daemon
+
+        if not Analytics._is_enabled(cmd):
+            return
+
+        analytics = Analytics()
+        analytics.collect_cmd(args, ret)
+        daemon(["analytics", analytics.dump()])
+
+    def send(self):
+        """Collect and send analytics."""
+        import requests
+
+        if not self._is_enabled():
+            return
+
+        self.collect()
+
+        logger.debug("Sending analytics: {}".format(self.info))
+
+        try:
+            requests.post(self.URL, json=self.info, timeout=self.TIMEOUT_POST)
+        except requests.exceptions.RequestException as exc:
+            logger.debug("Failed to send analytics: {}".format(str(exc)))

dvc/cache.py

@@ -1,39 +1,48 @@
+"""Manages cache of a dvc repo."""
+
+from __future__ import unicode_literals
+
 import os
 
 from dvc.config import Config
 
 
 class Cache(object):
-    CACHE_DIR = 'cache'
+    """Class that manages cache locations of a dvc repo.
 
-    def __init__(self, project):
+    Args:
+        repo (dvc.repo.Repo): repo instance that this cache belongs to.
+    """
+
+    CACHE_DIR = "cache"
+
+    def __init__(self, repo):
         from dvc.remote import Remote
 
-        self.project = project
+        self.repo = repo
 
-        config = project.config._config[Config.SECTION_CACHE]
+        config = repo.config.config[Config.SECTION_CACHE]
         local = config.get(Config.SECTION_CACHE_LOCAL)
 
         if local:
             name = Config.SECTION_REMOTE_FMT.format(local)
-            sect = project.config._config[name]
+            settings = repo.config.config[name]
         else:
-            cache_dir = config.get(Config.SECTION_CACHE_DIR, self.CACHE_DIR)
+            default_cache_dir = os.path.join(repo.dvc_dir, self.CACHE_DIR)
+            cache_dir = config.get(Config.SECTION_CACHE_DIR, default_cache_dir)
             cache_type = config.get(Config.SECTION_CACHE_TYPE)
             protected = config.get(Config.SECTION_CACHE_PROTECTED)
 
-            if not os.path.isabs(cache_dir):
-                cache_dir = os.path.join(project.dvc_dir, cache_dir)
-                cache_dir = os.path.abspath(os.path.realpath(cache_dir))
-
-            sect = {
+            settings = {
+                Config.PRIVATE_CWD: config.get(
+                    Config.PRIVATE_CWD, repo.dvc_dir
+                ),
                 Config.SECTION_REMOTE_URL: cache_dir,
                 Config.SECTION_CACHE_TYPE: cache_type,
-                Config.SECTION_CACHE_PROTECTED: protected
+                Config.SECTION_CACHE_PROTECTED: protected,
             }
 
-        self.local = Remote(project, sect)
-
+        self.local = Remote(repo, settings)
         self.s3 = self._get_remote(config, Config.SECTION_CACHE_S3)
         self.gs = self._get_remote(config, Config.SECTION_CACHE_GS)
         self.ssh = self._get_remote(config, Config.SECTION_CACHE_SSH)
@@ -41,12 +50,42 @@ class Cache(object):
         self.azure = self._get_remote(config, Config.SECTION_CACHE_AZURE)
 
     def _get_remote(self, config, name):
+        """
+        The config file is stored in a way that allows you to have a
+        cache for each remote.
+
+        This is needed when specifying external outputs
+        (as they require you to have an external cache location).
+
+        Imagine a config file like the following:
+
+                ['remote "dvc-storage"']
+                url = ssh://localhost/tmp
+                ask_password = true
+
+                [cache]
+                ssh = dvc-storage
+
+        This method resolves the name under the cache section into the
+        correct Remote instance.
+
+        Args:
+            config (dict): The cache section on the config file
+            name (str): Name of the section we are interested in to retrieve
+
+        Returns:
+            remote (dvc.Remote): Remote instance that the section is referring.
+                None when there's no remote with that name.
+
+        Example:
+            >>> _get_remote(config={'ssh': 'dvc-storage'}, name='ssh')
+        """
         from dvc.remote import Remote
 
-        remote = config.get(name, None)
+        remote = config.get(name)
+
         if not remote:
             return None
 
-        name = Config.SECTION_REMOTE_FMT.format(remote)
-        sect = self.project.config._config[name]
-        return Remote(self.project, sect)
+        settings = self.repo.config.get_remote_settings(remote)
+        return Remote(self.repo, settings)

dvc/command/add.py

@@ -1,14 +1,62 @@
+from __future__ import unicode_literals
+
+import argparse
+import logging
+
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase
-from dvc.logger import Logger
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdAdd(CmdBase):
     def run(self):
-        for target in self.args.targets:
-            try:
-                self.project.add(target, recursive=self.args.recursive)
-            except DvcException as ex:
-                Logger.error('Failed to add \'{}\''.format(target), ex)
-                return 1
+        try:
+            if len(self.args.targets) > 1 and self.args.file:
+                raise DvcException("can't use '--file' with multiple targets")
+
+            for target in self.args.targets:
+                self.repo.add(
+                    target,
+                    recursive=self.args.recursive,
+                    no_commit=self.args.no_commit,
+                    fname=self.args.file,
+                )
+
+        except DvcException:
+            logger.exception("failed to add file")
+            return 1
         return 0
+
+
+def add_parser(subparsers, parent_parser):
+    ADD_HELP = "Take data files or directories under DVC control."
+
+    add_parser = subparsers.add_parser(
+        "add",
+        parents=[parent_parser],
+        description=append_doc_link(ADD_HELP, "add"),
+        help=ADD_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    add_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="Recursively add each file under the directory.",
+    )
+    add_parser.add_argument(
+        "--no-commit",
+        action="store_true",
+        default=False,
+        help="Don't put files/directories into cache.",
+    )
+    add_parser.add_argument(
+        "-f", "--file", help="Specify name of the DVC file it generates."
+    )
+    add_parser.add_argument(
+        "targets", nargs="+", help="Input files/directories."
+    )
+    add_parser.set_defaults(func=CmdAdd)

dvc/command/base.py

@@ -1,48 +1,75 @@
-import os
+from __future__ import unicode_literals
 
-from dvc.logger import Logger
-from dvc.exceptions import DvcException
-from dvc.lock import LockError
+import colorama
+import logging
+
+
+logger = logging.getLogger(__name__)
+
+
+def fix_subparsers(subparsers):
+    """Workaround for bug in Python 3. See more info at:
+        https://bugs.python.org/issue16308
+        https://github.com/iterative/dvc/issues/769
+
+        Args:
+            subparsers: subparsers to fix.
+    """
+    from dvc.utils.compat import is_py3
+
+    if is_py3:  # pragma: no cover
+        subparsers.required = True
+        subparsers.dest = "cmd"
+
+
+def append_doc_link(help_message, path):
+    if not path:
+        return help_message
+    doc_base = "https://man.dvc.org/"
+    return "{message}\ndocumentation: {blue}{base}{path}{nc}".format(
+        message=help_message,
+        base=doc_base,
+        path=path,
+        blue=colorama.Fore.CYAN,
+        nc=colorama.Fore.RESET,
+    )
 
 
 class CmdBase(object):
     def __init__(self, args):
-        from dvc.project import Project
+        from dvc.repo import Repo
 
-        self.project = Project(self._find_root())
+        self.repo = Repo()
+        self.config = self.repo.config
         self.args = args
-        self._set_loglevel(args)
 
-    @staticmethod
-    def _set_loglevel(args):
-        if args.quiet:
-            Logger.be_quiet()
+    @property
+    def default_targets(self):
+        """Default targets for `dvc repro` and `dvc pipeline`."""
+        from dvc.stage import Stage
 
-        elif args.verbose:
-            Logger.be_verbose()
-
-    def _find_root(self):
-        from dvc.project import Project
-
-        root = os.getcwd()
-        while True:
-            dvc_dir = os.path.join(root, Project.DVC_DIR)
-            if os.path.isdir(dvc_dir):
-                return root
-            if os.path.ismount(root):
-                break
-            root = os.path.dirname(root)
-        msg = "Not a dvc repository (checked up to mount point {})"
-        raise DvcException(msg.format(root))
+        msg = "assuming default target '{}'.".format(Stage.STAGE_FILE)
+        logger.warning(msg)
+        return [Stage.STAGE_FILE]
 
     def run_cmd(self):
+        from dvc.lock import LockError
+
         try:
-            with self.project.lock:
+            with self.repo.lock:
                 return self.run()
-        except LockError as ex:
-            Logger.error('Failed to lock before running a command', ex)
+        except LockError:
+            logger.exception("failed to lock before running a command")
             return 1
 
     # Abstract methods that have to be implemented by any inheritance class
     def run(self):
         pass
+
+
+class CmdBaseNoRepo(CmdBase):
+    def __init__(self, args):
+        self.args = args
+
+    def run_cmd(self):
+        return self.run()

dvc/command/cache.py

@@ -0,0 +1,65 @@
+from __future__ import unicode_literals
+
+import argparse
+
+from dvc.command.base import fix_subparsers, append_doc_link
+from dvc.command.remote import CmdRemoteAdd
+from dvc.command.config import CmdConfig
+
+
+class CmdCacheDir(CmdConfig):
+    def run(self):
+        self.args.name = "cache.dir"
+        self.args.value = CmdRemoteAdd.resolve_path(
+            self.args.value, self.configobj.filename
+        )
+
+        return super(CmdCacheDir, self).run()
+
+
+def add_parser(subparsers, parent_parser):
+    from dvc.command.config import parent_config_parser
+
+    CACHE_HELP = "Manage cache settings."
+
+    cache_parser = subparsers.add_parser(
+        "cache",
+        parents=[parent_parser],
+        description=append_doc_link(CACHE_HELP, "cache"),
+        help=CACHE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    cache_subparsers = cache_parser.add_subparsers(
+        dest="cmd", help="Use dvc cache CMD --help for command-specific help."
+    )
+
+    fix_subparsers(cache_subparsers)
+
+    parent_cache_config_parser = argparse.ArgumentParser(
+        add_help=False, parents=[parent_config_parser]
+    )
+    CACHE_DIR_HELP = "Configure cache directory location."
+
+    cache_dir_parser = cache_subparsers.add_parser(
+        "dir",
+        parents=[parent_cache_config_parser],
+        description=append_doc_link(CACHE_HELP, "cache-dir"),
+        help=CACHE_DIR_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    cache_dir_parser.add_argument(
+        "-u",
+        "--unset",
+        default=False,
+        action="store_true",
+        help="Unset option.",
+    )
+    cache_dir_parser.add_argument(
+        "value",
+        default=None,
+        help="Path to cache directory. Relative paths are resolved relative "
+        "to the current directory and saved to config relative to the "
+        "config file location.",
+    )
+    cache_dir_parser.set_defaults(func=CmdCacheDir)

dvc/command/checkout.py

@@ -1,12 +1,55 @@
-from dvc.command.base import CmdBase
+from __future__ import unicode_literals
+
+import argparse
+
+from dvc.command.base import CmdBase, append_doc_link
 
 
 class CmdCheckout(CmdBase):
     def run(self):
         if not self.args.targets:
-            self.project.checkout()
+            self.repo.checkout(force=self.args.force)
         else:
             for target in self.args.targets:
-                self.project.checkout(target=target,
-                                      with_deps=self.args.with_deps)
+                self.repo.checkout(
+                    target=target,
+                    with_deps=self.args.with_deps,
+                    force=self.args.force,
+                    recursive=self.args.recursive,
+                )
         return 0
+
+
+def add_parser(subparsers, parent_parser):
+    CHECKOUT_HELP = "Checkout data files from cache."
+
+    checkout_parser = subparsers.add_parser(
+        "checkout",
+        parents=[parent_parser],
+        description=append_doc_link(CHECKOUT_HELP, "checkout"),
+        help=CHECKOUT_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    checkout_parser.add_argument(
+        "-d",
+        "--with-deps",
+        action="store_true",
+        default=False,
+        help="Checkout all dependencies of the specified target.",
+    )
+    checkout_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Do not prompt when removing working directory files.",
+    )
+    checkout_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="Checkout all subdirectories of the specified directory.",
+    )
+    checkout_parser.add_argument("targets", nargs="*", help="DVC files.")
+    checkout_parser.set_defaults(func=CmdCheckout)

dvc/command/commit.py

@@ -0,0 +1,70 @@
+from __future__ import unicode_literals
+
+import argparse
+import logging
+
+from dvc.exceptions import DvcException
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
+
+
+class CmdCommit(CmdBase):
+    def run(self):
+        if not self.args.targets:
+            self.args.targets = [None]
+
+        for target in self.args.targets:
+            try:
+                self.repo.commit(
+                    target,
+                    with_deps=self.args.with_deps,
+                    recursive=self.args.recursive,
+                    force=self.args.force,
+                )
+            except DvcException:
+                logger.exception(
+                    "failed to commit{}".format(
+                        (" " + target) if target else ""
+                    )
+                )
+                return 1
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    COMMIT_HELP = "Save changed data to cache and update DVC files."
+
+    commit_parser = subparsers.add_parser(
+        "commit",
+        parents=[parent_parser],
+        description=append_doc_link(COMMIT_HELP, "commit"),
+        help=COMMIT_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    commit_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Commit even if checksums for dependencies/outputs changed.",
+    )
+    commit_parser.add_argument(
+        "-d",
+        "--with-deps",
+        action="store_true",
+        default=False,
+        help="Commit all dependencies of the specified target.",
+    )
+    commit_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="Commit cache for subdirectories of the specified directory.",
+    )
+    commit_parser.add_argument(
+        "targets", nargs="*", default=None, help="DVC files."
+    )
+    commit_parser.set_defaults(func=CmdCommit)

dvc/command/config.py

@@ -1,102 +1,121 @@
+from __future__ import unicode_literals
+
 import os
-import configobj
+import argparse
+import logging
 
-from dvc.command.base import CmdBase
-from dvc.logger import Logger
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.config import Config
+from dvc.exceptions import DvcException
 
 
-class CmdConfig(CmdBase):
+logger = logging.getLogger(__name__)
+
+
+class CmdConfig(CmdBaseNoRepo):
     def __init__(self, args):
-        from dvc.project import Project
+        from dvc.repo import Repo, NotDvcRepoError
 
-        self.args = args
-        root_dir = self._find_root()
-        if args.local:
-            config = Config.CONFIG_LOCAL
-        else:
-            config = Config.CONFIG
-        self.config_file = os.path.join(root_dir, Project.DVC_DIR, config)
-        # Using configobj because it doesn't
-        # drop comments like configparser does.
-        self.configobj = configobj.ConfigObj(self.config_file)
+        super(CmdConfig, self).__init__(args)
 
-    def run_cmd(self):
-        return self.run()
-
-    def _get_key(self, d, name, add=False):
-        for k in d.keys():
-            if k.lower() == name.lower():
-                return k
-
-        if add:
-            d[name] = {}
-            return name
-
-        return None
-
-    def save(self):
         try:
-            self.configobj.write()
-        except Exception as exc:
-            msg = "Failed to write config '{}'".format(self.configobj.filename)
-            Logger.error(msg, exc)
+            dvc_dir = os.path.join(Repo.find_root(), Repo.DVC_DIR)
+            saved_exc = None
+        except NotDvcRepoError as exc:
+            dvc_dir = None
+            saved_exc = exc
+
+        self.config = Config(dvc_dir, validate=False)
+        if self.args.system:
+            self.configobj = self.config._system_config
+        elif self.args.glob:
+            self.configobj = self.config._global_config
+        elif self.args.local:
+            if dvc_dir is None:
+                raise saved_exc
+            self.configobj = self.config._local_config
+        else:
+            if dvc_dir is None:
+                raise saved_exc
+            self.configobj = self.config._repo_config
+
+    def _unset(self, section, opt=None, configobj=None):
+        if configobj is None:
+            configobj = self.configobj
+
+        try:
+            self.config.unset(configobj, section, opt)
+            self.config.save(configobj)
+        except DvcException:
+            logger.exception("failed to unset '{}'".format(self.args.name))
             return 1
         return 0
 
-    def unset(self, section, opt=None):
-        if section not in self.configobj.keys():
-            Logger.error("Section '{}' doesn't exist".format(section))
+    def _show(self, section, opt):
+        try:
+            self.config.show(self.configobj, section, opt)
+        except DvcException:
+            logger.exception("failed to show '{}'".format(self.args.name))
             return 1
-
-        if opt in self.configobj[section].keys():
-            del self.configobj[section][opt]
-
-        if len(self.configobj[section]) == 0 or opt is None:
-            del self.configobj[section]
-
-        return self.save()
-
-    def show(self):
-        Logger.info(self.configobj[self.section][self.opt])
         return 0
 
-    def set(self, section, opt, value):
-        if section not in self.configobj.keys():
-            self.configobj[section] = {}
-
-        self.configobj[section][opt] = value
-
-        return self.save()
-
-    def check_opt(self):
-        _section, _opt = self.args.name.strip().split('.', 1)
-        add = (self.args.value is not None and self.args.unset is False)
-
-        section = self._get_key(self.configobj, _section, add)
-
-        if not section:
-            Logger.error('Invalid option name {}'.format(_section))
+    def _set(self, section, opt, value):
+        try:
+            self.config.set(self.configobj, section, opt, value)
+            self.config.save(self.configobj)
+        except DvcException:
+            logger.exception(
+                "failed to set '{}.{}' to '{}'".format(section, opt, value)
+            )
             return 1
-
-        opt = self._get_key(self.configobj[section], _opt, add)
-        if not opt:
-            Logger.error('Invalid option value: {}'.format(_opt))
-            return 1
-
-        self.section = section
-        self.opt = opt
-
         return 0
 
     def run(self):
-        if self.check_opt() != 0:
-            return 1
+        section, opt = self.args.name.lower().strip().split(".", 1)
 
         if self.args.unset:
-            return self.unset(self.section, self.opt)
+            return self._unset(section, opt)
+        elif self.args.value is None:
+            return self._show(section, opt)
+        else:
+            return self._set(section, opt, self.args.value)
 
-        if self.args.value is None:
-            return self.show()
 
-        return self.set(self.section, self.opt, self.args.value)
+parent_config_parser = argparse.ArgumentParser(add_help=False)
+parent_config_parser.add_argument(
+    "--global",
+    dest="glob",
+    action="store_true",
+    default=False,
+    help="Use global config.",
+)
+parent_config_parser.add_argument(
+    "--system", action="store_true", default=False, help="Use system config."
+)
+parent_config_parser.add_argument(
+    "--local", action="store_true", default=False, help="Use local config."
+)
+
+
+def add_parser(subparsers, parent_parser):
+    CONFIG_HELP = "Get or set config options."
+
+    config_parser = subparsers.add_parser(
+        "config",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(CONFIG_HELP, "config"),
+        help=CONFIG_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    config_parser.add_argument(
+        "-u",
+        "--unset",
+        default=False,
+        action="store_true",
+        help="Unset option.",
+    )
+    config_parser.add_argument("name", help="Option name.")
+    config_parser.add_argument(
+        "value", nargs="?", default=None, help="Option value."
+    )
+    config_parser.set_defaults(func=CmdConfig)

dvc/command/daemon.py

@@ -0,0 +1,66 @@
+from __future__ import unicode_literals
+
+from dvc.command.base import CmdBaseNoRepo, fix_subparsers
+
+
+class CmdDaemonBase(CmdBaseNoRepo):
+    pass
+
+
+class CmdDaemonUpdater(CmdDaemonBase):
+    def run(self):
+        import os
+        from dvc.repo import Repo
+        from dvc.updater import Updater
+
+        root_dir = Repo.find_root()
+        dvc_dir = os.path.join(root_dir, Repo.DVC_DIR)
+        updater = Updater(dvc_dir)
+        updater.fetch(detach=False)
+
+        return 0
+
+
+class CmdDaemonAnalytics(CmdDaemonBase):
+    def run(self):
+        from dvc.analytics import Analytics
+
+        analytics = Analytics.load(self.args.target)
+        analytics.send()
+
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    DAEMON_HELP = "Service daemon."
+    daemon_parser = subparsers.add_parser(
+        "daemon",
+        parents=[parent_parser],
+        description=DAEMON_HELP,
+        add_help=False,
+    )
+
+    daemon_subparsers = daemon_parser.add_subparsers(
+        dest="cmd", help="Use dvc daemon CMD --help for command-specific help."
+    )
+
+    fix_subparsers(daemon_subparsers)
+
+    DAEMON_UPDATER_HELP = "Fetch latest available version."
+    daemon_updater_parser = daemon_subparsers.add_parser(
+        "updater",
+        parents=[parent_parser],
+        description=DAEMON_UPDATER_HELP,
+        help=DAEMON_UPDATER_HELP,
+    )
+    daemon_updater_parser.set_defaults(func=CmdDaemonUpdater)
+
+    DAEMON_ANALYTICS_HELP = "Send dvc usage analytics."
+    daemon_analytics_parser = daemon_subparsers.add_parser(
+        "analytics",
+        parents=[parent_parser],
+        description=DAEMON_ANALYTICS_HELP,
+        help=DAEMON_ANALYTICS_HELP,
+    )
+    daemon_analytics_parser.add_argument("target", help="Analytics file.")
+    daemon_analytics_parser.set_defaults(func=CmdDaemonAnalytics)

dvc/command/data_sync.py

@@ -1,7 +1,17 @@
-from dvc.command.base import CmdBase
+from __future__ import unicode_literals
+
+import argparse
+import logging
+
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdDataBase(CmdBase):
+    UP_TO_DATE_MSG = "Everything is up to date."
+
     def do_run(self, target):
         pass
 
@@ -15,52 +25,293 @@ class CmdDataBase(CmdBase):
                 ret = 1
         return ret
 
+    @classmethod
+    def check_up_to_date(cls, processed_files_count):
+        if processed_files_count == 0:
+            logger.info(cls.UP_TO_DATE_MSG)
+
 
 class CmdDataPull(CmdDataBase):
     def do_run(self, target=None):
         try:
-            self.project.pull(target=target,
-                              jobs=self.args.jobs,
-                              remote=self.args.remote,
-                              show_checksums=self.args.show_checksums,
-                              all_branches=self.args.all_branches,
-                              all_tags=self.args.all_tags,
-                              with_deps=self.args.with_deps)
-        except Exception as exc:
-            self.project.logger.error('Failed to pull data from the cloud',
-                                      exc)
+            processed_files_count = self.repo.pull(
+                target=target,
+                jobs=self.args.jobs,
+                remote=self.args.remote,
+                show_checksums=self.args.show_checksums,
+                all_branches=self.args.all_branches,
+                all_tags=self.args.all_tags,
+                with_deps=self.args.with_deps,
+                force=self.args.force,
+                recursive=self.args.recursive,
+            )
+        except Exception:
+            logger.exception("failed to pull data from the cloud")
             return 1
+        self.check_up_to_date(processed_files_count)
         return 0
 
 
 class CmdDataPush(CmdDataBase):
     def do_run(self, target=None):
         try:
-            self.project.push(target=target,
-                              jobs=self.args.jobs,
-                              remote=self.args.remote,
-                              show_checksums=self.args.show_checksums,
-                              all_branches=self.args.all_branches,
-                              all_tags=self.args.all_tags,
-                              with_deps=self.args.with_deps)
-        except Exception as exc:
-            self.project.logger.error('Failed to push data to the cloud', exc)
+            processed_files_count = self.repo.push(
+                target=target,
+                jobs=self.args.jobs,
+                remote=self.args.remote,
+                show_checksums=self.args.show_checksums,
+                all_branches=self.args.all_branches,
+                all_tags=self.args.all_tags,
+                with_deps=self.args.with_deps,
+                recursive=self.args.recursive,
+            )
+        except Exception:
+            logger.exception("failed to push data to the cloud")
             return 1
+        self.check_up_to_date(processed_files_count)
         return 0
 
 
 class CmdDataFetch(CmdDataBase):
     def do_run(self, target=None):
         try:
-            self.project.fetch(target=target,
-                               jobs=self.args.jobs,
-                               remote=self.args.remote,
-                               show_checksums=self.args.show_checksums,
-                               all_branches=self.args.all_branches,
-                               all_tags=self.args.all_tags,
-                               with_deps=self.args.with_deps)
-        except Exception as exc:
-            self.project.logger.error('Failed to fetch data from the cloud',
-                                      exc)
+            processed_files_count = self.repo.fetch(
+                target=target,
+                jobs=self.args.jobs,
+                remote=self.args.remote,
+                show_checksums=self.args.show_checksums,
+                all_branches=self.args.all_branches,
+                all_tags=self.args.all_tags,
+                with_deps=self.args.with_deps,
+                recursive=self.args.recursive,
+            )
+        except Exception:
+            logger.exception("failed to fetch data from the cloud")
             return 1
+        self.check_up_to_date(processed_files_count)
         return 0
+
+
+def shared_parent_parser():
+    from dvc.cli import get_parent_parser
+
+    # Parent parser used in pull/push/status
+    shared_parent_parser = argparse.ArgumentParser(
+        add_help=False, parents=[get_parent_parser()]
+    )
+    shared_parent_parser.add_argument(
+        "-j",
+        "--jobs",
+        type=int,
+        default=None,
+        help="Number of jobs to run simultaneously.",
+    )
+    shared_parent_parser.add_argument(
+        "--show-checksums",
+        action="store_true",
+        default=False,
+        help="Show checksums instead of file names.",
+    )
+    shared_parent_parser.add_argument(
+        "targets", nargs="*", default=None, help="DVC files."
+    )
+
+    return shared_parent_parser
+
+
+def add_parser(subparsers, _parent_parser):
+    from dvc.command.status import CmdDataStatus
+
+    # Pull
+    PULL_HELP = "Pull data files from a DVC remote storage."
+
+    pull_parser = subparsers.add_parser(
+        "pull",
+        parents=[shared_parent_parser()],
+        description=append_doc_link(PULL_HELP, "pull"),
+        help=PULL_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    pull_parser.add_argument(
+        "-r", "--remote", help="Remote repository to pull from."
+    )
+    pull_parser.add_argument(
+        "-a",
+        "--all-branches",
+        action="store_true",
+        default=False,
+        help="Fetch cache for all branches.",
+    )
+    pull_parser.add_argument(
+        "-T",
+        "--all-tags",
+        action="store_true",
+        default=False,
+        help="Fetch cache for all tags.",
+    )
+    pull_parser.add_argument(
+        "-d",
+        "--with-deps",
+        action="store_true",
+        default=False,
+        help="Fetch cache for all dependencies of the specified target.",
+    )
+    pull_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Do not prompt when removing working directory files.",
+    )
+    pull_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="Pull cache for subdirectories of the specified directory.",
+    )
+    pull_parser.set_defaults(func=CmdDataPull)
+
+    # Push
+    PUSH_HELP = "Push data files to a DVC remote storage."
+
+    push_parser = subparsers.add_parser(
+        "push",
+        parents=[shared_parent_parser()],
+        description=append_doc_link(PUSH_HELP, "push"),
+        help=PUSH_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    push_parser.add_argument(
+        "-r", "--remote", help="Remote repository to push to."
+    )
+    push_parser.add_argument(
+        "-a",
+        "--all-branches",
+        action="store_true",
+        default=False,
+        help="Push cache for all branches.",
+    )
+    push_parser.add_argument(
+        "-T",
+        "--all-tags",
+        action="store_true",
+        default=False,
+        help="Push cache for all tags.",
+    )
+    push_parser.add_argument(
+        "-d",
+        "--with-deps",
+        action="store_true",
+        default=False,
+        help="Push cache for all dependencies of the specified target.",
+    )
+    push_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="Push cache from subdirectories of specified directory.",
+    )
+    push_parser.set_defaults(func=CmdDataPush)
+
+    # Fetch
+    FETCH_HELP = "Fetch data files from a DVC remote storage."
+
+    fetch_parser = subparsers.add_parser(
+        "fetch",
+        parents=[shared_parent_parser()],
+        description=append_doc_link(FETCH_HELP, "fetch"),
+        help=FETCH_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    fetch_parser.add_argument(
+        "-r", "--remote", help="Remote repository to fetch from."
+    )
+    fetch_parser.add_argument(
+        "-a",
+        "--all-branches",
+        action="store_true",
+        default=False,
+        help="Fetch cache for all branches.",
+    )
+    fetch_parser.add_argument(
+        "-T",
+        "--all-tags",
+        action="store_true",
+        default=False,
+        help="Fetch cache for all tags.",
+    )
+    fetch_parser.add_argument(
+        "-d",
+        "--with-deps",
+        action="store_true",
+        default=False,
+        help="Fetch cache for all dependencies of the " "specified target.",
+    )
+    fetch_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="Fetch cache for subdirectories of specified directory.",
+    )
+    fetch_parser.set_defaults(func=CmdDataFetch)
+
+    # Status
+    STATUS_HELP = (
+        "Show changed stages, compare local cache and a remote storage."
+    )
+
+    status_parser = subparsers.add_parser(
+        "status",
+        parents=[shared_parent_parser()],
+        description=append_doc_link(STATUS_HELP, "status"),
+        help=STATUS_HELP,
+        conflict_handler="resolve",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    status_parser.add_argument(
+        "-q",
+        "--quiet",
+        action="store_true",
+        default=False,
+        help=(
+            "Suppresses all output."
+            " Exit with 0 if pipeline is up to date, otherwise 1."
+        ),
+    )
+    status_parser.add_argument(
+        "-c",
+        "--cloud",
+        action="store_true",
+        default=False,
+        help="Show status of a local cache compared to a remote repository.",
+    )
+    status_parser.add_argument(
+        "-r", "--remote", help="Remote repository to compare local cache to."
+    )
+    status_parser.add_argument(
+        "-a",
+        "--all-branches",
+        action="store_true",
+        default=False,
+        help="Show status of a local cache compared to a remote repository "
+        "for all branches.",
+    )
+    status_parser.add_argument(
+        "-T",
+        "--all-tags",
+        action="store_true",
+        default=False,
+        help="Show status of a local cache compared to a remote repository "
+        "for all tags.",
+    )
+    status_parser.add_argument(
+        "-d",
+        "--with-deps",
+        action="store_true",
+        default=False,
+        help="Show status for all dependencies of the specified target.",
+    )
+    status_parser.set_defaults(func=CmdDataStatus)

dvc/command/destroy.py

@@ -1,22 +1,54 @@
-from dvc.command.base import CmdBase
+from __future__ import unicode_literals
+
+import argparse
+import logging
+
+import dvc.prompt as prompt
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
 
+logger = logging.getLogger(__name__)
+
+
 class CmdDestroy(CmdBase):
     def run_cmd(self):
         try:
-            msg = u'This will destroy all information about your pipelines, ' \
-                  u'all data files, as well as cache in .dvc/cache.\n' \
-                  u'Are you sure you want to continue?'
+            statement = (
+                "This will destroy all information about your pipelines,"
+                " all data files, as well as cache in .dvc/cache."
+                "\n"
+                "Are you sure you want to continue?"
+            )
 
-            if not self.args.force \
-               and not self.project.prompt.prompt(msg, False):
-                msg = u'Cannot destroy without a confirmation from the ' \
-                      u'user. Use \'-f\' to force.'
-                raise DvcException(msg)
+            if not self.args.force and not prompt.confirm(statement):
+                raise DvcException(
+                    "cannot destroy without a confirmation from the user."
+                    " Use '-f' to force."
+                )
 
-            self.project.destroy()
-        except Exception as exc:
-            self.project.logger.error('Failed to destroy DVC', exc)
+            self.repo.destroy()
+        except Exception:
+            logger.exception("failed to destroy DVC")
             return 1
         return 0
+
+
+def add_parser(subparsers, parent_parser):
+    DESTROY_HELP = "Remove DVC files, local DVC config and data cache."
+
+    destroy_parser = subparsers.add_parser(
+        "destroy",
+        parents=[parent_parser],
+        description=append_doc_link(DESTROY_HELP, "destroy"),
+        help=DESTROY_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    destroy_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Force destruction.",
+    )
+    destroy_parser.set_defaults(func=CmdDestroy)

dvc/command/diff.py

@@ -0,0 +1,160 @@
+from __future__ import unicode_literals
+
+import argparse
+
+import humanize
+import inflect
+import logging
+
+from dvc.exceptions import DvcException
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.utils.collections import compact
+import dvc.repo.diff as diff
+
+
+logger = logging.getLogger(__name__)
+
+
+class CmdDiff(CmdBase):
+    def _print_size(self, size):
+        if size < 0:
+            change = "decreased by {}"
+        elif size > 0:
+            change = "increased by {}"
+        else:
+            change = "not changed"
+        natur_size = humanize.naturalsize(abs(size))
+        return change.format(natur_size)
+
+    def _get_md5_string(self, sign, file_name, checksum):
+        sample_msg = ""
+        if file_name:
+            sample_msg = "{}{} with md5 {}\n"
+            sample_msg = sample_msg.format(sign, file_name, checksum)
+        return sample_msg
+
+    def _get_dir_changes(self, dct):
+        engine = inflect.engine()
+        changes_msg = (
+            "{} {} not changed, {} {} modified, {} {} added, "
+            "{} {} deleted, size was {}"
+        )
+        changes_msg = changes_msg.format(
+            dct[diff.DIFF_IDENT],
+            engine.plural("file", dct[diff.DIFF_IDENT]),
+            dct[diff.DIFF_CHANGE],
+            engine.plural("file", dct[diff.DIFF_CHANGE]),
+            dct[diff.DIFF_NEW],
+            engine.plural("file", dct[diff.DIFF_NEW]),
+            dct[diff.DIFF_DEL],
+            engine.plural("file", dct[diff.DIFF_DEL]),
+            self._print_size(dct[diff.DIFF_SIZE]),
+        )
+        return changes_msg
+
+    def _get_file_changes(self, dct):
+        if (
+            dct.get(diff.DIFF_OLD_FILE)
+            and dct.get(diff.DIFF_NEW_FILE)
+            and dct[diff.DIFF_SIZE] == 0
+        ):
+            msg = "file size was not changed"
+        elif dct.get(diff.DIFF_NEW_FILE):
+            msg = "added file with size {}".format(
+                humanize.naturalsize(dct[diff.DIFF_SIZE])
+            )
+        elif dct.get(diff.DIFF_OLD_FILE):
+            msg = "deleted file with size {}".format(
+                humanize.naturalsize(abs(dct[diff.DIFF_SIZE]))
+            )
+        else:
+            msg = "file was modified, file size {}".format(
+                self._print_size(dct[diff.DIFF_SIZE])
+            )
+        return msg
+
+    def _get_royal_changes(self, dct):
+        if dct[diff.DIFF_SIZE] != diff.DIFF_SIZE_UNKNOWN:
+            if dct.get("is_dir"):
+                return self._get_dir_changes(dct)
+            else:
+                return self._get_file_changes(dct)
+        return "size is ?"
+
+    def _show(self, diff_dct):
+        msg = "dvc diff from {} to {}".format(
+            diff_dct[diff.DIFF_A_REF], diff_dct[diff.DIFF_B_REF]
+        )
+        if diff_dct.get(diff.DIFF_EQUAL):
+            logger.info(msg)
+            return
+        for dct in diff_dct[diff.DIFF_LIST]:
+            msg += "\n\ndiff for '{}'\n".format(dct[diff.DIFF_TARGET])
+            msg += self._get_md5_string(
+                "-",
+                dct.get(diff.DIFF_OLD_FILE),
+                dct.get(diff.DIFF_OLD_CHECKSUM),
+            )
+            msg += self._get_md5_string(
+                "+",
+                dct.get(diff.DIFF_NEW_FILE),
+                dct.get(diff.DIFF_NEW_CHECKSUM),
+            )
+            msg += "\n"
+            msg += self._get_royal_changes(dct)
+        logger.info(msg)
+        return msg
+
+    def run(self):
+        try:
+            msg = self.repo.diff(
+                self.args.a_ref, target=self.args.target, b_ref=self.args.b_ref
+            )
+            self._show(msg)
+        except DvcException:
+            msg = "failed to get 'diff {}'"
+            args = " ".join(
+                compact([self.args.target, self.args.a_ref, self.args.b_ref])
+            )
+            msg = msg.format(args)
+            logger.exception(msg)
+            return 1
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    DIFF_DESCRIPTION = (
+        "Show diff of a data file or a directory that is under DVC control.\n"
+        "Some basic statistics summary, how many files were deleted/changed."
+    )
+    DIFF_HELP = "Show a diff of a DVC controlled data file or a directory."
+    diff_parser = subparsers.add_parser(
+        "diff",
+        parents=[parent_parser],
+        description=append_doc_link(DIFF_DESCRIPTION, "diff"),
+        help=DIFF_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    diff_parser.add_argument(
+        "-t",
+        "--target",
+        default=None,
+        help=(
+            "Source path to a data file or directory. Default None. "
+            "If not specified, compares all files and directories "
+            "that are under DVC control in the current working space."
+        ),
+    )
+    diff_parser.add_argument(
+        "a_ref", help="Git reference from which diff calculates"
+    )
+    diff_parser.add_argument(
+        "b_ref",
+        help=(
+            "Git reference untill which diff calculates, if omitted "
+            "diff shows the difference between current HEAD and a_ref"
+        ),
+        nargs="?",
+        default=None,
+    )
+    diff_parser.set_defaults(func=CmdDiff)

dvc/command/gc.py

@@ -1,39 +1,114 @@
-import os
+from __future__ import unicode_literals
 
-from dvc.command.base import CmdBase
+import argparse
+import os
+import dvc.prompt as prompt
+import logging
+
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdGC(CmdBase):
     def run(self):
-        msg = 'This will remove all cache except the cache that is used in '
+        msg = "this will remove all cache except the cache that is used in "
         if not self.args.all_branches and not self.args.all_tags:
-            msg += 'the current git branch'
+            msg += "the current git branch"
         elif self.args.all_branches and not self.args.all_tags:
-            msg += 'all git branches'
+            msg += "all git branches"
         elif not self.args.all_branches and self.args.all_tags:
-            msg += 'all git tags'
+            msg += "all git tags"
         else:
-            msg += 'all git branches and all git tags'
+            msg += "all git branches and all git tags"
 
-        if self.args.projects is not None and len(self.args.projects) > 0:
-            msg += ' of the current and the following projects:'
+        if self.args.repos is not None and len(self.args.repos) > 0:
+            msg += " of the current and the following repos:"
 
-            for project_path in self.args.projects:
-                msg += '\n  - %s' % os.path.abspath(project_path)
+            for repo_path in self.args.repos:
+                msg += "\n  - %s" % os.path.abspath(repo_path)
         else:
-            msg += ' of the current project.'
+            msg += " of the current repo."
 
-        self.project.logger.warn(msg)
+        logger.warning(msg)
 
-        msg = 'Are you sure you want to proceed?'
-        if not self.args.force and not self.project.prompt.prompt(msg):
+        msg = "Are you sure you want to proceed?"
+        if not self.args.force and not prompt.confirm(msg):
             return 1
 
-        self.project.gc(all_branches=self.args.all_branches,
-                        all_tags=self.args.all_tags,
-                        cloud=self.args.cloud,
-                        remote=self.args.remote,
-                        force=self.args.force,
-                        jobs=self.args.jobs,
-                        projects=self.args.projects)
+        self.repo.gc(
+            all_branches=self.args.all_branches,
+            all_tags=self.args.all_tags,
+            cloud=self.args.cloud,
+            remote=self.args.remote,
+            force=self.args.force,
+            jobs=self.args.jobs,
+            repos=self.args.repos,
+        )
         return 0
+
+
+def add_parser(subparsers, parent_parser):
+    GC_HELP = "Collect unused data from DVC cache or a remote storage."
+    GC_DESCRIPTION = (
+        "Deletes all files in the cache or a remote which are not in\n"
+        "use by the specified git references (defaults to just HEAD)."
+    )
+    gc_parser = subparsers.add_parser(
+        "gc",
+        parents=[parent_parser],
+        description=append_doc_link(GC_DESCRIPTION, "gc"),
+        help=GC_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    gc_parser.add_argument(
+        "-a",
+        "--all-branches",
+        action="store_true",
+        default=False,
+        help="Keep data files for the tips of all git branches.",
+    )
+    gc_parser.add_argument(
+        "-T",
+        "--all-tags",
+        action="store_true",
+        default=False,
+        help="Keep data files for all git tags.",
+    )
+    gc_parser.add_argument(
+        "-c",
+        "--cloud",
+        action="store_true",
+        default=False,
+        help="Collect garbage in remote repository.",
+    )
+    gc_parser.add_argument(
+        "-r", "--remote", help="Remote storage to collect garbage in."
+    )
+    gc_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Force garbage collection - automatically agree to all prompts.",
+    )
+    gc_parser.add_argument(
+        "-j",
+        "--jobs",
+        type=int,
+        default=None,
+        help="Number of jobs to run simultaneously.",
+    )
+    gc_parser.add_argument(
+        "-p",
+        "--projects",
+        dest="repos",
+        type=str,
+        nargs="*",
+        default=None,
+        help="Keep data files required by these projects "
+        "in addition to the current one. "
+        "Useful if you share a single cache across repos.",
+    )
+    gc_parser.set_defaults(func=CmdGC)

dvc/command/imp.py

@@ -1,13 +1,70 @@
+from __future__ import unicode_literals
+
+import argparse
+import os
+import logging
+
+from dvc.utils.compat import urlparse
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase
-from dvc.logger import Logger
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdImport(CmdBase):
     def run(self):
         try:
-            self.project.imp(self.args.url, self.args.out)
-        except DvcException as ex:
-            Logger.error('Failed to import {}'.format(self.args.url), ex)
+            default_out = os.path.basename(urlparse(self.args.url).path)
+
+            out = self.args.out or default_out
+
+            self.repo.imp(
+                self.args.url, out, self.args.resume, fname=self.args.file
+            )
+        except DvcException:
+            logger.exception(
+                "failed to import {}. You could also try downloading "
+                "it manually and adding it with `dvc add` command.".format(
+                    self.args.url
+                )
+            )
             return 1
         return 0
+
+
+def add_parser(subparsers, parent_parser):
+    IMPORT_HELP = "Download or copy files from URL and take under DVC control."
+
+    import_parser = subparsers.add_parser(
+        "import",
+        parents=[parent_parser],
+        description=append_doc_link(IMPORT_HELP, "import"),
+        help=IMPORT_HELP,
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    import_parser.add_argument(
+        "url",
+        help="Supported urls:\n"
+        "/path/to/file\n"
+        "C:\\\\path\\to\\file\n"
+        "https://example.com/path/to/file\n"
+        "s3://bucket/path/to/file\n"
+        "gs://bucket/path/to/file\n"
+        "hdfs://example.com/path/to/file\n"
+        "ssh://example.com:/path/to/file\n"
+        "remote://myremote/path/to/file (see `dvc remote`)",
+    )
+    import_parser.add_argument(
+        "--resume",
+        action="store_true",
+        default=False,
+        help="Resume previously started download.",
+    )
+    import_parser.add_argument(
+        "out", nargs="?", help="Destination path to put files to."
+    )
+    import_parser.add_argument(
+        "-f", "--file", help="Specify name of the DVC file it generates."
+    )
+    import_parser.set_defaults(func=CmdImport)

dvc/command/init.py

@@ -1,16 +1,60 @@
-from dvc.logger import Logger
+from __future__ import unicode_literals
+
+import argparse
+import logging
+
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
 
 
-class CmdInit(object):
-    def __init__(self, args):
-        self.args = args
+logger = logging.getLogger(__name__)
 
-    def run_cmd(self):
-        from dvc.project import Project, InitError
+
+class CmdInit(CmdBaseNoRepo):
+    def run(self):
+        from dvc.repo import Repo
+        from dvc.exceptions import InitError
 
         try:
-            Project.init('.', no_scm=self.args.no_scm, force=self.args.force)
-        except InitError as e:
-            Logger.error('Failed to initiate dvc', e)
+            self.repo = Repo.init(
+                ".", no_scm=self.args.no_scm, force=self.args.force
+            )
+            self.config = self.repo.config
+        except InitError:
+            logger.exception("failed to initiate dvc")
             return 1
         return 0
+
+
+def add_parser(subparsers, parent_parser):
+    """Setup parser for `dvc init`."""
+    INIT_HELP = "Initialize DVC in the current directory."
+    INIT_DESCRIPTION = (
+        "Initialize DVC in the current directory. Expects directory\n"
+        "to be a Git repository unless --no-scm option is specified."
+    )
+
+    init_parser = subparsers.add_parser(
+        "init",
+        parents=[parent_parser],
+        description=append_doc_link(INIT_DESCRIPTION, "init"),
+        help=INIT_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    init_parser.add_argument(
+        "--no-scm",
+        action="store_true",
+        default=False,
+        help="Initiate dvc in directory that is "
+        "not tracked by any scm tool (e.g. git).",
+    )
+    init_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help=(
+            "Overwrite existing '.dvc' directory. "
+            "This operation removes local cache."
+        ),
+    )
+    init_parser.set_defaults(func=CmdInit)

dvc/command/install.py

@@ -1,12 +1,31 @@
-from dvc.logger import Logger
-from dvc.command.base import CmdBase
+from __future__ import unicode_literals
+
+import argparse
+import logging
+
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdInstall(CmdBase):
     def run_cmd(self):
         try:
-            self.project.install()
-        except Exception as e:
-            Logger.error('Failed to install dvc hooks', e)
+            self.repo.install()
+        except Exception:
+            logger.exception("failed to install dvc hooks")
             return 1
         return 0
+
+
+def add_parser(subparsers, parent_parser):
+    INSTALL_HELP = "Install DVC git hooks into the repository."
+    install_parser = subparsers.add_parser(
+        "install",
+        parents=[parent_parser],
+        description=append_doc_link(INSTALL_HELP, "install"),
+        help=INSTALL_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    install_parser.set_defaults(func=CmdInstall)

dvc/command/lock.py

@@ -1,25 +1,60 @@
+from __future__ import unicode_literals
+
+import argparse
+import logging
+
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdLockBase(CmdBase):
-    def run(self, unlock=False):
+    def _run(self, unlock):
         for target in self.args.targets:
             try:
-                self.project.lock_stage(target, unlock=unlock)
-            except DvcException as ex:
-                msg = 'Failed to {}lock \'{}\''
-                msg.format('un' if unlock else '', target)
-                self.project.logger.error(msg, ex)
+                self.repo.lock_stage(target, unlock=unlock)
+            except DvcException:
+                logger.exception(
+                    "failed to {}lock '{}'".format(
+                        "un" if unlock else "", target
+                    )
+                )
+
                 return 1
         return 0
 
 
 class CmdLock(CmdLockBase):
     def run(self):
-        return super(CmdLock, self).run(False)
+        return self._run(False)
 
 
 class CmdUnlock(CmdLockBase):
     def run(self):
-        return super(CmdUnlock, self).run(True)
+        return self._run(True)
+
+
+def add_parser(subparsers, parent_parser):
+    LOCK_HELP = "Lock DVC file."
+    lock_parser = subparsers.add_parser(
+        "lock",
+        parents=[parent_parser],
+        description=append_doc_link(LOCK_HELP, "lock"),
+        help=LOCK_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    lock_parser.add_argument("targets", nargs="+", help="DVC files.")
+    lock_parser.set_defaults(func=CmdLock)
+
+    UNLOCK_HELP = "Unlock DVC file."
+    unlock_parser = subparsers.add_parser(
+        "unlock",
+        parents=[parent_parser],
+        description=append_doc_link(UNLOCK_HELP, "unlock"),
+        help=UNLOCK_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    unlock_parser.add_argument("targets", nargs="+", help="DVC files.")
+    unlock_parser.set_defaults(func=CmdUnlock)

dvc/command/metrics.py

@@ -1,37 +1,55 @@
-from dvc.command.base import CmdBase
+from __future__ import unicode_literals
+
+import argparse
+import logging
+
 from dvc.exceptions import DvcException
+from dvc.command.base import CmdBase, fix_subparsers, append_doc_link
+
+
+logger = logging.getLogger(__name__)
+
+
+def show_metrics(metrics, all_branches=False, all_tags=False):
+    """
+    Args:
+        metrics (list): Where each element is either a `list`
+            if an xpath was specified, otherwise a `str`
+    """
+    for branch, val in metrics.items():
+        if all_branches or all_tags:
+            logger.info("{branch}:".format(branch=branch))
+
+        for fname, metric in val.items():
+            lines = metric if type(metric) is list else metric.splitlines()
+
+            if len(lines) > 1:
+                logger.info("\t{fname}:".format(fname=fname))
+
+                for line in lines:
+                    logger.info("\t\t{content}".format(content=line))
+
+            else:
+                logger.info("\t{}: {}".format(fname, metric))
 
 
 class CmdMetricsShow(CmdBase):
     def run(self):
+        typ = self.args.type
+        xpath = self.args.xpath
         try:
-            # backward compatibility
-            if self.args.json_path:
-                typ = 'json'
-                xpath = self.args.json_path
-            elif self.args.tsv_path:
-                typ = 'tsv'
-                xpath = self.args.tsv_path
-            elif self.args.htsv_path:
-                typ = 'htsv'
-                xpath = self.args.htsv_path
-            elif self.args.csv_path:
-                typ = 'csv'
-                xpath = self.args.csv_path
-            elif self.args.hcsv_path:
-                typ = 'hcsv'
-                xpath = self.args.hcsv_path
-            else:
-                typ = self.args.type
-                xpath = self.args.xpath
+            metrics = self.repo.metrics.show(
+                self.args.path,
+                typ=typ,
+                xpath=xpath,
+                all_branches=self.args.all_branches,
+                all_tags=self.args.all_tags,
+                recursive=self.args.recursive,
+            )
 
-            self.project.metrics_show(self.args.path,
-                                      typ=typ,
-                                      xpath=xpath,
-                                      all_branches=self.args.all_branches,
-                                      all_tags=self.args.all_tags)
-        except DvcException as exc:
-            self.project.logger.error('Failed to show metrics', exc)
+            show_metrics(metrics, self.args.all_branches, self.args.all_tags)
+        except DvcException:
+            logger.exception("failed to show metrics")
             return 1
 
         return 0
@@ -40,11 +58,11 @@ class CmdMetricsShow(CmdBase):
 class CmdMetricsModify(CmdBase):
     def run(self):
         try:
-            self.project.metrics_modify(self.args.path,
-                                        typ=self.args.type,
-                                        xpath=self.args.xpath)
-        except DvcException as exc:
-            self.project.logger.error('Failed to modify metrics', exc)
+            self.repo.metrics.modify(
+                self.args.path, typ=self.args.type, xpath=self.args.xpath
+            )
+        except DvcException:
+            logger.exception("failed to modify metric file settings")
             return 1
 
         return 0
@@ -53,12 +71,12 @@ class CmdMetricsModify(CmdBase):
 class CmdMetricsAdd(CmdBase):
     def run(self):
         try:
-            self.project.metrics_add(self.args.path,
-                                     self.args.type,
-                                     self.args.xpath)
-        except DvcException as exc:
-            msg = 'Failed to add metric file \'{}\''.format(self.args.path)
-            self.project.logger.error(msg, exc)
+            self.repo.metrics.add(
+                self.args.path, self.args.type, self.args.xpath
+            )
+        except DvcException:
+            msg = "failed to add metric file '{}'".format(self.args.path)
+            logger.exception(msg)
             return 1
 
         return 0
@@ -67,10 +85,123 @@ class CmdMetricsAdd(CmdBase):
 class CmdMetricsRemove(CmdBase):
     def run(self):
         try:
-            self.project.metrics_remove(self.args.path)
-        except DvcException as exc:
-            msg = 'Failed to remove metric file \'{}\''.format(self.args.path)
-            self.project.logger.error(msg, exc)
+            self.repo.metrics.remove(self.args.path)
+        except DvcException:
+            msg = "failed to remove metric file '{}'".format(self.args.path)
+            logger.exception(msg)
             return 1
 
         return 0
+
+
+def add_parser(subparsers, parent_parser):
+    METRICS_HELP = "Commands to add, manage, collect and display metrics."
+
+    metrics_parser = subparsers.add_parser(
+        "metrics",
+        parents=[parent_parser],
+        description=append_doc_link(METRICS_HELP, "metrics"),
+        help=METRICS_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    metrics_subparsers = metrics_parser.add_subparsers(
+        dest="cmd",
+        help="Use dvc metrics CMD --help to display command-specific help.",
+    )
+
+    fix_subparsers(metrics_subparsers)
+
+    METRICS_SHOW_HELP = "Output metric values."
+    metrics_show_parser = metrics_subparsers.add_parser(
+        "show",
+        parents=[parent_parser],
+        description=append_doc_link(METRICS_SHOW_HELP, "metrics-show"),
+        help=METRICS_SHOW_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    metrics_show_parser.add_argument(
+        "path", nargs="?", help="Path to a metric file or a directory."
+    )
+    metrics_show_parser.add_argument(
+        "-t",
+        "--type",
+        help=(
+            "Type of metrics (json/tsv/htsv/csv/hcsv). "
+            "It can be detected by the file extension automatically. "
+            "Unsupported types will be treated as raw."
+        ),
+    )
+    metrics_show_parser.add_argument(
+        "-x", "--xpath", help="json/tsv/htsv/csv/hcsv path."
+    )
+    metrics_show_parser.add_argument(
+        "-a",
+        "--all-branches",
+        action="store_true",
+        default=False,
+        help="Show metrics for all branches.",
+    )
+    metrics_show_parser.add_argument(
+        "-T",
+        "--all-tags",
+        action="store_true",
+        default=False,
+        help="Show metrics for all tags.",
+    )
+    metrics_show_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help=(
+            "If path is a directory, recursively search and process metric "
+            "files in path."
+        ),
+    )
+    metrics_show_parser.set_defaults(func=CmdMetricsShow)
+
+    METRICS_ADD_HELP = "Tag file as a metric file."
+    metrics_add_parser = metrics_subparsers.add_parser(
+        "add",
+        parents=[parent_parser],
+        description=append_doc_link(METRICS_ADD_HELP, "metrics-add"),
+        help=METRICS_ADD_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    metrics_add_parser.add_argument(
+        "-t", "--type", help="Type of metrics (raw/json/tsv/htsv/csv/hcsv)."
+    )
+    metrics_add_parser.add_argument(
+        "-x", "--xpath", help="json/tsv/htsv/csv/hcsv path."
+    )
+    metrics_add_parser.add_argument("path", help="Path to a metric file.")
+    metrics_add_parser.set_defaults(func=CmdMetricsAdd)
+
+    METRICS_MODIFY_HELP = "Modify metric file options."
+    metrics_modify_parser = metrics_subparsers.add_parser(
+        "modify",
+        parents=[parent_parser],
+        description=append_doc_link(METRICS_MODIFY_HELP, "metrics-modify"),
+        help=METRICS_MODIFY_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    metrics_modify_parser.add_argument(
+        "-t", "--type", help="Type of metrics (raw/json/tsv/htsv/csv/hcsv)."
+    )
+    metrics_modify_parser.add_argument(
+        "-x", "--xpath", help="json/tsv/htsv/csv/hcsv path."
+    )
+    metrics_modify_parser.add_argument("path", help="Path to a metric file.")
+    metrics_modify_parser.set_defaults(func=CmdMetricsModify)
+
+    METRICS_REMOVE_HELP = "Remove files's metric tag."
+    metrics_remove_parser = metrics_subparsers.add_parser(
+        "remove",
+        parents=[parent_parser],
+        description=append_doc_link(METRICS_REMOVE_HELP, "metrics-remove"),
+        help=METRICS_REMOVE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    metrics_remove_parser.add_argument("path", help="Path to a metric file.")
+    metrics_remove_parser.set_defaults(func=CmdMetricsRemove)

dvc/command/move.py

@@ -1,14 +1,45 @@
+from __future__ import unicode_literals
+
+import argparse
+import logging
+
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdMove(CmdBase):
     def run(self):
         try:
-            self.project.move(self.args.src, self.args.dst)
-        except DvcException as ex:
-            msg = 'Failed to move \'{}\' -> \'{}\''.format(self.args.src,
-                                                           self.args.dst)
-            self.project.logger.error(msg, ex)
+            self.repo.move(self.args.src, self.args.dst)
+        except DvcException:
+            msg = "failed to move '{}' -> '{}'".format(
+                self.args.src, self.args.dst
+            )
+            logger.exception(msg)
             return 1
         return 0
+
+
+def add_parser(subparsers, parent_parser):
+    MOVE_HELP = "Rename or move a DVC controlled data file or a directory."
+    MOVE_DESCRIPTION = (
+        "Rename or move a DVC controlled data file or a directory.\n"
+        "It renames and modifies the corresponding DVC file to reflect the"
+        " changes."
+    )
+
+    move_parser = subparsers.add_parser(
+        "move",
+        parents=[parent_parser],
+        description=append_doc_link(MOVE_DESCRIPTION, "move"),
+        help=MOVE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    move_parser.add_argument(
+        "src", help="Source path to a data file or directory."
+    )
+    move_parser.add_argument("dst", help="Destination path.")
+    move_parser.set_defaults(func=CmdMove)

dvc/command/pipeline.py

@@ -1,42 +1,55 @@
+from __future__ import unicode_literals
+
+from dvc.utils.compat import str
+
+import argparse
 import os
+import logging
 
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase
-from dvc.dagascii import Dagascii
+from dvc.command.base import CmdBase, fix_subparsers, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdPipelineShow(CmdBase):
-    def _show(self, target, commands, outs):
+    def _show(self, target, commands, outs, locked):
         import networkx
         from dvc.stage import Stage
 
-        stage = Stage.load(self.project, target)
-        G = self.project.graph()[0]
-        stages = networkx.get_node_attributes(G, 'stage')
-        node = os.path.relpath(stage.path, self.project.root_dir)
+        stage = Stage.load(self.repo, target)
+        G = self.repo.graph()[0]
+        stages = networkx.get_node_attributes(G, "stage")
+        node = os.path.relpath(stage.path, self.repo.root_dir)
+        nodes = networkx.dfs_postorder_nodes(G, node)
 
-        for n in networkx.dfs_postorder_nodes(G, node):
+        if locked:
+            nodes = [n for n in nodes if stages[n].locked]
+
+        for n in nodes:
             if commands:
-                self.project.logger.info(stages[n].cmd)
+                logger.info(stages[n].cmd)
             elif outs:
                 for out in stages[n].outs:
-                    self.project.logger.info(str(out))
+                    logger.info(str(out))
             else:
-                self.project.logger.info(n)
+                logger.info(n)
 
     def __build_graph(self, target, commands, outs):
         import networkx
         from dvc.stage import Stage
 
-        stage = Stage.load(self.project, target)
-        node = os.path.relpath(stage.path, self.project.root_dir)
+        stage = Stage.load(self.repo, target)
+        node = os.path.relpath(stage.path, self.repo.root_dir)
 
-        pipelines = list(filter(lambda g: node in g.nodes(),
-                                self.project.pipelines()))
+        pipelines = list(
+            filter(lambda g: node in g.nodes(), self.repo.pipelines())
+        )
 
         assert len(pipelines) == 1
         G = pipelines[0]
-        stages = networkx.get_node_attributes(G, 'stage')
+        stages = networkx.get_node_attributes(G, "stage")
 
         nodes = []
         for n in G.nodes():
@@ -62,52 +75,88 @@ class CmdPipelineShow(CmdBase):
             elif outs:
                 for from_out in from_stage.outs:
                     for to_out in to_stage.outs:
-                        edges.append((str(from_out),
-                                      str(to_out)))
+                        edges.append((str(from_out), str(to_out)))
             else:
                 edges.append((from_stage.relpath, to_stage.relpath))
 
-        return nodes, edges
+        return nodes, edges, networkx.is_tree(G)
 
     def _show_ascii(self, target, commands, outs):
-        nodes, edges = self.__build_graph(target, commands, outs)
+        from dvc.dagascii import draw
+
+        nodes, edges, _ = self.__build_graph(target, commands, outs)
 
         if not nodes:
             return
 
-        d = Dagascii(nodes, edges)
-        d.draw()
+        draw(nodes, edges)
+
+    def _show_dependencies_tree(self, target, commands, outs):
+        from treelib import Tree
+
+        nodes, edges, is_tree = self.__build_graph(target, commands, outs)
+        if not nodes:
+            return
+        if not is_tree:
+            raise DvcException(
+                "DAG is not a tree, can not print it in tree-structure way, "
+                "please use --ascii instead"
+            )
+
+        tree = Tree()
+        tree.create_node(target, target)  # Root node
+        observe_list = [target]
+        while len(observe_list) > 0:
+            current_root = observe_list[0]
+            for edge in edges:
+                if edge[0] == current_root:
+                    tree.create_node(edge[1], edge[1], parent=current_root)
+                    observe_list.append(edge[1])
+            observe_list.pop(0)
+        tree.show()
 
     def __write_dot(self, target, commands, outs, filename):
         import networkx
         from networkx.drawing.nx_pydot import write_dot
 
-        _, edges = self.__build_graph(target, commands, outs)
+        _, edges, _ = self.__build_graph(target, commands, outs)
         edges = [edge[::-1] for edge in edges]
 
         simple_g = networkx.DiGraph()
         simple_g.add_edges_from(edges)
         write_dot(simple_g, filename)
 
-    def run(self, unlock=False):
+    def run(self):
+        if not self.args.targets:
+            self.args.targets = self.default_targets
+
         for target in self.args.targets:
             try:
                 if self.args.ascii:
-                    self._show_ascii(target,
-                                     self.args.commands,
-                                     self.args.outs)
+                    self._show_ascii(
+                        target, self.args.commands, self.args.outs
+                    )
                 elif self.args.dot:
-                    self.__write_dot(target,
-                                     self.args.commands,
-                                     self.args.outs,
-                                     self.args.dot)
+                    self.__write_dot(
+                        target,
+                        self.args.commands,
+                        self.args.outs,
+                        self.args.dot,
+                    )
+                elif self.args.tree:
+                    self._show_dependencies_tree(
+                        target, self.args.commands, self.args.outs
+                    )
                 else:
-                    self._show(target,
-                               self.args.commands,
-                               self.args.outs)
-            except DvcException as ex:
-                msg = 'Failed to show pipeline for \'{}\''.format(target)
-                self.project.logger.error(msg, ex)
+                    self._show(
+                        target,
+                        self.args.commands,
+                        self.args.outs,
+                        self.args.locked,
+                    )
+            except DvcException:
+                msg = "failed to show pipeline for '{}'".format(target)
+                logger.exception(msg)
                 return 1
         return 0
 
@@ -116,13 +165,91 @@ class CmdPipelineList(CmdBase):
     def run(self):
         import networkx
 
-        pipelines = self.project.pipelines()
+        pipelines = self.repo.pipelines()
         for p in pipelines:
-            stages = networkx.get_node_attributes(p, 'stage')
+            stages = networkx.get_node_attributes(p, "stage")
             for stage in stages:
-                self.project.logger.info(stage)
+                logger.info(stage)
             if len(stages) != 0:
-                self.project.logger.info("="*80)
-        self.project.logger.info("{} pipeline(s) total".format(len(pipelines)))
+                logger.info("=" * 80)
+        logger.info("{} pipeline(s) total".format(len(pipelines)))
 
         return 0
+
+
+def add_parser(subparsers, parent_parser):
+    PIPELINE_HELP = "Manage pipelines."
+    pipeline_parser = subparsers.add_parser(
+        "pipeline",
+        parents=[parent_parser],
+        description=append_doc_link(PIPELINE_HELP, "pipeline"),
+        help=PIPELINE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    pipeline_subparsers = pipeline_parser.add_subparsers(
+        dest="cmd",
+        help="Use dvc pipeline CMD --help for command-specific help.",
+    )
+
+    fix_subparsers(pipeline_subparsers)
+
+    PIPELINE_SHOW_HELP = "Show pipeline."
+    pipeline_show_parser = pipeline_subparsers.add_parser(
+        "show",
+        parents=[parent_parser],
+        description=append_doc_link(PIPELINE_SHOW_HELP, "pipeline-show"),
+        help=PIPELINE_SHOW_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    pipeline_show_group = pipeline_show_parser.add_mutually_exclusive_group()
+    pipeline_show_group.add_argument(
+        "-c",
+        "--commands",
+        action="store_true",
+        default=False,
+        help="Print commands instead of paths to DVC files.",
+    )
+    pipeline_show_group.add_argument(
+        "-o",
+        "--outs",
+        action="store_true",
+        default=False,
+        help="Print output files instead of paths to DVC files.",
+    )
+    pipeline_show_parser.add_argument(
+        "-l",
+        "--locked",
+        action="store_true",
+        default=False,
+        help="Print locked DVC stages",
+    )
+    pipeline_show_parser.add_argument(
+        "--ascii",
+        action="store_true",
+        default=False,
+        help="Output DAG as ASCII.",
+    )
+    pipeline_show_parser.add_argument(
+        "--dot", help="Write DAG in .dot format."
+    )
+    pipeline_show_parser.add_argument(
+        "--tree",
+        action="store_true",
+        default=False,
+        help="Output DAG as Dependencies Tree.",
+    )
+    pipeline_show_parser.add_argument(
+        "targets", nargs="*", help="DVC files. 'Dvcfile' by default."
+    )
+    pipeline_show_parser.set_defaults(func=CmdPipelineShow)
+
+    PIPELINE_LIST_HELP = "List pipelines."
+    pipeline_list_parser = pipeline_subparsers.add_parser(
+        "list",
+        parents=[parent_parser],
+        description=append_doc_link(PIPELINE_LIST_HELP, "pipeline-list"),
+        help=PIPELINE_LIST_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    pipeline_list_parser.set_defaults(func=CmdPipelineList)

dvc/command/pkg.py

@@ -0,0 +1,89 @@
+from __future__ import unicode_literals
+
+import argparse
+import logging
+
+from dvc.exceptions import DvcException
+from dvc.command.base import CmdBase, fix_subparsers, append_doc_link
+
+
+logger = logging.getLogger(__name__)
+
+
+class CmdPkgInstall(CmdBase):
+    def run(self):
+        try:
+            self.repo.pkg.install(
+                self.args.address,
+                self.args.target_dir,
+                self.args.select,
+                self.args.file,
+            )
+            return 0
+        except DvcException:
+            logger.exception(
+                "failed to install package '{}'".format(self.args.address)
+            )
+            return 1
+
+
+def add_parser(subparsers, parent_parser):
+    from dvc.command.config import parent_config_parser
+
+    PKG_HELP = "Manage DVC packages."
+    pkg_parser = subparsers.add_parser(
+        "pkg",
+        parents=[parent_parser],
+        description=append_doc_link(PKG_HELP, "pkg"),
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        add_help=False,
+    )
+
+    pkg_subparsers = pkg_parser.add_subparsers(
+        dest="cmd", help="Use dvc pkg CMD --help for command-specific help."
+    )
+
+    fix_subparsers(pkg_subparsers)
+
+    PKG_INSTALL_HELP = "Install package."
+    pkg_install_parser = pkg_subparsers.add_parser(
+        "install",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(PKG_INSTALL_HELP, "pkg-install"),
+        help=PKG_INSTALL_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    pkg_install_parser.add_argument(
+        "address",
+        nargs="?",
+        default="",
+        help="Package address: git://<url> or https://github.com/...",
+    )
+    pkg_install_parser.add_argument(
+        "target_dir",
+        metavar="target",
+        nargs="?",
+        default=".",
+        help="Target directory to deploy package outputs. "
+        "Default value is the current dir.",
+    )
+    pkg_install_parser.add_argument(
+        "-s",
+        "--select",
+        metavar="OUT",
+        action="append",
+        default=[],
+        help="Select and persist only specified outputs from a package. "
+        "The parameter can be used multiple times. "
+        "All outputs will be selected by default.",
+    )
+    pkg_install_parser.add_argument(
+        "-f",
+        "--file",
+        help="Specify name of the stage file. It should be "
+        "either 'Dvcfile' or have a '.dvc' suffix (e.g. "
+        "'prepare.dvc', 'clean.dvc', etc). "
+        "By default the file has 'mod_' prefix and imported package name "
+        "followed by .dvc",
+    )
+    pkg_install_parser.set_defaults(func=CmdPkgInstall)

dvc/command/remote.py

@@ -1,75 +1,238 @@
+from __future__ import unicode_literals
+
+import argparse
 import os
 import re
-import configobj
+import logging
 
-from dvc.config import Config
+from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig
-from dvc.logger import Logger
+from dvc.config import Config
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdRemoteAdd(CmdConfig):
+    @staticmethod
+    def resolve_path(path, config_file):
+        """Resolve path relative to config file location.
+
+        Args:
+            path: Path to be resolved.
+            config_file: Path to config file, which `path` is specified
+                relative to.
+
+        Returns:
+            Path relative to the `config_file` location. If `path` is an
+            absolute path then it will be returned without change.
+
+        """
+        if os.path.isabs(path):
+            return path
+        return os.path.relpath(path, os.path.dirname(config_file))
+
     def run(self):
+        from dvc.remote import _get, RemoteLOCAL
+
+        remote = _get({Config.SECTION_REMOTE_URL: self.args.url})
+        if remote == RemoteLOCAL and not self.args.url.startswith("remote://"):
+            self.args.url = self.resolve_path(
+                self.args.url, self.configobj.filename
+            )
+
         section = Config.SECTION_REMOTE_FMT.format(self.args.name)
-        ret = self.set(section, Config.SECTION_REMOTE_URL, self.args.url)
+        if (section in self.configobj.keys()) and not self.args.force:
+            logger.error(
+                "Remote with name {} already exists. "
+                "Use -f (--force) to overwrite remote "
+                "with new value".format(self.args.name)
+            )
+            return 1
+
+        ret = self._set(section, Config.SECTION_REMOTE_URL, self.args.url)
         if ret != 0:
             return ret
 
         if self.args.default:
-            msg = 'Setting \'{}\' as a default remote.'.format(self.args.name)
-            Logger.info(msg)
-            ret = self.set(Config.SECTION_CORE,
-                           Config.SECTION_CORE_REMOTE,
-                           self.args.name)
+            msg = "Setting '{}' as a default remote.".format(self.args.name)
+            logger.info(msg)
+            ret = self._set(
+                Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, self.args.name
+            )
 
         return ret
 
 
 class CmdRemoteRemove(CmdConfig):
-    def _remove_default(self, config_file, remote):
-        path = os.path.join(os.path.dirname(self.config_file),
-                            config_file)
-        config = configobj.ConfigObj(path)
-
+    def _remove_default(self, config):
         core = config.get(Config.SECTION_CORE, None)
         if core is None:
-            return
+            return 0
 
         default = core.get(Config.SECTION_CORE_REMOTE, None)
         if default is None:
-            return
+            return 0
 
-        if default == remote:
-            del config[Config.SECTION_CORE][Config.SECTION_CORE_REMOTE]
-            if len(config[Config.SECTION_CORE]) == 0:
-                del config[Config.SECTION_CORE]
-
-        config.write()
+        if default == self.args.name:
+            return self._unset(
+                Config.SECTION_CORE,
+                opt=Config.SECTION_CORE_REMOTE,
+                configobj=config,
+            )
 
     def run(self):
         section = Config.SECTION_REMOTE_FMT.format(self.args.name)
-        ret = self.unset(section)
+        ret = self._unset(section)
         if ret != 0:
             return ret
 
-        self._remove_default(Config.CONFIG, self.args.name)
-        self._remove_default(Config.CONFIG_LOCAL, self.args.name)
+        for configobj in [
+            self.config._local_config,
+            self.config._repo_config,
+            self.config._global_config,
+            self.config._system_config,
+        ]:
+            self._remove_default(configobj)
+            self.config.save(configobj)
+            if configobj == self.configobj:
+                break
+
         return 0
 
 
 class CmdRemoteModify(CmdConfig):
     def run(self):
         section = Config.SECTION_REMOTE_FMT.format(self.args.name)
-        self.args.name = '{}.{}'.format(section, self.args.option)
+        self.args.name = "{}.{}".format(section, self.args.option)
         return super(CmdRemoteModify, self).run()
 
 
+class CmdRemoteDefault(CmdConfig):
+    def run(self):
+        self.args.value = self.args.name
+        self.args.name = "core.remote"
+        return super(CmdRemoteDefault, self).run()
+
+
 class CmdRemoteList(CmdConfig):
     def run(self):
         for section in self.configobj.keys():
             r = re.match(Config.SECTION_REMOTE_REGEX, section)
             if r:
-                name = r.group('name')
-                url = self.configobj[section].get(Config.SECTION_REMOTE_URL,
-                                                  '')
-                Logger.info('{}\t{}'.format(name, url))
+                name = r.group("name")
+                url = self.configobj[section].get(
+                    Config.SECTION_REMOTE_URL, ""
+                )
+                logger.info("{}\t{}".format(name, url))
         return 0
+
+
+def add_parser(subparsers, parent_parser):
+    from dvc.command.config import parent_config_parser
+
+    REMOTE_HELP = "Manage remote storage configuration."
+    remote_parser = subparsers.add_parser(
+        "remote",
+        parents=[parent_parser],
+        description=append_doc_link(REMOTE_HELP, "remote"),
+        help=REMOTE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    remote_subparsers = remote_parser.add_subparsers(
+        dest="cmd",
+        help="Use dvc remote CMD --help for " "command-specific help.",
+    )
+
+    fix_subparsers(remote_subparsers)
+
+    REMOTE_ADD_HELP = "Add remote."
+    remote_add_parser = remote_subparsers.add_parser(
+        "add",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(REMOTE_ADD_HELP, "remote-add"),
+        help=REMOTE_ADD_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    remote_add_parser.add_argument("name", help="Name.")
+    remote_add_parser.add_argument(
+        "url",
+        help="URL. See full list of supported urls at " "man.dvc.org/remote",
+    )
+    remote_add_parser.add_argument(
+        "-d",
+        "--default",
+        action="store_true",
+        default=False,
+        help="Set as default remote.",
+    )
+    remote_add_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Force overwriting existing configs",
+    )
+    remote_add_parser.set_defaults(func=CmdRemoteAdd)
+
+    REMOTE_DEFAULT_HELP = "Set/unset default remote."
+    remote_default_parser = remote_subparsers.add_parser(
+        "default",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(REMOTE_DEFAULT_HELP, "remote-default"),
+        help=REMOTE_DEFAULT_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    remote_default_parser.add_argument(
+        "name", nargs="?", help="Name of the remote."
+    )
+    remote_default_parser.add_argument(
+        "-u",
+        "--unset",
+        action="store_true",
+        default=False,
+        help="Unset default remote.",
+    )
+    remote_default_parser.set_defaults(func=CmdRemoteDefault)
+
+    REMOTE_REMOVE_HELP = "Remove remote."
+    remote_remove_parser = remote_subparsers.add_parser(
+        "remove",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(REMOTE_REMOVE_HELP, "remote-remove"),
+        help=REMOTE_REMOVE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    remote_remove_parser.add_argument("name", help="Name")
+    remote_remove_parser.set_defaults(func=CmdRemoteRemove)
+
+    REMOTE_MODIFY_HELP = "Modify remote."
+    remote_modify_parser = remote_subparsers.add_parser(
+        "modify",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(REMOTE_MODIFY_HELP, "remote-modify"),
+        help=REMOTE_MODIFY_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    remote_modify_parser.add_argument("name", help="Name.")
+    remote_modify_parser.add_argument("option", help="Option.")
+    remote_modify_parser.add_argument("value", nargs="?", help="Value.")
+    remote_modify_parser.add_argument(
+        "-u",
+        "--unset",
+        default=False,
+        action="store_true",
+        help="Unset option.",
+    )
+    remote_modify_parser.set_defaults(func=CmdRemoteModify)
+
+    REMOTE_LIST_HELP = "List available remotes."
+    remote_list_parser = remote_subparsers.add_parser(
+        "list",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(REMOTE_LIST_HELP, "remote-list"),
+        help=REMOTE_LIST_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    remote_list_parser.set_defaults(func=CmdRemoteList)

dvc/command/remove.py

@@ -1,6 +1,14 @@
+from __future__ import unicode_literals
+
+import argparse
+import logging
+
+import dvc.prompt as prompt
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase
-from dvc.logger import Logger
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdRemove(CmdBase):
@@ -11,27 +19,59 @@ class CmdRemove(CmdBase):
         if self.args.force:
             return False
 
-        msg = (
-            u'Are you sure you want to remove {} with its outputs?'
-            .format(target)
+        msg = "Are you sure you want to remove {} with its outputs?".format(
+            target
         )
 
-        confirmed = self.project.prompt.prompt(msg)
-
-        if confirmed:
+        if prompt.confirm(msg):
             return False
 
         raise DvcException(
-            u'Cannot purge without a confirmation from the user.'
-            u" Use '-f' to force."
+            "Cannot purge without a confirmation from the user."
+            " Use '-f' to force."
         )
 
     def run(self):
         for target in self.args.targets:
             try:
                 outs_only = self._is_outs_only(target)
-                self.project.remove(target, outs_only=outs_only)
-            except DvcException as ex:
-                Logger.error('Failed to remove {}'.format(target), ex)
+                self.repo.remove(target, outs_only=outs_only)
+            except DvcException:
+                logger.exception("failed to remove {}".format(target))
                 return 1
         return 0
+
+
+def add_parser(subparsers, parent_parser):
+    REMOVE_HELP = "Remove outputs of DVC file."
+    remove_parser = subparsers.add_parser(
+        "remove",
+        parents=[parent_parser],
+        description=append_doc_link(REMOVE_HELP, "remove"),
+        help=REMOVE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    remove_parser_group = remove_parser.add_mutually_exclusive_group()
+    remove_parser_group.add_argument(
+        "-o",
+        "--outs",
+        action="store_true",
+        default=True,
+        help="Only remove DVC file outputs.(default)",
+    )
+    remove_parser_group.add_argument(
+        "-p",
+        "--purge",
+        action="store_true",
+        default=False,
+        help="Remove DVC file and all its outputs.",
+    )
+    remove_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Force purge.",
+    )
+    remove_parser.add_argument("targets", nargs="+", help="DVC files.")
+    remove_parser.set_defaults(func=CmdRemove)

dvc/command/repro.py

@@ -1,10 +1,18 @@
-import os
+from __future__ import unicode_literals
 
-from dvc.command.base import CmdBase
+import argparse
+import os
+import logging
+
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.metrics import show_metrics
 from dvc.command.status import CmdDataStatus
 from dvc.exceptions import DvcException
 
 
+logger = logging.getLogger(__name__)
+
+
 class CmdRepro(CmdBase):
     def run(self):
         recursive = not self.args.single_item
@@ -12,28 +20,131 @@ class CmdRepro(CmdBase):
         if self.args.cwd:
             os.chdir(self.args.cwd)
 
+        # Dirty hack so the for loop below can at least enter once
+        if self.args.all_pipelines:
+            self.args.targets = [None]
+        elif not self.args.targets:
+            self.args.targets = self.default_targets
+
         ret = 0
         for target in self.args.targets:
             try:
-                stages = self.project.reproduce(
-                                       target,
-                                       recursive=recursive,
-                                       force=self.args.force,
-                                       dry=self.args.dry,
-                                       interactive=self.args.interactive,
-                                       pipeline=self.args.pipeline,
-                                       all_pipelines=self.args.all_pipelines)
+                stages = self.repo.reproduce(
+                    target,
+                    recursive=recursive,
+                    force=self.args.force,
+                    dry=self.args.dry,
+                    interactive=self.args.interactive,
+                    pipeline=self.args.pipeline,
+                    all_pipelines=self.args.all_pipelines,
+                    ignore_build_cache=self.args.ignore_build_cache,
+                    no_commit=self.args.no_commit,
+                    downstream=self.args.downstream,
+                )
 
                 if len(stages) == 0:
-                    self.project.logger.info(CmdDataStatus.UP_TO_DATE_MSG)
+                    logger.info(CmdDataStatus.UP_TO_DATE_MSG)
 
                 if self.args.metrics:
-                    self.project.metrics_show()
-            except DvcException as ex:
-                msg = 'Failed to reproduce \'{}\''.format(target)
-                self.project.logger.error(msg, ex)
+                    metrics = self.repo.metrics.show()
+                    show_metrics(metrics)
+            except DvcException:
+                logger.exception("")
                 ret = 1
                 break
 
         os.chdir(saved_dir)
         return ret
+
+
+def add_parser(subparsers, parent_parser):
+    REPRO_HELP = "Check for changes and reproduce DVC file and dependencies."
+    repro_parser = subparsers.add_parser(
+        "repro",
+        parents=[parent_parser],
+        description=append_doc_link(REPRO_HELP, "repro"),
+        help=REPRO_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    repro_parser.add_argument(
+        "targets",
+        nargs="*",
+        help="DVC file to reproduce (default - 'Dvcfile').",
+    )
+    repro_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Reproduce even if dependencies were not changed.",
+    )
+    repro_parser.add_argument(
+        "-s",
+        "--single-item",
+        action="store_true",
+        default=False,
+        help="Reproduce only single data item without recursive dependencies "
+        "check.",
+    )
+    repro_parser.add_argument(
+        "-c",
+        "--cwd",
+        default=os.path.curdir,
+        help="Directory within your repo to reproduce from.",
+    )
+    repro_parser.add_argument(
+        "-m",
+        "--metrics",
+        action="store_true",
+        default=False,
+        help="Show metrics after reproduction.",
+    )
+    repro_parser.add_argument(
+        "--dry",
+        action="store_true",
+        default=False,
+        help="Only print the commands that would be executed without "
+        "actually executing.",
+    )
+    repro_parser.add_argument(
+        "-i",
+        "--interactive",
+        action="store_true",
+        default=False,
+        help="Ask for confirmation before reproducing each stage.",
+    )
+    repro_parser.add_argument(
+        "-p",
+        "--pipeline",
+        action="store_true",
+        default=False,
+        help="Reproduce the whole pipeline that the specified stage file "
+        "belongs to.",
+    )
+    repro_parser.add_argument(
+        "-P",
+        "--all-pipelines",
+        action="store_true",
+        default=False,
+        help="Reproduce all pipelines in the repo.",
+    )
+    repro_parser.add_argument(
+        "--ignore-build-cache",
+        action="store_true",
+        default=False,
+        help="Reproduce all descendants of a changed stage even if their "
+        "direct dependencies didn't change.",
+    )
+    repro_parser.add_argument(
+        "--no-commit",
+        action="store_true",
+        default=False,
+        help="Don't put files/directories into cache.",
+    )
+    repro_parser.add_argument(
+        "--downstream",
+        action="store_true",
+        default=False,
+        help="Reproduce the pipeline starting from the specified stage.",
+    )
+    repro_parser.set_defaults(func=CmdRepro)

dvc/command/root.py

@@ -1,12 +1,29 @@
+from __future__ import unicode_literals
+
+import argparse
 import os
+import logging
 
-from dvc.command.base import CmdBase
+from dvc.repo import Repo
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
 
 
-class CmdRoot(CmdBase):
-    def run_cmd(self):
-        return self.run()
+logger = logging.getLogger(__name__)
 
+
+class CmdRoot(CmdBaseNoRepo):
     def run(self):
-        self.project.logger.info(os.path.relpath(self.project.root_dir))
+        logger.info(os.path.relpath(Repo.find_root()))
         return 0
+
+
+def add_parser(subparsers, parent_parser):
+    ROOT_HELP = "Relative path to the repository's directory."
+    root_parser = subparsers.add_parser(
+        "root",
+        parents=[parent_parser],
+        description=append_doc_link(ROOT_HELP, "root"),
+        help=ROOT_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    root_parser.set_defaults(func=CmdRoot)

dvc/command/run.py

@@ -1,23 +1,196 @@
-from dvc.command.base import CmdBase
+from __future__ import unicode_literals
+
+import argparse
+import logging
+
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
 
+logger = logging.getLogger(__name__)
+
+
 class CmdRun(CmdBase):
     def run(self):
-        try:
-            if self.args.yes:
-                self.project.prompt.default = True
+        overwrite = self.args.yes or self.args.overwrite_dvcfile
 
-            self.project.run(cmd=' '.join(self.args.command),
-                             outs=self.args.outs,
-                             outs_no_cache=self.args.outs_no_cache,
-                             metrics_no_cache=self.args.metrics_no_cache,
-                             deps=self.args.deps,
-                             fname=self.args.file,
-                             cwd=self.args.cwd,
-                             no_exec=self.args.no_exec)
-        except DvcException as ex:
-            self.project.logger.error('Failed to run command', ex)
+        if not any(
+            [
+                self.args.deps,
+                self.args.outs,
+                self.args.outs_no_cache,
+                self.args.metrics,
+                self.args.metrics_no_cache,
+                self.args.outs_persist,
+                self.args.outs_persist_no_cache,
+                self.args.command,
+            ]
+        ):  # pragma: no cover
+            logger.error(
+                "too few arguments. Specify at least one: '-d', '-o', '-O',"
+                " '-m', '-M', '--outs-persist', '--outs-persist-no-cache',"
+                " 'command'."
+            )
+            return 1
+
+        try:
+            self.repo.run(
+                cmd=self._parsed_cmd(),
+                outs=self.args.outs,
+                outs_no_cache=self.args.outs_no_cache,
+                metrics=self.args.metrics,
+                metrics_no_cache=self.args.metrics_no_cache,
+                deps=self.args.deps,
+                fname=self.args.file,
+                cwd=self.args.cwd,
+                wdir=self.args.wdir,
+                no_exec=self.args.no_exec,
+                overwrite=overwrite,
+                ignore_build_cache=self.args.ignore_build_cache,
+                remove_outs=self.args.remove_outs,
+                no_commit=self.args.no_commit,
+                outs_persist=self.args.outs_persist,
+                outs_persist_no_cache=self.args.outs_persist_no_cache,
+            )
+        except DvcException:
+            logger.exception("failed to run command")
             return 1
 
         return 0
+
+    def _parsed_cmd(self):
+        """
+        We need to take into account two cases:
+
+        - ['python code.py foo bar']: Used mainly with dvc as a library
+        - ['echo', 'foo bar']: List of arguments received from the CLI
+
+        The second case would need quoting, as it was passed through:
+                dvc run echo "foo bar"
+        """
+        if len(self.args.command) < 2:
+            return " ".join(self.args.command)
+
+        return " ".join(self._quote_argument(arg) for arg in self.args.command)
+
+    def _quote_argument(self, argument):
+        if " " not in argument or '"' in argument:
+            return argument
+
+        return '"{}"'.format(argument)
+
+
+def add_parser(subparsers, parent_parser):
+    RUN_HELP = "Generate a stage file from a command and execute the command."
+    run_parser = subparsers.add_parser(
+        "run",
+        parents=[parent_parser],
+        description=append_doc_link(RUN_HELP, "run"),
+        help=RUN_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    run_parser.add_argument(
+        "-d",
+        "--deps",
+        action="append",
+        default=[],
+        help="Declare dependencies for reproducible cmd.",
+    )
+    run_parser.add_argument(
+        "-o",
+        "--outs",
+        action="append",
+        default=[],
+        help="Declare output file or directory.",
+    )
+    run_parser.add_argument(
+        "-O",
+        "--outs-no-cache",
+        action="append",
+        default=[],
+        help="Declare output file or directory "
+        "(do not put into DVC cache).",
+    )
+    run_parser.add_argument(
+        "-m",
+        "--metrics",
+        action="append",
+        default=[],
+        help="Declare output metric file or directory.",
+    )
+    run_parser.add_argument(
+        "-M",
+        "--metrics-no-cache",
+        action="append",
+        default=[],
+        help="Declare output metric file or directory "
+        "(do not put into DVC cache).",
+    )
+    run_parser.add_argument(
+        "-f", "--file", help="Specify name of the DVC file it generates."
+    )
+    run_parser.add_argument(
+        "-c", "--cwd", default=None, help="Deprecated, use -w and -f instead."
+    )
+    run_parser.add_argument(
+        "-w",
+        "--wdir",
+        default=None,
+        help="Directory within your repo to run your command in.",
+    )
+    run_parser.add_argument(
+        "--no-exec",
+        action="store_true",
+        default=False,
+        help="Only create stage file without actually running it.",
+    )
+    run_parser.add_argument(
+        "-y",
+        "--yes",
+        action="store_true",
+        default=False,
+        help="Deprecated, use --overwrite-dvcfile instead",
+    )
+    run_parser.add_argument(
+        "--overwrite-dvcfile",
+        action="store_true",
+        default=False,
+        help="Overwrite existing dvc file without asking for confirmation.",
+    )
+    run_parser.add_argument(
+        "--ignore-build-cache",
+        action="store_true",
+        default=False,
+        help="Run this stage even if it has been already ran with the same "
+        "command/dependencies/outputs/etc before.",
+    )
+    run_parser.add_argument(
+        "--remove-outs",
+        action="store_true",
+        default=False,
+        help="Deprecated, this is now the default behavior",
+    )
+    run_parser.add_argument(
+        "--no-commit",
+        action="store_true",
+        default=False,
+        help="Don't put files/directories into cache.",
+    )
+    run_parser.add_argument(
+        "--outs-persist",
+        action="append",
+        default=[],
+        help="Declare output file or directory that will not be "
+        "removed upon repro.",
+    )
+    run_parser.add_argument(
+        "--outs-persist-no-cache",
+        action="append",
+        default=[],
+        help="Declare output file or directory that will not be "
+        "removed upon repro (do not put into DVC cache).",
+    )
+    run_parser.add_argument(
+        "command", nargs=argparse.REMAINDER, help="Command to execute."
+    )
+    run_parser.set_defaults(func=CmdRun)

dvc/command/status.py

@@ -1,44 +1,67 @@
+from __future__ import unicode_literals
+
+import logging
+
 from dvc.command.data_sync import CmdDataBase
+from dvc.utils.compat import str
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdDataStatus(CmdDataBase):
-    STATUS_LEN = 10
-    STATUS_INDENT = '\t'
+    STATUS_LEN = 20
+    STATUS_INDENT = "\t"
     UP_TO_DATE_MSG = "Pipeline is up to date. Nothing to reproduce."
 
     def _normalize(self, s):
-        s += ':'
+        s += ":"
         assert len(s) < self.STATUS_LEN
-        return s + (self.STATUS_LEN - len(s))*' '
+        return s + (self.STATUS_LEN - len(s)) * " "
 
     def _show(self, status, indent=0):
         ind = indent * self.STATUS_INDENT
 
+        if isinstance(status, str):
+            logger.info("{}{}".format(ind, status))
+            return
+
+        if isinstance(status, list):
+            for entry in status:
+                self._show(entry, indent)
+            return
+
+        assert isinstance(status, dict)
+
         for key, value in status.items():
-            if isinstance(value, dict):
-                self.project.logger.info('{}{}'.format(ind, key))
-                self._show(value, indent+1)
-            else:
-                msg = '{}{}{}'.format(ind, self._normalize(value), key)
-                self.project.logger.info(msg)
+            if isinstance(value, str):
+                logger.info("{}{}{}".format(ind, self._normalize(value), key))
+            elif value:
+                logger.info("{}{}:".format(ind, key))
+                self._show(value, indent + 1)
 
     def do_run(self, target=None):
         indent = 1 if self.args.cloud else 0
         try:
-            st = self.project.status(target=target,
-                                     jobs=self.args.jobs,
-                                     cloud=self.args.cloud,
-                                     show_checksums=self.args.show_checksums,
-                                     remote=self.args.remote,
-                                     all_branches=self.args.all_branches,
-                                     all_tags=self.args.all_tags,
-                                     with_deps=self.args.with_deps)
+            st = self.repo.status(
+                target=target,
+                jobs=self.args.jobs,
+                cloud=self.args.cloud,
+                show_checksums=self.args.show_checksums,
+                remote=self.args.remote,
+                all_branches=self.args.all_branches,
+                all_tags=self.args.all_tags,
+                with_deps=self.args.with_deps,
+            )
             if st:
-                self._show(st, indent)
+                if self.args.quiet:
+                    return 1
+                else:
+                    self._show(st, indent)
             else:
-                self.project.logger.info(self.UP_TO_DATE_MSG)
+                logger.info(self.UP_TO_DATE_MSG)
 
-        except Exception as exc:
-            self.project.logger.error('Failed to obtain data status', exc)
+        except Exception:
+            logger.exception("failed to obtain data status")
             return 1
         return 0

dvc/command/tag.py

@@ -0,0 +1,154 @@
+import logging
+
+from dvc.utils import to_yaml_string
+from dvc.exceptions import DvcException
+from dvc.command.base import CmdBase, fix_subparsers, append_doc_link
+
+
+logger = logging.getLogger(__name__)
+
+
+class CmdTagAdd(CmdBase):
+    def run(self):
+        for target in self.args.targets:
+            try:
+                self.repo.tag.add(
+                    self.args.tag,
+                    target=target,
+                    with_deps=self.args.with_deps,
+                    recursive=self.args.recursive,
+                )
+            except DvcException:
+                logger.exception("failed to add tag")
+                return 1
+        return 0
+
+
+class CmdTagRemove(CmdBase):
+    def run(self):
+        for target in self.args.targets:
+            try:
+                self.repo.tag.remove(
+                    self.args.tag,
+                    target=target,
+                    with_deps=self.args.with_deps,
+                    recursive=self.args.recursive,
+                )
+            except DvcException:
+                logger.exception("failed to remove tag")
+                return 1
+        return 0
+
+
+class CmdTagList(CmdBase):
+    def run(self):
+        for target in self.args.targets:
+            try:
+                tags = self.repo.tag.list(
+                    target,
+                    with_deps=self.args.with_deps,
+                    recursive=self.args.recursive,
+                )
+                if tags:
+                    logger.info(to_yaml_string(tags))
+            except DvcException:
+                logger.exception("failed list tags")
+                return 1
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    TAG_HELP = "A set of commands to manage DVC tags."
+    tag_parser = subparsers.add_parser(
+        "tag",
+        parents=[parent_parser],
+        description=append_doc_link(TAG_HELP, "tag"),
+        add_help=False,
+    )
+
+    tag_subparsers = tag_parser.add_subparsers(
+        dest="cmd",
+        help="Use DVC tag CMD --help to display command-specific help.",
+    )
+
+    fix_subparsers(tag_subparsers)
+
+    TAG_ADD_HELP = "Add DVC tag."
+    tag_add_parser = tag_subparsers.add_parser(
+        "add",
+        parents=[parent_parser],
+        description=append_doc_link(TAG_ADD_HELP, "tag-add"),
+        help=TAG_ADD_HELP,
+    )
+    tag_add_parser.add_argument("tag", help="Dvc tag.")
+    tag_add_parser.add_argument(
+        "targets", nargs="*", default=[None], help="Dvc files."
+    )
+    tag_add_parser.add_argument(
+        "-d",
+        "--with-deps",
+        action="store_true",
+        default=False,
+        help="Add tag for all dependencies of the specified DVC file.",
+    )
+    tag_add_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="Add tag for subdirectories of the specified directory.",
+    )
+    tag_add_parser.set_defaults(func=CmdTagAdd)
+
+    TAG_REMOVE_HELP = "Remove DVC tag."
+    tag_remove_parser = tag_subparsers.add_parser(
+        "remove",
+        parents=[parent_parser],
+        description=append_doc_link(TAG_REMOVE_HELP, "tag-remove"),
+        help=TAG_REMOVE_HELP,
+    )
+    tag_remove_parser.add_argument("tag", help="Dvc tag.")
+    tag_remove_parser.add_argument(
+        "targets", nargs="*", default=[None], help="Dvc files."
+    )
+    tag_remove_parser.add_argument(
+        "-d",
+        "--with-deps",
+        action="store_true",
+        default=False,
+        help="Remove tag for all dependencies of the specified DVC file.",
+    )
+    tag_remove_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="Remove tag for subdirectories of the specified directory.",
+    )
+    tag_remove_parser.set_defaults(func=CmdTagRemove)
+
+    TAG_LIST_HELP = "List DVC tags."
+    tag_list_parser = tag_subparsers.add_parser(
+        "list",
+        parents=[parent_parser],
+        description=append_doc_link(TAG_LIST_HELP, "tag-list"),
+        help=TAG_LIST_HELP,
+    )
+    tag_list_parser.add_argument(
+        "targets", nargs="*", default=[None], help="Dvc files."
+    )
+    tag_list_parser.add_argument(
+        "-d",
+        "--with-deps",
+        action="store_true",
+        default=False,
+        help="List tags for all dependencies of the specified DVC file.",
+    )
+    tag_list_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="List tags for subdirectories of the specified directory.",
+    )
+    tag_list_parser.set_defaults(func=CmdTagList)

dvc/command/unprotect.py

@@ -1,14 +1,37 @@
+from __future__ import unicode_literals
+
+import argparse
+import logging
+
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdUnprotect(CmdBase):
     def run(self):
         for target in self.args.targets:
             try:
-                self.project.unprotect(target)
-            except DvcException as ex:
-                msg = "Failed to unprotect '{}'".format(target)
-                self.project.logger.error(msg, ex)
+                self.repo.unprotect(target)
+            except DvcException:
+                msg = "failed to unprotect '{}'".format(target)
+                logger.exception(msg)
                 return 1
         return 0
+
+
+def add_parser(subparsers, parent_parser):
+    UNPROTECT_HELP = "Unprotect a data file/directory."
+    unprotect_parser = subparsers.add_parser(
+        "unprotect",
+        parents=[parent_parser],
+        description=append_doc_link(UNPROTECT_HELP, "unprotect"),
+        help=UNPROTECT_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    unprotect_parser.add_argument(
+        "targets", nargs="+", help="Data files/directory."
+    )
+    unprotect_parser.set_defaults(func=CmdUnprotect)

dvc/command/version.py

@@ -0,0 +1,39 @@
+from __future__ import unicode_literals
+
+import platform
+import argparse
+import logging
+
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.version import __version__
+
+
+logger = logging.getLogger(__name__)
+
+
+class CmdVersion(CmdBaseNoRepo):
+    def run(self):
+        info = (
+            "DVC version: {dvc_version}\n"
+            "Python version: {python_version}\n"
+            "Platform: {platform_type}"
+        ).format(
+            dvc_version=__version__,
+            python_version=platform.python_version(),
+            platform_type=platform.platform(),
+        )
+        logger.info(info)
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    VERSION_HELP = "Show DVC version and system/environment informaion."
+
+    version_parser = subparsers.add_parser(
+        "version",
+        parents=[parent_parser],
+        description=append_doc_link(VERSION_HELP, "version"),
+        help=VERSION_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    version_parser.set_defaults(func=CmdVersion)

dvc/config.py

@@ -1,80 +1,162 @@
-"""
-DVC config objects.
-"""
-import os
-import configobj
-from schema import Schema, Optional, And, Use, Regex
+"""DVC config objects."""
 
+from __future__ import unicode_literals
+
+from dvc.utils.compat import str, open, urlparse
+
+import os
+import errno
+import configobj
+import logging
+
+from schema import Schema, Optional, And, Use, Regex
 from dvc.exceptions import DvcException
 
+logger = logging.getLogger(__name__)
+
 
 class ConfigError(DvcException):
-    """ DVC config exception """
+    """DVC config exception.
+
+    Args:
+        msg (str): error message.
+        ex (Exception): optional exception that has caused this error.
+    """
+
     def __init__(self, msg, ex=None):
-        super(ConfigError, self).__init__('Config file error: {}'.format(msg),
-                                          ex)
-
-
-def supported_url(url):
-    from dvc.remote import supported_url as supported
-    return supported(url)
+        super(ConfigError, self).__init__(
+            "config file error: {}".format(msg), ex
+        )
 
 
 def supported_cache_type(types):
+    """Checks if link type config option has a valid value.
+
+    Args:
+        types (list/string): type(s) of links that dvc should try out.
+    """
     if isinstance(types, str):
-        types = [t.strip() for t in types.split(',')]
-    for t in types:
-        if t not in ['reflink', 'hardlink', 'symlink', 'copy']:
+        types = [typ.strip() for typ in types.split(",")]
+    for typ in types:
+        if typ not in ["reflink", "hardlink", "symlink", "copy"]:
             return False
     return True
 
 
 def supported_loglevel(level):
-    return level in ['info', 'debug', 'warning', 'error']
+    """Checks if log level config option has a valid value.
+
+    Args:
+        level (str): log level name.
+    """
+    return level in ["info", "debug", "warning", "error"]
 
 
 def supported_cloud(cloud):
-    return cloud in ['aws', 'gcp', 'local', '']
+    """Checks if obsoleted cloud option has a valid value.
+
+    Args:
+        cloud (str): cloud type name.
+    """
+    return cloud in ["aws", "gcp", "local", ""]
 
 
 def is_bool(val):
-    return val.lower() in ['true', 'false']
+    """Checks that value is a boolean.
+
+    Args:
+        val (str): string value verify.
+
+    Returns:
+        bool: True if value stands for boolean, False otherwise.
+    """
+    return val.lower() in ["true", "false"]
 
 
 def to_bool(val):
-    return val.lower() == 'true'
+    """Converts value to boolean.
+
+    Args:
+        val (str): string to convert to boolean.
+
+    Returns:
+        bool: True if value.lower() == 'true', False otherwise.
+    """
+    return val.lower() == "true"
 
 
 def is_whole(val):
+    """Checks that value is a whole integer.
+
+    Args:
+        val (str): number string to verify.
+
+    Returns:
+        bool: True if val is a whole number, False otherwise.
+    """
     return int(val) >= 0
 
 
 def is_percent(val):
+    """Checks that value is a percent.
+
+    Args:
+        val (str): number string to verify.
+
+    Returns:
+        bool: True if 0<=value<=100, False otherwise.
+    """
     return int(val) >= 0 and int(val) <= 100
 
 
-class Config(object):
-    CONFIG = 'config'
-    CONFIG_LOCAL = 'config.local'
+class Config(object):  # pylint: disable=too-many-instance-attributes
+    """Class that manages configuration files for a dvc repo.
 
-    SECTION_CORE = 'core'
-    SECTION_CORE_LOGLEVEL = 'loglevel'
+    Args:
+        dvc_dir (str): optional path to `.dvc` directory, that is used to
+            access repo-specific configs like .dvc/config and
+            .dvc/config.local.
+        validate (bool): optional flag to tell dvc if it should validate the
+            config or just load it as is. 'True' by default.
+
+
+    Raises:
+        ConfigError: thrown when config has an invalid format.
+    """
+
+    APPNAME = "dvc"
+    APPAUTHOR = "iterative"
+
+    # NOTE: used internally in RemoteLOCAL to know config
+    # location, that url should resolved relative to.
+    PRIVATE_CWD = "_cwd"
+
+    CONFIG = "config"
+    CONFIG_LOCAL = "config.local"
+
+    BOOL_SCHEMA = And(str, is_bool, Use(to_bool))
+
+    SECTION_CORE = "core"
+    SECTION_CORE_LOGLEVEL = "loglevel"
     SECTION_CORE_LOGLEVEL_SCHEMA = And(Use(str.lower), supported_loglevel)
-    SECTION_CORE_REMOTE = 'remote'
-    SECTION_CORE_INTERACTIVE_SCHEMA = And(str, is_bool, Use(to_bool))
-    SECTION_CORE_INTERACTIVE = 'interactive'
+    SECTION_CORE_REMOTE = "remote"
+    SECTION_CORE_INTERACTIVE_SCHEMA = BOOL_SCHEMA
+    SECTION_CORE_INTERACTIVE = "interactive"
+    SECTION_CORE_ANALYTICS = "analytics"
+    SECTION_CORE_ANALYTICS_SCHEMA = BOOL_SCHEMA
 
-    SECTION_CACHE = 'cache'
-    SECTION_CACHE_DIR = 'dir'
-    SECTION_CACHE_TYPE = 'type'
+    SECTION_CACHE = "cache"
+    SECTION_CACHE_DIR = "dir"
+    SECTION_CACHE_TYPE = "type"
     SECTION_CACHE_TYPE_SCHEMA = supported_cache_type
-    SECTION_CACHE_PROTECTED = 'protected'
-    SECTION_CACHE_LOCAL = 'local'
-    SECTION_CACHE_S3 = 's3'
-    SECTION_CACHE_GS = 'gs'
-    SECTION_CACHE_SSH = 'ssh'
-    SECTION_CACHE_HDFS = 'hdfs'
-    SECTION_CACHE_AZURE = 'azure'
+    SECTION_CACHE_PROTECTED = "protected"
+    SECTION_CACHE_LOCAL = "local"
+    SECTION_CACHE_S3 = "s3"
+    SECTION_CACHE_GS = "gs"
+    SECTION_CACHE_SSH = "ssh"
+    SECTION_CACHE_HDFS = "hdfs"
+    SECTION_CACHE_AZURE = "azure"
+    SECTION_CACHE_SLOW_LINK_WARNING = "slow_link_warning"
     SECTION_CACHE_SCHEMA = {
         Optional(SECTION_CACHE_LOCAL): str,
         Optional(SECTION_CACHE_S3): str,
@@ -82,92 +164,110 @@ class Config(object):
         Optional(SECTION_CACHE_HDFS): str,
         Optional(SECTION_CACHE_SSH): str,
         Optional(SECTION_CACHE_AZURE): str,
-
-        Optional(SECTION_CACHE_DIR, default='cache'): str,
+        Optional(SECTION_CACHE_DIR): str,
         Optional(SECTION_CACHE_TYPE, default=None): SECTION_CACHE_TYPE_SCHEMA,
-        Optional(SECTION_CACHE_PROTECTED,
-                 default=False): And(str, is_bool, Use(to_bool)),
+        Optional(SECTION_CACHE_PROTECTED, default=False): BOOL_SCHEMA,
+        Optional(PRIVATE_CWD): str,
+        Optional(SECTION_CACHE_SLOW_LINK_WARNING, default=True): BOOL_SCHEMA,
     }
 
     # backward compatibility
-    SECTION_CORE_CLOUD = 'cloud'
+    SECTION_CORE_CLOUD = "cloud"
     SECTION_CORE_CLOUD_SCHEMA = And(Use(str.lower), supported_cloud)
-    SECTION_CORE_STORAGEPATH = 'storagepath'
+    SECTION_CORE_STORAGEPATH = "storagepath"
 
     SECTION_CORE_SCHEMA = {
-        Optional(SECTION_CORE_LOGLEVEL,
-                 default='info'): And(str, Use(str.lower),
-                                      SECTION_CORE_LOGLEVEL_SCHEMA),
-        Optional(SECTION_CORE_REMOTE, default=''): And(str, Use(str.lower)),
-        Optional(SECTION_CORE_INTERACTIVE,
-                 default=False): SECTION_CORE_INTERACTIVE_SCHEMA,
-
+        Optional(SECTION_CORE_LOGLEVEL): And(
+            str, Use(str.lower), SECTION_CORE_LOGLEVEL_SCHEMA
+        ),
+        Optional(SECTION_CORE_REMOTE, default=""): And(str, Use(str.lower)),
+        Optional(
+            SECTION_CORE_INTERACTIVE, default=False
+        ): SECTION_CORE_INTERACTIVE_SCHEMA,
+        Optional(
+            SECTION_CORE_ANALYTICS, default=True
+        ): SECTION_CORE_ANALYTICS_SCHEMA,
         # backward compatibility
-        Optional(SECTION_CORE_CLOUD, default=''): SECTION_CORE_CLOUD_SCHEMA,
-        Optional(SECTION_CORE_STORAGEPATH, default=''): str,
+        Optional(SECTION_CORE_CLOUD, default=""): SECTION_CORE_CLOUD_SCHEMA,
+        Optional(SECTION_CORE_STORAGEPATH, default=""): str,
     }
 
     # backward compatibility
-    SECTION_AWS = 'aws'
-    SECTION_AWS_STORAGEPATH = 'storagepath'
-    SECTION_AWS_CREDENTIALPATH = 'credentialpath'
-    SECTION_AWS_ENDPOINT_URL = 'endpointurl'
-    SECTION_AWS_REGION = 'region'
-    SECTION_AWS_PROFILE = 'profile'
+    SECTION_AWS = "aws"
+    SECTION_AWS_STORAGEPATH = "storagepath"
+    SECTION_AWS_CREDENTIALPATH = "credentialpath"
+    SECTION_AWS_ENDPOINT_URL = "endpointurl"
+    SECTION_AWS_LIST_OBJECTS = "listobjects"
+    SECTION_AWS_REGION = "region"
+    SECTION_AWS_PROFILE = "profile"
+    SECTION_AWS_USE_SSL = "use_ssl"
     SECTION_AWS_SCHEMA = {
         SECTION_AWS_STORAGEPATH: str,
         Optional(SECTION_AWS_REGION): str,
         Optional(SECTION_AWS_PROFILE): str,
         Optional(SECTION_AWS_CREDENTIALPATH): str,
         Optional(SECTION_AWS_ENDPOINT_URL): str,
+        Optional(SECTION_AWS_LIST_OBJECTS, default=False): BOOL_SCHEMA,
+        Optional(SECTION_AWS_USE_SSL, default=True): BOOL_SCHEMA,
     }
 
     # backward compatibility
-    SECTION_GCP = 'gcp'
+    SECTION_GCP = "gcp"
     SECTION_GCP_STORAGEPATH = SECTION_AWS_STORAGEPATH
-    SECTION_GCP_PROJECTNAME = 'projectname'
+    SECTION_GCP_CREDENTIALPATH = SECTION_AWS_CREDENTIALPATH
+    SECTION_GCP_PROJECTNAME = "projectname"
     SECTION_GCP_SCHEMA = {
         SECTION_GCP_STORAGEPATH: str,
         Optional(SECTION_GCP_PROJECTNAME): str,
     }
 
     # backward compatibility
-    SECTION_LOCAL = 'local'
+    SECTION_LOCAL = "local"
     SECTION_LOCAL_STORAGEPATH = SECTION_AWS_STORAGEPATH
-    SECTION_LOCAL_SCHEMA = {
-        SECTION_LOCAL_STORAGEPATH: str,
-    }
+    SECTION_LOCAL_SCHEMA = {SECTION_LOCAL_STORAGEPATH: str}
+
+    SECTION_AZURE_CONNECTION_STRING = "connection_string"
+    # Alibabacloud oss options
+    SECTION_OSS_ACCESS_KEY_ID = "oss_key_id"
+    SECTION_OSS_ACCESS_KEY_SECRET = "oss_key_secret"
+    SECTION_OSS_ENDPOINT = "oss_endpoint"
 
     SECTION_REMOTE_REGEX = r'^\s*remote\s*"(?P<name>.*)"\s*$'
     SECTION_REMOTE_FMT = 'remote "{}"'
-    SECTION_REMOTE_URL = 'url'
-    SECTION_REMOTE_USER = 'user'
-    SECTION_REMOTE_PORT = 'port'
-    SECTION_REMOTE_KEY_FILE = 'keyfile'
-    SECTION_REMOTE_TIMEOUT = 'timeout'
-    SECTION_REMOTE_PASSWORD = 'password'
-    SECTION_REMOTE_ASK_PASSWORD = 'ask_password'
+    SECTION_REMOTE_URL = "url"
+    SECTION_REMOTE_USER = "user"
+    SECTION_REMOTE_PORT = "port"
+    SECTION_REMOTE_KEY_FILE = "keyfile"
+    SECTION_REMOTE_TIMEOUT = "timeout"
+    SECTION_REMOTE_PASSWORD = "password"
+    SECTION_REMOTE_ASK_PASSWORD = "ask_password"
     SECTION_REMOTE_SCHEMA = {
-        SECTION_REMOTE_URL: And(supported_url, error="Unsupported URL"),
+        SECTION_REMOTE_URL: str,
         Optional(SECTION_AWS_REGION): str,
         Optional(SECTION_AWS_PROFILE): str,
         Optional(SECTION_AWS_CREDENTIALPATH): str,
         Optional(SECTION_AWS_ENDPOINT_URL): str,
+        Optional(SECTION_AWS_LIST_OBJECTS, default=False): BOOL_SCHEMA,
+        Optional(SECTION_AWS_USE_SSL, default=True): BOOL_SCHEMA,
         Optional(SECTION_GCP_PROJECTNAME): str,
         Optional(SECTION_CACHE_TYPE): SECTION_CACHE_TYPE_SCHEMA,
-        Optional(SECTION_CACHE_PROTECTED,
-                 default=False): And(str, is_bool, Use(to_bool)),
+        Optional(SECTION_CACHE_PROTECTED, default=False): BOOL_SCHEMA,
         Optional(SECTION_REMOTE_USER): str,
         Optional(SECTION_REMOTE_PORT): Use(int),
         Optional(SECTION_REMOTE_KEY_FILE): str,
         Optional(SECTION_REMOTE_TIMEOUT): Use(int),
         Optional(SECTION_REMOTE_PASSWORD): str,
-        Optional(SECTION_REMOTE_ASK_PASSWORD): And(str, is_bool, Use(to_bool)),
+        Optional(SECTION_REMOTE_ASK_PASSWORD): BOOL_SCHEMA,
+        Optional(SECTION_AZURE_CONNECTION_STRING): str,
+        Optional(SECTION_OSS_ACCESS_KEY_ID): str,
+        Optional(SECTION_OSS_ACCESS_KEY_SECRET): str,
+        Optional(SECTION_OSS_ENDPOINT): str,
+        Optional(PRIVATE_CWD): str,
     }
 
-    SECTION_STATE = 'state'
-    SECTION_STATE_ROW_LIMIT = 'row_limit'
-    SECTION_STATE_ROW_CLEANUP_QUOTA = 'row_cleanup_quota'
+    SECTION_STATE = "state"
+    SECTION_STATE_ROW_LIMIT = "row_limit"
+    SECTION_STATE_ROW_CLEANUP_QUOTA = "row_cleanup_quota"
     SECTION_STATE_SCHEMA = {
         Optional(SECTION_STATE_ROW_LIMIT): And(Use(int), is_whole),
         Optional(SECTION_STATE_ROW_CLEANUP_QUOTA): And(Use(int), is_percent),
@@ -178,46 +278,329 @@ class Config(object):
         Optional(Regex(SECTION_REMOTE_REGEX)): SECTION_REMOTE_SCHEMA,
         Optional(SECTION_CACHE, default={}): SECTION_CACHE_SCHEMA,
         Optional(SECTION_STATE, default={}): SECTION_STATE_SCHEMA,
-
         # backward compatibility
         Optional(SECTION_AWS, default={}): SECTION_AWS_SCHEMA,
         Optional(SECTION_GCP, default={}): SECTION_GCP_SCHEMA,
         Optional(SECTION_LOCAL, default={}): SECTION_LOCAL_SCHEMA,
     }
 
-    def __init__(self, dvc_dir):
-        self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
-        self.config_file = os.path.join(dvc_dir, self.CONFIG)
-        self.config_local_file = os.path.join(dvc_dir, self.CONFIG_LOCAL)
+    def __init__(self, dvc_dir=None, validate=True):
+        self.system_config_file = os.path.join(
+            self.get_system_config_dir(), self.CONFIG
+        )
+        self.global_config_file = os.path.join(
+            self.get_global_config_dir(), self.CONFIG
+        )
 
+        if dvc_dir is not None:
+            self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
+            self.config_file = os.path.join(dvc_dir, self.CONFIG)
+            self.config_local_file = os.path.join(dvc_dir, self.CONFIG_LOCAL)
+        else:
+            self.dvc_dir = None
+            self.config_file = None
+            self.config_local_file = None
+
+        self._system_config = None
+        self._global_config = None
+        self._repo_config = None
+        self._local_config = None
+
+        self.config = None
+
+        self.load(validate=validate)
+
+    @staticmethod
+    def get_global_config_dir():
+        """Returns global config location. E.g. ~/.config/dvc/config.
+
+        Returns:
+            str: path to the global config directory.
+        """
+        from appdirs import user_config_dir
+
+        return user_config_dir(
+            appname=Config.APPNAME, appauthor=Config.APPAUTHOR
+        )
+
+    @staticmethod
+    def get_system_config_dir():
+        """Returns system config location. E.g. /etc/dvc.conf.
+
+        Returns:
+            str: path to the system config directory.
+        """
+        from appdirs import site_config_dir
+
+        return site_config_dir(
+            appname=Config.APPNAME, appauthor=Config.APPAUTHOR
+        )
+
+    @staticmethod
+    def init(dvc_dir):
+        """Initializes dvc config.
+
+        Args:
+            dvc_dir (str): path to .dvc directory.
+
+        Returns:
+            dvc.config.Config: config object.
+        """
+        config_file = os.path.join(dvc_dir, Config.CONFIG)
+        open(config_file, "w+").close()
+        return Config(dvc_dir)
+
+    def _load(self):
+        self._system_config = configobj.ConfigObj(self.system_config_file)
+        self._global_config = configobj.ConfigObj(self.global_config_file)
+
+        if self.config_file is not None:
+            self._repo_config = configobj.ConfigObj(self.config_file)
+        else:
+            self._repo_config = configobj.ConfigObj()
+
+        if self.config_local_file is not None:
+            self._local_config = configobj.ConfigObj(self.config_local_file)
+        else:
+            self._local_config = configobj.ConfigObj()
+
+        self.config = None
+
+    def _load_config(self, path):
+        config = configobj.ConfigObj(path)
+        config = self._lower(config)
+        self._resolve_paths(config, path)
+        return config
+
+    @staticmethod
+    def _resolve_path(path, config_file):
+        assert os.path.isabs(config_file)
+        config_dir = os.path.dirname(config_file)
+        return os.path.abspath(os.path.join(config_dir, path))
+
+    def _resolve_cache_path(self, config, fname):
+        cache = config.get(self.SECTION_CACHE)
+        if cache is None:
+            return
+
+        cache_dir = cache.get(self.SECTION_CACHE_DIR)
+        if cache_dir is None:
+            return
+
+        cache[self.PRIVATE_CWD] = os.path.dirname(fname)
+
+    def _resolve_paths(self, config, fname):
+        if fname is None:
+            return
+
+        self._resolve_cache_path(config, fname)
+        for section in config.values():
+            if self.SECTION_REMOTE_URL not in section.keys():
+                continue
+
+            section[self.PRIVATE_CWD] = os.path.dirname(fname)
+
+    def load(self, validate=True):
+        """Loads config from all the config files.
+
+        Args:
+            validate (bool): optional flag to tell dvc if it should validate
+                the config or just load it as is. 'True' by default.
+
+
+        Raises:
+            dvc.config.ConfigError: thrown if config has invalid format.
+        """
+        self._load()
         try:
-            self._config = configobj.ConfigObj(self.config_file)
-            local = configobj.ConfigObj(self.config_local_file)
+            self.config = self._load_config(self.system_config_file)
+            user = self._load_config(self.global_config_file)
+            config = self._load_config(self.config_file)
+            local = self._load_config(self.config_local_file)
 
             # NOTE: schema doesn't support ConfigObj.Section validation, so we
             # need to convert our config to dict before passing it to
-            self._config = self._lower(self._config)
-            local = self._lower(local)
-            self._config = self._merge(self._config, local)
+            for conf in [user, config, local]:
+                self.config = self._merge(self.config, conf)
 
-            self._config = Schema(self.SCHEMA).validate(self._config)
+            if validate:
+                self.config = Schema(self.SCHEMA).validate(self.config)
 
             # NOTE: now converting back to ConfigObj
-            self._config = configobj.ConfigObj(self._config,
-                                               write_empty_values=True)
-            self._config.filename = self.config_file
+            self.config = configobj.ConfigObj(
+                self.config, write_empty_values=True
+            )
+            self.config.filename = self.config_file
+            self._resolve_paths(self.config, self.config_file)
         except Exception as ex:
             raise ConfigError(ex)
 
+    @staticmethod
+    def _get_key(conf, name, add=False):
+        for k in conf.keys():
+            if k.lower() == name.lower():
+                return k
+
+        if add:
+            conf[name] = {}
+            return name
+
+        return None
+
+    def save(self, config=None):
+        """Saves config to config files.
+
+        Args:
+            config (configobj.ConfigObj): optional config object to save.
+
+        Raises:
+            dvc.config.ConfigError: thrown if failed to write config file.
+        """
+        if config is not None:
+            clist = [config]
+        else:
+            clist = [
+                self._system_config,
+                self._global_config,
+                self._repo_config,
+                self._local_config,
+            ]
+
+        for conf in clist:
+            if conf.filename is None:
+                continue
+
+            try:
+                logger.debug("Writing '{}'.".format(conf.filename))
+                dname = os.path.dirname(os.path.abspath(conf.filename))
+                try:
+                    os.makedirs(dname)
+                except OSError as exc:
+                    if exc.errno != errno.EEXIST:
+                        raise
+                conf.write()
+            except Exception as exc:
+                msg = "failed to write config '{}'".format(conf.filename)
+                raise ConfigError(msg, exc)
+
+    def get_remote_settings(self, name):
+        import posixpath
+
+        """
+        Args:
+            name (str): The name of the remote that we want to retrieve
+
+        Returns:
+            dict: The content beneath the given remote name.
+
+        Example:
+            >>> config = {'remote "server"': {'url': 'ssh://localhost/'}}
+            >>> get_remote_settings("server")
+            {'url': 'ssh://localhost/'}
+        """
+        settings = self.config[self.SECTION_REMOTE_FMT.format(name)]
+        parsed = urlparse(settings["url"])
+
+        # Support for cross referenced remotes.
+        # This will merge the settings, giving priority to the outer reference.
+        # For example, having:
+        #
+        #       dvc remote add server ssh://localhost
+        #       dvc remote modify server user root
+        #       dvc remote modify server ask_password true
+        #
+        #       dvc remote add images remote://server/tmp/pictures
+        #       dvc remote modify images user alice
+        #       dvc remote modify images ask_password false
+        #       dvc remote modify images password asdf1234
+        #
+        # Results on a config dictionary like:
+        #
+        #       {
+        #           "url": "ssh://localhost/tmp/pictures",
+        #           "user": "alice",
+        #           "password": "asdf1234",
+        #           "ask_password": False,
+        #       }
+        #
+        if parsed.scheme == "remote":
+            reference = self.get_remote_settings(parsed.netloc)
+            url = posixpath.join(reference["url"], parsed.path.lstrip("/"))
+            merged = reference.copy()
+            merged.update(settings)
+            merged["url"] = url
+            return merged
+
+        return settings
+
+    @staticmethod
+    def unset(config, section, opt=None):
+        """Unsets specified option and/or section in the config.
+
+        Args:
+            config (configobj.ConfigObj): config to work on.
+            section (str): section name.
+            opt (str): optional option name.
+        """
+        if section not in config.keys():
+            raise ConfigError("section '{}' doesn't exist".format(section))
+
+        if opt is None:
+            del config[section]
+            return
+
+        if opt not in config[section].keys():
+            raise ConfigError(
+                "option '{}.{}' doesn't exist".format(section, opt)
+            )
+        del config[section][opt]
+
+        if not config[section]:
+            del config[section]
+
+    @staticmethod
+    def set(config, section, opt, value):
+        """Sets specified option in the config.
+
+        Args:
+            config (configobj.ConfigObj): config to work on.
+            section (str): section name.
+            opt (str): option name.
+            value: value to set option to.
+        """
+        if section not in config.keys():
+            config[section] = {}
+
+        config[section][opt] = value
+
+    @staticmethod
+    def show(config, section, opt):
+        """Prints option value from the config.
+
+        Args:
+            config (configobj.ConfigObj): config to work on.
+            section (str): section name.
+            opt (str): option name.
+        """
+        if section not in config.keys():
+            raise ConfigError("section '{}' doesn't exist".format(section))
+
+        if opt not in config[section].keys():
+            raise ConfigError(
+                "option '{}.{}' doesn't exist".format(section, opt)
+            )
+
+        logger.info(config[section][opt])
+
     @staticmethod
     def _merge(first, second):
         res = {}
         sections = list(first.keys()) + list(second.keys())
         for section in sections:
-            f = first.get(section, {}).copy()
-            s = second.get(section, {}).copy()
-            f.update(s)
-            res[section] = f
+            first_copy = first.get(section, {}).copy()
+            second_copy = second.get(section, {}).copy()
+            first_copy.update(second_copy)
+            res[section] = first_copy
         return res
 
     @staticmethod
@@ -226,12 +609,6 @@ class Config(object):
         for s_key, s_value in config.items():
             new_s = {}
             for key, value in s_value.items():
-                new_s[key.lower()] = value
+                new_s[key.lower()] = str(value)
             new_config[s_key.lower()] = new_s
         return new_config
-
-    @staticmethod
-    def init(dvc_dir):
-        config_file = os.path.join(dvc_dir, Config.CONFIG)
-        open(config_file, 'w+').close()
-        return Config(dvc_dir)

dvc/daemon.py

@@ -0,0 +1,107 @@
+"""Launch `dvc daemon` command in a separate detached process."""
+
+from __future__ import unicode_literals
+
+import os
+import sys
+import inspect
+import logging
+from subprocess import Popen
+
+from dvc.env import DVC_DAEMON
+from dvc.utils import is_binary, fix_env
+from dvc.utils.compat import cast_bytes_py2
+
+
+logger = logging.getLogger(__name__)
+
+CREATE_NEW_PROCESS_GROUP = 0x00000200
+DETACHED_PROCESS = 0x00000008
+
+
+def _spawn_windows(cmd, env):
+    from subprocess import STARTUPINFO, STARTF_USESHOWWINDOW
+
+    creationflags = CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS
+
+    startupinfo = STARTUPINFO()
+    startupinfo.dwFlags |= STARTF_USESHOWWINDOW
+
+    Popen(
+        cmd,
+        env=env,
+        close_fds=True,
+        shell=False,
+        creationflags=creationflags,
+        startupinfo=startupinfo,
+    ).communicate()
+
+
+def _spawn_posix(cmd, env):
+    # NOTE: using os._exit instead of sys.exit, because dvc built
+    # with PyInstaller has trouble with SystemExit exeption and throws
+    # errors such as "[26338] Failed to execute script __main__"
+    try:
+        pid = os.fork()
+        if pid > 0:
+            return
+    except OSError:
+        logger.exception("failed at first fork")
+        os._exit(1)  # pylint: disable=protected-access
+
+    os.setsid()
+    os.umask(0)
+
+    try:
+        pid = os.fork()
+        if pid > 0:
+            os._exit(0)  # pylint: disable=protected-access
+    except OSError:
+        logger.exception("failed at second fork")
+        os._exit(1)  # pylint: disable=protected-access
+
+    sys.stdin.close()
+    sys.stdout.close()
+    sys.stderr.close()
+
+    Popen(cmd, env=env, close_fds=True, shell=False).communicate()
+
+    os._exit(0)  # pylint: disable=protected-access
+
+
+def _spawn(cmd, env):
+    logger.debug("Trying to spawn '{}'".format(cmd))
+
+    if os.name == "nt":
+        _spawn_windows(cmd, env)
+    elif os.name == "posix":
+        _spawn_posix(cmd, env)
+    else:
+        raise NotImplementedError
+
+    logger.debug("Spawned '{}'".format(cmd))
+
+
+def daemon(args):
+    """Launch a `dvc daemon` command in a detached process.
+
+    Args:
+        args (list): list of arguments to append to `dvc daemon` command.
+    """
+    if os.environ.get(DVC_DAEMON):
+        logger.debug("skipping launching a new daemon.")
+        return
+
+    cmd = [sys.executable]
+    if not is_binary():
+        cmd += ["-m", "dvc"]
+    cmd += ["daemon", "-q"] + args
+
+    env = fix_env()
+    file_path = os.path.abspath(inspect.stack()[0][1])
+    env[cast_bytes_py2("PYTHONPATH")] = cast_bytes_py2(
+        os.path.dirname(os.path.dirname(file_path))
+    )
+    env[cast_bytes_py2(DVC_DAEMON)] = cast_bytes_py2("1")
+
+    _spawn(cmd, env)

dvc/dagascii.py

@@ -1,13 +1,52 @@
+"""Draws DAG in ASCII."""
+
+from __future__ import unicode_literals
+from __future__ import print_function
+
 import sys
 import math
-import select
 
 from grandalf.graphs import Vertex, Edge, Graph
 from grandalf.layouts import SugiyamaLayout
 from grandalf.routing import route_with_lines, EdgeViewer
 
 
+class VertexViewer(object):
+    """Class to define vertex box boundaries that will be accounted for during
+    graph building by grandalf.
+
+    Args:
+        name (str): name of the vertex.
+    """
+
+    HEIGHT = 3  # top and bottom box edges + text
+
+    def __init__(self, name):
+        # pylint: disable=invalid-name
+        self._h = self.HEIGHT  # top and bottom box edges + text
+        self._w = len(name) + 2  # right and left bottom edges + text
+
+    @property
+    def h(self):  # pylint: disable=invalid-name
+        """Height of the box."""
+        return self._h
+
+    @property
+    def w(self):  # pylint: disable=invalid-name
+        """Width of the box."""
+        return self._w
+
+
 class AsciiCanvas(object):
+    """Class for drawing in ASCII.
+
+    Args:
+        cols (int): number of columns in the canvas. Should be > 1.
+        lines (int): number of lines in the canvas. Should be > 1.
+    """
+
+    TIMEOUT = 10
+
     def __init__(self, cols, lines):
         assert cols > 1
         assert lines > 1
@@ -15,17 +54,22 @@ class AsciiCanvas(object):
         self.cols = cols
         self.lines = lines
 
-        self.canvas = [[' '] * cols for l in range(lines)]
+        self.canvas = [[" "] * cols for l in range(lines)]
 
     def draw(self):
+        """Draws ASCII canvas on the screen."""
         if sys.stdout.isatty():  # pragma: no cover
             from asciimatics.screen import Screen
+
             Screen.wrapper(self._do_draw)
         else:
             for line in self.canvas:
-                print(''.join(line))
+                print("".join(line))
 
     def _do_draw(self, screen):  # pragma: no cover
+        # pylint: disable=too-many-locals
+        # pylint: disable=too-many-branches, too-many-statements
+        from dvc.system import System
         from asciimatics.event import KeyboardEvent
 
         offset_x = 0
@@ -52,42 +96,50 @@ class AsciiCanvas(object):
                 line = []
                 for x in range(smaxcol + 1):
                     x_index = offset_x + x
-                    if len(self.canvas) > y_index \
-                       and len(self.canvas[y_index]) > x_index:
-                            line.append(self.canvas[y_index][x_index])
+                    if (
+                        len(self.canvas) > y_index
+                        and len(self.canvas[y_index]) > x_index
+                    ):
+                        line.append(self.canvas[y_index][x_index])
                     else:
-                        line.append(' ')
+                        line.append(" ")
                 assert len(line) == (smaxcol + 1)
-                screen.print_at(''.join(line), 0, y)
+                screen.print_at("".join(line), 0, y)
 
             screen.refresh()
 
             # NOTE: get_event() doesn't block by itself,
             # so we have to do the blocking ourselves.
-            select.select([sys.stdin], [], [], None)
+            #
+            # NOTE: using this workaround while waiting for PR [1]
+            # to get merged and released. After that need to adjust
+            # asciimatics version requirements.
+            #
+            # [1] https://github.com/peterbrittain/asciimatics/pull/188
+            System.wait_for_input(self.TIMEOUT)
 
             event = screen.get_event()
             if not isinstance(event, KeyboardEvent):
                 continue
 
             k = event.key_code
-            if k == screen.KEY_DOWN or k == ord('s'):
+            if k == screen.KEY_DOWN or k == ord("s"):
                 offset_y += 1
-            elif k == screen.KEY_PAGE_DOWN or k == ord('S'):
+            elif k == screen.KEY_PAGE_DOWN or k == ord("S"):
                 offset_y += smaxrow
-            elif k == screen.KEY_UP or k == ord('w'):
+            elif k == screen.KEY_UP or k == ord("w"):
                 offset_y -= 1
-            elif k == screen.KEY_PAGE_UP or k == ord('W'):
+            elif k == screen.KEY_PAGE_UP or k == ord("W"):
                 offset_y -= smaxrow
-            elif k == screen.KEY_RIGHT or k == ord('d'):
+            elif k == screen.KEY_RIGHT or k == ord("d"):
                 offset_x += 1
-            elif k == ord('D'):
+            elif k == ord("D"):
                 offset_x += smaxcol
-            elif k == screen.KEY_LEFT or k == ord('a'):
+            elif k == screen.KEY_LEFT or k == ord("a"):
                 offset_x -= 1
-            elif k == ord('A'):
+            elif k == ord("A"):
                 offset_x -= smaxcol
-            elif k == ord('q') or k == ord('Q'):
+            elif k == ord("q") or k == ord("Q"):
                 break
 
             if offset_y > max_y:
@@ -101,6 +153,17 @@ class AsciiCanvas(object):
                 offset_x = 0
 
     def point(self, x, y, char):
+        """Create a point on ASCII canvas.
+
+        Args:
+            x (int): x coordinate. Should be >= 0 and < number of columns in
+                the canvas.
+            y (int): y coordinate. Should be >= 0 an < number of lines in the
+                canvas.
+            char (str): character to place in the specified point on the
+                canvas.
+        """
+        assert len(char) == 1
         assert x >= 0
         assert x < self.cols
         assert y >= 0
@@ -109,6 +172,16 @@ class AsciiCanvas(object):
         self.canvas[y][x] = char
 
     def line(self, x0, y0, x1, y1, char):
+        """Create a line on ASCII canvas.
+
+        Args:
+            x0 (int): x coordinate where the line should start.
+            y0 (int): y coordinate where the line should start.
+            x1 (int): x coordinate where the line should end.
+            y1 (int): y coordinate where the line should end.
+            char (str): character to draw the line with.
+        """
+        # pylint: disable=too-many-arguments, too-many-branches
         if x0 > x1:
             x1, x0 = x0, x1
             y1, y0 = y0, y1
@@ -141,138 +214,157 @@ class AsciiCanvas(object):
                 self.point(x, y, char)
 
     def text(self, x, y, text):
-        for i, c in enumerate(text):
-            self.point(x + i, y, c)
+        """Print a text on ASCII canvas.
 
-    def box(self, x0, y0, w, h):
-        assert w > 1
-        assert h > 1
+        Args:
+            x (int): x coordinate where the text should start.
+            y (int): y coordinate where the text should start.
+            text (str): string that should be printed.
+        """
+        for i, char in enumerate(text):
+            self.point(x + i, y, char)
 
-        w -= 1
-        h -= 1
+    def box(self, x0, y0, width, height):
+        """Create a box on ASCII canvas.
 
-        for x in range(x0, x0 + w):
-            self.point(x, y0, '-')
-            self.point(x, y0 + h, '-')
+        Args:
+            x0 (int): x coordinate of the box corner.
+            y0 (int): y coordinate of the box corner.
+            width (int): box width.
+            height (int): box height.
+        """
+        assert width > 1
+        assert height > 1
 
-        for y in range(y0, y0 + h):
-            self.point(x0, y, '|')
-            self.point(x0 + w, y, '|')
+        width -= 1
+        height -= 1
 
-        self.point(x0, y0, '+')
-        self.point(x0 + w, y0, '+')
-        self.point(x0, y0 + h, '+')
-        self.point(x0 + w, y0 + h, '+')
+        for x in range(x0, x0 + width):
+            self.point(x, y0, "-")
+            self.point(x, y0 + height, "-")
+
+        for y in range(y0, y0 + height):
+            self.point(x0, y, "|")
+            self.point(x0 + width, y, "|")
+
+        self.point(x0, y0, "+")
+        self.point(x0 + width, y0, "+")
+        self.point(x0, y0 + height, "+")
+        self.point(x0 + width, y0 + height, "+")
 
 
-class Dagascii(object):
-    def __init__(self, vertexes, edges):
-        #
-        # Just a reminder about naming conventions:
-        # +------------X
-        # |
-        # |
-        # |
-        # |
-        # Y
-        #
+def _build_sugiyama_layout(vertexes, edges):
+    #
+    # Just a reminder about naming conventions:
+    # +------------X
+    # |
+    # |
+    # |
+    # |
+    # Y
+    #
 
-        V = {v: Vertex(" {} ".format(v)) for v in vertexes}
-        # NOTE: reverting edges to correctly orientate the graph
-        E = [Edge(V[e], V[s]) for s, e in edges]
-        V = V.values()
-        g = Graph(V, E)
+    vertexes = {v: Vertex(" {} ".format(v)) for v in vertexes}
+    # NOTE: reverting edges to correctly orientate the graph
+    edges = [Edge(vertexes[e], vertexes[s]) for s, e in edges]
+    vertexes = vertexes.values()
+    graph = Graph(vertexes, edges)
 
-        class VertexViewer(object):
-            h = 3  # top and bottom box edges + text
+    for vertex in vertexes:
+        vertex.view = VertexViewer(vertex.data)
 
-            def __init__(self, name):
-                self.w = len(name) + 2  # right and left bottom edges + text
+    # NOTE: determine min box length to create the best layout
+    minw = min([v.view.w for v in vertexes])
 
-        for v in V:
-            v.view = VertexViewer(v.data)
+    for edge in edges:
+        edge.view = EdgeViewer()
 
-        # NOTE: determine min box length to create the best layout
-        minw = min([v.view.w for v in V])
+    sug = SugiyamaLayout(graph.C[0])
+    graph = graph.C[0]
+    roots = list(filter(lambda x: len(x.e_in()) == 0, graph.sV))
 
-        for e in E:
-            e.view = EdgeViewer()
+    sug.init_all(roots=roots, optimize=True)
 
-        sug = SugiyamaLayout(g.C[0])
-        gr = g.C[0]
-        r = list(filter(lambda x: len(x.e_in()) == 0, gr.sV))
+    sug.yspace = VertexViewer.HEIGHT
+    sug.xspace = minw
+    sug.route_edge = route_with_lines
 
-        sug.init_all(roots=r, optimize=True)
+    sug.draw()
 
-        sug.yspace = VertexViewer.h
-        sug.xspace = minw
-        sug.route_edge = route_with_lines
+    return sug
 
-        sug.draw()
 
-        self.sug = sug
+def draw(vertexes, edges):
+    """Build a DAG and draw it in ASCII.
 
-    def draw(self):
-        # NOTE: coordinates might me negative, so we need to shift
-        # everything to the positive plane before we actually draw it.
-        Xs = []
-        Ys = []
+    Args:
+        vertexes (list): list of graph vertexes.
+        edges (list): list of graph edges.
+    """
+    # pylint: disable=too-many-locals
+    # NOTE: coordinates might me negative, so we need to shift
+    # everything to the positive plane before we actually draw it.
+    Xs = []  # pylint: disable=invalid-name
+    Ys = []  # pylint: disable=invalid-name
 
-        for v in self.sug.g.sV:
-            # NOTE: moving boxes w/2 to the left
-            Xs.append(v.view.xy[0] - v.view.w/2.0)
-            Xs.append(v.view.xy[0] + v.view.w/2.0)
-            Ys.append(v.view.xy[1])
-            Ys.append(v.view.xy[1] + v.view.h)
+    sug = _build_sugiyama_layout(vertexes, edges)
 
-        for e in self.sug.g.sE:
-            for xy in e.view._pts:
-                Xs.append(xy[0])
-                Ys.append(xy[1])
+    for vertex in sug.g.sV:
+        # NOTE: moving boxes w/2 to the left
+        Xs.append(vertex.view.xy[0] - vertex.view.w / 2.0)
+        Xs.append(vertex.view.xy[0] + vertex.view.w / 2.0)
+        Ys.append(vertex.view.xy[1])
+        Ys.append(vertex.view.xy[1] + vertex.view.h)
 
-        minx = min(Xs)
-        miny = min(Ys)
-        maxx = max(Xs)
-        maxy = max(Ys)
+    for edge in sug.g.sE:
+        for x, y in edge.view._pts:  # pylint: disable=protected-access
+            Xs.append(x)
+            Ys.append(y)
 
-        canvas_cols = int(math.ceil(math.ceil(maxx) - math.floor(minx))) + 1
-        canvas_lines = int(round(maxy - miny))
+    minx = min(Xs)
+    miny = min(Ys)
+    maxx = max(Xs)
+    maxy = max(Ys)
 
-        canvas = AsciiCanvas(canvas_cols, canvas_lines)
+    canvas_cols = int(math.ceil(math.ceil(maxx) - math.floor(minx))) + 1
+    canvas_lines = int(round(maxy - miny))
 
-        # NOTE: first draw edges so that node boxes could overwrite them
-        for e in self.sug.g.sE:
-            assert len(e.view._pts) > 1
-            for index in range(1, len(e.view._pts)):
-                start = e.view._pts[index - 1]
-                end = e.view._pts[index]
+    canvas = AsciiCanvas(canvas_cols, canvas_lines)
 
-                start_x = int(round(start[0] - minx))
-                start_y = int(round(start[1] - miny))
-                end_x = int(round(end[0] - minx))
-                end_y = int(round(end[1] - miny))
+    # NOTE: first draw edges so that node boxes could overwrite them
+    for edge in sug.g.sE:
+        # pylint: disable=protected-access
+        assert len(edge.view._pts) > 1
+        for index in range(1, len(edge.view._pts)):
+            start = edge.view._pts[index - 1]
+            end = edge.view._pts[index]
 
-                assert start_x >= 0
-                assert start_y >= 0
-                assert end_x >= 0
-                assert end_y >= 0
+            start_x = int(round(start[0] - minx))
+            start_y = int(round(start[1] - miny))
+            end_x = int(round(end[0] - minx))
+            end_y = int(round(end[1] - miny))
 
-                canvas.line(start_x, start_y,
-                            end_x, end_y,
-                            '*')
+            assert start_x >= 0
+            assert start_y >= 0
+            assert end_x >= 0
+            assert end_y >= 0
 
-        for v in self.sug.g.sV:
-            # NOTE: moving boxes w/2 to the left
-            x = v.view.xy[0] - v.view.w/2.0
-            y = v.view.xy[1]
+            canvas.line(start_x, start_y, end_x, end_y, "*")
 
-            canvas.box(int(round(x - minx)),
-                       int(round(y - miny)),
-                       v.view.w,
-                       v.view.h)
+    for vertex in sug.g.sV:
+        # NOTE: moving boxes w/2 to the left
+        x = vertex.view.xy[0] - vertex.view.w / 2.0
+        y = vertex.view.xy[1]
 
-            canvas.text(int(round(x - minx)) + 1,
-                        int(round(y - miny)) + 1,
-                        v.data)
+        canvas.box(
+            int(round(x - minx)),
+            int(round(y - miny)),
+            vertex.view.w,
+            vertex.view.h,
+        )
 
-        canvas.draw()
+        canvas.text(
+            int(round(x - minx)) + 1, int(round(y - miny)) + 1, vertex.data
+        )
+
+    canvas.draw()

dvc/data_cloud.py

@@ -1,91 +1,107 @@
-from dvc.logger import Logger
-from dvc.config import Config, ConfigError
+"""Manages dvc remotes that user can use with push/pull/status comamnds."""
 
+from __future__ import unicode_literals
+
+import logging
+
+from dvc.config import Config, ConfigError
+from dvc.remote import Remote
 from dvc.remote.s3 import RemoteS3
 from dvc.remote.gs import RemoteGS
-from dvc.remote.azure import RemoteAzure
+from dvc.remote.azure import RemoteAZURE
+from dvc.remote.oss import RemoteOSS
 from dvc.remote.ssh import RemoteSSH
 from dvc.remote.hdfs import RemoteHDFS
 from dvc.remote.local import RemoteLOCAL
+from dvc.remote.http import RemoteHTTP
+
+
+logger = logging.getLogger(__name__)
 
 
 class DataCloud(object):
+    """Class that manages dvc remotes.
+
+    Args:
+        repo (dvc.repo.Repo): repo instance that belongs to the repo that
+            we are working on.
+        config (dict): config of the repo.
+
+    Raises:
+        config.ConfigError: thrown when config has invalid format.
+    """
+
     CLOUD_MAP = {
-        'aws': RemoteS3,
-        'gcp': RemoteGS,
-        'azure': RemoteAzure,
-        'ssh': RemoteSSH,
-        'hdfs': RemoteHDFS,
-        'local': RemoteLOCAL,
+        "aws": RemoteS3,
+        "gcp": RemoteGS,
+        "azure": RemoteAZURE,
+        "oss": RemoteOSS,
+        "ssh": RemoteSSH,
+        "hdfs": RemoteHDFS,
+        "local": RemoteLOCAL,
+        "http": RemoteHTTP,
+        "https": RemoteHTTP,
     }
 
-    def __init__(self, project, config=None):
-        self.project = project
+    def __init__(self, repo, config=None):
+        self.repo = repo
         self._config = config
         self._core = self._config[Config.SECTION_CORE]
 
-        remote = self._core.get(Config.SECTION_CORE_REMOTE, '')
-        if remote == '':
-            if self._core.get(Config.SECTION_CORE_CLOUD, None):
-                # backward compatibility
-                msg = 'Using obsoleted config format. Consider updating.'
-                Logger.warn(msg)
-                self._cloud = self.__init__compat()
-            else:
-                self._cloud = None
-            return
+    @property
+    def _cloud(self):
+        """Returns a Remote instance using the `core.remote` config"""
+        remote = self._core.get(Config.SECTION_CORE_REMOTE, "")
 
-        self._cloud = self._init_remote(remote)
+        if remote:
+            return self._init_remote(remote)
+
+        if self._core.get(Config.SECTION_CORE_CLOUD, None):
+            # backward compatibility
+            msg = "using obsoleted config format. Consider updating."
+            logger.warning(msg)
+            return self._init_compat()
 
-    @staticmethod
-    def supported(config):
-        for cloud in DataCloud.CLOUD_MAP.values():
-            if cloud.supported(config):
-                return cloud
         return None
 
     def _init_remote(self, remote):
-        section = Config.SECTION_REMOTE_FMT.format(remote)
+        section = Config.SECTION_REMOTE_FMT.format(remote).lower()
         cloud_config = self._config.get(section, None)
         if not cloud_config:
-            msg = "Can't find remote section '{}' in config"
+            msg = "can't find remote section '{}' in config"
             raise ConfigError(msg.format(section))
 
-        cloud_type = self.supported(cloud_config)
-        if not cloud_type:
-            raise ConfigError("Unsupported cloud '{}'".format(cloud_config))
+        return Remote(self.repo, cloud_config)
 
-        return self._init_cloud(cloud_config, cloud_type)
-
-    def __init__compat(self):
-        name = self._core.get(Config.SECTION_CORE_CLOUD, '').strip().lower()
-        if name == '':
+    def _init_compat(self):
+        name = self._core.get(Config.SECTION_CORE_CLOUD, "").strip().lower()
+        if name == "":
             self._cloud = None
-            return
+            return None
 
         cloud_type = self.CLOUD_MAP.get(name, None)
         if not cloud_type:
-            msg = "Wrong cloud type {} specified"
-            raise ConfigError(msg.format(name))
+            msg = "wrong cloud type '{}' specified".format(name)
+            raise ConfigError(msg)
 
         cloud_config = self._config.get(name, None)
         if not cloud_config:
-            msg = "Can't find cloud section '{}' in config"
-            raise ConfigError(msg.format(name))
+            msg = "can't find cloud section '{}' in config".format(name)
+            raise ConfigError(msg)
 
         # NOTE: check if the class itself has everything needed for operation.
         # E.g. all the imported packages.
         if not cloud_type.supported(cloud_type.compat_config(cloud_config)):
-            raise ConfigError("Unsupported cloud '{}'".format(name))
+            raise ConfigError("unsupported cloud '{}'".format(name))
 
         return self._init_cloud(cloud_config, cloud_type)
 
     def _init_cloud(self, cloud_config, cloud_type):
         global_storage_path = self._core.get(Config.SECTION_CORE_STORAGEPATH)
         if global_storage_path:
-            Logger.warn('Using obsoleted config format. Consider updating.')
+            logger.warning("using obsoleted config format. Consider updating.")
 
-        cloud = cloud_type(self.project, cloud_config)
+        cloud = cloud_type(self.repo, cloud_config)
         return cloud
 
     def _get_cloud(self, remote, cmd):
@@ -95,39 +111,62 @@ class DataCloud(object):
         if self._cloud:
             return self._cloud
 
-        msg = "No remote repository specified. Setup default " \
-              "repository with\n"\
-              "    dvc config core.remote <name>\n" \
-              "or use:\n" \
-              "    dvc {} -r <name>".format(cmd)
-        raise ConfigError(msg)
+        raise ConfigError(
+            "No remote repository specified. Setup default repository with\n"
+            "    dvc config core.remote <name>\n"
+            "or use:\n"
+            "    dvc {} -r <name>\n".format(cmd)
+        )
 
-    def push(self, targets, jobs=1, remote=None, show_checksums=False):
-        """
-        Push data items in a cloud-agnostic way.
-        """
-        return self.project.cache.local.push(targets,
-                                             jobs=jobs,
-                                             remote=self._get_cloud(remote,
-                                                                    'push'),
-                                             show_checksums=show_checksums)
+    def push(self, targets, jobs=None, remote=None, show_checksums=False):
+        """Push data items in a cloud-agnostic way.
 
-    def pull(self, targets, jobs=1, remote=None, show_checksums=False):
+        Args:
+            targets (list): list of targets to push to the cloud.
+            jobs (int): number of jobs that can be running simultaneously.
+            remote (dvc.remote.base.RemoteBASE): optional remote to push to.
+                By default remote from core.remote config option is used.
+            show_checksums (bool): show checksums instead of file names in
+                information messages.
         """
-        Pull data items in a cloud-agnostic way.
-        """
-        return self.project.cache.local.pull(targets,
-                                             jobs=jobs,
-                                             remote=self._get_cloud(remote,
-                                                                    'pull'),
-                                             show_checksums=show_checksums)
+        return self.repo.cache.local.push(
+            targets,
+            jobs=jobs,
+            remote=self._get_cloud(remote, "push"),
+            show_checksums=show_checksums,
+        )
 
-    def status(self, targets, jobs=1, remote=None, show_checksums=False):
+    def pull(self, targets, jobs=None, remote=None, show_checksums=False):
+        """Pull data items in a cloud-agnostic way.
+
+        Args:
+            targets (list): list of targets to pull from the cloud.
+            jobs (int): number of jobs that can be running simultaneously.
+            remote (dvc.remote.base.RemoteBASE): optional remote to pull from.
+                By default remote from core.remote config option is used.
+            show_checksums (bool): show checksums instead of file names in
+                information messages.
         """
-        Check status of data items in a cloud-agnostic way.
+        return self.repo.cache.local.pull(
+            targets,
+            jobs=jobs,
+            remote=self._get_cloud(remote, "pull"),
+            show_checksums=show_checksums,
+        )
+
+    def status(self, targets, jobs=None, remote=None, show_checksums=False):
+        """Check status of data items in a cloud-agnostic way.
+
+        Args:
+            targets (list): list of targets to check status for.
+            jobs (int): number of jobs that can be running simultaneously.
+            remote (dvc.remote.base.RemoteBASE): optional remote to compare
+                targets to. By default remote from core.remote config option
+                is used.
+            show_checksums (bool): show checksums instead of file names in
+                information messages.
         """
-        cloud = self._get_cloud(remote, 'status')
-        return self.project.cache.local.status(targets,
-                                               jobs=jobs,
-                                               remote=cloud,
-                                               show_checksums=show_checksums)
+        cloud = self._get_cloud(remote, "status")
+        return self.repo.cache.local.status(
+            targets, jobs=jobs, remote=cloud, show_checksums=show_checksums
+        )

dvc/dependency/__init__.py

@@ -1,63 +1,71 @@
+from __future__ import unicode_literals
+
 import schema
 
-try:
-    from urlparse import urlparse
-except ImportError:
-    from urllib.parse import urlparse
+from dvc.scheme import Schemes
 
-from dvc.exceptions import DvcException
-from dvc.config import Config
-
-from dvc.dependency.base import DependencyBase
+import dvc.output as output
+from dvc.output.base import OutputBase
 from dvc.dependency.s3 import DependencyS3
 from dvc.dependency.gs import DependencyGS
 from dvc.dependency.local import DependencyLOCAL
 from dvc.dependency.hdfs import DependencyHDFS
 from dvc.dependency.ssh import DependencySSH
+from dvc.dependency.http import DependencyHTTP
+from dvc.dependency.https import DependencyHTTPS
 
 from dvc.remote import Remote
-from dvc.remote.local import RemoteLOCAL
-from dvc.remote.s3 import RemoteS3
-from dvc.remote.hdfs import RemoteHDFS
 
-DEPS = [DependencyHDFS,
-        DependencyS3,
-        DependencyGS,
-        DependencySSH,
-        DependencyLOCAL]
+DEPS = [
+    DependencyGS,
+    DependencyHDFS,
+    DependencyHTTP,
+    DependencyHTTPS,
+    DependencyS3,
+    DependencySSH,
+    # NOTE: DependencyLOCAL is the default choice
+]
 
-DEP_MAP = {'': DependencyLOCAL,
-           'ssh': DependencySSH,
-           's3': DependencyS3,
-           'gs': DependencyGS,
-           'hdfs': DependencyHDFS}
-
-SCHEMA = {
-    DependencyBase.PARAM_PATH: str,
-    schema.Optional(RemoteLOCAL.PARAM_MD5): schema.Or(str, None),
-    schema.Optional(RemoteS3.PARAM_ETAG): schema.Or(str, None),
-    schema.Optional(RemoteHDFS.PARAM_CHECKSUM): schema.Or(str, None),
+DEP_MAP = {
+    Schemes.LOCAL: DependencyLOCAL,
+    Schemes.SSH: DependencySSH,
+    Schemes.S3: DependencyS3,
+    Schemes.GS: DependencyGS,
+    Schemes.HDFS: DependencyHDFS,
+    Schemes.HTTP: DependencyHTTP,
+    Schemes.HTTPS: DependencyHTTPS,
 }
 
 
+# NOTE: schema for dependencies is basically the same as for outputs, but
+# without output-specific entries like 'cache' (whether or not output is
+# cached, see -o and -O flags for `dvc run`) and 'metric' (whether or not
+# output is a metric file and how to parse it, see `-M` flag for `dvc run`).
+SCHEMA = output.SCHEMA.copy()
+del SCHEMA[schema.Optional(OutputBase.PARAM_CACHE)]
+del SCHEMA[schema.Optional(OutputBase.PARAM_METRIC)]
+
+
 def _get(stage, p, info):
+    from dvc.utils.compat import urlparse
+
     parsed = urlparse(p)
-    if parsed.scheme == 'remote':
-        name = Config.SECTION_REMOTE_FMT.format(parsed.netloc)
-        sect = stage.project.config._config[name]
-        remote = Remote(stage.project, sect)
+
+    if parsed.scheme == "remote":
+        settings = stage.repo.config.get_remote_settings(parsed.netloc)
+        remote = Remote(stage.repo, settings)
         return DEP_MAP[remote.scheme](stage, p, info, remote=remote)
 
     for d in DEPS:
         if d.supported(p):
             return d(stage, p, info)
-    raise DvcException('Dependency \'{}\' is not supported'.format(p))
+    return DependencyLOCAL(stage, p, info)
 
 
 def loadd_from(stage, d_list):
     ret = []
     for d in d_list:
-        p = d.pop(DependencyBase.PARAM_PATH)
+        p = d.pop(OutputBase.PARAM_PATH)
         ret.append(_get(stage, p, d))
     return ret
 

dvc/dependency/base.py

@@ -1,79 +1,22 @@
-import re
+from __future__ import unicode_literals
 
 from dvc.exceptions import DvcException
 
 
-class DependencyError(DvcException):
-    def __init__(self, path, msg):
-        msg = 'Dependency \'{}\' error: {}'
-        super(DependencyError, self).__init__(msg.format(path, msg))
-
-
-class DependencyDoesNotExistError(DependencyError):
+class DependencyDoesNotExistError(DvcException):
     def __init__(self, path):
-        msg = 'does not exist'
-        super(DependencyDoesNotExistError, self).__init__(path, msg)
+        msg = "dependency '{}' does not exist".format(path)
+        super(DependencyDoesNotExistError, self).__init__(msg)
 
 
-class DependencyIsNotFileOrDirError(DependencyError):
+class DependencyIsNotFileOrDirError(DvcException):
     def __init__(self, path):
-        msg = 'not a file or directory'
-        super(DependencyIsNotFileOrDirError, self).__init__(path, msg)
+        msg = "dependency '{}' is not a file or directory".format(path)
+        super(DependencyIsNotFileOrDirError, self).__init__(msg)
 
 
 class DependencyBase(object):
-    REGEX = None
+    IS_DEPENDENCY = True
 
-    PARAM_PATH = 'path'
-
-    def __init__(self, stage, path):
-        self.stage = stage
-        self.project = stage.project
-        self.path = path
-
-    def __str__(self):
-        return self.path
-
-    @classmethod
-    def match(cls, url):
-        return re.match(cls.REGEX, url)
-
-    def group(self, name):
-        match = self.match(self.path)
-        if not match:
-            return None
-        return match.group(name)
-
-    @classmethod
-    def supported(cls, url):
-        return cls.match(url) is not None
-
-    @property
-    def sep(self):
-        return '/'
-
-    @property
-    def exists(self):
-        return self.remote.exists([self.path_info])[0]
-
-    def changed(self):
-        raise NotImplementedError
-
-    def status(self):
-        if self.changed():
-            # FIXME better msgs
-            if self.path_info['scheme'] == 'local':
-                p = self.rel_path
-            else:
-                p = self.path
-            return {p: 'changed'}
-        return {}
-
-    def save(self):
-        raise NotImplementedError
-
-    def dumpd(self):
-        return {self.PARAM_PATH: self.path}
-
-    def download(self, to_info):
-        self.remote.download([self.path_info], [to_info])
+    DoesNotExistError = DependencyDoesNotExistError
+    IsNotFileOrDirError = DependencyIsNotFileOrDirError

dvc/dependency/gs.py

@@ -1,25 +1,8 @@
-import posixpath
+from __future__ import unicode_literals
 
-try:
-    from urlparse import urlparse
-except ImportError:
-    from urllib.parse import urlparse
-
-from dvc.dependency.s3 import DependencyS3
-from dvc.remote.gs import RemoteGS
+from dvc.output.gs import OutputGS
+from dvc.dependency.base import DependencyBase
 
 
-class DependencyGS(DependencyS3):
-    REGEX = RemoteGS.REGEX
-
-    def __init__(self, stage, path, info=None, remote=None):
-        super(DependencyGS, self).__init__(stage, path)
-        self.info = info
-        self.remote = remote if remote else RemoteGS(stage.project, {})
-        bucket = remote.bucket if remote else urlparse(path).netloc
-        key = urlparse(path).path.lstrip('/')
-        if remote:
-            key = posixpath.join(remote.prefix, key)
-        self.path_info = {'scheme': 'gs',
-                          'bucket': bucket,
-                          'key': key}
+class DependencyGS(DependencyBase, OutputGS):
+    pass

dvc/dependency/hdfs.py

@@ -1,35 +1,8 @@
-import posixpath
-
-try:
-    from urlparse import urlparse
-except ImportError:
-    from urllib.parse import urlparse
+from __future__ import unicode_literals
 
+from dvc.output.hdfs import OutputHDFS
 from dvc.dependency.base import DependencyBase
-from dvc.remote.hdfs import RemoteHDFS
 
 
-class DependencyHDFS(DependencyBase):
-    REGEX = RemoteHDFS.REGEX
-
-    def __init__(self, stage, path, info=None, remote=None):
-        super(DependencyHDFS, self).__init__(stage, path)
-        self.info = info
-        self.remote = remote if remote else RemoteHDFS(stage.project, {})
-        if remote:
-            path = posixpath.join(remote.url, urlparse(path).path.lstrip('/'))
-        user = remote.user if remote else self.group('user')
-        self.path_info = {'scheme': 'hdfs',
-                          'user': user,
-                          'url': path}
-
-    def changed(self):
-        return self.info != self.remote.save_info(self.path_info)
-
-    def save(self):
-        self.info = self.remote.save_info(self.path_info)
-
-    def dumpd(self):
-        ret = self.info
-        ret[self.PARAM_PATH] = self.path
-        return ret
+class DependencyHDFS(DependencyBase, OutputHDFS):
+    pass

dvc/dependency/http.py

@@ -0,0 +1,20 @@
+from __future__ import unicode_literals
+
+from dvc.path import Path
+from dvc.utils.compat import urlparse, urljoin
+from dvc.output.base import OutputBase
+from dvc.remote.http import RemoteHTTP
+from dvc.dependency.base import DependencyBase
+
+
+class DependencyHTTP(DependencyBase, OutputBase):
+    REMOTE = RemoteHTTP
+
+    def __init__(self, stage, path, info=None, remote=None):
+        super(DependencyHTTP, self).__init__(
+            stage, path, info=info, remote=remote
+        )
+        if path.startswith("remote"):
+            path = urljoin(self.remote.cache_dir, urlparse(path).path)
+
+        self.path_info = Path(self.scheme, url=self.url, path=path)

dvc/dependency/https.py

@@ -0,0 +1,8 @@
+from __future__ import unicode_literals
+
+from dvc.remote.https import RemoteHTTPS
+from .http import DependencyHTTP
+
+
+class DependencyHTTPS(DependencyHTTP):
+    REMOTE = RemoteHTTPS

dvc/dependency/local.py

@@ -1,89 +1,8 @@
-import os
-
-try:
-    from urlparse import urlparse
-except ImportError:
-    from urllib.parse import urlparse
+from __future__ import unicode_literals
 
 from dvc.dependency.base import DependencyBase
-from dvc.dependency.base import DependencyDoesNotExistError
-from dvc.dependency.base import DependencyIsNotFileOrDirError
-from dvc.logger import Logger
-from dvc.remote.local import RemoteLOCAL
+from dvc.output.local import OutputLOCAL
 
 
-class DependencyLOCAL(DependencyBase):
-    REGEX = r'^(?P<path>(/+|.:\\+)?[^:]*)$'
-
-    DoesNotExistError = DependencyDoesNotExistError
-    IsNotFileOrDirError = DependencyIsNotFileOrDirError
-
-    def __init__(self, stage, path, info=None, remote=None):
-        self.stage = stage
-        self.project = stage.project
-        self.info = info
-        if remote is not None:
-            self.remote = remote
-        else:
-            self.remote = RemoteLOCAL(stage.project, {})
-
-        if remote:
-            path = os.path.join(remote.prefix, urlparse(path).path.lstrip('/'))
-
-        if not os.path.isabs(path):
-            path = self.remote.ospath(path)
-            path = os.path.join(stage.cwd, path)
-        self.path = os.path.abspath(os.path.normpath(path))
-
-        self.path_info = {'scheme': 'local',
-                          'path': self.path}
-
-    def __str__(self):
-        return self.rel_path
-
-    @property
-    def is_local(self):
-        assert os.path.isabs(self.path)
-        assert os.path.isabs(self.project.root_dir)
-        return self.path.startswith(self.project.root_dir)
-
-    @property
-    def sep(self):
-        return os.sep
-
-    @property
-    def rel_path(self):
-        return os.path.relpath(self.path)
-
-    def changed(self):
-        if not self.exists:
-            return True
-
-        info = self.remote.save_info(self.path_info)
-
-        return self.info != info
-
-    def save(self):
-        if not self.exists:
-            raise self.DoesNotExistError(self.rel_path)
-
-        if not os.path.isfile(self.path) \
-           and not os.path.isdir(self.path):  # pragma: no cover
-            raise self.IsNotFileOrDirError(self.rel_path)
-
-        if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \
-           (os.path.isdir(self.path) and len(os.listdir(self.path)) == 0):
-            Logger.warn("File/directory '{}' is empty.".format(self.rel_path))
-
-        self.info = self.remote.save_info(self.path_info)
-
-    def dumpd(self):
-        if self.is_local:
-            path = self.remote.unixpath(os.path.relpath(self.path,
-                                                        self.stage.cwd))
-        else:
-            path = self.path
-
-        info = self.info.copy()
-        info[self.PARAM_PATH] = path
-        return info
+class DependencyLOCAL(DependencyBase, OutputLOCAL):
+    pass

dvc/dependency/s3.py

@@ -1,37 +1,8 @@
-import posixpath
-
-try:
-    from urlparse import urlparse
-except ImportError:
-    from urllib.parse import urlparse
+from __future__ import unicode_literals
 
+from dvc.output.s3 import OutputS3
 from dvc.dependency.base import DependencyBase
-from dvc.remote.s3 import RemoteS3
 
 
-class DependencyS3(DependencyBase):
-    REGEX = RemoteS3.REGEX
-
-    def __init__(self, stage, path, info=None, remote=None):
-        super(DependencyS3, self).__init__(stage, path)
-        self.info = info
-        self.remote = remote if remote else RemoteS3(stage.project, {})
-
-        bucket = remote.bucket if remote else urlparse(path).netloc
-        key = urlparse(path).path.lstrip('/')
-        if remote:
-            key = posixpath.join(remote.prefix, key)
-        self.path_info = {'scheme': 's3',
-                          'bucket': bucket,
-                          'key': key}
-
-    def changed(self):
-        return self.info != self.remote.save_info(self.path_info)
-
-    def save(self):
-        self.info = self.remote.save_info(self.path_info)
-
-    def dumpd(self):
-        ret = self.info
-        ret[self.PARAM_PATH] = self.path
-        return ret
+class DependencyS3(DependencyBase, OutputS3):
+    pass

dvc/dependency/ssh.py

@@ -1,44 +1,8 @@
-import posixpath
-
-try:
-    from urlparse import urlparse
-except ImportError:
-    from urllib.parse import urlparse
+from __future__ import unicode_literals
 
+from dvc.output.ssh import OutputSSH
 from dvc.dependency.base import DependencyBase
-from dvc.remote.ssh import RemoteSSH
 
 
-class DependencySSH(DependencyBase):
-    REGEX = RemoteSSH.REGEX
-
-    def __init__(self, stage, path, info=None, remote=None):
-        super(DependencySSH, self).__init__(stage, path)
-        self.info = info
-        self.remote = remote if remote else RemoteSSH(stage.project, {})
-
-        host = remote.host if remote else self.group('host')
-        port = remote.port if remote else RemoteSSH.DEFAULT_PORT
-        user = remote.user if remote else self.group('user')
-        if remote:
-            path = posixpath.join(remote.prefix,
-                                  urlparse(path).path.lstrip('/'))
-        else:
-            path = self.match(self.path).group('path')
-
-        self.path_info = {'scheme': 'ssh',
-                          'host': host,
-                          'port': port,
-                          'user': user,
-                          'path': path}
-
-    def changed(self):
-        return self.info != self.remote.save_info(self.path_info)
-
-    def save(self):
-        self.info = self.remote.save_info(self.path_info)
-
-    def dumpd(self):
-        ret = self.info
-        ret[self.PARAM_PATH] = self.path
-        return ret
+class DependencySSH(DependencyBase, OutputSSH):
+    pass

dvc/env.py

@@ -0,0 +1 @@
+DVC_DAEMON = "DVC_DAEMON"

dvc/exceptions.py

@@ -1,47 +1,266 @@
+"""Exceptions raised by the dvc."""
+
+from __future__ import unicode_literals
+
+from dvc.utils.compat import str, builtin_str
+
+import os
 import traceback
 
 
 class DvcException(Exception):
+    """Base class for all dvc exceptions.
+
+    Args:
+        msg (unicode): message for this exception.
+        cause (Exception): optional cause exception.
+    """
+
     def __init__(self, msg, cause=None):
+        # NOTE: unlike python 3, python 2 doesn't have built-in support
+        # for chained exceptions, so we are using our own implementation.
         self.cause = cause
-        self.case_tb = None
+        self.cause_tb = None
         if cause:
             try:
                 self.cause_tb = traceback.format_exc()
-            except Exception:  # pragma: no cover
+            except AttributeError:  # pragma: no cover
                 pass
         super(DvcException, self).__init__(msg)
 
 
-class UnsupportedRemoteError(DvcException):
-    def __init__(self, config):
-        msg = "Remote '{}' is not supported.".format(config)
-        super(UnsupportedRemoteError, self).__init__(msg)
-
-
 class OutputDuplicationError(DvcException):
+    """Thrown if a file/directory is specified as an output in more than one
+    stage.
+
+    Args:
+        output (unicode): path to the file/directory.
+        stages (list): list of paths to stages.
+    """
+
     def __init__(self, output, stages):
-        assert isinstance(output, str)
+        assert isinstance(output, str) or isinstance(output, builtin_str)
         assert isinstance(stages, list)
-        assert all(isinstance(stage, str) for stage in stages)
-        msg = "File '{}' is specified as an output in more than one stage:{}"
-        s = ""
-        for stage in stages:
-            s += "\n    " + stage
-        super(OutputDuplicationError, self).__init__(msg.format(output, s))
+        assert all(
+            isinstance(stage, str) or isinstance(stage, builtin_str)
+            for stage in stages
+        )
+        msg = (
+            "file/directory '{}' is specified as an output in more than one"
+            "stage: {}"
+        ).format(output, "\n    ".join(stages))
+        super(OutputDuplicationError, self).__init__(msg)
+
+
+class OutputNotFoundError(DvcException):
+    """Thrown if a file/directory not found in repository pipelines.
+
+    Args:
+        output (unicode): path to the file/directory.
+    """
+
+    def __init__(self, output):
+        super(OutputNotFoundError, self).__init__(
+            "unable to find stage file with output '{path}'".format(
+                path=os.path.relpath(output)
+            )
+        )
+
+
+class StagePathAsOutputError(DvcException):
+    """Thrown if directory that stage is going to be saved in is specified as
+    an output of another stage.
+
+    Args:
+        cwd (str): path to the directory.
+        fname (str): path to the stage file that has cwd specified as an
+            output.
+    """
+
+    def __init__(self, wdir, fname):
+        assert isinstance(wdir, str) or isinstance(wdir, builtin_str)
+        assert isinstance(fname, str) or isinstance(fname, builtin_str)
+        msg = (
+            "current working directory '{cwd}' is specified as an output in"
+            " '{fname}'. Use another CWD to prevent any data removal.".format(
+                cwd=wdir, fname=fname
+            )
+        )
+        super(StagePathAsOutputError, self).__init__(msg)
 
 
 class CircularDependencyError(DvcException):
+    """Thrown if a file/directory specified both as an output and as a
+    dependency.
+
+    Args:
+        dependency (str): path to the dependency.
+    """
+
     def __init__(self, dependency):
-        assert isinstance(dependency, str)
-        msg = "File '{}' is specified as an output and as a dependency."
+        assert isinstance(dependency, str) or isinstance(
+            dependency, builtin_str
+        )
+
+        msg = (
+            "file/directory '{}' is specified as an output and as a "
+            "dependency."
+        )
         super(CircularDependencyError, self).__init__(msg.format(dependency))
 
 
-class MoveNotDataSourceError(DvcException):
+class ArgumentDuplicationError(DvcException):
+    """Thrown if a file/directory is specified as a dependency/output more
+    than once.
+
+    Args:
+        path (str): path to the file/directory.
+    """
+
     def __init__(self, path):
-        msg = "Move is not permitted for stages that are not data sources. " \
-              "You need to either move '{path}' to a new location and edit " \
-              "it by hand, or remove '{path}' and create a new one at the " \
-              "desired location."
+        assert isinstance(path, str) or isinstance(path, builtin_str)
+        msg = "file '{}' is specified more than once."
+        super(ArgumentDuplicationError, self).__init__(msg.format(path))
+
+
+class MoveNotDataSourceError(DvcException):
+    """Thrown if attempted to move a file/directory that is not an output
+    in a data source stage.
+
+    Args:
+        path (str): path to the file/directory.
+    """
+
+    def __init__(self, path):
+        msg = (
+            "move is not permitted for stages that are not data sources. "
+            "You need to either move '{path}' to a new location and edit "
+            "it by hand, or remove '{path}' and create a new one at the "
+            "desired location."
+        )
         super(MoveNotDataSourceError, self).__init__(msg.format(path=path))
+
+
+class NotDvcRepoError(DvcException):
+    """Thrown if a directory is not a dvc repo.
+
+    Args:
+        root (str): path to the directory.
+    """
+
+    def __init__(self, root):
+        msg = (
+            "you are not inside of a dvc repository "
+            "(checked up to mount point '{}')"
+        )
+        super(NotDvcRepoError, self).__init__(msg.format(root))
+
+
+class DvcParserError(DvcException):
+    """Base class for CLI parser errors."""
+
+    def __init__(self):
+        super(DvcParserError, self).__init__("parser error")
+
+
+class CyclicGraphError(DvcException):
+    def __init__(self, stages):
+        assert isinstance(stages, list)
+        stages = "\n".join("\t- {}".format(stage) for stage in stages)
+        msg = (
+            "you've introduced a cycle in your pipeline that involves"
+            " the following stages:"
+            "\n"
+            "{stages}".format(stages=stages)
+        )
+        super(CyclicGraphError, self).__init__(msg)
+
+
+class ConfirmRemoveError(DvcException):
+    def __init__(self, path):
+        super(ConfirmRemoveError, self).__init__(
+            "unable to remove '{}' without a confirmation from the user. Use "
+            "'-f' to force.".format(path)
+        )
+
+
+class InitError(DvcException):
+    def __init__(self, msg):
+        super(InitError, self).__init__(msg)
+
+
+class ReproductionError(DvcException):
+    def __init__(self, dvc_file_name, ex):
+        self.path = dvc_file_name
+        msg = "failed to reproduce '{}'".format(dvc_file_name)
+        super(ReproductionError, self).__init__(msg, cause=ex)
+
+
+class BadMetricError(DvcException):
+    def __init__(self, path):
+        super(BadMetricError, self).__init__(
+            "'{}' does not exist, not a metric or is malformed".format(
+                os.path.relpath(path)
+            )
+        )
+
+
+class NoMetricsError(DvcException):
+    def __init__(self):
+        super(NoMetricsError, self).__init__(
+            "no metric files in this repository. "
+            "Use 'dvc metrics add' to add a metric file to track."
+        )
+
+
+class StageFileCorruptedError(DvcException):
+    def __init__(self, path, cause=None):
+        path = os.path.relpath(path)
+        super(StageFileCorruptedError, self).__init__(
+            "unable to read stage file: {} "
+            "YAML file structure is corrupted".format(path),
+            cause=cause,
+        )
+
+
+class RecursiveAddingWhileUsingFilename(DvcException):
+    def __init__(self):
+        super(RecursiveAddingWhileUsingFilename, self).__init__(
+            "using fname with recursive is not allowed."
+        )
+
+
+class OverlappingOutputPathsError(DvcException):
+    def __init__(self, out_1, out_2):
+        super(OverlappingOutputPathsError, self).__init__(
+            "Paths for outs:\n'{}'('{}')\n'{}'('{}')\noverlap. To avoid "
+            "unpredictable behaviour, rerun command with non overlapping outs "
+            "paths.".format(
+                str(out_1),
+                out_1.stage.relpath,
+                str(out_2),
+                out_2.stage.relpath,
+            )
+        )
+
+
+class TargetNotDirectoryError(DvcException):
+    def __init__(self, path):
+        super(TargetNotDirectoryError, self).__init__(
+            "Target: {} is not a directory".format(path)
+        )
+
+
+class CheckoutErrorSuggestGit(DvcException):
+    def __init__(self, target, cause):
+        super(CheckoutErrorSuggestGit, self).__init__(
+            "Did you mean 'git checkout {}'?".format(target), cause=cause
+        )
+
+
+class ETagMismatchError(DvcException):
+    def __init__(self, etag, cached_etag):
+        super(ETagMismatchError, self).__init__(
+            "ETag mismatch detected when copying file to cache! "
+            "(expected: '{}', actual: '{}')".format(etag, cached_etag)
+        )

dvc/ignore.py

@@ -0,0 +1,134 @@
+import os
+
+from dulwich.ignore import match_pattern, read_ignore_patterns
+from dvc.utils.compat import cast_bytes
+from dvc.utils.fs import get_parent_dirs_up_to
+
+
+class DvcIgnoreFileHandler(object):
+    def __init__(self, tree):
+        self.tree = tree
+
+    def read_patterns(self, path):
+        with self.tree.open(path, binary=True) as stream:
+            return self._read_patterns(stream)
+
+    def get_repo_root(self):
+        return self.tree.tree_root
+
+    def _read_patterns(self, binary_stream):
+        negate_patterns = []
+        patterns = []
+        for pattern in read_ignore_patterns(binary_stream):
+            if pattern.lstrip().startswith(b"!"):
+                negate_patterns.append(pattern)
+            else:
+                patterns.append(pattern)
+
+        return negate_patterns, patterns
+
+
+class DvcIgnore(object):
+    DVCIGNORE_FILE = ".dvcignore"
+
+    def __call__(self, root, dirs, files):
+        raise NotImplementedError
+
+
+class DvcIgnoreFromFile(DvcIgnore):
+    def __init__(self, ignore_file_path, ignore_handler):
+        self.ignore_file_path = ignore_file_path
+        self.dirname = os.path.normpath(os.path.dirname(ignore_file_path))
+        self.patterns = []
+        self.negate_patterns = []
+
+        self.negate_patterns, self.patterns = ignore_handler.read_patterns(
+            ignore_file_path
+        )
+
+    def __call__(self, root, dirs, files):
+        files = [f for f in files if not self.matches(root, f)]
+        dirs = [d for d in dirs if not self.matches(root, d)]
+
+        return dirs, files
+
+    def get_match(self, abs_path):
+        rel_path = os.path.relpath(abs_path, self.dirname)
+        if os.name == "nt":
+            rel_path = rel_path.replace("\\", "/")
+        rel_path = cast_bytes(rel_path, "utf-8")
+
+        for pattern in self.patterns:
+            if match_pattern(
+                rel_path, pattern
+            ) and self._no_negate_pattern_matches(rel_path):
+                return (abs_path, pattern, self.ignore_file_path)
+        return None
+
+    def matches(self, dirname, basename):
+        if self.get_match(os.path.join(dirname, basename)):
+            return True
+        return False
+
+    def _no_negate_pattern_matches(self, path):
+        return all([not match_pattern(path, p) for p in self.negate_patterns])
+
+    def __hash__(self):
+        return hash(self.ignore_file_path)
+
+
+class DvcIgnoreConstant(DvcIgnore):
+    def __init__(self, basename):
+        self.basename = basename
+
+
+class DvcIgnoreDir(DvcIgnoreConstant):
+    def __call__(self, root, dirs, files):
+        dirs = [d for d in dirs if not d == self.basename]
+
+        return dirs, files
+
+
+class DvcIgnoreFile(DvcIgnoreConstant):
+    def __call__(self, root, dirs, files):
+        files = [f for f in files if not f == self.basename]
+
+        return dirs, files
+
+
+class DvcIgnoreFilter(object):
+    def __init__(self, wdir, ignore_file_handler=None):
+        self.ignores = [
+            DvcIgnoreDir(".git"),
+            DvcIgnoreDir(".hg"),
+            DvcIgnoreDir(".dvc"),
+            DvcIgnoreFile(".dvcignore"),
+        ]
+
+        self.ignore_file_handler = ignore_file_handler
+        self._process_ignores_in_parent_dirs(wdir)
+
+    def _process_ignores_in_parent_dirs(self, wdir):
+        if self.ignore_file_handler:
+            wdir = os.path.normpath(os.path.abspath(wdir))
+            ignore_search_end_dir = self.ignore_file_handler.get_repo_root()
+            parent_dirs = get_parent_dirs_up_to(wdir, ignore_search_end_dir)
+            for d in parent_dirs:
+                self.update(d)
+
+    def update(self, wdir):
+        ignore_file_path = os.path.join(wdir, DvcIgnore.DVCIGNORE_FILE)
+        if os.path.exists(ignore_file_path):
+            file_ignore = DvcIgnoreFromFile(
+                ignore_file_path, ignore_handler=self.ignore_file_handler
+            )
+            self.ignores.append(file_ignore)
+
+    def __call__(self, root, dirs, files):
+        if self.ignore_file_handler:
+            self.update(root)
+
+        for ignore in self.ignores:
+            dirs, files = ignore(root, dirs, files)
+
+        return dirs, files

dvc/istextfile.py

@@ -1,19 +1,24 @@
+"""Use heuristics to guess if it is a text file or a binary file."""
+
+from __future__ import unicode_literals
+
+from dvc.utils.compat import is_py3, open
+
 # Based on https://eli.thegreenplace.net/2011/10/19/
 # perls-guess-if-file-is-text-or-binary-implemented-in-python
 
-import sys
-PY3 = sys.version_info[0] == 3
 
 # A function that takes an integer in the 8-bit range and returns
 # a single-character byte object in py3 / a single-character string
 # in py2.
 #
-int2byte = (lambda x: bytes((x,))) if PY3 else chr
+def _int2byte(i):
+    if is_py3:
+        return bytes((i,))
+    return chr(i)
 
 
-_text_characters = (
-        b''.join(int2byte(i) for i in range(32, 127)) +
-        b'\n\r\t\f\b')
+TEXT_CHARS = b"".join(_int2byte(i) for i in range(32, 127)) + b"\n\r\t\f\b"
 
 
 def istextfile(fname, blocksize=512):
@@ -22,17 +27,18 @@ def istextfile(fname, blocksize=512):
         If more than 30% of the chars in the block are non-text, or there
         are NUL ('\x00') bytes in the block, assume this is a binary file.
     """
-    with open(fname, 'rb') as fd:
-        block = fd.read(blocksize)
+    with open(fname, "rb") as fobj:
+        block = fobj.read(blocksize)
 
-    if b'\x00' in block:
-        # Files with null bytes are binary
-        return False
-    elif not block:
+    if not block:
         # An empty file is considered a valid text file
         return True
 
+    if b"\x00" in block:
+        # Files with null bytes are binary
+        return False
+
     # Use translate's 'deletechars' argument to efficiently remove all
-    # occurrences of _text_characters from the block
-    nontext = block.translate(None, _text_characters)
+    # occurrences of TEXT_CHARS from the block
+    nontext = block.translate(None, TEXT_CHARS)
     return float(len(nontext)) / len(block) <= 0.30

dvc/lock.py

@@ -1,3 +1,7 @@
+"""Manages dvc lock file."""
+
+from __future__ import unicode_literals
+
 import os
 import time
 import zc.lockfile
@@ -6,11 +10,19 @@ from dvc.exceptions import DvcException
 
 
 class LockError(DvcException):
-    pass
+    """Thrown when unable to acquire the lock for dvc repo."""
 
 
 class Lock(object):
-    LOCK_FILE = 'lock'
+    """Class for dvc repo lock.
+
+    Args:
+        dvc_dir (str): path to the directory that the lock should be created
+            in.
+        name (str): name of the lock file.
+    """
+
+    LOCK_FILE = "lock"
     TIMEOUT = 5
 
     def __init__(self, dvc_dir, name=LOCK_FILE):
@@ -21,10 +33,13 @@ class Lock(object):
         try:
             self._lock = zc.lockfile.LockFile(self.lock_file)
         except zc.lockfile.LockError:
-            raise LockError('Cannot perform the cmd since DVC is busy and '
-                            'locked. Please retry the cmd later.')
+            raise LockError(
+                "cannot perform the cmd since DVC is busy and "
+                "locked. Please retry the cmd later."
+            )
 
     def lock(self):
+        """Acquire lock for dvc repo."""
         try:
             self._do_lock()
             return
@@ -34,11 +49,12 @@ class Lock(object):
         self._do_lock()
 
     def unlock(self):
+        """Release lock for dvc repo."""
         self._lock.close()
         self._lock = None
 
     def __enter__(self):
         self.lock()
 
-    def __exit__(self, type, value, tb):
+    def __exit__(self, typ, value, tbck):
         self.unlock()

dvc/logger.py

@@ -1,230 +1,182 @@
-import sys
+"""Manages logging configuration for dvc repo."""
+
+from __future__ import unicode_literals
+
+from dvc.utils.compat import str, StringIO
+
 import logging
+import logging.handlers
+import logging.config
 import colorama
-import traceback
-import re
 
 
-colorama.init()
+class ExcludeErrorsFilter(logging.Filter):
+    def filter(self, record):
+        return record.levelno < logging.ERROR
 
 
-def visual_width(line):
-    """ Get the the number of columns required to display a string """
+class ColorFormatter(logging.Formatter):
+    """Enable color support when logging to a terminal that supports it.
 
-    return len(re.sub(colorama.ansitowin32.AnsiToWin32.ANSI_CSI_RE, '', line))
+    Color support on Windows versions that do not support ANSI color codes is
+    enabled by use of the colorama__ library.
+    See the colorama documentation for details.
 
+    __ https://pypi.python.org/pypi/colorama
 
-def visual_center(line, width):
-    """ Center align string according to it's visual width """
+    For records containing `exc_info`, it will use a custom `_walk_exc` to
+    retrieve the whole tracebak.
+    """
 
-    spaces = max(width - visual_width(line), 0)
-    left_padding = int(spaces / 2)
-    right_padding = (spaces - left_padding)
-
-    return (left_padding * ' ') + line + (right_padding * ' ')
-
-
-class Logger(object):
-    FMT = '%(message)s'
-    DEFAULT_LEVEL = logging.INFO
-
-    LEVEL_MAP = {
-        'debug': logging.DEBUG,
-        'info': logging.INFO,
-        'warn': logging.WARNING,
-        'error': logging.ERROR
+    color_code = {
+        "DEBUG": colorama.Fore.BLUE,
+        "INFO": "",
+        "WARNING": colorama.Fore.YELLOW,
+        "ERROR": colorama.Fore.RED,
+        "CRITICAL": colorama.Fore.RED,
     }
 
-    COLOR_MAP = {
-        'green': colorama.Fore.GREEN,
-        'yellow': colorama.Fore.YELLOW,
-        'blue': colorama.Fore.BLUE,
-        'red': colorama.Fore.RED,
-    }
+    footer = (
+        "{yellow}Having any troubles?{nc}."
+        " Hit us up at {blue}https://dvc.org/support{nc},"
+        " we are always happy to help!"
+    ).format(
+        blue=colorama.Fore.BLUE,
+        nc=colorama.Fore.RESET,
+        yellow=colorama.Fore.YELLOW,
+    )
 
-    LEVEL_COLOR_MAP = {
-        'debug': 'blue',
-        'warn': 'yellow',
-        'error': 'red',
-    }
+    def format(self, record):
+        if self._is_visible(record):
+            self._progress_aware()
 
-    def __init__(self, loglevel=None):
-        if loglevel:
-            Logger.set_level(loglevel)
+        if record.levelname == "INFO":
+            return record.msg
 
-    @staticmethod
-    def init():
+        if record.levelname == "ERROR" or record.levelname == "CRITICAL":
+            exception, stack_trace = self._parse_exc(record.exc_info)
 
-        class LogLevelFilter(logging.Filter):
-            def filter(self, record):
-                return record.levelno <= logging.WARNING
-
-        sh_out = logging.StreamHandler(sys.stdout)
-        sh_out.setFormatter(logging.Formatter(Logger.FMT))
-        sh_out.setLevel(logging.DEBUG)
-        sh_out.addFilter(LogLevelFilter())
-
-        sh_err = logging.StreamHandler(sys.stderr)
-        sh_err.setFormatter(logging.Formatter(Logger.FMT))
-        sh_err.setLevel(logging.ERROR)
-
-        Logger.logger().addHandler(sh_out)
-        Logger.logger().addHandler(sh_err)
-        Logger.set_level()
-
-    @staticmethod
-    def logger():
-        return logging.getLogger('dvc')
-
-    @staticmethod
-    def set_level(level=None):
-        if not level:
-            lvl = Logger.DEFAULT_LEVEL
-        else:
-            lvl = Logger.LEVEL_MAP.get(level.lower(), Logger.DEFAULT_LEVEL)
-        Logger.logger().setLevel(lvl)
-
-    @staticmethod
-    def be_quiet():
-        Logger.logger().setLevel(logging.CRITICAL)
-
-    @staticmethod
-    def be_verbose():
-        Logger.logger().setLevel(logging.DEBUG)
-
-    @staticmethod
-    def colorize(msg, color):
-        header = ''
-        footer = ''
-
-        if sys.stdout.isatty():  # pragma: no cover
-            header = Logger.COLOR_MAP.get(color.lower(), '')
-            footer = colorama.Style.RESET_ALL
-
-        return u'{}{}{}'.format(header, msg, footer)
-
-    @staticmethod
-    def parse_exc(exc, tb=None):
-        str_tb = tb if tb else None
-        str_exc = ': {}'.format(str(exc)) if exc else ""
-
-        if exc and hasattr(exc, 'cause') and exc.cause:
-            cause_tb = exc.cause_tb if hasattr(exc, 'cause_tb') else None
-            cause_str_exc, cause_str_tb = Logger.parse_exc(exc.cause, cause_tb)
-
-            str_tb = cause_str_tb
-            str_exc = '{}{}'.format(str_exc, cause_str_exc)
-
-        return (str_exc, str_tb)
-
-    @staticmethod
-    def _prefix(msg, typ):
-        color = Logger.LEVEL_COLOR_MAP.get(typ.lower(), '')
-        return Logger.colorize('{}: '.format(msg), color)
-
-    @staticmethod
-    def error_prefix():
-        return Logger._prefix('Error', 'error')
-
-    @staticmethod
-    def warning_prefix():
-        return Logger._prefix('Warning', 'warn')
-
-    @staticmethod
-    def debug_prefix():
-        return Logger._prefix('Debug', 'debug')
-
-    @staticmethod
-    def _with_progress(func, msg):
-        from dvc.progress import progress
-        with progress:
-            func(msg)
-
-    @staticmethod
-    def _error_exc(exc):
-        if exc is None:
-            return
-
-        if Logger.logger().getEffectiveLevel() != logging.DEBUG:
-            return
-
-        prefix = Logger.error_prefix()
-        str_exc, str_tb = Logger.parse_exc(exc)
-        str_tb = str_tb if str_tb else traceback.format_exc()
-        Logger._with_progress(Logger.logger().error, prefix + str_tb)
-
-    @staticmethod
-    def _with_exc(func, msg, suffix="", exc=None):
-        Logger._error_exc(exc)
-        msg = msg + Logger.parse_exc(exc)[0] + suffix
-        Logger._with_progress(func, msg)
-
-    @staticmethod
-    def error(msg, exc=None):
-        chat = "\n\nHaving any troubles? Hit us up at dvc.org/support, " \
-               "we are always happy to help!"
-        Logger._with_exc(Logger.logger().error,
-                         Logger.error_prefix() + msg,
-                         suffix=chat,
-                         exc=exc)
-
-    @classmethod
-    def warn(cls, msg, exc=None):
-        cls._with_exc(cls.logger().warning,
-                      cls.warning_prefix() + msg,
-                      exc=exc)
-
-    @classmethod
-    def debug(cls, msg, exc=None):
-        cls._with_exc(cls.logger().debug,
-                      cls.debug_prefix() + msg,
-                      exc=exc)
-
-    @staticmethod
-    def info(msg):
-        Logger._with_progress(Logger.logger().info, msg)
-
-    @classmethod
-    def box(cls, msg, border_color=''):
-        lines = msg.split('\n')
-        max_width = max(visual_width(line) for line in lines)
-
-        padding_horizontal = 5
-        padding_vertical = 1
-
-        box_size_horizontal = (max_width + (padding_horizontal * 2))
-
-        chars = {
-            'corner':     '+',
-            'horizontal': '-',
-            'vertical':   '|',
-            'empty':      ' ',
-        }
-
-        margin = "{corner}{line}{corner}\n".format(
-            corner=chars['corner'],
-            line=chars['horizontal'] * box_size_horizontal,
-        )
-
-        padding_lines = [
-            "{border}{space}{border}\n".format(
-                border=cls.colorize(chars['vertical'], border_color),
-                space=chars['empty'] * box_size_horizontal,
-            ) * padding_vertical
-        ]
-
-        content_lines = [
-            "{border}{space}{content}{space}{border}\n".format(
-                border=cls.colorize(chars['vertical'], border_color),
-                space=chars['empty'] * padding_horizontal,
-                content=visual_center(line, max_width),
-            ) for line in lines
-        ]
-
-        box = "{margin}{padding}{content}{padding}{margin}".format(
-                margin=cls.colorize(margin, border_color),
-                padding=''.join(padding_lines),
-                content=''.join(content_lines),
+            return (
+                "{color}{levelname}{nc}: {description}"
+                "{stack_trace}\n"
+                "\n"
+                "{footer}"
+            ).format(
+                color=self.color_code.get(record.levelname, ""),
+                nc=colorama.Fore.RESET,
+                levelname=record.levelname,
+                description=self._description(record.msg, exception),
+                msg=record.msg,
+                stack_trace=stack_trace,
+                footer=self.footer,
             )
 
-        print(box)
+        return "{color}{levelname}{nc}: {msg}".format(
+            color=self.color_code.get(record.levelname, ""),
+            nc=colorama.Fore.RESET,
+            levelname=record.levelname,
+            msg=record.msg,
+        )
+
+    def _current_level(self):
+        return logging.getLogger("dvc").getEffectiveLevel()
+
+    def _is_visible(self, record):
+        return record.levelno >= self._current_level()
+
+    def _description(self, message, exception):
+        description = ""
+
+        if exception and message:
+            description = "{message} - {exception}"
+        elif exception:
+            description = "{exception}"
+        elif message:
+            description = "{message}"
+
+        return description.format(message=message, exception=exception)
+
+    def _walk_exc(self, exc_info):
+        import traceback
+
+        buffer = StringIO()
+
+        traceback.print_exception(*exc_info, file=buffer)
+
+        exc = exc_info[1]
+        tb = buffer.getvalue()
+
+        exc_list = [str(exc)]
+        tb_list = [tb]
+
+        # NOTE: parsing chained exceptions. See dvc/exceptions.py for more info
+        while hasattr(exc, "cause") and exc.cause:
+            exc_list.append(str(exc.cause))
+            if hasattr(exc, "cause_tb") and exc.cause_tb:
+                tb_list.insert(0, str(exc.cause_tb))
+            exc = exc.cause
+
+        return exc_list, tb_list
+
+    def _parse_exc(self, exc_info):
+        if not exc_info:
+            return (None, "")
+
+        exc_list, tb_list = self._walk_exc(exc_info)
+
+        exception = ": ".join(exc_list)
+
+        if self._current_level() == logging.DEBUG:
+            stack_trace = (
+                "\n" "{red}{line}{nc}\n" "{stack_trace}" "{red}{line}{nc}"
+            ).format(
+                red=colorama.Fore.RED,
+                nc=colorama.Fore.RESET,
+                line="-" * 60,
+                stack_trace="\n".join(tb_list),
+            )
+        else:
+            stack_trace = ""
+
+        return (exception, stack_trace)
+
+    def _progress_aware(self):
+        """Add a new line if progress bar hasn't finished"""
+        from dvc.progress import progress
+
+        if not progress.is_finished:
+            progress._print()
+        progress.clearln()
+
+
+def setup(level=logging.INFO):
+    colorama.init()
+
+    logging.config.dictConfig(
+        {
+            "version": 1,
+            "filters": {"exclude_errors": {"()": ExcludeErrorsFilter}},
+            "formatters": {"color": {"()": ColorFormatter}},
+            "handlers": {
+                "console": {
+                    "class": "logging.StreamHandler",
+                    "level": "DEBUG",
+                    "formatter": "color",
+                    "stream": "ext://sys.stdout",
+                    "filters": ["exclude_errors"],
+                },
+                "console_errors": {
+                    "class": "logging.StreamHandler",
+                    "level": "ERROR",
+                    "formatter": "color",
+                    "stream": "ext://sys.stderr",
+                },
+            },
+            "loggers": {
+                "dvc": {
+                    "level": level,
+                    "handlers": ["console", "console_errors"],
+                }
+            },
+        }
+    )

dvc/main.py

@@ -1,27 +1,55 @@
-from dvc.logger import Logger
+"""Main entry point for dvc CLI."""
+
+from __future__ import unicode_literals
+
+import logging
+
 from dvc.cli import parse_args
-from dvc.command.base import CmdBase
+from dvc.analytics import Analytics
+from dvc.exceptions import NotDvcRepoError, DvcParserError
+
+
+logger = logging.getLogger("dvc")
 
 
 def main(argv=None):
-    Logger.init()
+    """Run dvc CLI command.
 
-    args = parse_args(argv)
+    Args:
+        argv: optional list of arguments to parse. sys.argv is used by default.
 
-    # Init loglevel early in case we'll run
-    # into errors before setting it properly
-    CmdBase._set_loglevel(args)
+    Returns:
+        int: command's return code.
+    """
+    args = None
+    cmd = None
 
+    outerLogLevel = logger.level
     try:
+        args = parse_args(argv)
+
+        if args.quiet:
+            logger.setLevel(logging.CRITICAL)
+
+        elif args.verbose:
+            logger.setLevel(logging.DEBUG)
+
         cmd = args.func(args)
-    except Exception as ex:
-        Logger.error('Initialization error', ex)
-        return 255
-
-    try:
         ret = cmd.run_cmd()
-    except Exception as ex:
-        Logger.error('Unexpected error', ex)
-        return 254
+    except KeyboardInterrupt:
+        logger.exception("interrupted by the user")
+        ret = 252
+    except NotDvcRepoError:
+        logger.exception("")
+        ret = 253
+    except DvcParserError:
+        ret = 254
+    except Exception:  # pylint: disable=broad-except
+        logger.exception("unexpected error")
+        ret = 255
+    finally:
+        logger.setLevel(outerLogLevel)
+
+    Analytics().send_cmd(cmd, args, ret)
 
     return ret

dvc/output/__init__.py

@@ -1,10 +1,11 @@
+from __future__ import unicode_literals
+
 import schema
 
-from dvc.exceptions import DvcException
-from dvc.config import Config
+from dvc.scheme import Schemes
+from dvc.utils.compat import urlparse, str
 
-from dvc.dependency import SCHEMA, urlparse
-from dvc.dependency.base import DependencyBase
+from dvc.output.base import OutputBase
 from dvc.output.s3 import OutputS3
 from dvc.output.gs import OutputGS
 from dvc.output.local import OutputLOCAL
@@ -12,51 +13,124 @@ from dvc.output.hdfs import OutputHDFS
 from dvc.output.ssh import OutputSSH
 
 from dvc.remote import Remote
+from dvc.remote.s3 import RemoteS3
+from dvc.remote.hdfs import RemoteHDFS
+from dvc.remote.local import RemoteLOCAL
+
+OUTS = [
+    OutputHDFS,
+    OutputS3,
+    OutputGS,
+    OutputSSH,
+    # NOTE: OutputLOCAL is the default choice
+]
+
+OUTS_MAP = {
+    Schemes.HDFS: OutputHDFS,
+    Schemes.S3: OutputS3,
+    Schemes.GS: OutputGS,
+    Schemes.SSH: OutputSSH,
+    Schemes.LOCAL: OutputLOCAL,
+}
+
+# NOTE: currently there are only 3 possible checksum names:
+#
+#    1) md5 (LOCAL, SSH, GS);
+#    2) etag (S3);
+#    3) checksum (HDFS);
+#
+# so when a few types of outputs share the same name, we only need
+# specify it once.
+CHECKSUM_SCHEMA = {
+    schema.Optional(RemoteLOCAL.PARAM_CHECKSUM): schema.Or(str, None),
+    schema.Optional(RemoteS3.PARAM_CHECKSUM): schema.Or(str, None),
+    schema.Optional(RemoteHDFS.PARAM_CHECKSUM): schema.Or(str, None),
+}
+
+TAGS_SCHEMA = {schema.Optional(str): CHECKSUM_SCHEMA}
+
+SCHEMA = CHECKSUM_SCHEMA.copy()
+SCHEMA[OutputBase.PARAM_PATH] = str
+SCHEMA[schema.Optional(OutputBase.PARAM_CACHE)] = bool
+SCHEMA[schema.Optional(OutputBase.PARAM_METRIC)] = OutputBase.METRIC_SCHEMA
+SCHEMA[schema.Optional(OutputBase.PARAM_TAGS)] = TAGS_SCHEMA
+SCHEMA[schema.Optional(OutputBase.PARAM_PERSIST)] = bool
 
 
-OUTS = [OutputHDFS, OutputS3, OutputGS, OutputSSH, OutputLOCAL]
-
-OUTS_MAP = {'hdfs': OutputHDFS,
-            's3': OutputS3,
-            'gs': OutputGS,
-            'ssh': OutputSSH,
-            '': OutputLOCAL}
-
-SCHEMA[schema.Optional(OutputLOCAL.PARAM_CACHE)] = bool
-SCHEMA[schema.Optional(OutputLOCAL.PARAM_METRIC)] = OutputLOCAL.METRIC_SCHEMA
-
-
-def _get(stage, p, info, cache, metric):
+def _get(stage, p, info, cache, metric, persist=False, tags=None):
     parsed = urlparse(p)
-    if parsed.scheme == 'remote':
-        name = Config.SECTION_REMOTE_FMT.format(parsed.netloc)
-        sect = stage.project.config._config[name]
-        remote = Remote(stage.project, sect)
-        return OUTS_MAP[remote.scheme](stage,
-                                       p,
-                                       info,
-                                       cache=cache,
-                                       remote=remote,
-                                       metric=metric)
+
+    if parsed.scheme == "remote":
+        settings = stage.repo.config.get_remote_settings(parsed.netloc)
+        remote = Remote(stage.repo, settings)
+        return OUTS_MAP[remote.scheme](
+            stage,
+            p,
+            info,
+            cache=cache,
+            remote=remote,
+            metric=metric,
+            persist=persist,
+            tags=tags,
+        )
 
     for o in OUTS:
         if o.supported(p):
-            return o(stage, p, info, cache=cache, remote=None, metric=metric)
-    raise DvcException('Output \'{}\' is not supported'.format(p))
+            return o(
+                stage,
+                p,
+                info,
+                cache=cache,
+                remote=None,
+                metric=metric,
+                persist=persist,
+                tags=tags,
+            )
+    return OutputLOCAL(
+        stage,
+        p,
+        info,
+        cache=cache,
+        remote=None,
+        metric=metric,
+        persist=persist,
+        tags=tags,
+    )
 
 
 def loadd_from(stage, d_list):
     ret = []
     for d in d_list:
-        p = d.pop(DependencyBase.PARAM_PATH)
-        cache = d.pop(OutputLOCAL.PARAM_CACHE, True)
-        metric = d.pop(OutputLOCAL.PARAM_METRIC, False)
-        ret.append(_get(stage, p, info=d, cache=cache, metric=metric))
+        p = d.pop(OutputBase.PARAM_PATH)
+        cache = d.pop(OutputBase.PARAM_CACHE, True)
+        metric = d.pop(OutputBase.PARAM_METRIC, False)
+        persist = d.pop(OutputBase.PARAM_PERSIST, False)
+        tags = d.pop(OutputBase.PARAM_TAGS, None)
+        ret.append(
+            _get(
+                stage,
+                p,
+                info=d,
+                cache=cache,
+                metric=metric,
+                persist=persist,
+                tags=tags,
+            )
+        )
     return ret
 
 
-def loads_from(stage, s_list, use_cache=True, metric=False):
+def loads_from(stage, s_list, use_cache=True, metric=False, persist=False):
     ret = []
     for s in s_list:
-        ret.append(_get(stage, s, info={}, cache=use_cache, metric=metric))
+        ret.append(
+            _get(
+                stage,
+                s,
+                info={},
+                cache=use_cache,
+                metric=metric,
+                persist=persist,
+            )
+        )
     return ret

dvc/output/base.py

@@ -1,24 +1,323 @@
+from __future__ import unicode_literals
+
+import re
+import logging
+from copy import copy
+
+from schema import Or, Optional
+
 from dvc.exceptions import DvcException
+from dvc.utils.compat import str
+from dvc.remote.base import RemoteBASE
 
 
-class OutputError(DvcException):
-    def __init__(self, path, msg):
-        msg = 'Output \'{}\' error: {}'.format(path, msg)
-        super(OutputError, self).__init__(msg)
+logger = logging.getLogger(__name__)
 
 
-class OutputDoesNotExistError(OutputError):
+class OutputDoesNotExistError(DvcException):
     def __init__(self, path):
-        super(OutputDoesNotExistError, self).__init__(path, 'does not exist')
+        msg = "output '{}' does not exist".format(path)
+        super(OutputDoesNotExistError, self).__init__(msg)
 
 
-class OutputIsNotFileOrDirError(OutputError):
+class OutputIsNotFileOrDirError(DvcException):
     def __init__(self, path):
-        msg = 'not a file or directory'
-        super(OutputIsNotFileOrDirError, self).__init__(path, msg)
+        msg = "output '{}' is not a file or directory".format(path)
+        super(OutputIsNotFileOrDirError, self).__init__(msg)
 
 
-class OutputAlreadyTrackedError(OutputError):
+class OutputAlreadyTrackedError(DvcException):
     def __init__(self, path):
-        msg = 'already tracked by scm(e.g. git)'
-        super(OutputAlreadyTrackedError, self).__init__(path, msg)
+        msg = "output '{}' is already tracked by scm (e.g. git)".format(path)
+        super(OutputAlreadyTrackedError, self).__init__(msg)
+
+
+class OutputBase(object):
+    IS_DEPENDENCY = False
+
+    REMOTE = RemoteBASE
+
+    PARAM_PATH = "path"
+    PARAM_CACHE = "cache"
+    PARAM_METRIC = "metric"
+    PARAM_METRIC_TYPE = "type"
+    PARAM_METRIC_XPATH = "xpath"
+    PARAM_PERSIST = "persist"
+
+    METRIC_SCHEMA = Or(
+        None,
+        bool,
+        {
+            Optional(PARAM_METRIC_TYPE): Or(str, None),
+            Optional(PARAM_METRIC_XPATH): Or(str, None),
+        },
+    )
+
+    PARAM_TAGS = "tags"
+
+    DoesNotExistError = OutputDoesNotExistError
+    IsNotFileOrDirError = OutputIsNotFileOrDirError
+
+    def __init__(
+        self,
+        stage,
+        path,
+        info=None,
+        remote=None,
+        cache=True,
+        metric=False,
+        persist=False,
+        tags=None,
+    ):
+        self.stage = stage
+        self.repo = stage.repo
+        self.url = path
+        self.info = info
+        self.remote = remote or self.REMOTE(self.repo, {})
+        self.use_cache = False if self.IS_DEPENDENCY else cache
+        self.metric = False if self.IS_DEPENDENCY else metric
+        self.persist = persist
+        self.tags = None if self.IS_DEPENDENCY else (tags or {})
+
+        if self.use_cache and self.cache is None:
+            raise DvcException(
+                "no cache location setup for '{}' outputs.".format(
+                    self.REMOTE.scheme
+                )
+            )
+
+    def __repr__(self):
+        return "{class_name}: '{url}'".format(
+            class_name=type(self).__name__, url=(self.url or "No url")
+        )
+
+    def __str__(self):
+        return self.url
+
+    @property
+    def is_in_repo(self):
+        return False
+
+    @property
+    def cache(self):
+        return getattr(self.repo.cache, self.scheme)
+
+    @property
+    def dir_cache(self):
+        return self.cache.get_dir_cache(self.checksum)
+
+    def assign_to_stage_file(self, target_repo):
+        raise DvcException(
+            "change repo is not supported for {}".format(self.scheme)
+        )
+
+    @classmethod
+    def match(cls, url):
+        return re.match(cls.REMOTE.REGEX, url)
+
+    def group(self, name):
+        match = self.match(self.url)
+        if not match:
+            return None
+        return match.group(name)
+
+    @classmethod
+    def supported(cls, url):
+        return cls.match(url) is not None
+
+    @property
+    def scheme(self):
+        return self.REMOTE.scheme
+
+    @property
+    def path(self):
+        return self.path_info.path
+
+    @property
+    def cache_path(self):
+        return self.cache.checksum_to_path(self.checksum)
+
+    @property
+    def sep(self):
+        return "/"
+
+    @property
+    def checksum(self):
+        return self.info.get(self.remote.PARAM_CHECKSUM)
+
+    @property
+    def is_dir_checksum(self):
+        return self.remote.is_dir_checksum(self.checksum)
+
+    @property
+    def exists(self):
+        return self.remote.exists(self.path_info)
+
+    def changed_checksum(self):
+        return (
+            self.checksum
+            != self.remote.save_info(self.path_info)[
+                self.remote.PARAM_CHECKSUM
+            ]
+        )
+
+    def changed_cache(self):
+        if not self.use_cache or not self.checksum:
+            return True
+
+        return self.cache.changed_cache(self.checksum)
+
+    def status(self):
+        if self.checksum and self.use_cache and self.changed_cache():
+            return {str(self): "not in cache"}
+
+        if not self.exists:
+            return {str(self): "deleted"}
+
+        if self.changed_checksum():
+            return {str(self): "modified"}
+
+        if not self.checksum:
+            return {str(self): "new"}
+
+        return {}
+
+    def changed(self):
+        status = self.status()
+        logger.debug(str(status))
+        return bool(status)
+
+    @property
+    def is_empty(self):
+        return self.remote.is_empty(self.path_info)
+
+    def isdir(self):
+        return self.remote.isdir(self.path_info)
+
+    def isfile(self):
+        return self.remote.isfile(self.path_info)
+
+    def save(self):
+        if not self.exists:
+            raise self.DoesNotExistError(self)
+
+        if not self.isfile and not self.isdir:
+            raise self.IsNotFileOrDirError(self)
+
+        if self.is_empty:
+            logger.warning("'{}' is empty.".format(self))
+
+        if not self.use_cache:
+            self.info = self.remote.save_info(self.path_info)
+            if self.metric:
+                self.verify_metric()
+            if not self.IS_DEPENDENCY:
+                logger.info(
+                    "Output '{}' doesn't use cache. Skipping saving.".format(
+                        self
+                    )
+                )
+            return
+
+        assert not self.IS_DEPENDENCY
+
+        if not self.changed():
+            logger.info(
+                "Output '{}' didn't change. Skipping saving.".format(self)
+            )
+            return
+
+        if self.is_in_repo:
+            if self.repo.scm.is_tracked(self.path):
+                raise OutputAlreadyTrackedError(self)
+
+            if self.use_cache:
+                self.repo.scm.ignore(self.path)
+
+        self.info = self.remote.save_info(self.path_info)
+
+    def commit(self):
+        if self.use_cache:
+            self.cache.save(self.path_info, self.info)
+
+    def dumpd(self):
+        ret = copy(self.info)
+        ret[self.PARAM_PATH] = self.url
+
+        if self.IS_DEPENDENCY:
+            return ret
+
+        ret[self.PARAM_CACHE] = self.use_cache
+
+        if isinstance(self.metric, dict):
+            if (
+                self.PARAM_METRIC_XPATH in self.metric
+                and not self.metric[self.PARAM_METRIC_XPATH]
+            ):
+                del self.metric[self.PARAM_METRIC_XPATH]
+
+        ret[self.PARAM_METRIC] = self.metric
+        ret[self.PARAM_PERSIST] = self.persist
+
+        if self.tags:
+            ret[self.PARAM_TAGS] = self.tags
+
+        return ret
+
+    def verify_metric(self):
+        raise DvcException(
+            "verify metric is not supported for {}".format(self.scheme)
+        )
+
+    def download(self, to_info, resume=False):
+        self.remote.download([self.path_info], [to_info], resume=resume)
+
+    def checkout(self, force=False, progress_callback=None, tag=None):
+        if not self.use_cache:
+            return
+
+        if tag:
+            info = self.tags[tag]
+        else:
+            info = self.info
+
+        self.cache.checkout(
+            self.path_info,
+            info,
+            force=force,
+            progress_callback=progress_callback,
+        )
+
+    def remove(self, ignore_remove=False):
+        self.remote.remove(self.path_info)
+        if self.scheme != "local":
+            return
+
+        if ignore_remove and self.use_cache and self.is_in_repo:
+            self.repo.scm.ignore_remove(self.path)
+
+    def move(self, out):
+        if self.scheme == "local" and self.use_cache and self.is_in_repo:
+            self.repo.scm.ignore_remove(self.path)
+
+        self.remote.move(self.path_info, out.path_info)
+        self.url = out.url
+        self.path_info = copy(out.path_info)
+        self.save()
+        self.commit()
+
+        if self.scheme == "local" and self.use_cache and self.is_in_repo:
+            self.repo.scm.ignore(self.path)
+
+    def get_files_number(self):
+        if not self.use_cache or not self.checksum:
+            return 0
+
+        if self.is_dir_checksum:
+            return len(self.dir_cache)
+
+        return 1
+
+    def unprotect(self):
+        if self.exists:
+            self.remote.unprotect(self.path_info)

dvc/output/gs.py

@@ -1,53 +1,8 @@
-from dvc.dependency.gs import DependencyGS
-from dvc.exceptions import DvcException
+from __future__ import unicode_literals
+
+from dvc.remote.gs import RemoteGS
+from dvc.output.s3 import OutputS3
 
 
-class OutputGS(DependencyGS):
-    PARAM_CACHE = 'cache'
-    PARAM_METRIC = 'metric'
-
-    def __init__(self,
-                 stage,
-                 path,
-                 info=None,
-                 remote=None,
-                 cache=True,
-                 metric=False):
-        super(OutputGS, self).__init__(stage, path, info, remote=remote)
-        self.use_cache = cache
-        self.metric = metric
-        if cache and self.project.cache.gs is None:
-            raise DvcException("No cache location setup for \'gs\' outputs.")
-
-    def dumpd(self):
-        ret = super(OutputGS, self).dumpd()
-        ret[self.PARAM_CACHE] = self.use_cache
-        ret[self.PARAM_METRIC] = self.metric
-        return ret
-
-    def changed(self):
-        if super(OutputGS, self).changed():
-            return True
-
-        if self.use_cache \
-           and self.project.cache.gs.changed(self.path_info, self.info):
-            return True
-
-        return False
-
-    def checkout(self):
-        if not self.use_cache:
-            return
-
-        self.project.cache.gs.checkout(self.path_info, self.info)
-
-    def save(self):
-        super(OutputGS, self).save()
-
-        if not self.use_cache:
-            return
-
-        self.info = self.project.cache.gs.save(self.path_info)
-
-    def remove(self, ignore_remove=False):
-        self.remote.remove(self.path_info)
+class OutputGS(OutputS3):
+    REMOTE = RemoteGS

dvc/output/hdfs.py

@@ -1,53 +1,38 @@
-from dvc.dependency.hdfs import DependencyHDFS
-from dvc.exceptions import DvcException
+from __future__ import unicode_literals
+
+import posixpath
+
+from dvc.path.hdfs import PathHDFS
+from dvc.utils.compat import urlparse
+from dvc.output.base import OutputBase
+from dvc.remote.hdfs import RemoteHDFS
 
 
-class OutputHDFS(DependencyHDFS):
-    PARAM_CACHE = 'cache'
-    PARAM_METRIC = 'metric'
+class OutputHDFS(OutputBase):
+    REMOTE = RemoteHDFS
 
-    def __init__(self,
-                 stage,
-                 path,
-                 info=None,
-                 remote=None,
-                 cache=True,
-                 metric=False):
-        super(OutputHDFS, self).__init__(stage, path, info, remote=remote)
-        self.use_cache = cache
-        self.metric = metric
-        if cache and self.project.cache.hdfs is None:
-            raise DvcException("No cache location setup for \'hdfs\' outputs.")
-
-    def dumpd(self):
-        ret = super(OutputHDFS, self).dumpd()
-        ret[self.PARAM_CACHE] = self.use_cache
-        ret[self.PARAM_METRIC] = self.metric
-        return ret
-
-    def changed(self):
-        if super(OutputHDFS, self).changed():
-            return True
-
-        if self.use_cache \
-           and self.project.cache.hdfs.changed(self.path_info, self.info):
-            return True
-
-        return False
-
-    def checkout(self):
-        if not self.use_cache:
-            return
-
-        self.project.cache.hdfs.checkout(self.path_info, self.info)
-
-    def save(self):
-        super(OutputHDFS, self).save()
-
-        if not self.use_cache:
-            return
-
-        self.info = self.project.cache.hdfs.save(self.path_info)
-
-    def remove(self, ignore_remove=False):
-        self.remote.remove(self.path_info)
+    def __init__(
+        self,
+        stage,
+        path,
+        info=None,
+        remote=None,
+        cache=True,
+        metric=False,
+        persist=False,
+        tags=None,
+    ):
+        super(OutputHDFS, self).__init__(
+            stage,
+            path,
+            info=info,
+            remote=remote,
+            cache=cache,
+            metric=metric,
+            persist=persist,
+            tags=tags,
+        )
+        if remote:
+            path = posixpath.join(remote.url, urlparse(path).path.lstrip("/"))
+        user = remote.user if remote else self.group("user")
+        self.path_info = PathHDFS(user=user, url=self.url, path=path)

dvc/output/local.py

@@ -1,130 +1,113 @@
+from __future__ import unicode_literals
+
 import os
-from schema import Optional, Or
+import logging
 
-from dvc.output.base import OutputDoesNotExistError, OutputIsNotFileOrDirError
-from dvc.output.base import OutputAlreadyTrackedError
-from dvc.dependency.local import DependencyLOCAL
-from dvc.exceptions import DvcException
+from dvc.path.local import PathLOCAL
+from dvc.utils.compat import urlparse
 from dvc.istextfile import istextfile
+from dvc.exceptions import DvcException
+from dvc.remote.local import RemoteLOCAL
+from dvc.output.base import OutputBase
 
 
-class OutputLOCAL(DependencyLOCAL):
-    PARAM_CACHE = 'cache'
-    PARAM_METRIC = 'metric'
-    PARAM_METRIC_TYPE = 'type'
-    PARAM_METRIC_XPATH = 'xpath'
+logger = logging.getLogger(__name__)
 
-    METRIC_SCHEMA = Or(None, bool,
-                       {Optional(PARAM_METRIC_TYPE): Or(str, None),
-                        Optional(PARAM_METRIC_XPATH): Or(str, None)})
 
-    DoesNotExistError = OutputDoesNotExistError
-    IsNotFileOrDirError = OutputIsNotFileOrDirError
+class OutputLOCAL(OutputBase):
+    REMOTE = RemoteLOCAL
 
-    def __init__(self,
-                 stage,
-                 path,
-                 info=None,
-                 remote=None,
-                 cache=True,
-                 metric=False):
-        super(OutputLOCAL, self).__init__(stage, path, info, remote=remote)
-        self.use_cache = cache
-        self.metric = metric
+    def __init__(
+        self,
+        stage,
+        path,
+        info=None,
+        remote=None,
+        cache=True,
+        metric=False,
+        persist=False,
+        tags=None,
+    ):
+        super(OutputLOCAL, self).__init__(
+            stage,
+            path,
+            info,
+            remote=remote,
+            cache=cache,
+            metric=metric,
+            persist=persist,
+            tags=tags,
+        )
+        if remote:
+            p = os.path.join(
+                remote.prefix, urlparse(self.url).path.lstrip("/")
+            )
+        else:
+            p = path
+
+        if not os.path.isabs(p):
+            p = self.remote.to_ospath(p)
+            p = os.path.join(stage.wdir, p)
+        p = os.path.abspath(os.path.normpath(p))
+
+        self.path_info = PathLOCAL(url=self.url, path=p)
+
+    def __str__(self):
+        return self.rel_path
 
     @property
-    def md5(self):
-        return self.info.get(self.project.cache.local.PARAM_MD5, None)
+    def is_in_repo(self):
+        return urlparse(self.url).scheme != "remote" and not os.path.isabs(
+            self.url
+        )
+
+    def assign_to_stage_file(self, stage):
+        from dvc.repo import Repo
+
+        fullpath = os.path.abspath(stage.wdir)
+        self.path_info.path = os.path.join(fullpath, self.stage_path)
+
+        self.repo = Repo(self.path)
+
+        self.stage = stage
+        return self
 
     @property
-    def cache(self):
-        return self.project.cache.local.get(self.md5)
+    def sep(self):
+        return os.sep
+
+    @property
+    def rel_path(self):
+        return os.path.relpath(self.path)
+
+    @property
+    def stage_path(self):
+        return os.path.relpath(self.path, self.stage.wdir)
 
     def dumpd(self):
         ret = super(OutputLOCAL, self).dumpd()
-        ret[self.PARAM_CACHE] = self.use_cache
+        if self.is_in_repo:
+            path = self.remote.to_posixpath(
+                os.path.relpath(self.path, self.stage.wdir)
+            )
+        else:
+            path = self.url
 
-        if isinstance(self.metric, dict):
-            if self.PARAM_METRIC_XPATH in self.metric and \
-               not self.metric[self.PARAM_METRIC_XPATH]:
-                del self.metric[self.PARAM_METRIC_XPATH]
-
-        if self.metric:
-            ret[self.PARAM_METRIC] = self.metric
+        ret[self.PARAM_PATH] = path
 
         return ret
 
-    def changed(self):
-        if not self.use_cache:
-            return super(OutputLOCAL, self).changed()
-
-        return self.project.cache.local.changed(self.path_info, self.info)
-
-    def checkout(self):
-        if not self.use_cache:
-            return
-
-        self.project.cache.local.checkout(self.path_info, self.info)
-
-    def _verify_metric(self):
+    def verify_metric(self):
         if not self.metric:
             return
 
+        if not os.path.exists(self.path):
+            return
+
         if os.path.isdir(self.path):
-            msg = 'Directory \'{}\' cannot be used as metrics.'
+            msg = "directory '{}' cannot be used as metrics."
             raise DvcException(msg.format(self.rel_path))
 
         if not istextfile(self.path):
-            msg = 'Binary file \'{}\' cannot be used as metrics.'
+            msg = "binary file '{}' cannot be used as metrics."
             raise DvcException(msg.format(self.rel_path))
-
-    def save(self):
-        if not self.use_cache:
-            super(OutputLOCAL, self).save()
-            self._verify_metric()
-            msg = 'Output \'{}\' doesn\'t use cache. Skipping saving.'
-            self.project.logger.info(msg.format(self.rel_path))
-            return
-
-        if not os.path.exists(self.path):
-            raise self.DoesNotExistError(self.rel_path)
-
-        if not os.path.isfile(self.path) \
-           and not os.path.isdir(self.path):  # pragma: no cover
-            raise self.IsNotFileOrDirError(self.rel_path)
-
-        if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \
-           (os.path.isdir(self.path) and len(os.listdir(self.path)) == 0):
-            msg = "File/directory '{}' is empty.".format(self.rel_path)
-            self.project.logger.warn(msg)
-
-        if not self.changed():
-            msg = 'Output \'{}\' didn\'t change. Skipping saving.'
-            self.project.logger.info(msg.format(self.rel_path))
-            return
-
-        if self.is_local:
-            if self.project.scm.is_tracked(self.path):
-                raise OutputAlreadyTrackedError(self.rel_path)
-
-            if self.use_cache:
-                self.project.scm.ignore(self.path)
-
-        self.info = self.project.cache.local.save(self.path_info)
-
-    def remove(self, ignore_remove=False):
-        self.remote.remove(self.path_info)
-        if ignore_remove and self.use_cache and self.is_local:
-            self.project.scm.ignore_remove(self.path)
-
-    def move(self, out):
-        if self.use_cache and self.is_local:
-            self.project.scm.ignore_remove(self.path)
-
-        self.remote.move(self.path_info, out.path_info)
-        self.path = out.path
-        self.path_info = out.path_info
-        self.save()
-
-        if self.use_cache and self.is_local:
-            self.project.scm.ignore(self.path)

dvc/output/s3.py

@@ -1,53 +1,42 @@
-from dvc.dependency.s3 import DependencyS3
-from dvc.exceptions import DvcException
+from __future__ import unicode_literals
+
+import posixpath
+
+from dvc.path import Path
+from dvc.remote.s3 import RemoteS3
+from dvc.utils.compat import urlparse
+from dvc.output.base import OutputBase
 
 
-class OutputS3(DependencyS3):
-    PARAM_CACHE = 'cache'
-    PARAM_METRIC = 'metric'
+class OutputS3(OutputBase):
+    REMOTE = RemoteS3
 
-    def __init__(self,
-                 stage,
-                 path,
-                 info=None,
-                 remote=None,
-                 cache=True,
-                 metric=False):
-        super(OutputS3, self).__init__(stage, path, info, remote=remote)
-        self.use_cache = cache
-        self.metric = metric
-        if cache and self.project.cache.s3 is None:
-            raise DvcException("No cache location setup for \'s3\' outputs.")
+    def __init__(
+        self,
+        stage,
+        path,
+        info=None,
+        remote=None,
+        cache=True,
+        metric=False,
+        persist=False,
+        tags=None,
+    ):
+        super(OutputS3, self).__init__(
+            stage,
+            path,
+            info=info,
+            remote=remote,
+            cache=cache,
+            metric=metric,
+            persist=persist,
+            tags=tags,
+        )
+        bucket = remote.bucket if remote else urlparse(path).netloc
+        path = urlparse(path).path.lstrip("/")
+        if remote:
+            path = posixpath.join(remote.prefix, path)
 
-    def dumpd(self):
-        ret = super(OutputS3, self).dumpd()
-        ret[self.PARAM_CACHE] = self.use_cache
-        ret[self.PARAM_METRIC] = self.metric
-        return ret
-
-    def changed(self):
-        if super(OutputS3, self).changed():
-            return True
-
-        if self.use_cache \
-           and self.project.cache.s3.changed(self.path_info, self.info):
-            return True
-
-        return False
-
-    def checkout(self):
-        if not self.use_cache:
-            return
-
-        self.project.cache.s3.checkout(self.path_info, self.info)
-
-    def save(self):
-        super(OutputS3, self).save()
-
-        if not self.use_cache:
-            return
-
-        self.info = self.project.cache.s3.save(self.path_info)
-
-    def remove(self, ignore_remove=False):
-        self.remote.remove(self.path_info)
+        self.path_info = Path(
+            self.scheme, bucket=bucket, path=path, url=self.url
+        )

dvc/output/ssh.py

@@ -1,53 +1,54 @@
-from dvc.dependency.ssh import DependencySSH
-from dvc.exceptions import DvcException
+from __future__ import unicode_literals
+
+import getpass
+import posixpath
+
+from dvc.path.ssh import PathSSH
+from dvc.utils.compat import urlparse
+from dvc.output.base import OutputBase
+from dvc.remote.ssh import RemoteSSH
 
 
-class OutputSSH(DependencySSH):
-    PARAM_CACHE = 'cache'
-    PARAM_METRIC = 'metric'
+class OutputSSH(OutputBase):
+    REMOTE = RemoteSSH
 
-    def __init__(self,
-                 stage,
-                 path,
-                 info=None,
-                 remote=None,
-                 cache=True,
-                 metric=False):
-        super(OutputSSH, self).__init__(stage, path, info, remote=remote)
-        self.use_cache = cache
-        self.metric = metric
-        if cache and self.project.cache.ssh is None:
-            raise DvcException("No cache location setup for \'ssh\' outputs.")
+    def __init__(
+        self,
+        stage,
+        path,
+        info=None,
+        remote=None,
+        cache=True,
+        metric=False,
+        persist=False,
+        tags=None,
+    ):
+        super(OutputSSH, self).__init__(
+            stage,
+            path,
+            info=info,
+            remote=remote,
+            cache=cache,
+            metric=metric,
+            persist=persist,
+            tags=tags,
+        )
+        parsed = urlparse(path)
+        host = remote.host if remote else parsed.hostname
+        port = (
+            remote.port if remote else (parsed.port or RemoteSSH.DEFAULT_PORT)
+        )
+        user = (
+            remote.user if remote else (parsed.username or getpass.getuser())
+        )
 
-    def dumpd(self):
-        ret = super(OutputSSH, self).dumpd()
-        ret[self.PARAM_CACHE] = self.use_cache
-        ret[self.PARAM_METRIC] = self.metric
-        return ret
+        if remote:
+            path = posixpath.join(
+                remote.prefix, urlparse(path).path.lstrip("/")
+            )
+        else:
+            path = parsed.path
 
-    def changed(self):
-        if super(OutputSSH, self).changed():
-            return True
-
-        if self.use_cache \
-           and self.project.cache.ssh.changed(self.path_info, self.info):
-            return True
-
-        return False
-
-    def checkout(self):
-        if not self.use_cache:
-            return
-
-        self.project.cache.ssh.checkout(self.path_info, self.info)
-
-    def save(self):
-        super(OutputSSH, self).save()
-
-        if not self.use_cache:
-            return
-
-        self.info = self.project.cache.ssh.save(self.path_info)
-
-    def remove(self, ignore_remove=False):
-        self.remote.remove(self.path_info)
+        self.path_info = PathSSH(
+            host=host, user=user, port=port, url=self.url, path=path
+        )

dvc/path/__init__.py

@@ -0,0 +1,29 @@
+from dvc.scheme import Schemes
+
+from dvc.path.azure import PathAZURE
+from dvc.path.gs import PathGS
+from dvc.path.hdfs import PathHDFS
+from dvc.path.http import PathHTTP
+from dvc.path.https import PathHTTPS
+from dvc.path.local import PathLOCAL
+from dvc.path.oss import PathOSS
+from dvc.path.s3 import PathS3
+from dvc.path.ssh import PathSSH
+
+
+PATH_MAP = {
+    Schemes.SSH: PathSSH,
+    Schemes.HDFS: PathHDFS,
+    Schemes.S3: PathS3,
+    Schemes.AZURE: PathAZURE,
+    Schemes.HTTP: PathHTTP,
+    Schemes.HTTPS: PathHTTPS,
+    Schemes.GS: PathGS,
+    Schemes.LOCAL: PathLOCAL,
+    Schemes.OSS: PathOSS,
+}
+
+
+def Path(scheme, *args, **kwargs):
+    cls = PATH_MAP[scheme]
+    return cls(*args, **kwargs)

dvc/path/azure.py

@@ -0,0 +1,6 @@
+from dvc.scheme import Schemes
+from .base import PathCloudBASE
+
+
+class PathAZURE(PathCloudBASE):
+    scheme = Schemes.AZURE

dvc/path/base.py

@@ -0,0 +1,23 @@
+from dvc.utils.compat import urlunsplit
+
+
+class PathBASE(object):
+    scheme = None
+
+    def __init__(self, url=None, path=None):
+        self.url = url
+        self.path = path
+
+    def __str__(self):
+        return self.url
+
+
+class PathCloudBASE(PathBASE):
+    def __init__(self, bucket, url=None, path=None):
+        super(PathCloudBASE, self).__init__(url, path)
+        self.bucket = bucket
+
+    def __str__(self):
+        if not self.url:
+            return urlunsplit((self.scheme, self.bucket, self.path, "", ""))
+        return self.url

dvc/path/gs.py

@@ -0,0 +1,6 @@
+from dvc.scheme import Schemes
+from .base import PathCloudBASE
+
+
+class PathGS(PathCloudBASE):
+    scheme = Schemes.GS

dvc/path/hdfs.py

@@ -0,0 +1,16 @@
+from dvc.utils.compat import urlunsplit
+from dvc.scheme import Schemes
+from .base import PathBASE
+
+
+class PathHDFS(PathBASE):
+    scheme = Schemes.HDFS
+
+    def __init__(self, user, url=None, path=None):
+        super(PathHDFS, self).__init__(url, path)
+        self.user = user
+
+    def __str__(self):
+        if not self.url:
+            return urlunsplit((self.scheme, self.user, self.path, "", ""))
+        return self.url

dvc/path/http.py

@@ -0,0 +1,13 @@
+from dvc.scheme import Schemes
+from dvc.utils.compat import urlunsplit
+
+from .base import PathBASE
+
+
+class PathHTTP(PathBASE):
+    scheme = Schemes.HTTP
+
+    def __str__(self):
+        if not self.url:
+            return urlunsplit((self.scheme, self.path, "", "", ""))
+        return self.url

dvc/path/https.py

@@ -0,0 +1,6 @@
+from dvc.scheme import Schemes
+from .http import PathHTTP
+
+
+class PathHTTPS(PathHTTP):
+    scheme = Schemes.HTTPS

dvc/path/local.py

@@ -0,0 +1,11 @@
+import os
+
+from dvc.scheme import Schemes
+from .base import PathBASE
+
+
+class PathLOCAL(PathBASE):
+    scheme = Schemes.LOCAL
+
+    def __str__(self):
+        return os.path.relpath(self.path)

dvc/path/oss.py

@@ -0,0 +1,6 @@
+from dvc.scheme import Schemes
+from .base import PathCloudBASE
+
+
+class PathOSS(PathCloudBASE):
+    scheme = Schemes.OSS

dvc/path/s3.py

@@ -0,0 +1,6 @@
+from dvc.scheme import Schemes
+from .base import PathCloudBASE
+
+
+class PathS3(PathCloudBASE):
+    scheme = Schemes.S3

dvc/path/ssh.py

@@ -0,0 +1,26 @@
+from dvc.utils.compat import urlunsplit
+from dvc.scheme import Schemes
+from .base import PathBASE
+
+
+class PathSSH(PathBASE):
+    scheme = Schemes.SSH
+
+    def __init__(self, host, user, port, url=None, path=None):
+        super(PathSSH, self).__init__(url, path)
+        self.host = host
+        self.user = user
+        self.port = port
+
+    def __str__(self):
+        if not self.url:
+            return urlunsplit(
+                (
+                    self.scheme,
+                    "{}@{}:{}".format(self.user, self.host, self.port),
+                    self.path,
+                    "",
+                    "",
+                )
+            )
+        return self.url

dvc/progress.py

@@ -1,12 +1,21 @@
+"""Manages progress bars for dvc repo."""
+
 from __future__ import print_function
+from __future__ import unicode_literals
+
+from dvc.utils.compat import str
+
 import sys
 import threading
 
+CLEARLINE_PATTERN = "\r\x1b[K"
+
 
 class Progress(object):
     """
     Simple multi-target progress bar.
     """
+
     def __init__(self):
         self._n_total = 0
         self._n_finished = 0
@@ -14,22 +23,31 @@ class Progress(object):
         self._line = None
 
     def set_n_total(self, total):
+        """Sets total number of targets."""
         self._n_total = total
         self._n_finished = 0
 
     @property
     def is_finished(self):
+        """Returns if all targets have finished."""
         return self._n_total == self._n_finished
 
-    def _clearln(self):
-        print('\r\x1b[K', end='')
+    def clearln(self):
+        self._print(CLEARLINE_PATTERN, end="")
 
     def _writeln(self, line):
-        self._clearln()
-        print(line, end='')
+        self.clearln()
+        self._print(line, end="")
         sys.stdout.flush()
 
+    def reset(self):
+        with self._lock:
+            self._n_total = 0
+            self._n_finished = 0
+            self._line = None
+
     def refresh(self, line=None):
+        """Refreshes progress bar."""
         # Just go away if it is locked. Will update next time
         if not self._lock.acquire(False):
             return
@@ -44,17 +62,19 @@ class Progress(object):
         self._lock.release()
 
     def update_target(self, name, current, total):
+        """Updates progress bar for a specified target."""
         self.refresh(self._bar(name, current, total))
 
     def finish_target(self, name):
+        """Finishes progress bar for a specified target."""
         # We have to write a msg about finished target
         with self._lock:
-            bar = self._bar(name, 100, 100)
+            pbar = self._bar(name, 100, 100)
 
             if sys.stdout.isatty():
-                self._clearln()
+                self.clearln()
 
-            print(bar)
+            self._print(pbar)
 
             self._n_finished += 1
             self._line = None
@@ -67,33 +87,68 @@ class Progress(object):
         bar_len = 30
 
         if total is None:
-            progress = 0
+            state = 0
             percent = "?% "
         else:
             total = int(total)
-            progress = int((100 * current)/total) if current < total else 100
-            percent = str(progress) + "% "
+            state = int((100 * current) / total) if current < total else 100
+            percent = str(state) + "% "
 
         if self._n_total > 1:
             num = "({}/{}): ".format(self._n_finished + 1, self._n_total)
         else:
             num = ""
 
-        n_sh = int((progress * bar_len)/100)
+        n_sh = int((state * bar_len) / 100)
         n_sp = bar_len - n_sh
-        bar = "[" + '#'*n_sh + ' '*n_sp + "] "
+        pbar = "[" + "#" * n_sh + " " * n_sp + "] "
 
-        return num + bar + percent + target_name
+        return num + pbar + percent + target_name
+
+    @staticmethod
+    def _print(*args, **kwargs):
+        import logging
+
+        logger = logging.getLogger(__name__)
+
+        if logger.getEffectiveLevel() == "CRITICAL":
+            return
+
+        print(*args, **kwargs)
 
     def __enter__(self):
         self._lock.acquire(True)
         if self._line is not None:
-            self._clearln()
+            self.clearln()
 
-    def __exit__(self, type, value, tb):
+    def __exit__(self, typ, value, tbck):
         if self._line is not None:
             self.refresh()
         self._lock.release()
 
+    def __call__(self, seq, name="", total=None):
+        if total is None:
+            total = len(seq)
 
-progress = Progress()
+        self.update_target(name, 0, total)
+        for done, item in enumerate(seq, start=1):
+            yield item
+            self.update_target(name, done, total)
+        self.finish_target(name)
+
+
+class ProgressCallback(object):
+    def __init__(self, total):
+        self.total = total
+        self.current = 0
+        progress.reset()
+
+    def update(self, name, progress_to_add=1):
+        self.current += progress_to_add
+        progress.update_target(name, self.current, self.total)
+
+    def finish(self, name):
+        progress.finish_target(name)
+
+
+progress = Progress()  # pylint: disable=invalid-name

dvc/prompt.py

@@ -1,40 +1,62 @@
+"""Manages user prompts."""
+
+from __future__ import unicode_literals
+from __future__ import print_function
+
 import sys
+import logging
+from getpass import getpass
 
-from dvc.progress import progress
-
-try:
-    # NOTE: in Python3 raw_input() was renamed to input()
-    input = raw_input
-except NameError:
-    pass
+from dvc.utils.compat import input
 
 
-class Prompt(object):
-    def __init__(self):
-        self.default = None
+logger = logging.getLogger(__name__)
 
-    def prompt(self, msg, default=False):  # pragma: no cover
-        if self.default is not None:
-            return self.default
 
-        if not sys.stdout.isatty():
-            return default
+def _ask(prompt, limited_to=None):
+    if not sys.stdout.isatty():
+        return None
 
-        answer = input(msg + u' (y/n)\n').lower()
-        while answer not in ['yes', 'no', 'y', 'n']:
-            answer = input('Enter \'yes\' or \'no\'.\n').lower()
-
-        return answer[0] == "y"
-
-    def prompt_password(self, msg):  # pragma: no cover
-        import getpass
-
-        if not sys.stdout.isatty():
+    while True:
+        try:
+            answer = input(prompt + " ").lower()
+        except EOFError:
             return None
 
-        msg = 'Enter password for {}:\n'.format(msg)
+        if not limited_to:
+            return answer
 
-        if not progress.is_finished:
-            msg = u'\n' + msg
+        if answer in limited_to:
+            return answer
 
-        return getpass.getpass(msg)
+        logger.info(
+            "Your response must be one of: {options}. "
+            "Please try again.".format(options=limited_to)
+        )
+
+
+def confirm(statement):
+    """Ask the user for confirmation about the specified statement.
+
+    Args:
+        statement (unicode): statement to ask the user confirmation about.
+
+    Returns:
+        bool: whether or not specified statement was confirmed.
+    """
+    prompt = "{statement} [y/n]".format(statement=statement)
+    answer = _ask(prompt, limited_to=["yes", "no", "y", "n"])
+    return answer and answer.startswith("y")
+
+
+def password(statement):
+    """Ask the user for a password.
+
+    Args:
+        statement (str): string to prompt the user with.
+
+    Returns:
+        str: password entered by the user.
+    """
+    logger.info("{statement}: ".format(statement=statement))
+    return getpass("")

dvc/remote/__init__.py

@@ -1,24 +1,35 @@
-from dvc.remote.local import RemoteLOCAL
-from dvc.remote.s3 import RemoteS3
+from __future__ import unicode_literals
+
+from dvc.remote.azure import RemoteAZURE
 from dvc.remote.gs import RemoteGS
 from dvc.remote.hdfs import RemoteHDFS
+from dvc.remote.local import RemoteLOCAL
+from dvc.remote.s3 import RemoteS3
 from dvc.remote.ssh import RemoteSSH
-from dvc.remote.azure import RemoteAzure
-
-from dvc.config import Config
-from dvc.exceptions import UnsupportedRemoteError
+from dvc.remote.http import RemoteHTTP
+from dvc.remote.https import RemoteHTTPS
+from dvc.remote.oss import RemoteOSS
 
 
-REMOTES = [RemoteHDFS, RemoteSSH, RemoteS3, RemoteGS, RemoteAzure, RemoteLOCAL]
+REMOTES = [
+    RemoteAZURE,
+    RemoteGS,
+    RemoteHDFS,
+    RemoteHTTP,
+    RemoteHTTPS,
+    RemoteS3,
+    RemoteSSH,
+    RemoteOSS,
+    # NOTE: RemoteLOCAL is the default
+]
 
 
-def supported_url(url):
-    config = {Config.SECTION_REMOTE_URL: url}
-    return any(remote.supported(config) for remote in REMOTES)
-
-
-def Remote(project, config):
+def _get(config):
     for remote in REMOTES:
         if remote.supported(config):
-            return remote(project, config)
-    raise UnsupportedRemoteError(str(config))
+            return remote
+    return RemoteLOCAL
+
+
+def Remote(repo, config):
+    return _get(config)(repo, config)

dvc/remote/azure.py

@@ -1,17 +1,27 @@
 from __future__ import absolute_import
+from __future__ import unicode_literals
+
 import os
 import re
-import posixpath
+import logging
+
+from dvc.scheme import Schemes
+from dvc.path.azure import PathAZURE
 
 try:
     from azure.storage.blob import BlockBlobService
+    from azure.common import AzureMissingResourceHttpError
 except ImportError:
     BlockBlobService = None
 
-from dvc.logger import Logger
+from dvc.utils import tmp_fname, move
+from dvc.utils.compat import urlparse, makedirs
 from dvc.progress import progress
 from dvc.config import Config
-from dvc.remote.base import RemoteBase
+from dvc.remote.base import RemoteBASE
+
+
+logger = logging.getLogger(__name__)
 
 
 class Callback(object):
@@ -22,230 +32,171 @@ class Callback(object):
         progress.update_target(self.name, current, total)
 
 
-class RemoteAzure(RemoteBase):
-    scheme = 'azure'
-    REGEX = (r'^azure://'
-             r'(ContainerName=(?P<container_name>[^;]+);?)?'
-             r'(?P<connection_string>.+)?$')
-    REQUIRES = {'azure-storage-blob': BlockBlobService}
-    PARAM_ETAG = 'etag'
+class RemoteAZURE(RemoteBASE):
+    scheme = Schemes.AZURE
+    REGEX = (
+        r"azure://((?P<path>[^=;]*)?|("
+        # backward compatibility
+        r"(ContainerName=(?P<container_name>[^;]+);?)?"
+        r"(?P<connection_string>.+)?)?)$"
+    )
+    REQUIRES = {"azure-storage-blob": BlockBlobService}
+    PARAM_CHECKSUM = "etag"
     COPY_POLL_SECONDS = 5
 
-    def __init__(self, project, config):
-        super(RemoteAzure, self).__init__(project, config)
-        self.project = project
+    def __init__(self, repo, config):
+        super(RemoteAZURE, self).__init__(repo, config)
 
-        self.url = config.get(Config.SECTION_REMOTE_URL)
-        match = re.match(self.REGEX, self.url)
+        self.url = config.get(Config.SECTION_REMOTE_URL, "azure://")
+        match = re.match(self.REGEX, self.url)  # backward compatibility
 
+        path = match.group("path")
         self.bucket = (
-            match.group('container_name')
-            or os.getenv('AZURE_STORAGE_CONTAINER_NAME'))
+            urlparse(self.url if path else "").netloc
+            or match.group("container_name")  # backward compatibility
+            or os.getenv("AZURE_STORAGE_CONTAINER_NAME")
+        )
+
+        self.prefix = urlparse(self.url).path.lstrip("/") if path else ""
 
         self.connection_string = (
-            match.group('connection_string')
-            or os.getenv('AZURE_STORAGE_CONNECTION_STRING'))
+            config.get(Config.SECTION_AZURE_CONNECTION_STRING)
+            or match.group("connection_string")  # backward compatibility
+            or os.getenv("AZURE_STORAGE_CONNECTION_STRING")
+        )
 
         if not self.bucket:
-            raise ValueError('Azure Storage container name missing')
+            raise ValueError("azure storage container name missing")
 
         if not self.connection_string:
-            raise ValueError('Azure Storage connection string missing')
+            raise ValueError("azure storage connection string missing")
 
         self.__blob_service = None
+        self.path_info = PathAZURE(bucket=self.bucket)
 
     @property
     def blob_service(self):
         if self.__blob_service is None:
+            logger.debug("URL {}".format(self.url))
+            logger.debug("Connection string {}".format(self.connection_string))
             self.__blob_service = BlockBlobService(
-                connection_string=self.connection_string)
-            self.__blob_service.create_container(self.bucket)
+                connection_string=self.connection_string
+            )
+            logger.debug("Container name {}".format(self.bucket))
+            try:  # verify that container exists
+                self.__blob_service.list_blobs(
+                    self.bucket, delimiter="/", num_results=1
+                )
+            except AzureMissingResourceHttpError:
+                self.__blob_service.create_container(self.bucket)
         return self.__blob_service
 
-# FIXME: temporarily disabled because of the lack of test for external azure
-# dependencies/outputs/cache.
-#
-#    def _get_etag(self, bucket, key):
-#        try:
-#            blob = self.blob_service.get_blob_properties(bucket, key)
-#            return blob.properties.etag
-#        except Exception:
-#            return None
-#
-#    def save_info(self, path_info):
-#        if path_info['scheme'] != self.scheme:
-#            raise NotImplementedError
-#
-#        return {self.PARAM_ETAG: self._get_etag(
-#            path_info['bucket'], path_info['key'])}
-#
-#    def save(self, path_info):
-#        if path_info['scheme'] != self.scheme:
-#            raise NotImplementedError
-#
-#        etag = self._get_etag(path_info['bucket'], path_info['key'])
-#        dest_key = '{}/{}'.format(etag[0:2], etag[2:])
-#
-#        self._copy_blob(to_bucket=self.bucket,
-#                        to_key=dest_key,
-#                        from_bucket=path_info['bucket'],
-#                        from_key=path_info['key'])
-#
-#        return {self.PARAM_ETAG: etag}
-#
-#    def changed(self, path_info, checksum_info):
-#        return checksum_info != self.save_info(path_info)
-#
-#    def _copy_blob(self, to_bucket, to_key, from_bucket, from_key):
-#        source = self.blob_service.make_blob_url(from_bucket, from_key)
-#
-#        copy = self.blob_service.copy_blob(to_bucket, to_key, source)
-#
-#        if self.COPY_POLL_SECONDS <= 0:
-#            return
-#
-#        while copy.status != 'success':
-#            time.sleep(self.COPY_POLL_SECONDS)
-#            copy = self.blob_service.get_blob_properties(
-#                to_bucket, to_key).properties.copy
-#
-#    def checkout(self, path_info, checksum_info):
-#        if path_info['scheme'] != self.scheme:
-#            raise NotImplementedError
-#
-#        etag = checksum_info.get(self.PARAM_ETAG, None)
-#        if not etag:
-#            return
-#
-#        self._copy_blob(to_bucket=path_info['bucket'],
-#                        to_key=path_info['key'],
-#                        from_bucket=self.bucket,
-#                        from_key='{}/{}'.format(etag[0:2], etag[2:]))
-#
     def remove(self, path_info):
-        if path_info['scheme'] != self.scheme:
+        if path_info.scheme != self.scheme:
             raise NotImplementedError
 
-        Logger.debug('Removing azure://{}/{}'.format(path_info['bucket'],
-                                                     path_info['key']))
+        logger.debug(
+            "Removing azure://{}/{}".format(path_info.bucket, path_info.path)
+        )
 
-        self.blob_service.delete_blob(path_info['bucket'], path_info['key'])
+        self.blob_service.delete_blob(path_info.bucket, path_info.path)
 
-    def md5s_to_path_infos(self, md5s):
-        return [{
-            'scheme': self.scheme,
-            'bucket': self.bucket,
-            'key': '{}/{}'.format(md5[0:2], md5[2:])
-        } for md5 in md5s]
+    def _list_paths(self, bucket, prefix):
+        blob_service = self.blob_service
+        next_marker = None
+        while True:
+            blobs = blob_service.list_blobs(
+                bucket, prefix=prefix, marker=next_marker
+            )
 
-    def _all_keys(self):
-        return {blob.name
-                for blob in self.blob_service.list_blobs(self.bucket)}
+            for blob in blobs:
+                yield blob.name
 
-    def exists(self, path_infos):
-        keys = self._all_keys()
-        ret = []
-        for path_info in path_infos:
-            if path_info['scheme'] != self.scheme:
-                raise NotImplementedError
+            if not blobs.next_marker:
+                break
 
-            key = path_info['key']
-            ret.append(key in keys)
+            next_marker = blobs.next_marker
 
-        return ret
+    def list_cache_paths(self):
+        return self._list_paths(self.bucket, self.prefix)
 
-    def upload(self, from_infos, to_infos, names=None):
+    def upload(self, from_infos, to_infos, names=None, no_progress_bar=False):
         names = self._verify_path_args(to_infos, from_infos, names)
 
         for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info['scheme'] != self.scheme:
+            if to_info.scheme != self.scheme:
                 raise NotImplementedError
 
-            if from_info['scheme'] != 'local':
+            if from_info.scheme != "local":
                 raise NotImplementedError
 
-            bucket = to_info['bucket']
-            key = to_info['key']
+            bucket = to_info.bucket
+            path = to_info.path
 
-            Logger.debug("Uploading '{}' to '{}/{}'".format(
-                from_info['path'], bucket, key))
+            logger.debug(
+                "Uploading '{}' to '{}/{}'".format(
+                    from_info.path, bucket, path
+                )
+            )
 
             if not name:
-                name = os.path.basename(from_info['path'])
-
-            cb = Callback(name)
-
-            try:
-                self.blob_service.create_blob_from_path(
-                    bucket, key, from_info['path'], progress_callback=cb)
-            except Exception as ex:
-                msg = "Failed to upload '{}'".format(from_info['path'])
-                Logger.warn(msg, ex)
-            else:
-                progress.finish_target(name)
-
-    def download(self,
-                 from_infos,
-                 to_infos,
-                 no_progress_bar=False,
-                 names=None):
-        names = self._verify_path_args(from_infos, to_infos, names)
-
-        for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info['scheme'] != self.scheme:
-                raise NotImplementedError
-
-            if to_info['scheme'] != 'local':
-                raise NotImplementedError
-
-            bucket = from_info['bucket']
-            key = from_info['key']
-
-            Logger.debug("Downloading '{}/{}' to '{}'".format(
-                bucket, key, to_info['path']))
-
-            tmp_file = self.tmp_file(to_info['path'])
-            if not name:
-                name = os.path.basename(to_info['path'])
+                name = os.path.basename(from_info.path)
 
             cb = None if no_progress_bar else Callback(name)
 
-            self._makedirs(to_info['path'])
+            try:
+                self.blob_service.create_blob_from_path(
+                    bucket, path, from_info.path, progress_callback=cb
+                )
+            except Exception:
+                msg = "failed to upload '{}'".format(from_info.path)
+                logger.warning(msg)
+            else:
+                progress.finish_target(name)
+
+    def download(
+        self,
+        from_infos,
+        to_infos,
+        no_progress_bar=False,
+        names=None,
+        resume=False,
+    ):
+        names = self._verify_path_args(from_infos, to_infos, names)
+
+        for to_info, from_info, name in zip(to_infos, from_infos, names):
+            if from_info.scheme != self.scheme:
+                raise NotImplementedError
+
+            if to_info.scheme != "local":
+                raise NotImplementedError
+
+            bucket = from_info.bucket
+            path = from_info.path
+
+            logger.debug(
+                "Downloading '{}/{}' to '{}'".format(
+                    bucket, path, to_info.path
+                )
+            )
+
+            tmp_file = tmp_fname(to_info.path)
+            if not name:
+                name = os.path.basename(to_info.path)
+
+            cb = None if no_progress_bar else Callback(name)
+
+            makedirs(os.path.dirname(to_info.path), exist_ok=True)
 
             try:
                 self.blob_service.get_blob_to_path(
-                    bucket, key, tmp_file, progress_callback=cb)
-            except Exception as exc:
-                msg = "Failed to download '{}/{}'".format(bucket, key)
-                Logger.warn(msg, exc)
+                    bucket, path, tmp_file, progress_callback=cb
+                )
+            except Exception:
+                msg = "failed to download '{}/{}'".format(bucket, path)
+                logger.warning(msg)
             else:
-                os.rename(tmp_file, to_info['path'])
+                move(tmp_file, to_info.path)
 
                 if not no_progress_bar:
                     progress.finish_target(name)
-
-    def _path_to_etag(self, path):
-        assert len(path.split('/')) == 2
-        return posixpath.dirname(path) + posixpath.basename(path)
-
-    def _all(self):
-        keys = self._all_keys()
-        return [self._path_to_etag(key) for key in keys]
-
-    def gc(self, cinfos):
-        from dvc.remote.local import RemoteLOCAL
-
-        used = [info[self.PARAM_ETAG] for info in cinfos['azure']]
-        used += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
-
-        removed = False
-        for etag in self._all():
-            if etag in used:
-                continue
-            path_info = {'scheme': self.scheme,
-                         'key': posixpath.join(etag[0:2], etag[2:]),
-                         'bucket': self.bucket}
-            self.remove(path_info)
-            removed = True
-
-        return removed

dvc/remote/base.py

@@ -1,14 +1,33 @@
+from __future__ import unicode_literals
+
+from copy import copy
+
+from dvc.path.local import PathLOCAL
+from dvc.utils.compat import str
+
 import os
 import re
-import errno
+import json
+import ntpath
+import logging
+import tempfile
+import posixpath
+from operator import itemgetter
 from multiprocessing import cpu_count
 
+import dvc.prompt as prompt
 from dvc.config import Config
-from dvc.logger import Logger
-from dvc.exceptions import DvcException
+from dvc.exceptions import DvcException, ConfirmRemoveError
+from dvc.progress import progress
+from dvc.utils import LARGE_DIR_SIZE, tmp_fname
+from dvc.state import StateBase
+
+
+logger = logging.getLogger(__name__)
 
 
 STATUS_OK = 1
+STATUS_MISSING = 2
 STATUS_NEW = 3
 STATUS_DELETED = 4
 
@@ -16,7 +35,7 @@ STATUS_DELETED = 4
 STATUS_MAP = {
     # (local_exists, remote_exists)
     (True, True): STATUS_OK,
-    (False, False): STATUS_OK,
+    (False, False): STATUS_MISSING,
     (True, False): STATUS_NEW,
     (False, True): STATUS_DELETED,
 }
@@ -24,17 +43,75 @@ STATUS_MAP = {
 
 class DataCloudError(DvcException):
     """ Data Cloud exception """
+
     def __init__(self, msg):
-        super(DataCloudError, self).__init__('Data sync error: {}'.format(msg))
+        super(DataCloudError, self).__init__("Data sync error: {}".format(msg))
 
 
-class RemoteBase(object):
+class RemoteCmdError(DvcException):
+    def __init__(self, remote, cmd, ret, err):
+        super(RemoteCmdError, self).__init__(
+            "{remote} command '{cmd}' finished with non-zero return code"
+            " {ret}': {err}".format(remote=remote, cmd=cmd, ret=ret, err=err)
+        )
+
+
+class RemoteActionNotImplemented(DvcException):
+    def __init__(self, action, scheme):
+        m = "{} is not supported by {} remote".format(action, scheme)
+        super(RemoteActionNotImplemented, self).__init__(m)
+
+
+class RemoteMissingDepsError(DvcException):
+    pass
+
+
+class RemoteBASE(object):
+    scheme = None
     REGEX = None
     REQUIRES = {}
     JOBS = 4 * cpu_count()
 
-    def __init__(self, project, config):
-        pass
+    PARAM_RELPATH = "relpath"
+    CHECKSUM_DIR_SUFFIX = ".dir"
+
+    def __init__(self, repo, config):
+        self.repo = repo
+        deps_ok = all(self.REQUIRES.values())
+        if not deps_ok:
+            missing = [k for k, v in self.REQUIRES.items() if v is None]
+            url = config.get(
+                Config.SECTION_REMOTE_URL, "{}://".format(self.scheme)
+            )
+            msg = (
+                "URL '{}' is supported but requires these missing "
+                "dependencies: {}. If you have installed dvc using pip, "
+                "choose one of these options to proceed: \n"
+                "\n"
+                "    1) Install specific missing dependencies:\n"
+                "        pip install {}\n"
+                "    2) Install dvc package that includes those missing "
+                "dependencies: \n"
+                "        pip install dvc[{}]\n"
+                "    3) Install dvc package with all possible "
+                "dependencies included: \n"
+                "        pip install dvc[all]\n"
+                "\n"
+                "If you have installed dvc from a binary package and you "
+                "are still seeing this message, please report it to us "
+                "using https://github.com/iterative/dvc/issues. Thank you!"
+            ).format(url, missing, " ".join(missing), self.scheme)
+            raise RemoteMissingDepsError(msg)
+
+        self.protected = False
+        self.state = StateBase()
+
+        self._dir_info = {}
+
+    def __repr__(self):
+        return "{class_name}: '{url}'".format(
+            class_name=type(self).__name__, url=(self.url or "No url")
+        )
 
     def compat_config(config):
         return config.copy()
@@ -42,30 +119,28 @@ class RemoteBase(object):
     @classmethod
     def supported(cls, config):
         url = config[Config.SECTION_REMOTE_URL]
-        url_ok = cls.match(url) is not None
-        deps_ok = all(cls.REQUIRES.values())
-        if url_ok and not deps_ok:
-            missing = [k for k, v in cls.REQUIRES.items() if v is None]
-            msg = "URL \'{}\' is supported but requires these missing " \
-                  "dependencies: {}. If you have installed dvc using pip, " \
-                  "choose one of these options to proceed: \n" \
-                  "\n" \
-                  "    1) Install specific missing dependencies:\n" \
-                  "        pip install {}\n" \
-                  "    2) Install dvc package that includes those missing " \
-                  "dependencies: \n" \
-                  "        pip install dvc[{}]\n" \
-                  "    3) Install dvc package with all possible " \
-                  "dependencies included: \n" \
-                  "        pip install dvc[all]\n" \
-                  "\n" \
-                  "If you have installed dvc from a binary package and you " \
-                  "are still seeing this message, please report it to us " \
-                  "using https://github.com/iterative/dvc/issues. Thank you!"
-            msg = msg.format(url, missing, " ".join(missing), cls.scheme)
-            Logger.warn(msg)
+        return cls.match(url) is not None
 
-        return url_ok and deps_ok
+    @staticmethod
+    def _replace_sep(path, from_sep, to_sep):
+        assert not ntpath.isabs(path)
+        assert not posixpath.isabs(path)
+        return path.replace(from_sep, to_sep)
+
+    @classmethod
+    def to_posixpath(cls, path):
+        return cls._replace_sep(path, "\\", "/")
+
+    @classmethod
+    def to_ntpath(cls, path):
+        return cls._replace_sep(path, "/", "\\")
+
+    def to_ospath(self, path):
+        if self.ospath == ntpath:
+            return self.to_ntpath(path)
+        elif self.ospath == posixpath:
+            return self.to_posixpath(path)
+        assert False
 
     @classmethod
     def match(cls, url):
@@ -77,48 +152,287 @@ class RemoteBase(object):
             return None
         return m.group(name)
 
-    @staticmethod
-    def tmp_file(fname):
-        """ Temporary name for a partial download """
-        import uuid
-        return fname + '.' + str(uuid.uuid4())
+    @property
+    def cache(self):
+        return getattr(self.repo.cache, self.scheme)
 
-    def save_info(self, path_info):
+    def get_file_checksum(self, path_info):
         raise NotImplementedError
 
-    def changed(self, path_info, checksum_info):
-        raise NotImplementedError
+    def _collect_dir(self, path_info):
+        dir_info = []
 
-    def save(self, path_info):
-        raise NotImplementedError
+        p_info = copy(path_info)
+        dpath = p_info.path
+        for root, dirs, files in self.walk(path_info):
+            if len(files) > LARGE_DIR_SIZE:
+                msg = (
+                    "Computing md5 for a large directory {}. "
+                    "This is only done once."
+                )
+                relpath = self.ospath.relpath(root)
+                logger.info(msg.format(relpath))
+                files = progress(files, name=relpath)
 
-    def checkout(self, path_info, checksum_info):
-        raise NotImplementedError
+            for fname in files:
+                path = self.ospath.join(root, fname)
+                p_info.path = path
+                relpath = self.to_posixpath(self.ospath.relpath(path, dpath))
 
-    def download(self, from_infos, to_infos, no_progress_bar=False, name=None):
-        raise NotImplementedError
+                checksum = self.get_file_checksum(p_info)
+                dir_info.append(
+                    {
+                        self.PARAM_RELPATH: relpath,
+                        self.PARAM_CHECKSUM: checksum,
+                    }
+                )
 
-    def upload(self, from_infos, to_infos, path_info, name=None):
-        raise NotImplementedError
+        # NOTE: sorting the list by path to ensure reproducibility
+        return sorted(dir_info, key=itemgetter(self.PARAM_RELPATH))
 
-    def remove(self, path_info):
-        raise NotImplementedError
+    def get_dir_checksum(self, path_info):
+        dir_info = self._collect_dir(path_info)
+        checksum, from_info, to_info = self._get_dir_info_checksum(
+            dir_info, path_info
+        )
+        if self.cache.changed_cache_file(checksum):
+            self.cache.move(from_info, to_info)
 
-    def move(self, path_info):
-        raise NotImplementedError
+        self.state.save(path_info, checksum)
+        self.state.save(to_info, checksum)
 
-    def _makedirs(self, fname):
-        dname = os.path.dirname(fname)
+        return checksum
+
+    def _get_dir_info_checksum(self, dir_info, path_info):
+        to_info = copy(path_info)
+        to_info.path = self.cache.ospath.join(self.cache.prefix, tmp_fname(""))
+
+        tmp = tempfile.NamedTemporaryFile(delete=False).name
+        with open(tmp, "w+") as fobj:
+            json.dump(dir_info, fobj, sort_keys=True)
+
+        from_info = PathLOCAL(path=tmp)
+        self.cache.upload([from_info], [to_info], no_progress_bar=True)
+
+        checksum = self.get_file_checksum(to_info) + self.CHECKSUM_DIR_SUFFIX
+        from_info = copy(to_info)
+        to_info.path = self.cache.checksum_to_path(checksum)
+        return checksum, from_info, to_info
+
+    def get_dir_cache(self, checksum):
+        assert checksum
+
+        dir_info = self._dir_info.get(checksum)
+        if dir_info:
+            return dir_info
+
+        dir_info = self.load_dir_cache(checksum)
+        self._dir_info[checksum] = dir_info
+        return dir_info
+
+    def load_dir_cache(self, checksum):
+        path_info = self.checksum_to_path_info(checksum)
+
+        fobj = tempfile.NamedTemporaryFile(delete=False)
+        path = fobj.name
+        to_info = PathLOCAL(path=path)
+        self.cache.download([path_info], [to_info], no_progress_bar=True)
 
         try:
-            os.makedirs(dname)
-        except OSError as e:
-            if e.errno != errno.EEXIST:
-                raise
+            with open(path, "r") as fobj:
+                d = json.load(fobj)
+        except ValueError:
+            logger.exception("Failed to load dir cache '{}'".format(path_info))
+            return []
+        finally:
+            os.unlink(path)
 
-    def md5s_to_path_infos(self, md5s):
+        if not isinstance(d, list):
+            msg = "dir cache file format error '{}' [skipping the file]"
+            logger.error(msg.format(os.path.relpath(path)))
+            return []
+
+        for info in d:
+            info[self.PARAM_RELPATH] = self.to_ospath(info[self.PARAM_RELPATH])
+
+        return d
+
+    @classmethod
+    def is_dir_checksum(cls, checksum):
+        return checksum.endswith(cls.CHECKSUM_DIR_SUFFIX)
+
+    def get_checksum(self, path_info):
+        if not self.exists(path_info):
+            return None
+
+        checksum = self.state.get(path_info)
+        if checksum:
+            return checksum
+
+        if self.isdir(path_info):
+            checksum = self.get_dir_checksum(path_info)
+        else:
+            checksum = self.get_file_checksum(path_info)
+
+        if checksum:
+            self.state.save(path_info, checksum)
+
+        return checksum
+
+    def save_info(self, path_info):
+        assert path_info.scheme == self.scheme
+        return {self.PARAM_CHECKSUM: self.get_checksum(path_info)}
+
+    def changed(self, path_info, checksum_info):
+        """Checks if data has changed.
+
+        A file is considered changed if:
+            - It doesn't exist on the working directory (was unlinked)
+            - Checksum is not computed (saving a new file)
+            - The checkusm stored in the State is different from the given one
+            - There's no file in the cache
+
+        Args:
+            path_info: dict with path information.
+            checksum: expected checksum for this data.
+
+        Returns:
+            bool: True if data has changed, False otherwise.
+        """
+
+        logger.debug(
+            "checking if '{}'('{}') has changed.".format(
+                path_info, checksum_info
+            )
+        )
+
+        if not self.exists(path_info):
+            logger.debug("'{}' doesn't exist.".format(path_info))
+            return True
+
+        checksum = checksum_info.get(self.PARAM_CHECKSUM)
+        if checksum is None:
+            logger.debug("checksum for '{}' is missing.".format(path_info))
+            return True
+
+        if self.changed_cache(checksum):
+            logger.debug(
+                "cache for '{}'('{}') has changed.".format(path_info, checksum)
+            )
+            return True
+
+        actual = self.save_info(path_info)[self.PARAM_CHECKSUM]
+        if checksum != actual:
+            logger.debug(
+                "checksum '{}'(actual '{}') for '{}' has changed.".format(
+                    checksum, actual, path_info
+                )
+            )
+            return True
+
+        logger.debug("'{}' hasn't changed.".format(path_info))
+        return False
+
+    def link(self, from_info, to_info):
+        self.copy(from_info, to_info)
+
+    def _save_file(self, path_info, checksum, save_link=True):
+        assert checksum
+
+        cache_info = self.checksum_to_path_info(checksum)
+        if self.changed_cache(checksum):
+            self.move(path_info, cache_info)
+        else:
+            self.remove(path_info)
+
+        self.link(cache_info, path_info)
+
+        if save_link:
+            self.state.save_link(path_info)
+
+        # we need to update path and cache, since in case of reflink,
+        # or copy cache type moving original file results in updates on
+        # next executed command, which causes md5 recalculation
+        self.state.save(path_info, checksum)
+        self.state.save(cache_info, checksum)
+
+    def _save_dir(self, path_info, checksum):
+        cache_info = self.checksum_to_path_info(checksum)
+        dir_info = self.get_dir_cache(checksum)
+
+        entry_info = copy(path_info)
+        for entry in dir_info:
+            entry_checksum = entry[self.PARAM_CHECKSUM]
+            entry_info.path = self.ospath.join(
+                path_info.path, entry[self.PARAM_RELPATH]
+            )
+
+            self._save_file(entry_info, entry_checksum, save_link=False)
+
+        self.state.save_link(path_info)
+        self.state.save(cache_info, checksum)
+        self.state.save(path_info, checksum)
+
+    def is_empty(self, path_info):
+        return False
+
+    def isfile(self, path_info):
         raise NotImplementedError
 
+    def isdir(self, path_info):
+        return False
+
+    def walk(self, path_info):
+        raise NotImplementedError
+
+    def protect(self, path_info):
+        pass
+
+    def save(self, path_info, checksum_info):
+        if path_info.scheme != self.scheme:
+            raise RemoteActionNotImplemented(
+                "save {} -> {}".format(path_info.scheme, self.scheme),
+                self.scheme,
+            )
+
+        checksum = checksum_info[self.PARAM_CHECKSUM]
+        if not self.changed_cache(checksum):
+            self._checkout(path_info, checksum)
+            return
+
+        self._save(path_info, checksum)
+
+    def _save(self, path_info, checksum):
+        to_info = self.checksum_to_path_info(checksum)
+        logger.info("Saving '{}' to '{}'.".format(path_info, to_info))
+        if self.isdir(path_info):
+            self._save_dir(path_info, checksum)
+            return
+        self._save_file(path_info, checksum)
+
+    def download(
+        self,
+        from_infos,
+        to_infos,
+        no_progress_bar=False,
+        name=None,
+        resume=False,
+    ):
+        raise RemoteActionNotImplemented("download", self.scheme)
+
+    def upload(self, from_infos, to_infos, names=None, no_progress_bar=False):
+        raise RemoteActionNotImplemented("upload", self.scheme)
+
+    def remove(self, path_info):
+        raise RemoteActionNotImplemented("remove", self.scheme)
+
+    def move(self, from_info, to_info):
+        self.copy(from_info, to_info)
+        self.remove(from_info)
+
+    def copy(self, from_info, to_info):
+        raise RemoteActionNotImplemented("copy", self.scheme)
+
     def exists(self, path_infos):
         raise NotImplementedError
 
@@ -135,3 +449,251 @@ class RemoteBase(object):
             assert len(names) == len(to_infos)
 
         return names
+
+    @property
+    def ospath(self):
+        return posixpath
+
+    def checksum_to_path(self, checksum):
+        assert checksum
+        return self.ospath.join(self.prefix, checksum[0:2], checksum[2:])
+
+    def path_to_checksum(self, path):
+        relpath = self.ospath.relpath(path, self.prefix)
+        return self.ospath.dirname(relpath) + self.ospath.basename(relpath)
+
+    def checksum_to_path_info(self, checksum):
+        path_info = copy(self.path_info)
+        path_info.path = self.checksum_to_path(checksum)
+        return path_info
+
+    def md5s_to_path_infos(self, md5s):
+        return [self.checksum_to_path_info(md5) for md5 in md5s]
+
+    def list_cache_paths(self):
+        raise NotImplementedError
+
+    def all(self):
+        # NOTE: The list might be way too big(e.g. 100M entries, md5 for each
+        # is 32 bytes, so ~3200Mb list) and we don't really need all of it at
+        # the same time, so it makes sense to use a generator to gradually
+        # iterate over it, without keeping all of it in memory.
+        return (
+            self.path_to_checksum(path) for path in self.list_cache_paths()
+        )
+
+    def gc(self, cinfos):
+        from dvc.remote.local import RemoteLOCAL
+
+        used = {info[RemoteLOCAL.PARAM_CHECKSUM] for info in cinfos["local"]}
+
+        if self.scheme != "":
+            used |= {
+                info[self.PARAM_CHECKSUM]
+                for info in cinfos.get(self.scheme, [])
+            }
+
+        removed = False
+        for checksum in self.all():
+            if checksum in used:
+                continue
+            path_info = self.checksum_to_path_info(checksum)
+            self.remove(path_info)
+            removed = True
+        return removed
+
+    def changed_cache_file(self, checksum):
+        cache_info = self.checksum_to_path_info(checksum)
+        actual = self.get_checksum(cache_info)
+
+        logger.debug(
+            "cache '{}' expected '{}' actual '{}'".format(
+                str(cache_info), checksum, actual
+            )
+        )
+
+        if not checksum or not actual:
+            return True
+
+        if actual.split(".")[0] == checksum.split(".")[0]:
+            return False
+
+        if self.exists(cache_info):
+            logger.warning("corrupted cache file '{}'.".format(cache_info))
+            self.remove(cache_info)
+
+        return True
+
+    def _changed_cache_dir(self):
+        # NOTE: only implemented for RemoteLOCAL
+        return True
+
+    def _changed_dir_cache(self, checksum):
+        if not self._changed_cache_dir():
+            return False
+
+        if self.changed_cache_file(checksum):
+            return True
+
+        for entry in self.get_dir_cache(checksum):
+            checksum = entry[self.PARAM_CHECKSUM]
+            if self.changed_cache_file(checksum):
+                return True
+
+        return False
+
+    def changed_cache(self, checksum):
+        if self.is_dir_checksum(checksum):
+            return self._changed_dir_cache(checksum)
+        return self.changed_cache_file(checksum)
+
+    def cache_exists(self, checksums):
+        # NOTE: The reason for such an odd logic is that most of the remotes
+        # take much shorter time to just retrieve everything they have under
+        # a certain prefix(e.g. s3, gs, ssh, hdfs). Other remotes that can
+        # check if particular file exists much quicker, use their own
+        # implementation of cache_exists(see http, local).
+        #
+        # Result of all() might be way too big, so we should walk through
+        # it in one pass.
+        return list(filter(lambda checksum: checksum in checksums, self.all()))
+
+    def already_cached(self, path_info):
+        current = self.get_checksum(path_info)
+
+        if not current:
+            return False
+
+        return not self.changed_cache(current)
+
+    def safe_remove(self, path_info, force=False):
+        if not self.exists(path_info):
+            return
+
+        if not force and not self.already_cached(path_info):
+            msg = (
+                "file '{}' is going to be removed."
+                " Are you sure you want to proceed?".format(str(path_info))
+            )
+
+            if not prompt.confirm(msg):
+                raise ConfirmRemoveError(str(path_info))
+
+        self.remove(path_info)
+
+    def _checkout_file(
+        self, path_info, checksum, force, progress_callback=None
+    ):
+        cache_info = self.checksum_to_path_info(checksum)
+        if self.exists(path_info):
+            msg = "data '{}' exists. Removing before checkout."
+            logger.warning(msg.format(str(path_info)))
+            self.safe_remove(path_info, force=force)
+
+        self.link(cache_info, path_info)
+        self.state.save_link(path_info)
+        self.state.save(path_info, checksum)
+        if progress_callback:
+            progress_callback.update(path_info.url)
+
+    def makedirs(self, path_info):
+        raise NotImplementedError
+
+    def _checkout_dir(
+        self, path_info, checksum, force, progress_callback=None
+    ):
+        # Create dir separately so that dir is created
+        # even if there are no files in it
+        if not self.exists(path_info):
+            self.makedirs(path_info)
+
+        dir_info = self.get_dir_cache(checksum)
+
+        logger.debug("Linking directory '{}'.".format(path_info))
+
+        entry_info = copy(path_info)
+        for entry in dir_info:
+            relpath = entry[self.PARAM_RELPATH]
+            entry_checksum = entry[self.PARAM_CHECKSUM]
+            entry_cache_info = self.checksum_to_path_info(entry_checksum)
+            entry_info.url = self.ospath.join(path_info.url, relpath)
+            entry_info.path = self.ospath.join(path_info.path, relpath)
+
+            entry_checksum_info = {self.PARAM_CHECKSUM: entry_checksum}
+            if self.changed(entry_info, entry_checksum_info):
+                if self.exists(entry_info):
+                    self.safe_remove(entry_info, force=force)
+                self.link(entry_cache_info, entry_info)
+                self.state.save(entry_info, entry_checksum)
+            if progress_callback:
+                progress_callback.update(entry_info.url)
+
+        self._remove_redundant_files(path_info, dir_info, force)
+
+        self.state.save_link(path_info)
+        self.state.save(path_info, checksum)
+
+    def _remove_redundant_files(self, path_info, dir_info, force):
+        existing_files = set(
+            self.ospath.join(root, fname)
+            for root, _, files in self.walk(path_info)
+            for fname in files
+        )
+
+        needed_files = set(
+            self.ospath.join(path_info.path, entry[self.PARAM_RELPATH])
+            for entry in dir_info
+        )
+
+        delta = existing_files - needed_files
+
+        d_info = copy(path_info)
+        for path in delta:
+            d_info.path = path
+            self.safe_remove(d_info, force)
+
+    def checkout(
+        self, path_info, checksum_info, force=False, progress_callback=None
+    ):
+        if (
+            path_info.scheme not in ["local"]
+            and path_info.scheme != self.scheme
+        ):
+            raise NotImplementedError
+
+        checksum = checksum_info.get(self.PARAM_CHECKSUM)
+        if not checksum:
+            msg = "No checksum info for '{}'."
+            logger.debug(msg.format(str(path_info)))
+            return
+
+        if not self.changed(path_info, checksum_info):
+            msg = "Data '{}' didn't change."
+            logger.debug(msg.format(str(path_info)))
+            return
+
+        if self.changed_cache(checksum):
+            msg = "Cache '{}' not found. File '{}' won't be created."
+            logger.warning(msg.format(checksum, str(path_info)))
+            self.safe_remove(path_info, force=force)
+            return
+
+        msg = "Checking out '{}' with cache '{}'."
+        logger.debug(msg.format(str(path_info), checksum))
+
+        return self._checkout(path_info, checksum, force, progress_callback)
+
+    def _checkout(
+        self, path_info, checksum, force=False, progress_callback=None
+    ):
+        if not self.is_dir_checksum(checksum):
+            return self._checkout_file(
+                path_info, checksum, force, progress_callback=progress_callback
+            )
+        return self._checkout_dir(
+            path_info, checksum, force, progress_callback=progress_callback
+        )
+
+    @staticmethod
+    def unprotect(path_info):
+        pass

dvc/remote/gs.py

@@ -1,300 +1,206 @@
+from __future__ import unicode_literals
+
 import os
-import posixpath
+import logging
+
+from dvc.scheme import Schemes
+from dvc.path.gs import PathGS
 
 try:
     from google.cloud import storage
 except ImportError:
     storage = None
 
-try:
-    from urlparse import urlparse
-except ImportError:
-    from urllib.parse import urlparse
-
-from dvc.logger import Logger
-from dvc.remote.base import RemoteBase
-from dvc.remote.local import RemoteLOCAL
+from dvc.utils import tmp_fname, move
+from dvc.utils.compat import urlparse, makedirs
+from dvc.remote.base import RemoteBASE
 from dvc.config import Config
 from dvc.progress import progress
 from dvc.exceptions import DvcException
 
 
-class RemoteGS(RemoteBase):
-    scheme = 'gs'
-    REGEX = r'^gs://(?P<path>.*)$'
-    REQUIRES = {'google.cloud.storage': storage}
-    PARAM_ETAG = 'etag'
+logger = logging.getLogger(__name__)
 
-    def __init__(self, project, config):
-        self.project = project
-        storagepath = 'gs://'
-        storagepath += config.get(Config.SECTION_AWS_STORAGEPATH, '/')
-        storagepath.lstrip('/')
+
+class RemoteGS(RemoteBASE):
+    scheme = Schemes.GS
+    REGEX = r"^gs://(?P<path>.*)$"
+    REQUIRES = {"google.cloud.storage": storage}
+    PARAM_CHECKSUM = "md5"
+
+    def __init__(self, repo, config):
+        super(RemoteGS, self).__init__(repo, config)
+        storagepath = "gs://"
+        storagepath += config.get(Config.SECTION_AWS_STORAGEPATH, "/")
+        storagepath.lstrip("/")
         self.url = config.get(Config.SECTION_REMOTE_URL, storagepath)
         self.projectname = config.get(Config.SECTION_GCP_PROJECTNAME, None)
+        self.credentialpath = config.get(Config.SECTION_GCP_CREDENTIALPATH)
+
+        parsed = urlparse(self.url)
+        self.bucket = parsed.netloc
+        self.prefix = parsed.path.lstrip("/")
+
+        self.path_info = PathGS(bucket=self.bucket)
 
     @staticmethod
     def compat_config(config):
         ret = config.copy()
-        url = 'gs://' + ret.pop(Config.SECTION_AWS_STORAGEPATH, '').lstrip('/')
+        url = "gs://" + ret.pop(Config.SECTION_AWS_STORAGEPATH, "").lstrip("/")
         ret[Config.SECTION_REMOTE_URL] = url
         return ret
 
-    @property
-    def bucket(self):
-        return urlparse(self.url).netloc
-
-    @property
-    def prefix(self):
-        return urlparse(self.url).path.lstrip('/')
-
     @property
     def gs(self):
-        return storage.Client()
+        return (
+            storage.Client.from_service_account_json(self.credentialpath)
+            if self.credentialpath
+            else storage.Client(self.projectname)
+        )
 
-    def get_etag(self, bucket, key):
-        blob = self.gs.bucket(bucket).get_blob(key)
+    def get_file_checksum(self, path_info):
+        import base64
+        import codecs
+
+        bucket = path_info.bucket
+        path = path_info.path
+        blob = self.gs.bucket(bucket).get_blob(path)
         if not blob:
             return None
 
-        return blob.etag
+        b64_md5 = blob.md5_hash
+        md5 = base64.b64decode(b64_md5)
+        return codecs.getencoder("hex")(md5)[0].decode("utf-8")
 
-    def save_info(self, path_info):
-        if path_info['scheme'] != 'gs':
-            raise NotImplementedError
-
-        return {self.PARAM_ETAG: self.get_etag(path_info['bucket'],
-                                               path_info['key'])}
-
-    @staticmethod
-    def to_string(path_info):
-        return "{}://{}/{}".format(path_info['scheme'],
-                                   path_info['bucket'],
-                                   path_info['key'])
-
-    def changed_cache(self, etag):
-        key = posixpath.join(self.prefix, etag[0:2], etag[2:])
-        cache = {'scheme': 'gs', 'bucket': self.bucket, 'key': key}
-
-        if {self.PARAM_ETAG: etag} != self.save_info(cache):
-            if self.exists([cache])[0]:
-                msg = 'Corrupted cache file {}'
-                Logger.warn(msg.format(self.to_string(cache)))
-                self.remove(cache)
-            return True
-        return False
-
-    def changed(self, path_info, checksum_info):
-        if not self.exists([path_info])[0]:
-            return True
-
-        etag = checksum_info.get(self.PARAM_ETAG, None)
-        if etag is None:
-            return True
-
-        if self.changed_cache(etag):
-            return True
-
-        return checksum_info != self.save_info(path_info)
-
-    def _copy(self, from_info, to_info, gs=None):
+    def copy(self, from_info, to_info, gs=None):
         gs = gs if gs else self.gs
 
-        blob = gs.bucket(from_info['bucket']).get_blob(from_info['key'])
+        blob = gs.bucket(from_info.bucket).get_blob(from_info.path)
         if not blob:
-            msg = '{} doesn\'t exist in the cloud'
-            raise DvcException(msg.format(from_info['key']))
+            msg = "'{}' doesn't exist in the cloud".format(from_info.path)
+            raise DvcException(msg)
 
-        bucket = self.gs.bucket(to_info['bucket'])
-        bucket.copy_blob(blob,
-                         self.gs.bucket(to_info['bucket']),
-                         new_name=to_info['key'])
-
-    def save(self, path_info):
-        if path_info['scheme'] != 'gs':
-            raise NotImplementedError
-
-        etag = self.get_etag(path_info['bucket'], path_info['key'])
-        key = posixpath.join(self.prefix, etag[0:2], etag[2:])
-        to_info = {'scheme': 'gs', 'bucket': self.bucket, 'key': key}
-
-        self._copy(path_info, to_info)
-
-        return {self.PARAM_ETAG: etag}
-
-    def checkout(self, path_info, checksum_info):
-        if path_info['scheme'] != 'gs':
-            raise NotImplementedError
-
-        etag = checksum_info.get(self.PARAM_ETAG, None)
-        if not etag:
-            return
-
-        if not self.changed(path_info, checksum_info):
-            msg = "Data '{}' didn't change."
-            Logger.info(msg.format(self.to_string(path_info)))
-            return
-
-        if self.changed_cache(etag):
-            msg = "Cache '{}' not found. File '{}' won't be created."
-            Logger.warn(msg.format(etag, self.to_string(path_info)))
-            return
-
-        if self.exists([path_info])[0]:
-            msg = "Data '{}' exists. Removing before checkout."
-            Logger.warn(msg.format(self.to_string(path_info)))
-            self.remove(path_info)
-            return
-
-        msg = "Checking out '{}' with cache '{}'."
-        Logger.info(msg.format(self.to_string(path_info), etag))
-
-        key = posixpath.join(self.prefix, etag[0:2], etag[2:])
-        from_info = {'scheme': 'gs', 'bucket': self.bucket, 'key': key}
-
-        self._copy(from_info, path_info)
+        bucket = self.gs.bucket(to_info.bucket)
+        bucket.copy_blob(
+            blob, self.gs.bucket(to_info.bucket), new_name=to_info.path
+        )
 
     def remove(self, path_info):
-        if path_info['scheme'] != 'gs':
+        if path_info.scheme != "gs":
             raise NotImplementedError
 
-        Logger.debug("Removing gs://{}/{}".format(path_info['bucket'],
-                                                  path_info['key']))
+        logger.debug(
+            "Removing gs://{}/{}".format(path_info.bucket, path_info.path)
+        )
 
-        blob = self.gs.bucket(path_info['bucket']).get_blob(path_info['key'])
+        blob = self.gs.bucket(path_info.bucket).get_blob(path_info.path)
         if not blob:
             return
 
         blob.delete()
 
-    def md5s_to_path_infos(self, md5s):
-        return [{'scheme': 'gs',
-                 'bucket': self.bucket,
-                 'key': posixpath.join(self.prefix,
-                                       md5[0:2], md5[2:])} for md5 in md5s]
+    def _list_paths(self, bucket, prefix):
+        for blob in self.gs.bucket(bucket).list_blobs(prefix=prefix):
+            yield blob.name
 
-    def exists(self, path_infos):
-        ret = []
-        gs = self.gs
+    def list_cache_paths(self):
+        return self._list_paths(self.bucket, self.prefix)
 
-        bucket = gs.bucket(self.bucket)
-        keys = [blob.name for blob in bucket.list_blobs(prefix=self.prefix)]
+    def exists(self, path_info):
+        assert not isinstance(path_info, list)
+        assert path_info.scheme == "gs"
 
-        for path_info in path_infos:
-            exists = False
-            if path_info['key'] in keys:
-                exists = True
-            ret.append(exists)
-        return ret
+        paths = self._list_paths(path_info.bucket, path_info.path)
+        return any(path_info.path == path for path in paths)
 
-    def upload(self, from_infos, to_infos, names=None):
+    def upload(self, from_infos, to_infos, names=None, no_progress_bar=False):
         names = self._verify_path_args(to_infos, from_infos, names)
 
         gs = self.gs
 
         for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info['scheme'] != 'gs':
+            if to_info.scheme != "gs":
                 raise NotImplementedError
 
-            if from_info['scheme'] != 'local':
+            if from_info.scheme != "local":
                 raise NotImplementedError
 
-            Logger.debug("Uploading '{}' to '{}/{}'".format(from_info['path'],
-                                                            to_info['bucket'],
-                                                            to_info['key']))
+            logger.debug(
+                "Uploading '{}' to '{}/{}'".format(
+                    from_info.path, to_info.bucket, to_info.path
+                )
+            )
 
             if not name:
-                name = os.path.basename(from_info['path'])
+                name = os.path.basename(from_info.path)
 
-            progress.update_target(name, 0, None)
+            if not no_progress_bar:
+                progress.update_target(name, 0, None)
 
             try:
-                bucket = gs.bucket(to_info['bucket'])
-                blob = bucket.blob(to_info['key'])
-                blob.upload_from_filename(from_info['path'])
-            except Exception as exc:
-                msg = "Failed to upload '{}' to '{}/{}'"
-                Logger.warn(msg.format(from_info['path'],
-                                       to_info['bucket'],
-                                       to_info['key']), exc)
+                bucket = gs.bucket(to_info.bucket)
+                blob = bucket.blob(to_info.path)
+                blob.upload_from_filename(from_info.path)
+            except Exception:
+                msg = "failed to upload '{}' to '{}/{}'"
+                logger.exception(
+                    msg.format(from_info.path, to_info.bucket, to_info.path)
+                )
                 continue
 
             progress.finish_target(name)
 
-    def download(self,
-                 from_infos,
-                 to_infos,
-                 no_progress_bar=False,
-                 names=None):
+    def download(
+        self,
+        from_infos,
+        to_infos,
+        no_progress_bar=False,
+        names=None,
+        resume=False,
+    ):
         names = self._verify_path_args(from_infos, to_infos, names)
 
         gs = self.gs
 
         for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info['scheme'] != 'gs':
+            if from_info.scheme != "gs":
                 raise NotImplementedError
 
-            if to_info['scheme'] == 'gs':
-                self._copy(from_info, to_info, gs=gs)
+            if to_info.scheme == "gs":
+                self.copy(from_info, to_info, gs=gs)
                 continue
 
-            if to_info['scheme'] != 'local':
+            if to_info.scheme != "local":
                 raise NotImplementedError
 
-            msg = "Downloading '{}/{}' to '{}'".format(from_info['bucket'],
-                                                       from_info['key'],
-                                                       to_info['path'])
-            Logger.debug(msg)
+            msg = "Downloading '{}/{}' to '{}'".format(
+                from_info.bucket, from_info.path, to_info.path
+            )
+            logger.debug(msg)
 
-            tmp_file = self.tmp_file(to_info['path'])
+            tmp_file = tmp_fname(to_info.path)
             if not name:
-                name = os.path.basename(to_info['path'])
+                name = os.path.basename(to_info.path)
 
             if not no_progress_bar:
                 # percent_cb is not available for download_to_filename, so
-                # lets at least update progress at keypoints(start, finish)
+                # lets at least update progress at pathpoints(start, finish)
                 progress.update_target(name, 0, None)
 
-            self._makedirs(to_info['path'])
+            makedirs(os.path.dirname(to_info.path), exist_ok=True)
 
             try:
-                bucket = gs.bucket(from_info['bucket'])
-                blob = bucket.get_blob(from_info['key'])
+                bucket = gs.bucket(from_info.bucket)
+                blob = bucket.get_blob(from_info.path)
                 blob.download_to_filename(tmp_file)
-            except Exception as exc:
-                msg = "Failed to download '{}/{}' to '{}'"
-                Logger.warn(msg.format(from_info['bucket'],
-                                       from_info['key'],
-                                       to_info['path']), exc)
+            except Exception:
+                msg = "failed to download '{}/{}' to '{}'"
+                logger.exception(
+                    msg.format(from_info.bucket, from_info.path, to_info.path)
+                )
                 continue
 
-            os.rename(tmp_file, to_info['path'])
+            move(tmp_file, to_info.path)
 
             if not no_progress_bar:
                 progress.finish_target(name)
-
-    def _path_to_etag(self, path):
-        relpath = posixpath.relpath(path, self.prefix)
-        return posixpath.dirname(relpath) + posixpath.basename(relpath)
-
-    def _all_etags(self):
-        blobs = self.gs.bucket(self.bucket).list_blobs(prefix=self.prefix)
-        blobs = list(blobs)
-        return [self._path_to_etag(blob.name) for blob in blobs]
-
-    def gc(self, cinfos):
-        used = [info[self.PARAM_ETAG] for info in cinfos['gs']]
-        used += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
-
-        removed = False
-        for etag in self._all_etags():
-            if etag in used:
-                continue
-            path_info = {'scheme': 'gs',
-                         'bucket': self.bucket,
-                         'key': posixpath.join(self.prefix,
-                                               etag[0:2], etag[2:])}
-            self.remove(path_info)
-            removed = True
-
-        return removed

dvc/remote/hdfs.py

@@ -1,52 +1,63 @@
+from __future__ import unicode_literals
+
 import os
 import re
 import getpass
 import posixpath
+import logging
 from subprocess import Popen, PIPE
 
 from dvc.config import Config
-from dvc.remote.base import RemoteBase
-from dvc.remote.local import RemoteLOCAL
-from dvc.exceptions import DvcException
-from dvc.logger import Logger
-from dvc.utils import fix_env
+from dvc.scheme import Schemes
+from dvc.path.hdfs import PathHDFS
+from dvc.remote.base import RemoteBASE, RemoteCmdError
+from dvc.utils import fix_env, tmp_fname
 
 
-class RemoteHDFS(RemoteBase):
-    scheme = 'hdfs'
-    REGEX = r'^hdfs://((?P<user>.*)@)?.*$'
-    PARAM_CHECKSUM = 'checksum'
+logger = logging.getLogger(__name__)
 
-    def __init__(self, project, config):
-        self.project = project
-        self.url = config.get(Config.SECTION_REMOTE_URL, '/')
-        self.user = self.group('user')
+
+class RemoteHDFS(RemoteBASE):
+    scheme = Schemes.HDFS
+    REGEX = r"^hdfs://((?P<user>.*)@)?.*$"
+    PARAM_CHECKSUM = "checksum"
+
+    def __init__(self, repo, config):
+        super(RemoteHDFS, self).__init__(repo, config)
+        self.url = config.get(Config.SECTION_REMOTE_URL, "/")
+        self.prefix = self.url
+        self.user = self.group("user")
         if not self.user:
-            self.user = config.get(Config.SECTION_REMOTE_USER,
-                                   getpass.getuser())
+            self.user = config.get(
+                Config.SECTION_REMOTE_USER, getpass.getuser()
+            )
+
+        self.path_info = PathHDFS(user=self.user)
 
     def hadoop_fs(self, cmd, user=None):
-        cmd = 'hadoop fs -' + cmd
+        cmd = "hadoop fs -" + cmd
         if user:
-            cmd = 'HADOOP_USER_NAME={} '.format(user) + cmd
+            cmd = "HADOOP_USER_NAME={} ".format(user) + cmd
 
         # NOTE: close_fds doesn't work with redirected stdin/stdout/stderr.
         # See https://github.com/iterative/dvc/issues/1197.
-        close_fds = (os.name != 'nt')
+        close_fds = os.name != "nt"
 
-        executable = os.getenv('SHELL') if os.name != 'nt' else None
-        p = Popen(cmd,
-                  shell=True,
-                  close_fds=close_fds,
-                  executable=executable,
-                  env=fix_env(os.environ),
-                  stdin=PIPE,
-                  stdout=PIPE,
-                  stderr=PIPE)
+        executable = os.getenv("SHELL") if os.name != "nt" else None
+        p = Popen(
+            cmd,
+            shell=True,
+            close_fds=close_fds,
+            executable=executable,
+            env=fix_env(os.environ),
+            stdin=PIPE,
+            stdout=PIPE,
+            stderr=PIPE,
+        )
         out, err = p.communicate()
         if p.returncode != 0:
-            raise DvcException('HDFS command failed: {}: {}'.format(cmd, err))
-        return out.decode('utf-8')
+            raise RemoteCmdError(self.scheme, cmd, p.returncode, err)
+        return out.decode("utf-8")
 
     @staticmethod
     def _group(regex, s, gname):
@@ -54,218 +65,115 @@ class RemoteHDFS(RemoteBase):
         assert match is not None
         return match.group(gname)
 
-    def checksum(self, path_info):
-        regex = r'.*\t.*\t(?P<checksum>.*)'
-        stdout = self.hadoop_fs('checksum {}'.format(path_info['url']),
-                                user=path_info['user'])
-        return self._group(regex, stdout, 'checksum')
+    def get_file_checksum(self, path_info):
+        regex = r".*\t.*\t(?P<checksum>.*)"
+        stdout = self.hadoop_fs(
+            "checksum {}".format(path_info.path), user=path_info.user
+        )
+        return self._group(regex, stdout, "checksum")
 
-    def cp(self, from_info, to_info):
-        self.hadoop_fs('mkdir -p {}'.format(posixpath.dirname(to_info['url'])),
-                       user=to_info['user'])
-        self.hadoop_fs('cp -f {} {}'.format(from_info['url'], to_info['url']),
-                       user=to_info['user'])
+    def copy(self, from_info, to_info):
+        dname = posixpath.dirname(to_info.path)
+        self.hadoop_fs("mkdir -p {}".format(dname), user=to_info.user)
+        self.hadoop_fs(
+            "cp -f {} {}".format(from_info.path, to_info.path),
+            user=to_info.user,
+        )
 
     def rm(self, path_info):
-        self.hadoop_fs('rm {}'.format(path_info['url']),
-                       user=path_info['user'])
-
-    def save_info(self, path_info):
-        if path_info['scheme'] != 'hdfs':
-            raise NotImplementedError
-
-        assert path_info.get('url')
-
-        return {self.PARAM_CHECKSUM: self.checksum(path_info)}
-
-    @staticmethod
-    def to_string(path_info):
-        return "{}://{}".format(path_info['scheme'],
-                                path_info['url'])
-
-    def changed_cache(self, checksum):
-        cache = {}
-        cache['scheme'] = 'hdfs'
-        cache['user'] = self.user
-        cache['url'] = posixpath.join(self.url, checksum[0:2], checksum[2:])
-
-        if {self.PARAM_CHECKSUM: checksum} != self.save_info(cache):
-            if self.exists([cache])[0]:
-                msg = 'Corrupted cache file {}'
-                Logger.warn(msg.format(self.to_string(cache)))
-                self.remove(cache)
-            return True
-
-        return False
-
-    def changed(self, path_info, checksum_info):
-        if not self.exists([path_info])[0]:
-            return True
-
-        checksum = checksum_info.get(self.PARAM_CHECKSUM, None)
-        if checksum is None:
-            return True
-
-        if self.changed_cache(self, checksum):
-            return True
-
-        return checksum_info != self.save_info(path_info)
-
-    def save(self, path_info):
-        if path_info['scheme'] != 'hdfs':
-            raise NotImplementedError
-
-        assert path_info.get('url')
-
-        checksum = self.checksum(path_info)
-        dest = path_info.copy()
-        dest['url'] = posixpath.join(self.url, checksum[0:2], checksum[2:])
-
-        self.cp(path_info, dest)
-
-        return {self.PARAM_CHECKSUM: checksum}
-
-    def checkout(self, path_info, checksum_info):
-        if path_info['scheme'] != 'hdfs':
-            raise NotImplementedError
-
-        assert path_info.get('url')
-
-        checksum = checksum_info.get(self.PARAM_CHECKSUM, None)
-        if not checksum:
-            return
-
-        if not self.changed(path_info, checksum_info):
-            msg = "Data '{}' didn't change."
-            Logger.info(msg.format(self.to_string(path_info)))
-            return
-
-        if self.changed_cache(checksum):
-            msg = "Cache '{}' not found. File '{}' won't be created."
-            Logger.warn(msg.format(checksum, self.to_string(path_info)))
-            return
-
-        if self.exists([path_info])[0]:
-            msg = "Data '{}' exists. Removing before checkout."
-            Logger.warn(msg.format(self.to_string(path_info)))
-            self.remove(path_info)
-            return
-
-        msg = "Checking out '{}' with cache '{}'."
-        Logger.info(msg.format(self.to_string(path_info), checksum))
-
-        src = path_info.copy()
-        src['url'] = posixpath.join(self.url, checksum[0:2], checksum[2:])
-
-        self.cp(src, path_info)
+        self.hadoop_fs("rm -f {}".format(path_info.path), user=path_info.user)
 
     def remove(self, path_info):
-        if path_info['scheme'] != 'hdfs':
+        if path_info.scheme != "hdfs":
             raise NotImplementedError
 
-        assert path_info.get('url')
+        assert path_info.path
 
-        Logger.debug('Removing {}'.format(path_info['url']))
+        logger.debug("Removing {}".format(path_info.path))
 
         self.rm(path_info)
 
-    def md5s_to_path_infos(self, md5s):
-        return [{'scheme': 'hdfs',
-                 'user': self.user,
-                 'url': posixpath.join(self.url,
-                                       md5[0:2], md5[2:])} for md5 in md5s]
+    def exists(self, path_info):
+        assert not isinstance(path_info, list)
+        assert path_info.scheme == "hdfs"
 
-    def exists(self, path_infos):
         try:
-            stdout = self.hadoop_fs('ls -R {}'.format(self.url))
-        except DvcException:
-            return len(path_infos) * [False]
+            self.hadoop_fs("test -e {}".format(path_info.path))
+            return True
+        except RemoteCmdError:
+            return False
 
-        lines = stdout.split('\n')
-        lurl = []
-        for line in lines:
-            if not line.startswith('-'):
-                continue
-            lurl.append(line.split()[-1])
-
-        ret = []
-        for path_info in path_infos:
-            exists = False
-            if path_info['url'] in lurl:
-                exists = True
-            ret.append(exists)
-
-        return ret
-
-    def upload(self, from_infos, to_infos, names=None):
+    def upload(self, from_infos, to_infos, names=None, no_progress_bar=False):
         names = self._verify_path_args(to_infos, from_infos, names)
 
         for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info['scheme'] != 'hdfs':
+            if to_info.scheme != "hdfs":
                 raise NotImplementedError
 
-            if from_info['scheme'] != 'local':
+            if from_info.scheme != "local":
                 raise NotImplementedError
 
-            cmd = 'mkdir -p {}'.format(posixpath.dirname(to_info['url']))
-            self.hadoop_fs(cmd, user=to_info['user'])
+            self.hadoop_fs(
+                "mkdir -p {}".format(posixpath.dirname(to_info.path)),
+                user=to_info.user,
+            )
 
-            cmd = 'copyFromLocal {} {}'.format(from_info['path'],
-                                               to_info['url'])
-            self.hadoop_fs(cmd, user=to_info['user'])
+            tmp_file = tmp_fname(to_info.path)
 
-    def download(self,
-                 from_infos,
-                 to_infos,
-                 no_progress_bar=False,
-                 names=None):
+            self.hadoop_fs(
+                "copyFromLocal {} {}".format(from_info.path, tmp_file),
+                user=to_info.user,
+            )
+
+            self.hadoop_fs(
+                "mv {} {}".format(tmp_file, to_info.path), user=to_info.user
+            )
+
+    def download(
+        self,
+        from_infos,
+        to_infos,
+        no_progress_bar=False,
+        names=None,
+        resume=False,
+    ):
         names = self._verify_path_args(from_infos, to_infos, names)
 
         for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info['scheme'] != 'hdfs':
+            if from_info.scheme != "hdfs":
                 raise NotImplementedError
 
-            if to_info['scheme'] == 'hdfs':
-                self.cp(from_info, to_info)
+            if to_info.scheme == "hdfs":
+                self.copy(from_info, to_info)
                 continue
 
-            if to_info['scheme'] != 'local':
+            if to_info.scheme != "local":
                 raise NotImplementedError
 
-            dname = os.path.dirname(to_info['path'])
+            dname = os.path.dirname(to_info.path)
             if not os.path.exists(dname):
                 os.makedirs(dname)
 
-            cmd = 'copyToLocal {} {}'.format(from_info['url'], to_info['path'])
-            self.hadoop_fs(cmd, user=from_info['user'])
+            tmp_file = tmp_fname(to_info.path)
 
-    def _path_to_checksum(self, path):
-        relpath = posixpath.relpath(path, self.url)
-        return posixpath.dirname(relpath) + posixpath.basename(relpath)
+            self.hadoop_fs(
+                "copyToLocal {} {}".format(from_info.path, tmp_file),
+                user=from_info.user,
+            )
 
-    def _all_checksums(self):
-        stdout = self.hadoop_fs('ls -R {}'.format(self.url))
-        lines = stdout.split('\n')
+            os.rename(tmp_file, to_info.path)
+
+    def list_cache_paths(self):
+        try:
+            self.hadoop_fs("test -e {}".format(self.prefix))
+        except RemoteCmdError:
+            return []
+
+        stdout = self.hadoop_fs("ls -R {}".format(self.prefix))
+        lines = stdout.split("\n")
         flist = []
         for line in lines:
-            if not line.startswith('-'):
+            if not line.startswith("-"):
                 continue
             flist.append(line.split()[-1])
-        return [self._path_to_checksum(path) for path in flist]
-
-    def gc(self, cinfos):
-        used = [info[self.PARAM_CHECKSUM] for info in cinfos['hdfs']]
-        used += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
-
-        removed = False
-        for checksum in self._all_checksums():
-            if checksum in used:
-                continue
-            path_info = {'scheme': 'hdfs',
-                         'user': self.user,
-                         'url': posixpath.join(self.url,
-                                               checksum[0:2], checksum[2:])}
-            self.remove(path_info)
-            removed = True
-
-        return removed
+        return flist

dvc/remote/http.py

@@ -0,0 +1,195 @@
+from __future__ import unicode_literals
+
+from dvc.scheme import Schemes
+from dvc.path import Path
+from dvc.utils.compat import open, makedirs
+
+import os
+import threading
+import requests
+import logging
+
+from dvc.progress import progress
+from dvc.exceptions import DvcException
+from dvc.config import Config
+from dvc.remote.base import RemoteBASE
+from dvc.utils import move
+
+
+logger = logging.getLogger(__name__)
+
+
+class ProgressBarCallback(object):
+    def __init__(self, name, total):
+        self.name = name
+        self.total = total
+        self.current = 0
+        self.lock = threading.Lock()
+
+    def __call__(self, byts):
+        with self.lock:
+            self.current += byts
+            progress.update_target(self.name, self.current, self.total)
+
+
+class RemoteHTTP(RemoteBASE):
+    scheme = Schemes.HTTP
+    REGEX = r"^http://.*$"
+    REQUEST_TIMEOUT = 10
+    CHUNK_SIZE = 2 ** 16
+    PARAM_CHECKSUM = "etag"
+
+    def __init__(self, repo, config):
+        super(RemoteHTTP, self).__init__(repo, config)
+        self.cache_dir = config.get(Config.SECTION_REMOTE_URL)
+        self.url = self.cache_dir
+
+        self.path_info = Path(self.scheme)
+
+    @property
+    def prefix(self):
+        return self.cache_dir
+
+    def download(
+        self,
+        from_infos,
+        to_infos,
+        no_progress_bar=False,
+        names=None,
+        resume=False,
+    ):
+        names = self._verify_path_args(to_infos, from_infos, names)
+
+        for to_info, from_info, name in zip(to_infos, from_infos, names):
+            if from_info.scheme != self.scheme:
+                raise NotImplementedError
+
+            if to_info.scheme != "local":
+                raise NotImplementedError
+
+            msg = "Downloading '{}' to '{}'".format(
+                from_info.path, to_info.path
+            )
+            logger.debug(msg)
+
+            if not name:
+                name = os.path.basename(to_info.path)
+
+            makedirs(os.path.dirname(to_info.path), exist_ok=True)
+
+            total = self._content_length(from_info.path)
+
+            if no_progress_bar or not total:
+                cb = None
+            else:
+                cb = ProgressBarCallback(name, total)
+
+            try:
+                self._download_to(
+                    from_info.path, to_info.path, callback=cb, resume=resume
+                )
+
+            except Exception:
+                msg = "failed to download '{}'".format(from_info.path)
+                logger.exception(msg)
+                continue
+
+            if not no_progress_bar:
+                progress.finish_target(name)
+
+    def exists(self, path_info):
+        assert not isinstance(path_info, list)
+        assert path_info.scheme == self.scheme
+        return bool(self._request("HEAD", path_info.path))
+
+    def cache_exists(self, md5s):
+        assert isinstance(md5s, list)
+
+        def func(md5):
+            return bool(self._request("HEAD", self.checksum_to_path(md5)))
+
+        return list(filter(func, md5s))
+
+    def _content_length(self, url):
+        return self._request("HEAD", url).headers.get("Content-Length")
+
+    def get_file_checksum(self, path_info):
+        url = path_info.path
+        etag = self._request("HEAD", url).headers.get("ETag") or self._request(
+            "HEAD", url
+        ).headers.get("Content-MD5")
+
+        if not etag:
+            raise DvcException(
+                "could not find an ETag or "
+                "Content-MD5 header for '{url}'".format(url=url)
+            )
+
+        if etag.startswith("W/"):
+            raise DvcException(
+                "Weak ETags are not supported."
+                " (Etag: '{etag}', URL: '{url}')".format(etag=etag, url=url)
+            )
+
+        return etag
+
+    def _download_to(self, url, target_file, callback=None, resume=False):
+        request = self._request("GET", url, stream=True)
+        partial_file = target_file + ".part"
+
+        mode, transferred_bytes = self._determine_mode_get_transferred_bytes(
+            partial_file, resume
+        )
+
+        self._validate_existing_file_size(transferred_bytes, partial_file)
+
+        self._write_request_content(
+            mode, partial_file, request, transferred_bytes, callback
+        )
+
+        move(partial_file, target_file)
+
+    def _write_request_content(
+        self, mode, partial_file, request, transferred_bytes, callback=None
+    ):
+        with open(partial_file, mode) as fd:
+
+            for index, chunk in enumerate(
+                request.iter_content(chunk_size=self.CHUNK_SIZE)
+            ):
+                chunk_number = index + 1
+                if chunk_number * self.CHUNK_SIZE > transferred_bytes:
+                    fd.write(chunk)
+                    fd.flush()
+                    transferred_bytes += len(chunk)
+
+                if callback:
+                    callback(transferred_bytes)
+
+    def _validate_existing_file_size(self, bytes_transferred, partial_file):
+        if bytes_transferred % self.CHUNK_SIZE != 0:
+            raise DvcException(
+                "File {}, might be corrupted, please remove "
+                "it and retry importing".format(partial_file)
+            )
+
+    def _determine_mode_get_transferred_bytes(self, partial_file, resume):
+        if os.path.exists(partial_file) and resume:
+            mode = "ab"
+            bytes_transfered = os.path.getsize(partial_file)
+        else:
+            mode = "wb"
+            bytes_transfered = 0
+        return mode, bytes_transfered
+
+    def _request(self, method, url, **kwargs):
+        kwargs.setdefault("allow_redirects", True)
+        kwargs.setdefault("timeout", self.REQUEST_TIMEOUT)
+
+        try:
+            return requests.request(method, url, **kwargs)
+        except requests.exceptions.RequestException:
+            raise DvcException("could not perform a {} request".format(method))
+
+    def gc(self):
+        raise NotImplementedError

dvc/remote/https.py

@@ -0,0 +1,10 @@
+from __future__ import unicode_literals
+
+from dvc.scheme import Schemes
+
+from .http import RemoteHTTP
+
+
+class RemoteHTTPS(RemoteHTTP):
+    scheme = Schemes.HTTPS
+    REGEX = r"^https://.*$"

dvc/remote/local.py

@@ -1,736 +0,0 @@
-import os
-import stat
-import uuid
-import json
-import ntpath
-import shutil
-import posixpath
-from operator import itemgetter
-
-from dvc.system import System
-from dvc.remote.base import RemoteBase, STATUS_MAP
-from dvc.logger import Logger
-from dvc.utils import remove, move, copyfile, dict_md5, to_chunks
-from dvc.utils import LARGE_DIR_SIZE
-from dvc.config import Config
-from dvc.exceptions import DvcException
-from dvc.progress import progress
-from concurrent.futures import ThreadPoolExecutor
-
-
-class RemoteLOCAL(RemoteBase):
-    scheme = ''
-    REGEX = r'^(?P<path>(/+|.:\\+).*)$'
-    PARAM_MD5 = 'md5'
-    PARAM_PATH = 'path'
-    PARAM_RELPATH = 'relpath'
-    MD5_DIR_SUFFIX = '.dir'
-
-    CACHE_TYPES = ['reflink', 'hardlink', 'symlink', 'copy']
-    CACHE_TYPE_MAP = {
-        'copy': shutil.copyfile,
-        'symlink': System.symlink,
-        'hardlink': System.hardlink,
-        'reflink': System.reflink,
-    }
-
-    def __init__(self, project, config):
-        self.project = project
-        self.state = self.project.state
-        self.protected = config.get(Config.SECTION_CACHE_PROTECTED, False)
-        storagepath = config.get(Config.SECTION_AWS_STORAGEPATH, None)
-        self.cache_dir = config.get(Config.SECTION_REMOTE_URL, storagepath)
-
-        types = config.get(Config.SECTION_CACHE_TYPE, None)
-        if types:
-            if isinstance(types, str):
-                types = [t.strip() for t in types.split(',')]
-            self.cache_types = types
-        else:
-            self.cache_types = self.CACHE_TYPES
-
-        if self.cache_dir is not None and not os.path.exists(self.cache_dir):
-            os.mkdir(self.cache_dir)
-
-    @staticmethod
-    def compat_config(config):
-        ret = config.copy()
-        url = ret.pop(Config.SECTION_AWS_STORAGEPATH, '')
-        ret[Config.SECTION_REMOTE_URL] = url
-        return ret
-
-    @property
-    def url(self):
-        return self.cache_dir
-
-    @property
-    def prefix(self):
-        return self.cache_dir
-
-    def all(self):
-        clist = []
-        for entry in os.listdir(self.cache_dir):
-            subdir = os.path.join(self.cache_dir, entry)
-            if not os.path.isdir(subdir):
-                continue
-
-            for cache in os.listdir(subdir):
-                path = os.path.join(subdir, cache)
-                clist.append(self.path_to_md5(path))
-
-        return clist
-
-    def get(self, md5):
-        if not md5:
-            return None
-
-        return os.path.join(self.cache_dir, md5[0:2], md5[2:])
-
-    def path_to_md5(self, path):
-        relpath = os.path.relpath(path, self.cache_dir)
-        return os.path.dirname(relpath) + os.path.basename(relpath)
-
-    def changed_cache_file(self, md5):
-        cache = self.get(md5)
-        if self.state.changed(cache, md5=md5):
-            if os.path.exists(cache):
-                msg = 'Corrupted cache file {}.'
-                Logger.warn(msg.format(os.path.relpath(cache)))
-                remove(cache)
-            return True
-        return False
-
-    def changed_cache(self, md5):
-        cache = self.get(md5)
-        clist = [(cache, md5)]
-
-        while True:
-            if len(clist) == 0:
-                break
-
-            cache, md5 = clist.pop()
-            if self.changed_cache_file(md5):
-                return True
-
-            if not self.is_dir_cache(cache):
-                continue
-
-            for entry in self.load_dir_cache(md5):
-                md5 = entry[self.PARAM_MD5]
-                cache = self.get(md5)
-                clist.append((cache, md5))
-
-        return False
-
-    def link(self, cache, path):
-        assert os.path.isfile(cache)
-
-        dname = os.path.dirname(path)
-        if not os.path.exists(dname):
-            os.makedirs(dname)
-
-        # NOTE: just create an empty file for an empty cache
-        if os.path.getsize(cache) == 0:
-            open(path, 'w+').close()
-
-            msg = "Created empty file: {} -> {}".format(
-                os.path.relpath(cache),
-                os.path.relpath(path),
-            )
-
-            Logger.debug(msg)
-            return
-
-        i = len(self.cache_types)
-        while i > 0:
-            try:
-                self.CACHE_TYPE_MAP[self.cache_types[0]](cache, path)
-
-                if self.protected:
-                    os.chmod(path, stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH)
-
-                msg = "Created {}'{}': {} -> {}".format(
-                    'protected ' if self.protected else '',
-                    self.cache_types[0],
-                    os.path.relpath(cache),
-                    os.path.relpath(path)
-                )
-
-                Logger.debug(msg)
-                return
-            except DvcException as exc:
-                msg = 'Cache type \'{}\' is not supported: {}'
-                Logger.debug(msg.format(self.cache_types[0], str(exc)))
-                del self.cache_types[0]
-                i -= 1
-
-        raise DvcException('No possible cache types left to try out.')
-
-    @classmethod
-    def ospath(cls, path):
-        if os.name == 'nt':
-            return cls.ntpath(path)
-        return cls.unixpath(path)
-
-    @staticmethod
-    def unixpath(path):
-        assert not ntpath.isabs(path)
-        assert not posixpath.isabs(path)
-        return path.replace('\\', '/')
-
-    @staticmethod
-    def ntpath(path):
-        assert not ntpath.isabs(path)
-        assert not posixpath.isabs(path)
-        return path.replace('/', '\\')
-
-    def collect_dir_cache(self, dname):
-        dir_info = []
-
-        for root, dirs, files in os.walk(dname):
-            bar = False
-
-            if len(files) > LARGE_DIR_SIZE:
-                msg = "Computing md5 for a large directory {}. " \
-                      "This is only done once."
-                Logger.info(msg.format(os.path.relpath(root)))
-                bar = True
-                title = os.path.relpath(root)
-                processed = 0
-                total = len(files)
-                progress.update_target(title, 0, total)
-
-            for fname in files:
-                path = os.path.join(root, fname)
-                relpath = self.unixpath(os.path.relpath(path, dname))
-
-                if bar:
-                    progress.update_target(title, processed, total)
-                    processed += 1
-
-                md5 = self.state.update(path)
-                dir_info.append({self.PARAM_RELPATH: relpath,
-                                 self.PARAM_MD5: md5})
-
-            if bar:
-                progress.finish_target(title)
-
-        # NOTE: sorting the list by path to ensure reproducibility
-        dir_info = sorted(dir_info, key=itemgetter(self.PARAM_RELPATH))
-
-        md5 = dict_md5(dir_info) + self.MD5_DIR_SUFFIX
-        if self.changed_cache(md5):
-            self.dump_dir_cache(md5, dir_info)
-
-        return (md5, dir_info)
-
-    def load_dir_cache(self, md5):
-        path = self.get(md5)
-
-        assert self.is_dir_cache(path)
-
-        try:
-            with open(path, 'r') as fd:
-                d = json.load(fd)
-        except Exception as exc:
-            msg = u'Failed to load dir cache \'{}\''
-            Logger.error(msg.format(os.path.relpath(path)), exc)
-            return []
-
-        if not isinstance(d, list):
-            msg = u'Dir cache file format error \'{}\': skipping the file'
-            Logger.error(msg.format(os.path.relpath(path)))
-            return []
-
-        for info in d:
-            info['relpath'] = self.ospath(info['relpath'])
-
-        return d
-
-    def dump_dir_cache(self, md5, dir_info):
-        path = self.get(md5)
-        dname = os.path.dirname(path)
-
-        assert self.is_dir_cache(path)
-        assert isinstance(dir_info, list)
-
-        if not os.path.isdir(dname):
-            os.makedirs(dname)
-
-        # NOTE: Writing first and renaming after that
-        # to make sure that the operation is atomic.
-        tmp = '{}.{}'.format(path, str(uuid.uuid4()))
-        with open(tmp, 'w+') as fd:
-            json.dump(dir_info, fd, sort_keys=True)
-        move(tmp, path)
-
-    @classmethod
-    def is_dir_cache(cls, cache):
-        return cache.endswith(cls.MD5_DIR_SUFFIX)
-
-    def checkout(self, path_info, checksum_info):
-        path = path_info['path']
-        md5 = checksum_info.get(self.PARAM_MD5)
-        cache = self.get(md5)
-
-        if not cache:
-            msg = 'No cache info for \'{}\'. Skipping checkout.'
-            Logger.warn(msg.format(os.path.relpath(path)))
-            return
-
-        if not self.changed(path_info, checksum_info):
-            msg = "Data '{}' didn't change."
-            Logger.info(msg.format(os.path.relpath(path)))
-            return
-
-        if self.changed_cache(md5):
-            msg = u'Cache \'{}\' not found. File \'{}\' won\'t be created.'
-            Logger.warn(msg.format(md5, os.path.relpath(path)))
-            remove(path)
-            return
-
-        msg = u'Checking out \'{}\' with cache \'{}\'.'
-        Logger.info(msg.format(os.path.relpath(path), md5))
-
-        if not self.is_dir_cache(cache):
-
-            if os.path.exists(path):
-                msg = u'File \'{}\' exists. Removing before checkout.'
-                Logger.warn(msg.format(os.path.relpath(path)))
-                remove(path)
-
-            self.link(cache, path)
-            self.state.update_link(path)
-            return
-
-        # Create dir separately so that dir is created
-        # even if there are no files in it
-        if not os.path.exists(path):
-            os.makedirs(path)
-
-        dir_info = self.load_dir_cache(md5)
-        dir_relpath = os.path.relpath(path)
-        dir_size = len(dir_info)
-        bar = dir_size > LARGE_DIR_SIZE
-
-        Logger.info("Linking directory '{}'.".format(dir_relpath))
-
-        for processed, entry in enumerate(dir_info):
-            relpath = entry[self.PARAM_RELPATH]
-            m = entry[self.PARAM_MD5]
-            p = os.path.join(path, relpath)
-            c = self.get(m)
-
-            entry_info = {
-                'scheme': path_info['scheme'],
-                self.PARAM_PATH: p,
-            }
-
-            entry_checksum_info = {
-                self.PARAM_MD5: m,
-            }
-
-            if self.changed(entry_info, entry_checksum_info):
-                remove(p)
-                self.link(c, p)
-
-            if bar:
-                progress.update_target(dir_relpath, processed, dir_size)
-
-        self._discard_working_directory_changes(path, dir_info)
-
-        self.state.update_link(path)
-
-        if bar:
-            progress.finish_target(dir_relpath)
-
-    def _discard_working_directory_changes(self, path, dir_info):
-        working_dir_files = set(
-            os.path.join(root, file)
-            for root, _, files in os.walk(path)
-            for file in files
-        )
-
-        cached_files = set(
-            os.path.join(path, file['relpath'])
-            for file in dir_info
-        )
-
-        delta = working_dir_files - cached_files
-
-        for file in delta:
-            remove(file)
-
-    def _move(self, inp, outp):
-        # moving in two stages to make last the move atomic in
-        # case inp and outp are in different filesystems
-        tmp = '{}.{}'.format(outp, str(uuid.uuid4()))
-        move(inp, tmp)
-        move(tmp, outp)
-
-    def _save_file(self, path_info):
-        path = path_info['path']
-        md5 = self.state.update(path)
-        assert md5 is not None
-
-        cache = self.get(md5)
-
-        if self.changed_cache(md5):
-            self._move(path, cache)
-        else:
-            remove(path)
-
-        self.link(cache, path)
-        self.state.update_link(path)
-
-        return {self.PARAM_MD5: md5}
-
-    def _save_dir(self, path_info):
-        path = path_info['path']
-        md5, dir_info = self.state.update_info(path)
-        dir_relpath = os.path.relpath(path)
-        dir_size = len(dir_info)
-        bar = dir_size > LARGE_DIR_SIZE
-
-        Logger.info("Linking directory '{}'.".format(dir_relpath))
-
-        for processed, entry in enumerate(dir_info):
-            relpath = entry[self.PARAM_RELPATH]
-            m = entry[self.PARAM_MD5]
-            p = os.path.join(path, relpath)
-            c = self.get(m)
-
-            if self.changed_cache(m):
-                self._move(p, c)
-            else:
-                remove(p)
-
-            self.link(c, p)
-
-            if bar:
-                progress.update_target(dir_relpath, processed, dir_size)
-
-        self.state.update_link(path)
-
-        if bar:
-            progress.finish_target(dir_relpath)
-
-        return {self.PARAM_MD5: md5}
-
-    def save(self, path_info):
-        if path_info['scheme'] != 'local':
-            raise NotImplementedError
-
-        path = path_info['path']
-
-        msg = "Saving '{}' to cache '{}'."
-        Logger.info(msg.format(os.path.relpath(path),
-                               os.path.relpath(self.cache_dir)))
-
-        if os.path.isdir(path):
-            return self._save_dir(path_info)
-        else:
-            return self._save_file(path_info)
-
-    def save_info(self, path_info):
-        if path_info['scheme'] != 'local':
-            raise NotImplementedError
-
-        return {self.PARAM_MD5: self.state.update(path_info['path'])}
-
-    def changed(self, path_info, checksum_info):
-        if not self.exists([path_info])[0]:
-            return True
-
-        md5 = checksum_info.get(self.PARAM_MD5, None)
-        if md5 is None:
-            return True
-
-        if self.changed_cache(md5):
-            return True
-
-        return checksum_info != self.save_info(path_info)
-
-    def remove(self, path_info):
-        if path_info['scheme'] != 'local':
-            raise NotImplementedError
-
-        remove(path_info['path'])
-
-    def move(self, from_info, to_info):
-        if from_info['scheme'] != 'local' or to_info['scheme'] != 'local':
-            raise NotImplementedError
-
-        move(from_info['path'], to_info['path'])
-
-    def md5s_to_path_infos(self, md5s):
-        return [{'scheme': 'local',
-                 'path': os.path.join(self.prefix,
-                                      md5[0:2], md5[2:])} for md5 in md5s]
-
-    def exists(self, path_infos):
-        ret = []
-        for path_info in path_infos:
-            ret.append(os.path.exists(path_info['path']))
-        return ret
-
-    def upload(self, from_infos, to_infos, names=None):
-        names = self._verify_path_args(to_infos, from_infos, names)
-
-        for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info['scheme'] != 'local':
-                raise NotImplementedError
-
-            if from_info['scheme'] != 'local':
-                raise NotImplementedError
-
-            Logger.debug("Uploading '{}' to '{}'".format(from_info['path'],
-                                                         to_info['path']))
-
-            if not name:
-                name = os.path.basename(from_info['path'])
-
-            self._makedirs(to_info['path'])
-
-            try:
-                copyfile(from_info['path'], to_info['path'], name=name)
-            except Exception as exc:
-                msg = "Failed to upload '{}' tp '{}'"
-                Logger.warn(msg.format(from_info['path'],
-                                       to_info['path']), exc)
-
-    def download(self,
-                 from_infos,
-                 to_infos,
-                 no_progress_bar=False,
-                 names=None):
-        names = self._verify_path_args(from_infos, to_infos, names)
-
-        for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info['scheme'] != 'local':
-                raise NotImplementedError
-
-            if to_info['scheme'] != 'local':
-                raise NotImplementedError
-
-            Logger.debug("Downloading '{}' to '{}'".format(from_info['path'],
-                                                           to_info['path']))
-
-            if not name:
-                name = os.path.basename(to_info['path'])
-
-            self._makedirs(to_info['path'])
-            tmp_file = self.tmp_file(to_info['path'])
-            try:
-                copyfile(from_info['path'],
-                         tmp_file,
-                         no_progress_bar=no_progress_bar,
-                         name=name)
-            except Exception as exc:
-                msg = "Failed to download '{}' to '{}'"
-                Logger.warn(msg.format(from_info['path'],
-                                       to_info['path']), exc)
-                continue
-
-            os.rename(tmp_file, to_info['path'])
-
-    def _group(self, checksum_infos, show_checksums=False):
-        by_md5 = {}
-
-        for info in checksum_infos:
-            md5 = info[self.PARAM_MD5]
-
-            if show_checksums:
-                by_md5[md5] = md5
-                continue
-
-            name = info[self.PARAM_PATH]
-            branch = info.get('branch')
-            if branch:
-                name += '({})'.format(branch)
-
-            if md5 not in by_md5.keys():
-                by_md5[md5] = ''
-            else:
-                by_md5[md5] += ' '
-
-            by_md5[md5] += name
-
-        return list(by_md5.keys()), list(by_md5.values())
-
-    def gc(self, checksum_infos):
-        checksum_infos = checksum_infos['local']
-        used_md5s = [info[self.PARAM_MD5] for info in checksum_infos]
-
-        removed = False
-        for md5 in self.all():
-            if md5 in used_md5s:
-                continue
-            remove(self.get(md5))
-            removed = True
-
-        return removed
-
-    def status(self, checksum_infos, remote, jobs=None, show_checksums=False):
-        Logger.info("Preparing to pull data from {}".format(remote.url))
-        title = "Collecting information"
-
-        progress.set_n_total(1)
-        progress.update_target(title, 0, 100)
-
-        progress.update_target(title, 10, 100)
-
-        md5s, names = self._group(checksum_infos,
-                                  show_checksums=show_checksums)
-
-        progress.update_target(title, 20, 100)
-
-        path_infos = remote.md5s_to_path_infos(md5s)
-
-        progress.update_target(title, 30, 100)
-
-        remote_exists = remote.exists(path_infos)
-
-        progress.update_target(title, 90, 100)
-
-        local_exists = [not self.changed_cache_file(md5) for md5 in md5s]
-
-        progress.finish_target(title)
-
-        return [(name, STATUS_MAP[l, r]) for name, l, r in zip(names,
-                                                               local_exists,
-                                                               remote_exists)]
-
-    def pull(self,
-             checksum_infos,
-             remote,
-             jobs=None,
-             show_checksums=False):
-        title = "Collecting information"
-
-        progress.set_n_total(1)
-        progress.update_target(title, 0, 100)
-
-        grouped = zip(*self._group(checksum_infos,
-                                   show_checksums=show_checksums))
-
-        progress.update_target(title, 10, 100)
-
-        md5s = []
-        names = []
-        # NOTE: filter files that are not corrupted
-        for md5, name in grouped:
-            if self.changed_cache_file(md5):
-                md5s.append(md5)
-                names.append(name)
-
-        progress.update_target(title, 30, 100)
-
-        cache = [{'scheme': 'local', 'path': self.get(md5)} for md5 in md5s]
-
-        progress.update_target(title, 50, 100)
-
-        path_infos = remote.md5s_to_path_infos(md5s)
-
-        progress.update_target(title, 60, 100)
-
-        # NOTE: dummy call to try to establish a connection
-        # to see if we need to ask user for a password.
-        remote.exists(remote.md5s_to_path_infos(['000']))
-
-        progress.update_target(title, 70, 100)
-
-        assert len(path_infos) == len(cache) == len(md5s) == len(names)
-
-        if jobs is None:
-            jobs = remote.JOBS
-
-        chunks = list(zip(to_chunks(path_infos, jobs),
-                          to_chunks(cache, jobs),
-                          to_chunks(names, jobs)))
-
-        progress.finish_target(title)
-
-        progress.set_n_total(len(names))
-
-        if len(chunks) == 0:
-            return
-
-        futures = []
-        with ThreadPoolExecutor(max_workers=len(chunks)) as executor:
-            for from_infos, to_infos, names in chunks:
-                res = executor.submit(remote.download,
-                                      from_infos,
-                                      to_infos,
-                                      names=names)
-                futures.append(res)
-
-        for f in futures:
-            f.result()
-
-    def push(self, checksum_infos, remote, jobs=None, show_checksums=False):
-        Logger.info("Preparing to push data to {}".format(remote.url))
-        title = "Collecting information"
-
-        progress.set_n_total(1)
-        progress.update_target(title, 0, 100)
-
-        # NOTE: verifying that our cache is not corrupted
-        def func(info):
-            return not self.changed_cache_file(info[self.PARAM_MD5])
-        checksum_infos = list(filter(func, checksum_infos))
-
-        progress.update_target(title, 20, 100)
-
-        # NOTE: filter files that are already uploaded
-        md5s = [i[self.PARAM_MD5] for i in checksum_infos]
-        exists = remote.exists(remote.md5s_to_path_infos(md5s))
-
-        progress.update_target(title, 30, 100)
-
-        def func(entry):
-            return not entry[0]
-
-        assert len(exists) == len(checksum_infos)
-        infos_exist = list(filter(func, zip(exists, checksum_infos)))
-        checksum_infos = [i for e, i in infos_exist]
-
-        progress.update_target(title, 70, 100)
-
-        md5s, names = self._group(checksum_infos,
-                                  show_checksums=show_checksums)
-        cache = [{'scheme': 'local', 'path': self.get(md5)} for md5 in md5s]
-
-        progress.update_target(title, 80, 100)
-
-        path_infos = remote.md5s_to_path_infos(md5s)
-
-        assert len(path_infos) == len(cache) == len(md5s) == len(names)
-
-        progress.update_target(title, 90, 100)
-
-        if jobs is None:
-            jobs = remote.JOBS
-
-        chunks = list(zip(to_chunks(path_infos, jobs),
-                          to_chunks(cache, jobs),
-                          to_chunks(names, jobs)))
-
-        progress.finish_target(title)
-
-        progress.set_n_total(len(names))
-
-        if len(chunks) == 0:
-            return
-
-        futures = []
-        with ThreadPoolExecutor(max_workers=len(chunks)) as executor:
-            for to_infos, from_infos, names in chunks:
-                res = executor.submit(remote.upload,
-                                      from_infos,
-                                      to_infos,
-                                      names=names)
-                futures.append(res)
-
-        for f in futures:
-            f.result()

dvc/remote/local/__init__.py

@@ -0,0 +1,577 @@
+from __future__ import unicode_literals
+
+from copy import copy
+
+from dvc.scheme import Schemes
+from dvc.path.base import PathBASE
+from dvc.path.local import PathLOCAL
+from dvc.remote.local.slow_link_detection import slow_link_guard
+from dvc.utils.compat import str, makedirs
+
+import os
+import stat
+import uuid
+import ntpath
+import shutil
+import posixpath
+import logging
+
+from dvc.system import System
+from dvc.remote.base import (
+    RemoteBASE,
+    STATUS_MAP,
+    STATUS_NEW,
+    STATUS_DELETED,
+    STATUS_MISSING,
+)
+from dvc.utils import (
+    remove,
+    move,
+    copyfile,
+    to_chunks,
+    tmp_fname,
+    file_md5,
+    walk_files,
+)
+from dvc.config import Config
+from dvc.exceptions import DvcException
+from dvc.progress import progress
+from concurrent.futures import ThreadPoolExecutor
+
+from dvc.utils.fs import get_mtime_and_size, get_inode
+
+
+logger = logging.getLogger(__name__)
+
+
+class RemoteLOCAL(RemoteBASE):
+    scheme = Schemes.LOCAL
+    REGEX = r"^(?P<path>.*)$"
+    PARAM_CHECKSUM = "md5"
+    PARAM_PATH = "path"
+
+    DEFAULT_CACHE_TYPES = ["reflink", "copy"]
+    CACHE_TYPE_MAP = {
+        "copy": shutil.copyfile,
+        "symlink": System.symlink,
+        "hardlink": System.hardlink,
+        "reflink": System.reflink,
+    }
+
+    def __init__(self, repo, config):
+        super(RemoteLOCAL, self).__init__(repo, config)
+        self.state = self.repo.state if self.repo else None
+        self.protected = config.get(Config.SECTION_CACHE_PROTECTED, False)
+        storagepath = config.get(Config.SECTION_AWS_STORAGEPATH, None)
+        self.cache_dir = config.get(Config.SECTION_REMOTE_URL, storagepath)
+
+        if self.cache_dir is not None and not os.path.isabs(self.cache_dir):
+            cwd = config[Config.PRIVATE_CWD]
+            self.cache_dir = os.path.abspath(os.path.join(cwd, self.cache_dir))
+
+        types = config.get(Config.SECTION_CACHE_TYPE, None)
+        if types:
+            if isinstance(types, str):
+                types = [t.strip() for t in types.split(",")]
+            self.cache_types = types
+        else:
+            self.cache_types = copy(self.DEFAULT_CACHE_TYPES)
+
+        if self.cache_dir is not None and not os.path.exists(self.cache_dir):
+            os.mkdir(self.cache_dir)
+
+        self._dir_info = {}
+        self.path_info = PathLOCAL()
+
+    @staticmethod
+    def compat_config(config):
+        ret = config.copy()
+        url = ret.pop(Config.SECTION_AWS_STORAGEPATH, "")
+        ret[Config.SECTION_REMOTE_URL] = url
+        return ret
+
+    @property
+    def url(self):
+        return self.cache_dir
+
+    @property
+    def prefix(self):
+        return self.cache_dir
+
+    def list_cache_paths(self):
+        clist = []
+        for entry in os.listdir(self.cache_dir):
+            subdir = os.path.join(self.cache_dir, entry)
+            if not os.path.isdir(subdir):
+                continue
+
+            for cache in os.listdir(subdir):
+                clist.append(os.path.join(subdir, cache))
+
+        return clist
+
+    def get(self, md5):
+        if not md5:
+            return None
+
+        return self.checksum_to_path(md5)
+
+    def exists(self, path_info):
+        assert isinstance(path_info, PathBASE)
+        assert path_info.scheme == "local"
+        return os.path.lexists(path_info.path)
+
+    def makedirs(self, path_info):
+        if not self.exists(path_info):
+            os.makedirs(path_info.path)
+
+    @slow_link_guard
+    def link(self, cache_info, path_info):
+        cache = cache_info.path
+        path = path_info.path
+
+        assert os.path.isfile(cache)
+
+        dname = os.path.dirname(path)
+        if not os.path.exists(dname):
+            os.makedirs(dname)
+
+        # NOTE: just create an empty file for an empty cache
+        if os.path.getsize(cache) == 0:
+            open(path, "w+").close()
+
+            msg = "Created empty file: {} -> {}".format(cache, path)
+            logger.debug(msg)
+            return
+
+        i = len(self.cache_types)
+        while i > 0:
+            try:
+                self.CACHE_TYPE_MAP[self.cache_types[0]](cache, path)
+
+                if self.protected:
+                    self.protect(path_info)
+
+                msg = "Created {}'{}': {} -> {}".format(
+                    "protected " if self.protected else "",
+                    self.cache_types[0],
+                    cache,
+                    path,
+                )
+
+                logger.debug(msg)
+                return
+
+            except DvcException as exc:
+                msg = "Cache type '{}' is not supported: {}"
+                logger.debug(msg.format(self.cache_types[0], str(exc)))
+                del self.cache_types[0]
+                i -= 1
+
+        raise DvcException("no possible cache types left to try out.")
+
+    @property
+    def ospath(self):
+        if os.name == "nt":
+            return ntpath
+        return posixpath
+
+    def already_cached(self, path_info):
+        assert path_info.scheme in ["", "local"]
+
+        current_md5 = self.get_checksum(path_info)
+
+        if not current_md5:
+            return False
+
+        return not self.changed_cache(current_md5)
+
+    def is_empty(self, path_info):
+        path = path_info.path
+
+        if self.isfile(path_info) and os.path.getsize(path) == 0:
+            return True
+
+        if self.isdir(path_info) and len(os.listdir(path)) == 0:
+            return True
+
+        return False
+
+    def isfile(self, path_info):
+        return os.path.isfile(path_info.path)
+
+    def isdir(self, path_info):
+        return os.path.isdir(path_info.path)
+
+    def walk(self, path_info):
+        return os.walk(path_info.path)
+
+    def get_file_checksum(self, path_info):
+        return file_md5(path_info.path)[0]
+
+    def remove(self, path_info):
+        if path_info.scheme != "local":
+            raise NotImplementedError
+
+        remove(path_info.path)
+
+    def move(self, from_info, to_info):
+        if from_info.scheme != "local" or to_info.scheme != "local":
+            raise NotImplementedError
+
+        inp = from_info.path
+        outp = to_info.path
+
+        # moving in two stages to make the whole operation atomic in
+        # case inp and outp are in different filesystems and actual
+        # physical copying of data is happening
+        tmp = "{}.{}".format(outp, str(uuid.uuid4()))
+        move(inp, tmp)
+        move(tmp, outp)
+
+    def cache_exists(self, md5s):
+        assert isinstance(md5s, list)
+        return list(filter(lambda md5: not self.changed_cache_file(md5), md5s))
+
+    def upload(self, from_infos, to_infos, names=None, no_progress_bar=False):
+        names = self._verify_path_args(to_infos, from_infos, names)
+
+        for from_info, to_info, name in zip(from_infos, to_infos, names):
+            if to_info.scheme != "local":
+                raise NotImplementedError
+
+            if from_info.scheme != "local":
+                raise NotImplementedError
+
+            logger.debug(
+                "Uploading '{}' to '{}'".format(from_info.path, to_info.path)
+            )
+
+            if not name:
+                name = os.path.basename(from_info.path)
+
+            makedirs(os.path.dirname(to_info.path), exist_ok=True)
+            tmp_file = tmp_fname(to_info.path)
+
+            try:
+                copyfile(
+                    from_info.path,
+                    tmp_file,
+                    name=name,
+                    no_progress_bar=no_progress_bar,
+                )
+                os.rename(tmp_file, to_info.path)
+            except Exception:
+                logger.exception(
+                    "failed to upload '{}' to '{}'".format(
+                        from_info.path, to_info.path
+                    )
+                )
+
+    def download(
+        self,
+        from_infos,
+        to_infos,
+        no_progress_bar=False,
+        names=None,
+        resume=False,
+    ):
+        names = self._verify_path_args(from_infos, to_infos, names)
+
+        for to_info, from_info, name in zip(to_infos, from_infos, names):
+            if from_info.scheme != "local":
+                raise NotImplementedError
+
+            if to_info.scheme != "local":
+                raise NotImplementedError
+
+            logger.debug(
+                "Downloading '{}' to '{}'".format(from_info.path, to_info.path)
+            )
+
+            if not name:
+                name = os.path.basename(to_info.path)
+
+            makedirs(os.path.dirname(to_info.path), exist_ok=True)
+            tmp_file = tmp_fname(to_info.path)
+            try:
+                copyfile(
+                    from_info.path,
+                    tmp_file,
+                    no_progress_bar=no_progress_bar,
+                    name=name,
+                )
+
+                move(tmp_file, to_info.path)
+            except Exception:
+                logger.exception(
+                    "failed to download '{}' to '{}'".format(
+                        from_info.path, to_info.path
+                    )
+                )
+
+                continue
+
+    def _group(self, checksum_infos, show_checksums=False):
+        by_md5 = {}
+
+        for info in checksum_infos:
+            md5 = info[self.PARAM_CHECKSUM]
+
+            if show_checksums:
+                by_md5[md5] = {"name": md5}
+                continue
+
+            name = info[self.PARAM_PATH]
+            branch = info.get("branch")
+            if branch:
+                name += "({})".format(branch)
+
+            if md5 not in by_md5.keys():
+                by_md5[md5] = {"name": name}
+            else:
+                by_md5[md5]["name"] += " " + name
+
+        return by_md5
+
+    def status(
+        self,
+        checksum_infos,
+        remote,
+        jobs=None,
+        show_checksums=False,
+        download=False,
+    ):
+        logger.info("Preparing to collect status from {}".format(remote.url))
+        title = "Collecting information"
+
+        ret = {}
+
+        progress.set_n_total(1)
+        progress.update_target(title, 0, 100)
+
+        progress.update_target(title, 10, 100)
+
+        ret = self._group(checksum_infos, show_checksums=show_checksums)
+        md5s = list(ret.keys())
+
+        progress.update_target(title, 30, 100)
+
+        local_exists = self.cache_exists(md5s)
+
+        progress.update_target(title, 40, 100)
+
+        # This is a performance optimization. We can safely assume that,
+        # if the resources that we want to fetch are already cached,
+        # there's no need to check the remote storage for the existance of
+        # those files.
+        if download and sorted(local_exists) == sorted(md5s):
+            remote_exists = local_exists
+        else:
+            remote_exists = list(remote.cache_exists(md5s))
+
+        progress.update_target(title, 90, 100)
+
+        progress.finish_target(title)
+
+        self._fill_statuses(ret, local_exists, remote_exists)
+
+        self._log_missing_caches(ret)
+
+        return ret
+
+    def _fill_statuses(self, checksum_info_dir, local_exists, remote_exists):
+        # Using sets because they are way faster for lookups
+        local = set(local_exists)
+        remote = set(remote_exists)
+
+        for md5, info in checksum_info_dir.items():
+            status = STATUS_MAP[(md5 in local, md5 in remote)]
+            info["status"] = status
+
+    def _get_chunks(self, download, remote, status_info, status, jobs):
+        title = "Analysing status."
+
+        progress.set_n_total(1)
+        total = len(status_info)
+        current = 0
+
+        cache = []
+        path_infos = []
+        names = []
+        for md5, info in status_info.items():
+            if info["status"] == status:
+                cache.append(self.checksum_to_path_info(md5))
+                path_infos.append(remote.checksum_to_path_info(md5))
+                names.append(info["name"])
+            current += 1
+            progress.update_target(title, current, total)
+
+        progress.finish_target(title)
+
+        progress.set_n_total(len(names))
+
+        if download:
+            to_infos = cache
+            from_infos = path_infos
+        else:
+            to_infos = path_infos
+            from_infos = cache
+
+        return list(
+            zip(
+                to_chunks(from_infos, jobs),
+                to_chunks(to_infos, jobs),
+                to_chunks(names, jobs),
+            )
+        )
+
+    def _process(
+        self,
+        checksum_infos,
+        remote,
+        jobs=None,
+        show_checksums=False,
+        download=False,
+    ):
+        msg = "Preparing to {} data {} '{}'"
+        logger.info(
+            msg.format(
+                "download" if download else "upload",
+                "from" if download else "to",
+                remote.url,
+            )
+        )
+
+        if download:
+            func = remote.download
+            status = STATUS_DELETED
+        else:
+            func = remote.upload
+            status = STATUS_NEW
+
+        if jobs is None:
+            jobs = remote.JOBS
+
+        status_info = self.status(
+            checksum_infos,
+            remote,
+            jobs=jobs,
+            show_checksums=show_checksums,
+            download=download,
+        )
+
+        chunks = self._get_chunks(download, remote, status_info, status, jobs)
+
+        if len(chunks) == 0:
+            return 0
+
+        futures = []
+        with ThreadPoolExecutor(max_workers=jobs) as executor:
+            for from_infos, to_infos, names in chunks:
+                res = executor.submit(func, from_infos, to_infos, names=names)
+                futures.append(res)
+
+        for f in futures:
+            f.result()
+
+        return len(chunks)
+
+    def push(self, checksum_infos, remote, jobs=None, show_checksums=False):
+        return self._process(
+            checksum_infos,
+            remote,
+            jobs=jobs,
+            show_checksums=show_checksums,
+            download=False,
+        )
+
+    def pull(self, checksum_infos, remote, jobs=None, show_checksums=False):
+        return self._process(
+            checksum_infos,
+            remote,
+            jobs=jobs,
+            show_checksums=show_checksums,
+            download=True,
+        )
+
+    def _changed_cache_dir(self):
+        mtime, size = get_mtime_and_size(self.cache_dir)
+        inode = get_inode(self.cache_dir)
+
+        existing_record = self.state.get_state_record_for_inode(inode)
+        if existing_record:
+            cached_mtime, cached_size, _, _ = existing_record
+            changed = not (mtime == cached_mtime and size == cached_size)
+        else:
+            changed = True
+
+        return changed
+
+    def _log_missing_caches(self, checksum_info_dict):
+        missing_caches = [
+            (md5, info)
+            for md5, info in checksum_info_dict.items()
+            if info["status"] == STATUS_MISSING
+        ]
+        if missing_caches:
+            missing_desc = "".join(
+                [
+                    "\nname: {}, md5: {}".format(info["name"], md5)
+                    for md5, info in missing_caches
+                ]
+            )
+            msg = (
+                "Some of the cache files do not exist neither locally "
+                "nor on remote. Missing cache files: {}".format(missing_desc)
+            )
+            logger.warning(msg)
+
+    @staticmethod
+    def _unprotect_file(path):
+        if System.is_symlink(path) or System.is_hardlink(path):
+            logger.debug("Unprotecting '{}'".format(path))
+            tmp = os.path.join(os.path.dirname(path), "." + str(uuid.uuid4()))
+
+            # The operations order is important here - if some application
+            # would access the file during the process of copyfile then it
+            # would get only the part of file. So, at first, the file should be
+            # copied with the temporary name, and then original file should be
+            # replaced by new.
+            copyfile(
+                path,
+                tmp,
+                name="Unprotecting '{}'".format(os.path.relpath(path)),
+            )
+            remove(path)
+            os.rename(tmp, path)
+
+        else:
+            logger.debug(
+                "Skipping copying for '{}', since it is not "
+                "a symlink or a hardlink.".format(path)
+            )
+
+        os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
+
+    @staticmethod
+    def _unprotect_dir(path):
+        for path in walk_files(path):
+            RemoteLOCAL._unprotect_file(path)
+
+    @staticmethod
+    def unprotect(path_info):
+        path = path_info.path
+        if not os.path.exists(path):
+            raise DvcException(
+                "can't unprotect non-existing data '{}'".format(path)
+            )
+
+        if os.path.isdir(path):
+            RemoteLOCAL._unprotect_dir(path)
+        else:
+            RemoteLOCAL._unprotect_file(path)
+
+    @staticmethod
+    def protect(path_info):
+        os.chmod(path_info.path, stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH)

dvc/remote/local/slow_link_detection.py

@@ -0,0 +1,63 @@
+import logging
+import time
+
+import colorama
+from dvc.config import Config
+
+logger = logging.getLogger(__name__)
+
+
+class SlowLinkDetectorDecorator(object):
+    LINKING_TIMEOUT_SECONDS = 10.0
+    was_displayed = False
+
+    @classmethod
+    def should_display(cls):
+        if not cls.was_displayed:
+            cls.was_displayed = True
+            return True
+        return False
+
+    def __init__(self, method):
+        self.method = method
+
+    def __call__(self, *args, **kwargs):
+        start = time.time()
+        result = self.method(*args, **kwargs)
+        execution_time_seconds = time.time() - start
+
+        if (
+            execution_time_seconds >= self.LINKING_TIMEOUT_SECONDS
+            and self.should_display()
+        ):
+            msg = (
+                "You can cut execution time considerably. Check:\n"
+                "{blue}https://dvc.org/doc/commands-reference/config#cache{"
+                "reset}"
+                "\nfor "
+                "more information.\nTo disable this message, run:\n'dvc "
+                "config "
+                "cache.slow_link_warning False'".format(
+                    blue=colorama.Fore.BLUE, reset=colorama.Fore.RESET
+                )
+            )
+            logger.warning(msg)
+
+        return result
+
+
+def slow_link_guard(method):
+    def call(remote_local, *args, **kwargs):
+        cache_config = remote_local.repo.config.config.get(
+            Config.SECTION_CACHE
+        )
+        should_warn = cache_config.get(
+            Config.SECTION_CACHE_SLOW_LINK_WARNING, True
+        ) and not cache_config.get(Config.SECTION_CACHE_TYPE, None)
+
+        if should_warn:
+            decorated = SlowLinkDetectorDecorator(method)
+            return decorated(remote_local, *args, **kwargs)
+        return method(remote_local, *args, **kwargs)
+
+    return call

dvc/remote/oss.py

@@ -0,0 +1,192 @@
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+import os
+import logging
+
+from dvc.scheme import Schemes
+from dvc.path.oss import PathOSS
+
+try:
+    import oss2
+except ImportError:
+    oss2 = None
+
+from dvc.utils import tmp_fname, move
+from dvc.utils.compat import urlparse, makedirs
+from dvc.progress import progress
+from dvc.config import Config
+from dvc.remote.base import RemoteBASE
+from dvc.remote.azure import Callback
+
+
+logger = logging.getLogger(__name__)
+
+
+class RemoteOSS(RemoteBASE):
+    """
+    oss2 document:
+    https://www.alibabacloud.com/help/doc-detail/32026.htm
+
+
+    Examples
+    ----------
+    $ dvc remote add myremote oss://my-bucket/path
+    Set key id, key secret and endpoint using modify command
+    $ dvc remote modify myremote oss_key_id my-key-id
+    $ dvc remote modify myremote oss_key_secret my-key-secret
+    $ dvc remote modify myremote oss_endpoint endpoint
+    or environment variables
+    $ export OSS_ACCESS_KEY_ID="my-key-id"
+    $ export OSS_ACCESS_KEY_SECRET="my-key-secret"
+    $ export OSS_ENDPOINT="endpoint"
+    """
+
+    scheme = Schemes.OSS
+    REGEX = r"^oss://(?P<path>.*)?$"
+    REQUIRES = {"oss2": oss2}
+    PARAM_CHECKSUM = "etag"
+    COPY_POLL_SECONDS = 5
+
+    def __init__(self, repo, config):
+        super(RemoteOSS, self).__init__(repo, config)
+
+        self.url = config.get(Config.SECTION_REMOTE_URL)
+        parsed = urlparse(self.url)
+        self.bucket = parsed.netloc
+        self.prefix = parsed.path.lstrip("/")
+
+        self.endpoint = config.get(Config.SECTION_OSS_ENDPOINT) or os.getenv(
+            "OSS_ENDPOINT"
+        )
+
+        self.key_id = (
+            config.get(Config.SECTION_OSS_ACCESS_KEY_ID)
+            or os.getenv("OSS_ACCESS_KEY_ID")
+            or "defaultId"
+        )
+
+        self.key_secret = (
+            config.get(Config.SECTION_OSS_ACCESS_KEY_SECRET)
+            or os.getenv("OSS_ACCESS_KEY_SECRET")
+            or "defaultSecret"
+        )
+
+        self._bucket = None
+        self.path_info = PathOSS(bucket=self.bucket)
+
+    @property
+    def oss_service(self):
+        if self._bucket is None:
+            logger.debug("URL {}".format(self.url))
+            logger.debug("key id {}".format(self.key_id))
+            logger.debug("key secret {}".format(self.key_secret))
+            auth = oss2.Auth(self.key_id, self.key_secret)
+            logger.debug("bucket name {}".format(self.bucket))
+            self._bucket = oss2.Bucket(auth, self.endpoint, self.bucket)
+            try:  # verify that bucket exists
+                self._bucket.get_bucket_info()
+            except oss2.exceptions.NoSuchBucket:
+                self._bucket.create_bucket(
+                    oss2.BUCKET_ACL_PUBLIC_READ,
+                    oss2.models.BucketCreateConfig(
+                        oss2.BUCKET_STORAGE_CLASS_STANDARD
+                    ),
+                )
+        return self._bucket
+
+    def remove(self, path_info):
+        if path_info.scheme != self.scheme:
+            raise NotImplementedError
+
+        logger.debug(
+            "Removing oss://{}/{}".format(path_info.bucket, path_info.path)
+        )
+
+        self.oss_service.delete_object(path_info.path)
+
+    def _list_paths(self, prefix):
+        for blob in oss2.ObjectIterator(self.oss_service, prefix=prefix):
+            yield blob.key
+
+    def list_cache_paths(self):
+        return self._list_paths(self.prefix)
+
+    def upload(self, from_infos, to_infos, names=None, no_progress_bar=False):
+        names = self._verify_path_args(to_infos, from_infos, names)
+
+        for from_info, to_info, name in zip(from_infos, to_infos, names):
+            if to_info.scheme != self.scheme:
+                raise NotImplementedError
+
+            if from_info.scheme != "local":
+                raise NotImplementedError
+
+            bucket = to_info.bucket
+            path = to_info.path
+
+            logger.debug(
+                "Uploading '{}' to 'oss://{}/{}'".format(
+                    from_info.path, bucket, path
+                )
+            )
+
+            if not name:
+                name = os.path.basename(from_info.path)
+
+            cb = None if no_progress_bar else Callback(name)
+
+            try:
+                self.oss_service.put_object_from_file(
+                    path, from_info.path, progress_callback=cb
+                )
+            except Exception:
+                msg = "failed to upload '{}'".format(from_info.path)
+                logger.warning(msg)
+            else:
+                progress.finish_target(name)
+
+    def download(
+        self,
+        from_infos,
+        to_infos,
+        names=None,
+        no_progress_bar=False,
+        resume=False,
+    ):
+        names = self._verify_path_args(from_infos, to_infos, names)
+        for to_info, from_info, name in zip(to_infos, from_infos, names):
+            if from_info.scheme != self.scheme:
+                raise NotImplementedError
+            if to_info.scheme != "local":
+                raise NotImplementedError
+
+            bucket = from_info.bucket
+            path = from_info.path
+
+            logger.debug(
+                "Downloading 'oss://{}/{}' to '{}'".format(
+                    bucket, path, to_info.path
+                )
+            )
+
+            tmp_file = tmp_fname(to_info.path)
+            if not name:
+                name = os.path.basename(to_info.path)
+
+            cb = None if no_progress_bar else Callback(name)
+
+            makedirs(os.path.dirname(to_info.path), exist_ok=True)
+
+            try:
+                self.oss_service.get_object_to_file(
+                    path, tmp_file, progress_callback=cb
+                )
+            except Exception:
+                msg = "failed to download 'oss://{}/{}'".format(bucket, path)
+                logger.warning(msg)
+            else:
+                move(tmp_file, to_info.path)
+
+                if not no_progress_bar:
+                    progress.finish_target(name)

dvc/remote/s3.py

@@ -1,23 +1,25 @@
+from __future__ import unicode_literals
+
 import os
 import threading
-import posixpath
+import logging
+
+from dvc.scheme import Schemes
+from dvc.path.s3 import PathS3
 
 try:
     import boto3
 except ImportError:
     boto3 = None
 
-try:
-    from urlparse import urlparse
-except ImportError:
-    from urllib.parse import urlparse
-
-from dvc.logger import Logger
+from dvc.utils import tmp_fname, move
+from dvc.utils.compat import urlparse, makedirs
 from dvc.progress import progress
 from dvc.config import Config
-from dvc.remote.base import RemoteBase
-from dvc.remote.local import RemoteLOCAL
-from dvc.exceptions import DvcException
+from dvc.remote.base import RemoteBASE
+from dvc.exceptions import DvcException, ETagMismatchError
+
+logger = logging.getLogger(__name__)
 
 
 class Callback(object):
@@ -33,321 +35,291 @@ class Callback(object):
             progress.update_target(self.name, self.current, self.total)
 
 
-class RemoteS3(RemoteBase):
-    scheme = 's3'
-    REGEX = r'^s3://(?P<path>.*)$'
-    REQUIRES = {'boto3': boto3}
-    PARAM_ETAG = 'etag'
+class RemoteS3(RemoteBASE):
+    scheme = Schemes.S3
+    REGEX = r"^s3://(?P<path>.*)$"
+    REQUIRES = {"boto3": boto3}
+    PARAM_CHECKSUM = "etag"
 
-    def __init__(self, project, config):
-        self.project = project
+    def __init__(self, repo, config):
+        super(RemoteS3, self).__init__(repo, config)
 
-        storagepath = 's3://{}'.format(
-            config.get(Config.SECTION_AWS_STORAGEPATH, '').lstrip('/')
+        storagepath = "s3://{}".format(
+            config.get(Config.SECTION_AWS_STORAGEPATH, "").lstrip("/")
         )
 
         self.url = config.get(Config.SECTION_REMOTE_URL, storagepath)
 
-        self.region = (
-            os.environ.get('AWS_DEFAULT_REGION') or
-            config.get(Config.SECTION_AWS_REGION)
+        self.region = os.environ.get("AWS_DEFAULT_REGION") or config.get(
+            Config.SECTION_AWS_REGION
         )
 
-        self.profile = (
-            os.environ.get('AWS_PROFILE') or
-            config.get(Config.SECTION_AWS_PROFILE)
+        self.profile = os.environ.get("AWS_PROFILE") or config.get(
+            Config.SECTION_AWS_PROFILE
         )
 
         self.endpoint_url = config.get(Config.SECTION_AWS_ENDPOINT_URL)
 
+        self.list_objects = config.get(Config.SECTION_AWS_LIST_OBJECTS)
+
+        self.use_ssl = config.get(Config.SECTION_AWS_USE_SSL, True)
+
         shared_creds = config.get(Config.SECTION_AWS_CREDENTIALPATH)
         if shared_creds:
-            os.environ.setdefault('AWS_SHARED_CREDENTIALS_FILE', shared_creds)
+            os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", shared_creds)
+
+        parsed = urlparse(self.url)
+        self.bucket = parsed.netloc
+        self.prefix = parsed.path.lstrip("/")
+
+        self.path_info = PathS3(bucket=self.bucket)
 
     @staticmethod
     def compat_config(config):
         ret = config.copy()
-        url = 's3://' + ret.pop(Config.SECTION_AWS_STORAGEPATH, '').lstrip('/')
+        url = "s3://" + ret.pop(Config.SECTION_AWS_STORAGEPATH, "").lstrip("/")
         ret[Config.SECTION_REMOTE_URL] = url
         return ret
 
-    @property
-    def bucket(self):
-        return urlparse(self.url).netloc
-
-    @property
-    def prefix(self):
-        return urlparse(self.url).path.lstrip('/')
-
     @property
     def s3(self):
-        session = boto3.session.Session(profile_name=self.profile,
-                                        region_name=self.region)
+        session = boto3.session.Session(
+            profile_name=self.profile, region_name=self.region
+        )
 
-        return session.client('s3', endpoint_url=self.endpoint_url)
+        return session.client(
+            "s3", endpoint_url=self.endpoint_url, use_ssl=self.use_ssl
+        )
 
-    def get_etag(self, bucket, key):
-        try:
-            obj = self.s3.head_object(Bucket=bucket, Key=key)
-        except Exception:
-            raise DvcException('s3://{}/{} does not exist'.format(bucket, key))
+    @classmethod
+    def get_etag(cls, s3, bucket, path):
+        obj = cls.get_head_object(s3, bucket, path)
 
-        return obj['ETag'].strip('"')
+        return obj["ETag"].strip('"')
 
-    def save_info(self, path_info):
-        if path_info['scheme'] != 's3':
-            raise NotImplementedError
-
-        return {self.PARAM_ETAG: self.get_etag(path_info['bucket'],
-                                               path_info['key'])}
-
-    def changed(self, path_info, checksum_info):
-        if not self.exists([path_info])[0]:
-            return True
-
-        etag = checksum_info.get(self.PARAM_ETAG, None)
-        if etag is None:
-            return True
-
-        if self.changed_cache(etag):
-            return True
-
-        return checksum_info != self.save_info(path_info)
-
-    def _copy(self, from_info, to_info, s3=None):
-        s3 = s3 if s3 else self.s3
-
-        source = {'Bucket': from_info['bucket'],
-                  'Key': from_info['key']}
-        self.s3.copy(source, to_info['bucket'], to_info['key'])
-
-    def save(self, path_info):
-        if path_info['scheme'] != 's3':
-            raise NotImplementedError
-
-        etag = self.get_etag(path_info['bucket'], path_info['key'])
-        key = posixpath.join(self.prefix, etag[0:2], etag[2:])
-        to_info = {'scheme': 's3', 'bucket': self.bucket, 'key': key}
-
-        self._copy(path_info, to_info)
-
-        return {self.PARAM_ETAG: etag}
+    def get_file_checksum(self, path_info):
+        return self.get_etag(self.s3, path_info.bucket, path_info.path)
 
     @staticmethod
-    def to_string(path_info):
-        return "s3://{}/{}".format(path_info['bucket'], path_info['key'])
+    def get_head_object(s3, bucket, path, *args, **kwargs):
 
-    def changed_cache(self, etag):
-        key = posixpath.join(self.prefix, etag[0:2], etag[2:])
-        cache = {'scheme': 's3', 'bucket': self.bucket, 'key': key}
+        try:
+            obj = s3.head_object(Bucket=bucket, Key=path, *args, **kwargs)
+        except Exception as exc:
+            raise DvcException(
+                "s3://{}/{} does not exist".format(bucket, path), exc
+            )
+        return obj
 
-        if {self.PARAM_ETAG: etag} != self.save_info(cache):
-            if self.exists([cache])[0]:
-                msg = 'Corrupted cache file {}'
-                Logger.warn(msg.format(self.to_string(cache)))
-                self.remove(cache)
-            return True
+    @classmethod
+    def _copy_multipart(cls, s3, from_info, to_info, size, n_parts):
+        mpu = s3.create_multipart_upload(
+            Bucket=to_info.bucket, Key=to_info.path
+        )
+        mpu_id = mpu["UploadId"]
 
-        return False
+        parts = []
+        byte_position = 0
+        for i in range(1, n_parts + 1):
+            obj = cls.get_head_object(
+                s3, from_info.bucket, from_info.path, PartNumber=i
+            )
+            part_size = obj["ContentLength"]
+            lastbyte = byte_position + part_size - 1
+            if lastbyte > size:
+                lastbyte = size - 1
 
-    def checkout(self, path_info, checksum_info):
-        if path_info['scheme'] != 's3':
-            raise NotImplementedError
+            srange = "bytes={}-{}".format(byte_position, lastbyte)
 
-        etag = checksum_info.get(self.PARAM_ETAG, None)
-        if not etag:
-            return
+            part = s3.upload_part_copy(
+                Bucket=to_info.bucket,
+                Key=to_info.path,
+                PartNumber=i,
+                UploadId=mpu_id,
+                CopySourceRange=srange,
+                CopySource={"Bucket": from_info.bucket, "Key": from_info.path},
+            )
+            parts.append(
+                {"PartNumber": i, "ETag": part["CopyPartResult"]["ETag"]}
+            )
+            byte_position += part_size
 
-        if not self.changed(path_info, checksum_info):
-            msg = "Data '{}' didn't change."
-            Logger.info(msg.format(self.to_string(path_info)))
-            return
+        assert n_parts == len(parts)
 
-        if self.changed_cache(etag):
-            msg = "Cache '{}' not found. File '{}' won't be created."
-            Logger.warn(msg.format(etag, self.to_string(path_info)))
-            return
+        s3.complete_multipart_upload(
+            Bucket=to_info.bucket,
+            Key=to_info.path,
+            UploadId=mpu_id,
+            MultipartUpload={"Parts": parts},
+        )
 
-        if self.exists([path_info])[0]:
-            msg = "Data '{}' exists. Removing before checkout."
-            Logger.warn(msg.format(self.to_string(path_info)))
-            self.remove(path_info)
-            return
+    @classmethod
+    def _copy(cls, s3, from_info, to_info):
+        # NOTE: object's etag depends on the way it was uploaded to s3 or the
+        # way it was copied within the s3. More specifically, it depends on
+        # the chunk size that was used to transfer it, which would affect
+        # whether an object would be uploaded as a single part or as a
+        # multipart.
+        #
+        # If an object's etag looks like '8978c98bb5a48c2fb5f2c4c905768afa',
+        # then it was transfered as a single part, which means that the chunk
+        # size used to transfer it was greater or equal to the ContentLength
+        # of that object. So to preserve that tag over the next transfer, we
+        # could use any value >= ContentLength.
+        #
+        # If an object's etag looks like '50d67013a5e1a4070bef1fc8eea4d5f9-13',
+        # then it was transfered as a multipart, which means that the chunk
+        # size used to transfer it was less than ContentLength of that object.
+        # Unfortunately, in general, it doesn't mean that the chunk size was
+        # the same throughout the transfer, so it means that in order to
+        # preserve etag, we need to transfer each part separately, so the
+        # object is transfered in the same chunks as it was originally.
 
-        msg = "Checking out '{}' with cache '{}'."
-        Logger.info(msg.format(self.to_string(path_info), etag))
+        obj = cls.get_head_object(s3, from_info.bucket, from_info.path)
+        etag = obj["ETag"].strip('"')
+        size = obj["ContentLength"]
 
-        key = posixpath.join(self.prefix, etag[0:2], etag[2:])
-        from_info = {'scheme': 's3', 'bucket': self.bucket, 'key': key}
+        _, _, parts_suffix = etag.partition("-")
+        if parts_suffix:
+            n_parts = int(parts_suffix)
+            cls._copy_multipart(s3, from_info, to_info, size, n_parts)
+        else:
+            source = {"Bucket": from_info.bucket, "Key": from_info.path}
+            s3.copy(source, to_info.bucket, to_info.path)
 
-        self._copy(from_info, path_info)
+        cached_etag = cls.get_etag(s3, to_info.bucket, to_info.path)
+        if etag != cached_etag:
+            raise ETagMismatchError(etag, cached_etag)
+
+    def copy(self, from_info, to_info, s3=None):
+        s3 = s3 if s3 else self.s3
+        self._copy(s3, from_info, to_info)
 
     def remove(self, path_info):
-        if path_info['scheme'] != 's3':
+        if path_info.scheme != "s3":
             raise NotImplementedError
 
-        Logger.debug('Removing s3://{}/{}'.format(path_info['bucket'],
-                                                  path_info['key']))
+        logger.debug(
+            "Removing s3://{}/{}".format(path_info.bucket, path_info.path)
+        )
 
-        self.s3.delete_object(Bucket=path_info['bucket'],
-                              Key=path_info['key'])
+        self.s3.delete_object(Bucket=path_info.bucket, Key=path_info.path)
 
-    def md5s_to_path_infos(self, md5s):
-        return [{'scheme': self.scheme,
-                 'bucket': self.bucket,
-                 'key': posixpath.join(self.prefix,
-                                       md5[0:2], md5[2:])} for md5 in md5s]
-
-    def _all_keys(self):
+    def _list_paths(self, bucket, prefix):
+        """ Read config for list object api, paginate through list objects."""
         s3 = self.s3
-
-        keys = []
-        kwargs = {'Bucket': self.bucket,
-                  'Prefix': self.prefix}
-        while True:
-            resp = s3.list_objects_v2(**kwargs)
-            contents = resp.get('Contents', None)
+        kwargs = {"Bucket": bucket, "Prefix": prefix}
+        if self.list_objects:
+            list_objects_api = "list_objects"
+        else:
+            list_objects_api = "list_objects_v2"
+        paginator = s3.get_paginator(list_objects_api)
+        for page in paginator.paginate(**kwargs):
+            contents = page.get("Contents", None)
             if not contents:
-                break
+                continue
+            for item in contents:
+                yield item["Key"]
 
-            for obj in contents:
-                keys.append(obj['Key'])
+    def list_cache_paths(self):
+        return self._list_paths(self.bucket, self.prefix)
 
-            token = resp.get('NextContinuationToken', None)
-            if not token:
-                break
+    def exists(self, path_info):
+        assert not isinstance(path_info, list)
+        assert path_info.scheme == "s3"
 
-            kwargs['ContinuationToken'] = token
+        paths = self._list_paths(path_info.bucket, path_info.path)
+        return any(path_info.path == path for path in paths)
 
-        return keys
-
-    def exists(self, path_infos):
-        # NOTE: We mostly use exists() method when filtering a bulk of cache
-        # files to decide if we need to download/upload them and in s3
-        # list_objects_v2() is much-much faster than trying to check keys
-        # one-by-one.
-        ret = []
-
-        keys = self._all_keys()
-
-        for path_info in path_infos:
-            exists = False
-            if path_info['key'] in keys:
-                exists = True
-            ret.append(exists)
-
-        return ret
-
-    def upload(self, from_infos, to_infos, names=None):
+    def upload(self, from_infos, to_infos, names=None, no_progress_bar=False):
         names = self._verify_path_args(to_infos, from_infos, names)
 
         s3 = self.s3
 
         for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info['scheme'] != 's3':
+            if to_info.scheme != "s3":
                 raise NotImplementedError
 
-            if from_info['scheme'] != 'local':
+            if from_info.scheme != "local":
                 raise NotImplementedError
 
-            Logger.debug("Uploading '{}' to '{}/{}'".format(from_info['path'],
-                                                            to_info['bucket'],
-                                                            to_info['key']))
+            logger.debug(
+                "Uploading '{}' to '{}/{}'".format(
+                    from_info.path, to_info.bucket, to_info.path
+                )
+            )
 
             if not name:
-                name = os.path.basename(from_info['path'])
+                name = os.path.basename(from_info.path)
 
-            total = os.path.getsize(from_info['path'])
-            cb = Callback(name, total)
+            total = os.path.getsize(from_info.path)
+            cb = None if no_progress_bar else Callback(name, total)
 
             try:
-                s3.upload_file(from_info['path'],
-                               to_info['bucket'],
-                               to_info['key'],
-                               Callback=cb)
-            except Exception as exc:
-                msg = "Failed to upload '{}'".format(from_info['path'])
-                Logger.warn(msg, exc)
+                s3.upload_file(
+                    from_info.path, to_info.bucket, to_info.path, Callback=cb
+                )
+            except Exception:
+                msg = "failed to upload '{}'".format(from_info.path)
+                logger.exception(msg)
                 continue
 
             progress.finish_target(name)
 
-    def download(self,
-                 from_infos,
-                 to_infos,
-                 no_progress_bar=False,
-                 names=None):
+    def download(
+        self,
+        from_infos,
+        to_infos,
+        no_progress_bar=False,
+        names=None,
+        resume=False,
+    ):
         names = self._verify_path_args(from_infos, to_infos, names)
 
         s3 = self.s3
 
         for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info['scheme'] != 's3':
+            if from_info.scheme != "s3":
                 raise NotImplementedError
 
-            if to_info['scheme'] == 's3':
-                self._copy(from_info, to_info, s3=s3)
+            if to_info.scheme == "s3":
+                self.copy(from_info, to_info, s3=s3)
                 continue
 
-            if to_info['scheme'] != 'local':
+            if to_info.scheme != "local":
                 raise NotImplementedError
 
-            msg = "Downloading '{}/{}' to '{}'".format(from_info['bucket'],
-                                                       from_info['key'],
-                                                       to_info['path'])
-            Logger.debug(msg)
+            msg = "Downloading '{}/{}' to '{}'".format(
+                from_info.bucket, from_info.path, to_info.path
+            )
+            logger.debug(msg)
 
-            tmp_file = self.tmp_file(to_info['path'])
+            tmp_file = tmp_fname(to_info.path)
             if not name:
-                name = os.path.basename(to_info['path'])
+                name = os.path.basename(to_info.path)
 
-            self._makedirs(to_info['path'])
+            makedirs(os.path.dirname(to_info.path), exist_ok=True)
 
             try:
                 if no_progress_bar:
                     cb = None
                 else:
                     total = s3.head_object(
-                                    Bucket=from_info['bucket'],
-                                    Key=from_info['key'])['ContentLength']
+                        Bucket=from_info.bucket, Key=from_info.path
+                    )["ContentLength"]
                     cb = Callback(name, total)
 
-                s3.download_file(from_info['bucket'],
-                                 from_info['key'],
-                                 tmp_file,
-                                 Callback=cb)
-            except Exception as exc:
-                msg = "Failed to download '{}/{}'".format(from_info['bucket'],
-                                                          from_info['key'])
-                Logger.warn(msg, exc)
+                s3.download_file(
+                    from_info.bucket, from_info.path, tmp_file, Callback=cb
+                )
+            except Exception:
+                msg = "failed to download '{}/{}'".format(
+                    from_info.bucket, from_info.path
+                )
+                logger.exception(msg)
                 continue
 
-            os.rename(tmp_file, to_info['path'])
+            move(tmp_file, to_info.path)
 
             if not no_progress_bar:
                 progress.finish_target(name)
-
-    def _path_to_etag(self, path):
-        relpath = posixpath.relpath(path, self.prefix)
-        return posixpath.dirname(relpath) + posixpath.basename(relpath)
-
-    def _all(self):
-        keys = self._all_keys()
-        return [self._path_to_etag(key) for key in keys]
-
-    def gc(self, cinfos):
-        used_etags = [info[self.PARAM_ETAG] for info in cinfos['s3']]
-        used_etags += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
-
-        removed = False
-        for etag in self._all():
-            if etag in used_etags:
-                continue
-            path_info = {'scheme': 's3',
-                         'key': posixpath.join(self.prefix,
-                                               etag[0:2], etag[2:]),
-                         'bucket': self.bucket}
-            self.remove(path_info)
-            removed = True
-
-        return removed

dvc/remote/ssh.py

@@ -1,435 +0,0 @@
-import os
-import getpass
-import posixpath
-
-try:
-    import paramiko
-except ImportError:
-    paramiko = None
-
-from dvc.logger import Logger
-from dvc.progress import progress
-from dvc.remote.base import RemoteBase
-from dvc.remote.local import RemoteLOCAL
-from dvc.config import Config
-from dvc.exceptions import DvcException
-
-
-def sizeof_fmt(num, suffix='B'):
-    """ Convert number of bytes to human-readable string """
-    for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
-        if abs(num) < 1024.0:
-            return "%3.1f%s%s" % (num, unit, suffix)
-        num /= 1024.0
-    return "%.1f%s%s" % (num, 'Y', suffix)
-
-
-def percent_cb(name, complete, total):
-    """ Callback for updating target progress """
-    Logger.debug('{}: {} transferred out of {}'.format(name,
-                                                       sizeof_fmt(complete),
-                                                       sizeof_fmt(total)))
-    progress.update_target(name, complete, total)
-
-
-def create_cb(name):
-    """ Create callback function for multipart object """
-    return (lambda cur, tot: percent_cb(name, cur, tot))
-
-
-class RemoteSSH(RemoteBase):
-    scheme = 'ssh'
-
-    # NOTE: temporarily only absolute paths are allowed
-    REGEX = r'^ssh://((?P<user>.*)@)?(?P<host>[^/]*):(?P<path>/.*)$'
-
-    REQUIRES = {'paramiko': paramiko}
-
-    JOBS = 4
-
-    PARAM_MD5 = 'md5'
-
-    DEFAULT_PORT = 22
-    TIMEOUT = 1800
-
-    def __init__(self, project, config):
-        self.project = project
-        self.url = config.get(Config.SECTION_REMOTE_URL, '/')
-        self.host = self.group('host')
-        self.user = self.group('user')
-        if not self.user:
-            self.user = config.get(Config.SECTION_REMOTE_USER,
-                                   getpass.getuser())
-        self.prefix = self.group('path')
-        self.port = config.get(Config.SECTION_REMOTE_PORT, self.DEFAULT_PORT)
-        self.keyfile = config.get(Config.SECTION_REMOTE_KEY_FILE, None)
-        self.timeout = config.get(Config.SECTION_REMOTE_TIMEOUT, self.TIMEOUT)
-        self.password = config.get(Config.SECTION_REMOTE_PASSWORD, None)
-        self.ask_password = config.get(Config.SECTION_REMOTE_ASK_PASSWORD,
-                                       False)
-
-    def md5s_to_path_infos(self, md5s):
-        return [{'scheme': 'ssh',
-                 'host': self.host,
-                 'user': self.user,
-                 'port': self.port,
-                 'path': posixpath.join(self.prefix,
-                                        md5[0:2], md5[2:])} for md5 in md5s]
-
-    def ssh(self, host=None, user=None, port=None):
-        msg = "Establishing ssh connection with '{}' " \
-              "through port '{}' as user '{}'"
-        Logger.debug(msg.format(host, port, user))
-
-        ssh = paramiko.SSHClient()
-
-        ssh.load_system_host_keys()
-        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-
-        if self.ask_password and self.password is None:
-            msg = "host '{}' port '{}' user '{}'".format(host, port, user)
-            self.password = self.project.prompt.prompt_password(msg)
-
-        ssh.connect(host,
-                    username=user,
-                    port=port,
-                    key_filename=self.keyfile,
-                    timeout=self.timeout,
-                    password=self.password)
-
-        return ssh
-
-    def exists(self, path_infos):
-        ret = []
-        ssh = self.ssh(host=self.host,
-                       user=self.user,
-                       port=self.port)
-        cmd = 'find {} -type f -follow -print'.format(self.prefix)
-        stdout = self._exec(ssh, cmd)
-        plist = stdout.split()
-        ssh.close()
-
-        for path_info in path_infos:
-            exists = False
-            if path_info['path'] in plist:
-                exists = True
-            ret.append(exists)
-
-        return ret
-
-    def _exec(self, ssh, cmd):
-        stdin, stdout, stderr = ssh.exec_command(cmd)
-        channel = stdout.channel
-
-        stdin.close()
-        channel.shutdown_write()
-
-        stdout_chunks = []
-        stderr_chunks = []
-        while (not channel.closed
-               or channel.recv_ready()
-               or channel.recv_stderr_ready()):
-            import select
-            got_chunk = False
-            readq, _, _ = select.select([stdout.channel], [], [], self.timeout)
-            for c in readq:
-                if c.recv_ready():
-                    stdout_chunks.append(stdout.channel.recv(len(c.in_buffer)))
-                    got_chunk = True
-
-                if c.recv_stderr_ready():
-                    stderr_len = len(c.in_stderr_buffer)
-                    s = stderr.channel.recv_stderr(stderr_len)
-                    stderr_chunks.append(s)
-                    got_chunk = True
-
-            if not got_chunk \
-               and stdout.channel.exit_status_ready() \
-               and not stderr.channel.recv_stderr_ready() \
-               and not stdout.channel.recv_ready():
-                stdout.channel.shutdown_read()
-                stdout.channel.close()
-                break
-
-        stdout.close()
-        stderr.close()
-
-        if stdout.channel.recv_exit_status() != 0:
-            err = b''.join(stderr_chunks).decode('utf-8')
-            msg = 'SSH command \'{}\' failed: {}'.format(cmd, err)
-            raise DvcException(msg)
-
-        return b''.join(stdout_chunks).decode('utf-8')
-
-    def md5(self, path_info):
-        if path_info['scheme'] != 'ssh':
-            raise NotImplementedError
-
-        ssh = self.ssh(host=path_info['host'],
-                       user=path_info['user'],
-                       port=path_info['port'])
-
-        # Use different md5 commands depending on os
-        stdout = self._exec(ssh, 'uname').strip()
-        if stdout == 'Darwin':
-            md5cmd = 'md5'
-            index = -1
-        elif stdout == 'Linux':
-            md5cmd = 'md5sum'
-            index = 0
-        else:
-            msg = '\'{}\' is not supported as a remote'.format(stdout)
-            raise DvcException(msg)
-
-        stdout = self._exec(ssh, '{} {}'.format(md5cmd, path_info['path']))
-        md5 = stdout.split()[index]
-        ssh.close()
-
-        assert len(md5) == 32
-
-        return md5
-
-    def cp(self, from_info, to_info, ssh=None):
-        if from_info['scheme'] != 'ssh' or to_info['scheme'] != 'ssh':
-            raise NotImplementedError
-
-        assert from_info['host'] == to_info['host']
-        assert from_info['user'] == to_info['user']
-
-        s = ssh if ssh else self.ssh(host=from_info['host'],
-                                     user=from_info['user'],
-                                     port=from_info['port'])
-
-        dname = posixpath.dirname(to_info['path'])
-        self._exec(s, 'mkdir -p {}'.format(dname))
-        self._exec(s, 'cp {} {}'.format(from_info['path'], to_info['path']))
-
-        if not ssh:
-            s.close()
-
-    def save_info(self, path_info):
-        if path_info['scheme'] != 'ssh':
-            raise NotImplementedError
-
-        return {self.PARAM_MD5: self.md5(path_info)}
-
-    @staticmethod
-    def to_string(path_info):
-        return "{}://{}@{}:{}".format(path_info['scheme'],
-                                      path_info['user'],
-                                      path_info['host'],
-                                      path_info['path'])
-
-    def changed_cache(self, md5):
-        cache = {}
-        cache['scheme'] = 'ssh'
-        cache['host'] = self.host
-        cache['port'] = self.port
-        cache['user'] = self.user
-        cache['path'] = posixpath.join(self.prefix, md5[0:2], md5[2:])
-
-        if {self.PARAM_MD5: md5} != self.save_info(cache):
-            if self.exists([cache])[0]:
-                msg = 'Corrupted cache file {}'
-                Logger.warn(msg.format(self.to_string(cache)))
-                self.remove(cache)
-            return True
-
-        return False
-
-    def changed(self, path_info, checksum_info):
-        if not self.exists([path_info])[0]:
-            return True
-
-        md5 = checksum_info.get(self.PARAM_MD5, None)
-        if md5 is None:
-            return True
-
-        if self.changed_code(md5):
-            return True
-
-        return checksum_info != self.save_info(path_info)
-
-    def save(self, path_info):
-        if path_info['scheme'] != 'ssh':
-            raise NotImplementedError
-
-        md5 = self.md5(path_info)
-        dest = path_info.copy()
-        dest['path'] = posixpath.join(self.prefix, md5[0:2], md5[2:])
-
-        self.cp(path_info, dest)
-
-        return {self.PARAM_MD5: md5}
-
-    def checkout(self, path_info, checksum_info):
-        if path_info['scheme'] != 'ssh':
-            raise NotImplementedError
-
-        md5 = checksum_info.get(self.PARAM_MD5, None)
-        if not md5:
-            return
-
-        if not self.changed(path_info, checksum_info):
-            msg = "Data '{}' didn't change."
-            Logger.info(msg.format(self.to_string(path_info)))
-            return
-
-        if self.changed_cache(md5):
-            msg = "Cache '{}' not found. File '{}' won't be created."
-            Logger.warn(msg.format(md5, self.to_string(path_info)))
-            return
-
-        if self.exists([path_info])[0]:
-            msg = "Data '{}' exists. Removing before checkout."
-            Logger.warn(msg.format(self.to_string(path_info)))
-            self.remove(path_info)
-            return
-
-        msg = "Checking out '{}' with cache '{}'."
-        Logger.info(msg.format(self.to_string(path_info), md5))
-
-        src = path_info.copy()
-        src['path'] = posixpath.join(self.prefix, md5[0:2], md5[2:])
-
-        self.cp(src, path_info)
-
-    def remove(self, path_info):
-        if path_info['scheme'] != 'ssh':
-            raise NotImplementedError
-
-        Logger.debug('Removing ssh://{}@{}/{}'.format(path_info['user'],
-                                                      path_info['host'],
-                                                      path_info['path']))
-
-        ssh = self.ssh(host=path_info['host'],
-                       user=path_info['user'],
-                       port=path_info['port'])
-        ssh.open_sftp().remove(path_info['path'])
-        ssh.close()
-
-    def download(self,
-                 from_infos,
-                 to_infos,
-                 no_progress_bar=False,
-                 names=None):
-        names = self._verify_path_args(from_infos, to_infos, names)
-
-        ssh = self.ssh(host=from_infos[0]['host'],
-                       user=from_infos[0]['user'],
-                       port=from_infos[0]['port'])
-
-        for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info['scheme'] != 'ssh':
-                raise NotImplementedError
-
-            if to_info['scheme'] == 'ssh':
-                assert from_info['host'] == to_info['host']
-                assert from_info['port'] == to_info['port']
-                assert from_info['user'] == to_info['user']
-                self.cp(from_info, to_info, ssh=ssh)
-                continue
-
-            if to_info['scheme'] != 'local':
-                raise NotImplementedError
-
-            msg = "Downloading '{}/{}' to '{}'".format(from_info['host'],
-                                                       from_info['path'],
-                                                       to_info['path'])
-            Logger.debug(msg)
-
-            if not name:
-                name = os.path.basename(to_info['path'])
-
-            self._makedirs(to_info['path'])
-            tmp_file = self.tmp_file(to_info['path'])
-            try:
-                ssh.open_sftp().get(from_info['path'],
-                                    tmp_file,
-                                    callback=create_cb(name))
-            except Exception as exc:
-                msg = "Failed to download '{}/{}' to '{}'"
-                Logger.warn(msg.format(from_info['host'],
-                                       from_info['path'],
-                                       to_info['path']), exc)
-                continue
-
-            os.rename(tmp_file, to_info['path'])
-            progress.finish_target(name)
-
-        ssh.close()
-
-    def upload(self, from_infos, to_infos, names=None):
-        names = self._verify_path_args(to_infos, from_infos, names)
-
-        ssh = self.ssh(host=to_infos[0]['host'],
-                       user=to_infos[0]['user'],
-                       port=to_infos[0]['port'])
-        sftp = ssh.open_sftp()
-
-        for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info['scheme'] != 'ssh':
-                raise NotImplementedError
-
-            if from_info['scheme'] != 'local':
-                raise NotImplementedError
-
-            Logger.debug("Uploading '{}' to '{}/{}'".format(from_info['path'],
-                                                            to_info['host'],
-                                                            to_info['path']))
-
-            if not name:
-                name = os.path.basename(from_info['path'])
-
-            dname = posixpath.dirname(to_info['path'])
-            self._exec(ssh, 'mkdir -p {}'.format(dname))
-
-            try:
-                sftp.put(from_info['path'],
-                         to_info['path'],
-                         callback=create_cb(name))
-            except Exception as exc:
-                msg = "Failed to upload '{}' to '{}/{}'"
-                Logger.warn(msg.format(from_info['path'],
-                                       to_info['host'],
-                                       to_info['path'], exc))
-                continue
-
-            progress.finish_target(name)
-
-        sftp.close()
-        ssh.close()
-
-    def _path_to_md5(self, path):
-        relpath = posixpath.relpath(path, self.prefix)
-        return posixpath.dirname(relpath) + posixpath.basename(relpath)
-
-    def _all_md5s(self):
-        ssh = self.ssh(host=self.host,
-                       user=self.user,
-                       port=self.port)
-        cmd = 'find {} -type f -follow -print'.format(self.prefix)
-        stdout = self._exec(ssh, cmd)
-        flist = stdout.split()
-        ssh.close()
-
-        return [self._path_to_md5(path) for path in flist]
-
-    def gc(self, cinfos):
-        used = [info[self.PARAM_MD5] for info in cinfos['ssh']]
-        used += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
-
-        removed = False
-        for md5 in self._all_md5s():
-            if md5 in used:
-                continue
-            path_info = {'scheme': 'ssh',
-                         'user': self.user,
-                         'host': self.host,
-                         'port': self.port,
-                         'path': posixpath.join(self.prefix,
-                                                md5[0:2], md5[2:])}
-            self.remove(path_info)
-            removed = True
-
-        return removed