dvc: bump to 0.31.1

add: cleanup .gitignore decorator

.appveyor.yml

@@ -3,6 +3,10 @@ branches:
     - master
     - devel
 
+cache:
+  - C:\ProgramData\chocolatey\bin -> appveyor.yml
+  - C:\ProgramData\chocolatey\lib -> appveyor.yml
+
 platform: x86
 
 environment:
@@ -10,7 +14,6 @@ environment:
     secure: veTfhOrd4wUpG7NlRsvS3tufuaFfST8i7SZa2HVH6Vc=
   aws_secret_access_key:
     secure: nCQcN/Mzce0erwcNOU3yAuWgF6ZuTvPBEqTF3XFp1poGcDbSIg+s67eVxsxWCXQj
-  GOOGLE_APPLICATION_CREDENTIALS: gcp-creds.json
   GCP_CREDS:
     secure: 96fJ3r2i2GohbXHwnSs5N4EplQ7q8YmLpPWM0AC+f4s=
   CODECOV_TOKEN:
@@ -54,7 +57,7 @@ install:
   - cinst gcloudsdk
   - refreshenv
   - cinst gsutil
-  - cinst openssl.light
+  - cinst openssl.light --version 1.1.1
   - python -m pip install -U pip setuptools wheel
   - pip install -r requirements.txt
   - python setup.py install
@@ -66,7 +69,7 @@ before_test:
   - aws configure set aws_secret_access_key "%aws_secret_access_key%"
   - aws configure set region us-east-2
   - openssl enc -d -aes-256-cbc -md md5 -k "%GCP_CREDS%" -in scripts\ci\gcp-creds.json.enc -out scripts\ci\gcp-creds.json & exit 0
-  - pip install -r test-requirements.txt
+  - pip install -r tests/requirements.txt
 
 test_script:
   - python -mtests

.codeclimate.yml

@@ -1,14 +1,5 @@
 languages:
     Python: true
 
-checks:
-    argument-count:
-        enabled: false
-    method-complexity:
-        enabled: false
-    method-lines:
-        config:
-            threshold: 50
-
 exclude_paths:
     - tests/

.codecov.yml

@@ -1 +1 @@
-comment: false
+

.coveragerc

@@ -7,3 +7,5 @@ exclude_lines =
     if __name__ == .__main__.:
     pass
     raise NotImplementedError
+    # pragma: no cover
+    #  pragma: no cover

.dvc/.gitignore

@@ -1,8 +1,8 @@
-link.state
-link.state.lock
-lock
-config.local
-updater
-cache
-state
-state.lock
+/state
+/lock
+/config.local
+/updater
+/updater.lock
+/state-journal
+/state-wal
+/cache

.flake8

@@ -0,0 +1,7 @@
+[flake8]
+ignore =
+    E203, # Whitespace before ':'
+    E266, # Too many leading '#' for block comment
+    W503, # Line break occurred before a binary operator
+max-line-length = 79
+select = B,C,E,F,W,T4,B9

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,10 @@
+---
+name: Bug report
+about: Create a report to help us improve
+
+---
+
+
+**Please provide information about your setup**
+DVC version(i.e. `dvc --version`), Platform and method of installation (pip, homebrew, pkg Mac, exe (Windows), DEB(Linux), RPM(Linux))
+

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,6 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+
+---
+

.gitignore

@@ -3,6 +3,7 @@ neatlynx/__pycache__
 .idea
 cache
 *.pyc
+.env/
 
 cache
 .dvc.conf.lock

.pre-commit-config.yaml

@@ -0,0 +1,11 @@
+repos:
+-   repo: https://github.com/ambv/black
+    rev: stable
+    hooks:
+    - id: black
+      language_version: python3
+-   repo: https://gitlab.com/pycqa/flake8
+    rev: master
+    hooks:
+    - id: flake8
+      language_version: python3

.pylintrc

@@ -0,0 +1,568 @@
+[MASTER]
+
+# A comma-separated list of package or module names from where C extensions may
+# be loaded. Extensions are loading into the active Python interpreter and may
+# run arbitrary code.
+extension-pkg-whitelist=
+
+# Add files or directories to the blacklist. They should be base names, not
+# paths.
+ignore=CVS
+
+# Add files or directories matching the regex patterns to the blacklist. The
+# regex matches against base names, not paths.
+ignore-patterns=
+
+# Python code to execute, usually for sys.path manipulation such as
+# pygtk.require().
+#init-hook=
+
+# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
+# number of processors available to use.
+jobs=1
+
+# Control the amount of potential inferred values when inferring a single
+# object. This can help the performance when dealing with large functions or
+# complex, nested conditions.
+limit-inference-results=100
+
+# List of plugins (as comma separated values of python modules names) to load,
+# usually to register additional checkers.
+load-plugins=
+
+# Pickle collected data for later comparisons.
+persistent=yes
+
+# Specify a configuration file.
+#rcfile=
+
+# When enabled, pylint would attempt to guess common misconfiguration and emit
+# user-friendly hints instead of false-positive error messages.
+suggestion-mode=yes
+
+# Allow loading of arbitrary C extensions. Extensions are imported into the
+# active Python interpreter and may run arbitrary code.
+unsafe-load-any-extension=no
+
+
+[MESSAGES CONTROL]
+
+# Only show warnings with the listed confidence levels. Leave empty to show
+# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED.
+confidence=
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifiers separated by comma (,) or put this
+# option multiple times (only on the command line, not in the configuration
+# file where it should appear only once). You can also use "--disable=all" to
+# disable everything first and then reenable specific checks. For example, if
+# you want to run only the similarities checker, you can use "--disable=all
+# --enable=similarities". If you want to run only the classes checker, but have
+# no Warning level messages displayed, use "--disable=all --enable=classes
+# --disable=W".
+disable=print-statement,
+        parameter-unpacking,
+        unpacking-in-except,
+        old-raise-syntax,
+        backtick,
+        long-suffix,
+        old-ne-operator,
+        old-octal-literal,
+        import-star-module-level,
+        non-ascii-bytes-literal,
+        raw-checker-failed,
+        bad-inline-option,
+        locally-disabled,
+        file-ignored,
+        suppressed-message,
+        useless-suppression,
+        deprecated-pragma,
+        use-symbolic-message-instead,
+        apply-builtin,
+        basestring-builtin,
+        buffer-builtin,
+        cmp-builtin,
+        coerce-builtin,
+        execfile-builtin,
+        file-builtin,
+        long-builtin,
+        raw_input-builtin,
+        reduce-builtin,
+        standarderror-builtin,
+        unicode-builtin,
+        xrange-builtin,
+        coerce-method,
+        delslice-method,
+        getslice-method,
+        setslice-method,
+        no-absolute-import,
+        old-division,
+        dict-iter-method,
+        dict-view-method,
+        next-method-called,
+        metaclass-assignment,
+        indexing-exception,
+        raising-string,
+        reload-builtin,
+        oct-method,
+        hex-method,
+        nonzero-method,
+        cmp-method,
+        input-builtin,
+        round-builtin,
+        intern-builtin,
+        unichr-builtin,
+        map-builtin-not-iterating,
+        zip-builtin-not-iterating,
+        range-builtin-not-iterating,
+        filter-builtin-not-iterating,
+        using-cmp-argument,
+        eq-without-hash,
+        div-method,
+        idiv-method,
+        rdiv-method,
+        exception-message-attribute,
+        invalid-str-codec,
+        sys-max-int,
+        bad-python3-import,
+        deprecated-string-function,
+        deprecated-str-translate-call,
+        deprecated-itertools-function,
+        deprecated-types-field,
+        next-method-defined,
+        dict-items-not-iterating,
+        dict-keys-not-iterating,
+        dict-values-not-iterating,
+        deprecated-operator-function,
+        deprecated-urllib-function,
+        xreadlines-attribute,
+        deprecated-sys-function,
+        exception-escape,
+        comprehension-escape,
+        useless-object-inheritance
+
+# Enable the message, report, category or checker with the given id(s). You can
+# either give multiple identifier separated by comma (,) or put this option
+# multiple time (only on the command line, not in the configuration file where
+# it should appear only once). See also the "--disable" option for examples.
+enable=c-extension-no-member
+
+
+[REPORTS]
+
+# Python expression which should return a note less than 10 (10 is the highest
+# note). You have access to the variables errors warning, statement which
+# respectively contain the number of errors / warnings messages and the total
+# number of statements analyzed. This is used by the global evaluation report
+# (RP0004).
+evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
+
+# Template used to display messages. This is a python new-style format string
+# used to format the message information. See doc for all details.
+#msg-template=
+
+# Set the output format. Available formats are text, parseable, colorized, json
+# and msvs (visual studio). You can also give a reporter class, e.g.
+# mypackage.mymodule.MyReporterClass.
+output-format=text
+
+# Tells whether to display a full report or only the messages.
+reports=no
+
+# Activate the evaluation score.
+score=yes
+
+
+[REFACTORING]
+
+# Maximum number of nested blocks for function / method body
+max-nested-blocks=5
+
+# Complete name of functions that never returns. When checking for
+# inconsistent-return-statements if a never returning function is called then
+# it will be considered as an explicit return statement and no message will be
+# printed.
+never-returning-functions=sys.exit
+
+
+[LOGGING]
+
+# Format style used to check logging format string. `old` means using %
+# formatting, while `new` is for `{}` formatting.
+logging-format-style=old
+
+# Logging modules to check that the string format arguments are in logging
+# function parameter format.
+logging-modules=logging
+
+
+[SPELLING]
+
+# Limits count of emitted suggestions for spelling mistakes.
+max-spelling-suggestions=4
+
+# Spelling dictionary name. Available dictionaries: none. To make it working
+# install python-enchant package..
+spelling-dict=
+
+# List of comma separated words that should not be checked.
+spelling-ignore-words=
+
+# A path to a file that contains private dictionary; one word per line.
+spelling-private-dict-file=
+
+# Tells whether to store unknown words to indicated private dictionary in
+# --spelling-private-dict-file option instead of raising a message.
+spelling-store-unknown-words=no
+
+
+[TYPECHECK]
+
+# List of decorators that produce context managers, such as
+# contextlib.contextmanager. Add to this list to register other decorators that
+# produce valid context managers.
+contextmanager-decorators=contextlib.contextmanager
+
+# List of members which are set dynamically and missed by pylint inference
+# system, and so shouldn't trigger E1101 when accessed. Python regular
+# expressions are accepted.
+generated-members=
+
+# Tells whether missing members accessed in mixin class should be ignored. A
+# mixin class is detected if its name ends with "mixin" (case insensitive).
+ignore-mixin-members=yes
+
+# Tells whether to warn about missing members when the owner of the attribute
+# is inferred to be None.
+ignore-none=yes
+
+# This flag controls whether pylint should warn about no-member and similar
+# checks whenever an opaque object is returned when inferring. The inference
+# can return multiple potential results while evaluating a Python object, but
+# some branches might not be evaluated, which results in partial inference. In
+# that case, it might be useful to still emit no-member and other checks for
+# the rest of the inferred objects.
+ignore-on-opaque-inference=yes
+
+# List of class names for which member attributes should not be checked (useful
+# for classes with dynamically set attributes). This supports the use of
+# qualified names.
+ignored-classes=optparse.Values,thread._local,_thread._local
+
+# List of module names for which member attributes should not be checked
+# (useful for modules/projects where namespaces are manipulated during runtime
+# and thus existing member attributes cannot be deduced by static analysis. It
+# supports qualified module names, as well as Unix pattern matching.
+ignored-modules=
+
+# Show a hint with possible names when a member name was not found. The aspect
+# of finding the hint is based on edit distance.
+missing-member-hint=yes
+
+# The minimum edit distance a name should have in order to be considered a
+# similar match for a missing member name.
+missing-member-hint-distance=1
+
+# The total number of similar names that should be taken in consideration when
+# showing a hint for a missing member.
+missing-member-max-choices=1
+
+
+[VARIABLES]
+
+# List of additional names supposed to be defined in builtins. Remember that
+# you should avoid defining new builtins when possible.
+additional-builtins=
+
+# Tells whether unused global variables should be treated as a violation.
+allow-global-unused-variables=yes
+
+# List of strings which can identify a callback function by name. A callback
+# name must start or end with one of those strings.
+callbacks=cb_,
+          _cb
+
+# A regular expression matching the name of dummy variables (i.e. expected to
+# not be used).
+dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_
+
+# Argument names that match this expression will be ignored. Default to name
+# with leading underscore.
+ignored-argument-names=_.*|^ignored_|^unused_
+
+# Tells whether we should check for unused import in __init__ files.
+init-import=no
+
+# List of qualified module names which can have objects that can redefine
+# builtins.
+redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io
+
+
+[FORMAT]
+
+# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
+expected-line-ending-format=
+
+# Regexp for a line that is allowed to be longer than the limit.
+ignore-long-lines=^\s*(# )?<?https?://\S+>?$
+
+# Number of spaces of indent required inside a hanging or continued line.
+indent-after-paren=4
+
+# String used as indentation unit. This is usually "    " (4 spaces) or "\t" (1
+# tab).
+indent-string='    '
+
+# Maximum number of characters on a single line.
+max-line-length=100
+
+# Maximum number of lines in a module.
+max-module-lines=1000
+
+# List of optional constructs for which whitespace checking is disabled. `dict-
+# separator` is used to allow tabulation in dicts, etc.: {1  : 1,\n222: 2}.
+# `trailing-comma` allows a space between comma and closing bracket: (a, ).
+# `empty-line` allows space-only lines.
+no-space-check=trailing-comma,
+               dict-separator
+
+# Allow the body of a class to be on the same line as the declaration if body
+# contains single statement.
+single-line-class-stmt=no
+
+# Allow the body of an if to be on the same line as the test if there is no
+# else.
+single-line-if-stmt=no
+
+
+[BASIC]
+
+# Naming style matching correct argument names.
+argument-naming-style=snake_case
+
+# Regular expression matching correct argument names. Overrides argument-
+# naming-style.
+#argument-rgx=
+
+# Naming style matching correct attribute names.
+attr-naming-style=snake_case
+
+# Regular expression matching correct attribute names. Overrides attr-naming-
+# style.
+#attr-rgx=
+
+# Bad variable names which should always be refused, separated by a comma.
+bad-names=foo,
+          bar,
+          baz,
+          toto,
+          tutu,
+          tata
+
+# Naming style matching correct class attribute names.
+class-attribute-naming-style=any
+
+# Regular expression matching correct class attribute names. Overrides class-
+# attribute-naming-style.
+#class-attribute-rgx=
+
+# Naming style matching correct class names.
+class-naming-style=PascalCase
+
+# Regular expression matching correct class names. Overrides class-naming-
+# style.
+#class-rgx=
+
+# Naming style matching correct constant names.
+const-naming-style=UPPER_CASE
+
+# Regular expression matching correct constant names. Overrides const-naming-
+# style.
+#const-rgx=
+
+# Minimum line length for functions/classes that require docstrings, shorter
+# ones are exempt.
+docstring-min-length=-1
+
+# Naming style matching correct function names.
+function-naming-style=snake_case
+
+# Regular expression matching correct function names. Overrides function-
+# naming-style.
+#function-rgx=
+
+# Good variable names which should always be accepted, separated by a comma.
+good-names=i,
+           j,
+           k,
+           ex,
+           Run,
+           _,
+           s3,
+           gs,
+           x, y,
+           x0, y0,
+           x1, y1,
+           dx, dy
+
+# Include a hint for the correct naming format with invalid-name.
+include-naming-hint=no
+
+# Naming style matching correct inline iteration names.
+inlinevar-naming-style=any
+
+# Regular expression matching correct inline iteration names. Overrides
+# inlinevar-naming-style.
+#inlinevar-rgx=
+
+# Naming style matching correct method names.
+method-naming-style=snake_case
+
+# Regular expression matching correct method names. Overrides method-naming-
+# style.
+#method-rgx=
+
+# Naming style matching correct module names.
+module-naming-style=snake_case
+
+# Regular expression matching correct module names. Overrides module-naming-
+# style.
+#module-rgx=
+
+# Colon-delimited sets of names that determine each other's naming style when
+# the name regexes allow several styles.
+name-group=
+
+# Regular expression which should only match function or class names that do
+# not require a docstring.
+no-docstring-rgx=^_
+
+# List of decorators that produce properties, such as abc.abstractproperty. Add
+# to this list to register other decorators that produce valid properties.
+# These decorators are taken in consideration only for invalid-name.
+property-classes=abc.abstractproperty
+
+# Naming style matching correct variable names.
+variable-naming-style=snake_case
+
+# Regular expression matching correct variable names. Overrides variable-
+# naming-style.
+#variable-rgx=
+
+
+[SIMILARITIES]
+
+# Ignore comments when computing similarities.
+ignore-comments=yes
+
+# Ignore docstrings when computing similarities.
+ignore-docstrings=yes
+
+# Ignore imports when computing similarities.
+ignore-imports=no
+
+# Minimum lines number of a similarity.
+min-similarity-lines=4
+
+
+[MISCELLANEOUS]
+
+# List of note tags to take in consideration, separated by a comma.
+notes=FIXME,
+      XXX,
+      TODO
+
+
+[DESIGN]
+
+# Maximum number of arguments for function / method.
+max-args=5
+
+# Maximum number of attributes for a class (see R0902).
+max-attributes=7
+
+# Maximum number of boolean expressions in an if statement.
+max-bool-expr=5
+
+# Maximum number of branch for function / method body.
+max-branches=12
+
+# Maximum number of locals for function / method body.
+max-locals=15
+
+# Maximum number of parents for a class (see R0901).
+max-parents=7
+
+# Maximum number of public methods for a class (see R0904).
+max-public-methods=20
+
+# Maximum number of return / yield for function / method body.
+max-returns=6
+
+# Maximum number of statements in function / method body.
+max-statements=50
+
+# Minimum number of public methods for a class (see R0903).
+min-public-methods=2
+
+
+[CLASSES]
+
+# List of method names used to declare (i.e. assign) instance attributes.
+defining-attr-methods=__init__,
+                      __new__,
+                      setUp
+
+# List of member names, which should be excluded from the protected access
+# warning.
+exclude-protected=_asdict,
+                  _fields,
+                  _replace,
+                  _source,
+                  _make
+
+# List of valid names for the first argument in a class method.
+valid-classmethod-first-arg=cls
+
+# List of valid names for the first argument in a metaclass class method.
+valid-metaclass-classmethod-first-arg=cls
+
+
+[IMPORTS]
+
+# Allow wildcard imports from modules that define __all__.
+allow-wildcard-with-all=no
+
+# Analyse import fallback blocks. This can be used to support both Python 2 and
+# 3 compatible code, which means that the block might have code that exists
+# only in one or another interpreter, leading to false positives when analysed.
+analyse-fallback-blocks=no
+
+# Deprecated modules which should not be used, separated by a comma.
+deprecated-modules=optparse,tkinter.tix
+
+# Create a graph of external dependencies in the given file (report RP0402 must
+# not be disabled).
+ext-import-graph=
+
+# Create a graph of every (i.e. internal and external) dependencies in the
+# given file (report RP0402 must not be disabled).
+import-graph=
+
+# Create a graph of internal dependencies in the given file (report RP0402 must
+# not be disabled).
+int-import-graph=
+
+# Force import order to recognize a module as part of the standard
+# compatibility libraries.
+known-standard-library=
+
+# Force import order to recognize a module as part of a third party library.
+known-third-party=enchant
+
+
+[EXCEPTIONS]
+
+# Exceptions that will emit a warning when being caught. Defaults to
+# "Exception".
+overgeneral-exceptions=Exception

.travis.yml

@@ -1,12 +1,16 @@
 language: python
-sudo: required
-dist: trusty
-group: deprecated-2017Q4
-python:
-  - "2.7"
-  - "3.4"
-  - "3.5"
-  - "3.6"
+cache:
+  - pip
+  - directories:
+    - /opt/python
+    - $HOME/.pyenv
+sudo: true
+dist: xenial
+python: "3.7"
+env:
+  global:
+    - PYENV_ROOT="$HOME/.pyenv"
+    - PATH="$PYENV_ROOT/bin:$PATH"
 # Travis doesn't support testing python on Mac yet, so we need
 # to workaround it by installing it directly with brew.
 #
@@ -15,74 +19,76 @@ python:
 # version, that causes fpm to crash.
 matrix:
   include:
+  - name: "2.7"
+    env:
+    - PYTHON_VER=2.7.13
+  - name: "3.4"
+    env:
+    - PYTHON_VER=3.4.6
+  - name: "3.5"
+    env:
+    - PYTHON_VER=3.5.3
+  - name: "3.6"
+    env:
+    - PYTHON_VER=3.6.2
+  - name: "3.7"
+    env:
+    # NOTE: using default python
+    - PYTHON_VER=
   - if: type = cron OR tag is present
     os: osx
-    osx_image: xcode9.3
+    osx_image: xcode7.3
     language: ruby
     rvm: 2.4.0
     env:
     - PYTHON_VER=2.7.10
-    - PYENV_ROOT="$HOME/.pyenv"
-    - PATH="$PYENV_ROOT/bin:$PATH"
+    - CPPFLAGS="-I/usr/local/opt/openssl/include $CPPFLAGS"
+    - LDFLAGS="-L/usr/local/opt/openssl/lib $LDFLAGS"
+    - PYTHON_CONFIGURE_OPTS="--enable-framework"
   - if: type = cron OR tag is present
     os: osx
-    osx_image: xcode9.3
+    osx_image: xcode7.3
     language: ruby
     rvm: 2.4.0
     env:
     - PYTHON_VER=3.4.3
-    - PYENV_ROOT="$HOME/.pyenv"
-    - PATH="$PYENV_ROOT/bin:$PATH"
+    - PYTHON_CONFIGURE_OPTS="--enable-framework"
   - if: type = cron OR tag is present
     os: osx
-    osx_image: xcode9.3
+    osx_image: xcode7.3
     language: ruby
     rvm: 2.4.0
     env:
     - PYTHON_VER=3.5.0
-    - PYENV_ROOT="$HOME/.pyenv"
-    - PATH="$PYENV_ROOT/bin:$PATH"
-  # 3.7 needs Xenial image because of https://github.com/travis-ci/travis-ci/issues/9069
-  - python: "3.7"
-    dist: xenial
-    sudo: true
+    - PYTHON_CONFIGURE_OPTS="--enable-framework"
+  - if: type = cron OR tag is present
+    os: osx
+    osx_image: xcode10.1
+    language: ruby
+    rvm: 2.4.0
+    env:
+    - PYTHON_VER=2.7.10
+    - CPPFLAGS="-I/usr/local/opt/openssl/include $CPPFLAGS"
+    - LDFLAGS="-L/usr/local/opt/openssl/lib $LDFLAGS"
+    - PYTHON_CONFIGURE_OPTS="--enable-framework"
+  - if: type = cron OR tag is present
+    os: osx
+    osx_image: xcode10.1
+    language: ruby
+    rvm: 2.4.0
+    env:
+    - PYTHON_VER=3.5.0
+    - PYTHON_CONFIGURE_OPTS="--enable-framework"
 before_install:
   - bash ./scripts/ci/before_install.sh
-  - if  [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-      eval "$(pyenv init -)";
-      PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing --keep --verbose "$PYTHON_VER";
-      pyenv shell "$PYTHON_VER";
-      brew cask install google-cloud-sdk;
-    fi
-  - which python
-  - which pip
+  - source env.sh
   - python --version
-  - if [[ "$TRAVIS_OS_NAME" != "osx" ]]; then
-      bash ./scripts/ci/install_hadoop.sh;
-      source ~/.bashrc;
-      hadoop fs -help;
-      pushd ~;
-      sudo mkdir -p /home/hadoop/hadoopinfra/hdfs/namenode/;
-      sudo chmod 777 /home/hadoop/hadoopinfra/hdfs/namenode/;
-      sudo chmod 777 -R /usr/local/hadoop;
-      hdfs namenode -format;
-      start-dfs.sh;
-      jps;
-      hadoop fs -ls /;
-      hadoop fs -ls hdfs://127.0.0.1/;
-      hadoop fs -mkdir hdfs://127.0.0.1/ololo;
-      popd;
-    fi
 install:
   - bash ./scripts/ci/install.sh
-  - pip install flake8
-before_script:
-  # stop the build if there are any flake8 errors
-  - flake8 dvc
 script:
   - bash ./scripts/ci/script.sh
 deploy:
-  condition: "[[ $(python -c 'import sys; print(sys.version_info[0])') == '2' ]]"
+  condition: "[[ $(./scripts/ci/deploy_condition.sh) == 'true' ]]"
   provider: releases
   api_key:
     secure: nyo17gF4Xl6DFL/KVkMewC805Cv+eK99L7SiYCU+WUAYDIzzFL3YJm7fm8bznPeor7DmpO7uSLSveMnJETBtdurW6JWhCTUss4yD66yFzpk6Z5cvHf/1UA5pdMHkkMM54fRaTRt6L2Yal7RPwGEQTBECLXW1Xmf78Y6+0l8lTvXQ9jXOQSRO/63VDKI5Zww2+XytVicUFlr0hLMGKUuaCjjXcQlAPzBgQz3/5Iz6EtzeTf2o+gTPjocbRbSFPikfiBXUn8XlJ8jBj7i114XJRJffw59KxYGDzxp+enfLBULGNoW0Np2hZqkKkPUIdB08C8XVbRXlFYX8cEYBczjqhv8A05p5F5CnltnqQ0KazcoUP2awPVqn68dWNRF+kqrnMBus8euYKIZ4dy/6bVEKJRa84ESJXZVDcwXTAXRqyovSxOlXqg9ZgAImMKBfKn/fpDX2xIvQPaXcnJxKbmcePfdbC1d0HYJ5jkbNaxKE/T15VvhzIho7mJ2WAv1r8QouQUuzqtpzIGbRn/KXHRUzCtyP0k4+eMKMLFOPzg848WsDarP3v8VYiSy4nFY7ZP7DRd3KttvW5bgVbSeO4jPmTgS6TnnpRwual9BJXSG7bjbfpIMii3bNZ7jWL4zpKT9Jfm8w9RGKLcX9HqkZ3+y60+LtYdhEfBvqsIlWQoIJBXg=

CODE_OF_CONDUCT.md

@@ -0,0 +1,77 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, sex characteristics, gender identity and expression,
+level of experience, education, socio-economic status, nationality, personal
+appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+  advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at info@dvc.org. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq
+

README.rst

@@ -1,5 +1,16 @@
-.. image:: https://travis-ci.org/iterative/dvc.svg?branch=master
-  :target: https://travis-ci.org/iterative/dvc
+.. image:: https://dvc.org/static/img/logo-owl-readme.png
+  :target: https://dvc.org
+  :alt: DVC logo
+
+`Website <https://dvc.org>`_
+• `Docs <https://dvc.org/doc>`_
+• `Twitter <https://twitter.com/iterativeai>`_
+• `Chat (Community & Support) <https://dvc.org/chat>`_
+• `Tutorial <https://dvc.org/doc/tutorial>`_
+• `Mailing List <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_
+
+.. image:: https://travis-ci.com/iterative/dvc.svg?branch=master
+  :target: https://travis-ci.com/iterative/dvc
   :alt: Travis
 
 .. image:: https://ci.appveyor.com/api/projects/status/github/iterative/dvc?branch=master&svg=true
@@ -14,22 +25,44 @@
   :target: https://codecov.io/gh/iterative/dvc
   :alt: Codecov
 
-`Website <https://dvc.org>`_ • `Docs <https://dvc.org/doc>`_ • `Discuss <https://discuss.dvc.org/>`_ •
-`Blog <https://blog.dataversioncontrol.com/>`_ • `Twitter <https://twitter.com/iterativeai>`_ •
-`Tutorial <https://dvc.org/tutorial>`_
+|
 
-.. image:: https://dvc.org/static/img/logo-github-readme.png
-  :align: center
+**Data Science Version Control** or **DVC** is an **open-source** tool for data science and
+machine learning projects. With a simple and flexible Git-like architecture and interface it
+helps data scientists:
 
-Data Science Version Control or DVC is an open source tool for data science projects.
-It helps data scientists manage their code and data together in a simple form of Git-like commands.
+#. manage **machine learning models** - versioning, including data sets and transformations (scripts) that were
+   used to generate models;
+
+#. make projects **reproducible**;
+
+#. make projects **shareable**;
+
+#. manage experiments with branching and **metrics** tracking;
+
+It aims to replace tools like Excel and Docs that are being commonly used as a knowledge repo and
+a ledger for the team, ad-hoc scripts to track and move deploy different model versions, ad-hoc
+data file suffixes and prefixes.
 
 .. contents:: **Contents**
   :backlinks: none
 
-Get started
+How DVC works
+=============
+
+DVC is compatible with Git for storing code and the dependency graph (DAG), but not data files cache.
+To store and share data files cache DVC supports remotes - any cloud (S3, Azure, Google Cloud, etc) or any on-premise
+network storage (via SSH, for example).
+
+.. image:: https://dvc.org/static/img/flow.gif
+   :target: https://dvc.org/static/img/flow.gif
+   :alt: how_dvc_works
+
+Quick start
 ===========
 
+Please read `Get Started <https://dvc.org/doc/get-started>`_ for the full version. Common workflow commands include:
+
 +-----------------------------------+-------------------------------------------------------------------+
 | Step                              | Command                                                           |
 +===================================+===================================================================+
@@ -47,26 +80,29 @@ Get started
 |                                   | | ``$ git push``                                                  |
 +-----------------------------------+-------------------------------------------------------------------+
 | Share data and ML models          | | ``$ dvc remote add myremote s3://mybucket/image_cnn``           |
-|                                   | | ``$ dvc core.remote myremote``                                  |
+|                                   | | ``$ dvc config core.remote myremote``                           |
 |                                   | | ``$ dvc push``                                                  |
 +-----------------------------------+-------------------------------------------------------------------+
 
-Follow this link to learn more and get more details: `get started <https://dvc.org/doc/get-started>`_.
-
 Installation
 ============
 
-It could be installed using ``pip``, Homebrew or an OS-specific package.
+There are three options to install DVC: ``pip``, Homebrew, or an OS-specific package:
 
+pip (PyPI)
+----------
+
+Stable
+^^^^^^
 .. code-block:: bash
 
    pip install dvc
 
-Package
--------
+Development
+^^^^^^^^^^^
+.. code-block:: bash
 
-Self-contained packages for Windows, Linux, Mac are available. The latest version of the packages can be found at
-GitHub `releases page <https://github.com/iterative/dvc/releases>`_.
+   pip install git+git://github.com/iterative/dvc
 
 Homebrew
 --------
@@ -81,6 +117,37 @@ or:
 
    brew cask install iterative/homebrew-dvc/dvc
 
+Package
+-------
+
+Self-contained packages for Windows, Linux, Mac are available. The latest version of the packages can be found at
+GitHub `releases page <https://github.com/iterative/dvc/releases>`_.
+
+Ubuntu / Debian (apt)
+^^^^^^^^^^^^^^^^^^^^^
+.. code-block:: bash
+
+   sudo wget https://dvc.org/deb/dvc.list -O /etc/apt/sources.list.d/dvc.list
+   sudo apt-get update
+   sudo apt-get install dvc
+
+Fedora / CentOS (rpm)
+^^^^^^^^^^^^^^^^^^^^^
+.. code-block:: bash
+
+   sudo wget https://dvc.org/rpm/dvc.repo -O /etc/yum.repos.d/dvc.repo
+   sudo yum update
+   sudo yum install dvc
+
+Arch linux (AUR)
+^^^^^^^^^^^^^^^^
+*Unofficial package*, any inquiries regarding the AUR package,
+`refer to the maintainer <https://github.com/mroutis/pkgbuilds>`_.
+
+.. code-block:: bash
+
+   yay -S dvc
+
 Related technologies
 ====================
 
@@ -88,33 +155,59 @@ Related technologies
    don't want to see in your Git repository) in a local key-value store and uses file hardlinks/symlinks instead of the
    copying actual files.
 
-#. `Git-LFS <https://git-lfs.github.com/>`_. - DVC is compatible with any remote storage (S3, Google Cloud, Azure, SSH,
+#. `Git-LFS <https://git-lfs.github.com/>`_ - DVC is compatible with any remote storage (S3, Google Cloud, Azure, SSH,
    etc). DVC utilizes reflinks or hardlinks to avoid copy operation on checkouts which makes much more efficient for
    large data files.
 
-#. Makefile (and it's analogues). DVC tracks dependencies (DAG).
+#. *Makefile* (and its analogues). DVC tracks dependencies (DAG).
 
 #. `Workflow Management Systems <https://en.wikipedia.org/wiki/Workflow_management_system>`_. DVC is a workflow
    management system designed specifically to manage machine learning experiments. DVC is built on top of Git.
 
-DVC is compatible with Git for storing code and the dependency graph (DAG), but not data files cache.
-To store and share data files cache DVC supports remotes - any cloud (S3, Azure, Google Cloud, etc) or any on-premise
-network storage (via SSH, for example).
-
-How DVC works
-=============
-
-
-.. image:: https://dvc.org/static/img/flow.png
-   :target: https://dvc.org/static/img/flow.png
-   :alt: how_dvc_works
-
+#. `DAGsHub <https://dagshub.com/>`_ Is a Github equivalent for DVC - pushing your Git+DVC based repo to DAGsHub will give you a high level dashboard of your project, including DVC pipeline and metrics visualizations, as well as links to DVC managed files if they are in cloud storage.
 
 Contributing
 ============
 Contributions are welcome! Please see our `Contributing Guide <https://dvc.org/doc/user-guide/contributing/>`_ for more
 details.
 
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/0
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/0
+  :alt: 0
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/1
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/1
+  :alt: 1
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/2
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/2
+  :alt: 2
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/3
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/3
+  :alt: 3
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/4
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/4
+  :alt: 4
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/5
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/5
+  :alt: 5
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/6
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/6
+  :alt: 6
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/7
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/7
+  :alt: 7
+
+Mailing List
+============
+
+Want to stay up to date? Want to help improve DVC by participating in our occasional polls? Subscribe to our `mailing list <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_. No spam, really low traffic.
+
 Copyright
 =========
 

dvc/__init__.py

@@ -3,47 +3,80 @@ DVC
 ----
 Make your data science projects reproducible and shareable.
 """
-import os
 
-VERSION_BASE = '0.15.1'
+from __future__ import unicode_literals
+
+import os
+import warnings
+
+
+VERSION_BASE = "0.31.1"
 __version__ = VERSION_BASE
 
 PACKAGEPATH = os.path.abspath(os.path.dirname(__file__))
 HOMEPATH = os.path.dirname(PACKAGEPATH)
-VERSIONPATH = os.path.join(PACKAGEPATH, 'version.py')
+VERSIONPATH = os.path.join(PACKAGEPATH, "version.py")
 
-if os.path.exists(os.path.join(HOMEPATH, 'setup.py')):
+
+def _update_version_file():
+    """Dynamically update version file."""
+    try:
+        from git import Repo
+        from git.exc import InvalidGitRepositoryError
+    except ImportError:
+        return __version__
+
+    try:
+        repo = Repo(HOMEPATH)
+    except InvalidGitRepositoryError:
+        return __version__
+
+    sha = repo.head.object.hexsha
+    short_sha = repo.git.rev_parse(sha, short=6)
+    dirty = ".mod" if repo.is_dirty() else ""
+    ver = "{}+{}{}".format(__version__, short_sha, dirty)
+
+    # Write a helper file, that will be installed with the package
+    # and will provide a true version of the installed dvc
+    with open(VERSIONPATH, "w+") as fobj:
+        fobj.write("# AUTOGENERATED by dvc/__init__.py\n")
+        fobj.write('version = "{}"\n'.format(ver))
+
+    return ver
+
+
+def _remove_version_file():
+    """Remove version.py so that it doesn't get into the release."""
+    if os.path.exists(VERSIONPATH):
+        os.unlink(VERSIONPATH)
+
+
+if os.path.exists(os.path.join(HOMEPATH, "setup.py")):
     # dvc is run directly from source without installation or
     # __version__ is called from setup.py
-    if os.getenv('APPVEYOR_REPO_TAG', '').lower() != 'true' \
-       and os.getenv('TRAVIS_TAG', '') == '':
-        # Dynamically update version
-        try:
-            import git
-            repo = git.Repo(HOMEPATH)
-            sha = repo.head.object.hexsha
-            short_sha = repo.git.rev_parse(sha, short=6)
-            dirty = '.mod' if repo.is_dirty() else ''
-            __version__ = '{}+{}{}'.format(__version__, short_sha, dirty)
-
-            # Write a helper file, that will be installed with the package
-            # and will provide a true version of the installed dvc
-            with open(VERSIONPATH, 'w+') as fd:
-                fd.write('# AUTOGENERATED by dvc/__init__.py\n')
-                fd.write('version = "{}"\n'.format(__version__))
-        except Exception:
-            pass
-    else:
-        # Remove version.py so that it doesn't get into the release
-        if os.path.exists(VERSIONPATH):
-            os.unlink(VERSIONPATH)
-else:
+    if (
+        os.getenv("APPVEYOR_REPO_TAG", "").lower() != "true"
+        and os.getenv("TRAVIS_TAG", "") == ""
+    ):
+        __version__ = _update_version_file()
+    else:  # pragma: no cover
+        _remove_version_file()
+else:  # pragma: no cover
     # dvc was installed with pip or something. Hopefully we have our
     # auto-generated version.py to help us provide a true version
     try:
         from dvc.version import version
+
         __version__ = version
     except Exception:
         pass
 
 VERSION = __version__
+
+
+# Ignore numpy's runtime warnings: https://github.com/numpy/numpy/pull/432.
+# We don't directly import numpy, but our dependency networkx does, causing
+# these warnings in some environments. Luckily these warnings are benign and
+# we can simply ignore them so that they don't show up when you are using dvc.
+warnings.filterwarnings("ignore", message="numpy.dtype size changed")
+warnings.filterwarnings("ignore", message="numpy.ufunc size changed")

dvc/__main__.py

@@ -1,6 +1,10 @@
+"""Main entry point for dvc command line tool."""
+
+from __future__ import unicode_literals
+
 import sys
 
 from dvc.main import main
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main(sys.argv[1:])

dvc/analytics.py

@@ -0,0 +1,267 @@
+"""Collect and send usage analytics"""
+
+from __future__ import unicode_literals
+
+from dvc.utils.compat import str
+
+import os
+import json
+import errno
+
+import dvc.logger as logger
+from dvc import VERSION
+
+
+class Analytics(object):
+    """Class for collecting and sending usage analytics.
+
+    Args:
+        info (dict): optional existing analytics report.
+    """
+
+    URL = "https://analytics.dvc.org"
+    TIMEOUT_POST = 5
+
+    USER_ID_FILE = "user_id"
+
+    PARAM_DVC_VERSION = "dvc_version"
+    PARAM_USER_ID = "user_id"
+    PARAM_SYSTEM_INFO = "system_info"
+
+    PARAM_OS = "os"
+
+    PARAM_WINDOWS_VERSION_MAJOR = "windows_version_major"
+    PARAM_WINDOWS_VERSION_MINOR = "windows_version_minor"
+    PARAM_WINDOWS_VERSION_BUILD = "windows_version_build"
+    PARAM_WINDOWS_VERSION_SERVICE_PACK = "windows_version_service_pack"
+
+    PARAM_MAC_VERSION = "mac_version"
+
+    PARAM_LINUX_DISTRO = "linux_distro"
+    PARAM_LINUX_DISTRO_VERSION = "linux_distro_version"
+    PARAM_LINUX_DISTRO_LIKE = "linux_distro_like"
+
+    PARAM_SCM_CLASS = "scm_class"
+    PARAM_IS_BINARY = "is_binary"
+    PARAM_CMD_CLASS = "cmd_class"
+    PARAM_CMD_RETURN_CODE = "cmd_return_code"
+
+    def __init__(self, info=None):
+        from dvc.config import Config
+        from dvc.lock import Lock
+
+        if info is None:
+            info = {}
+
+        self.info = info
+
+        cdir = Config.get_global_config_dir()
+        try:
+            os.makedirs(cdir)
+        except OSError as exc:
+            if exc.errno != errno.EEXIST:
+                raise
+
+        self.user_id_file = os.path.join(cdir, self.USER_ID_FILE)
+        self.user_id_file_lock = Lock(cdir, self.USER_ID_FILE + ".lock")
+
+    @staticmethod
+    def load(path):
+        """Loads analytics report from json file specified by path.
+
+        Args:
+            path (str): path to json file with analytics report.
+        """
+        with open(path, "r") as fobj:
+            analytics = Analytics(info=json.load(fobj))
+        os.unlink(path)
+        return analytics
+
+    def _write_user_id(self):
+        import uuid
+
+        with open(self.user_id_file, "w+") as fobj:
+            user_id = str(uuid.uuid4())
+            info = {self.PARAM_USER_ID: user_id}
+            json.dump(info, fobj)
+            return user_id
+
+    def _read_user_id(self):
+        if not os.path.exists(self.user_id_file):
+            return None
+
+        with open(self.user_id_file, "r") as fobj:
+            try:
+                info = json.load(fobj)
+                return info[self.PARAM_USER_ID]
+            except json.JSONDecodeError as exc:
+                logger.debug("Failed to load user_id: {}".format(exc))
+                return None
+
+    def _get_user_id(self):
+        from dvc.lock import LockError
+
+        try:
+            with self.user_id_file_lock:
+                user_id = self._read_user_id()
+                if user_id is None:
+                    user_id = self._write_user_id()
+                return user_id
+        except LockError:
+            msg = "Failed to acquire '{}'"
+            logger.debug(msg.format(self.user_id_file_lock.lock_file))
+
+    def _collect_windows(self):
+        import sys
+
+        version = sys.getwindowsversion()  # pylint: disable=no-member
+        info = {}
+        info[self.PARAM_OS] = "windows"
+        info[self.PARAM_WINDOWS_VERSION_MAJOR] = version.major
+        info[self.PARAM_WINDOWS_VERSION_MINOR] = version.minor
+        info[self.PARAM_WINDOWS_VERSION_BUILD] = version.build
+        info[self.PARAM_WINDOWS_VERSION_SERVICE_PACK] = version.service_pack
+        return info
+
+    def _collect_darwin(self):
+        import platform
+
+        info = {}
+        info[self.PARAM_OS] = "mac"
+        info[self.PARAM_MAC_VERSION] = platform.mac_ver()[0]
+        return info
+
+    def _collect_linux(self):
+        import distro
+
+        info = {}
+        info[self.PARAM_OS] = "linux"
+        info[self.PARAM_LINUX_DISTRO] = distro.id()
+        info[self.PARAM_LINUX_DISTRO_VERSION] = distro.version()
+        info[self.PARAM_LINUX_DISTRO_LIKE] = distro.like()
+        return info
+
+    def _collect_system_info(self):
+        import platform
+
+        system = platform.system()
+
+        if system == "Windows":
+            return self._collect_windows()
+
+        if system == "Darwin":
+            return self._collect_darwin()
+
+        if system == "Linux":
+            return self._collect_linux()
+
+        raise NotImplementedError
+
+    def collect(self):
+        """Collect analytics report."""
+        from dvc.scm import SCM
+        from dvc.utils import is_binary
+        from dvc.repo import Repo
+        from dvc.exceptions import NotDvcRepoError
+
+        self.info[self.PARAM_DVC_VERSION] = VERSION
+        self.info[self.PARAM_IS_BINARY] = is_binary()
+        self.info[self.PARAM_USER_ID] = self._get_user_id()
+
+        self.info[self.PARAM_SYSTEM_INFO] = self._collect_system_info()
+
+        try:
+            scm = SCM(root_dir=Repo.find_root())
+            self.info[self.PARAM_SCM_CLASS] = type(scm).__name__
+        except NotDvcRepoError:
+            pass
+
+    def collect_cmd(self, args, ret):
+        """Collect analytics info from a CLI command."""
+        from dvc.command.daemon import CmdDaemonAnalytics
+
+        assert isinstance(ret, int) or ret is None
+
+        if ret is not None:
+            self.info[self.PARAM_CMD_RETURN_CODE] = ret
+
+        if args is not None and hasattr(args, "func"):
+            assert args.func != CmdDaemonAnalytics
+            self.info[self.PARAM_CMD_CLASS] = args.func.__name__
+
+    def dump(self):
+        """Save analytics report to a temporary file.
+
+        Returns:
+            str: path to the temporary file that contains the analytics report.
+        """
+        import tempfile
+
+        with tempfile.NamedTemporaryFile(delete=False, mode="w") as fobj:
+            json.dump(self.info, fobj)
+            return fobj.name
+
+    @staticmethod
+    def _is_enabled(cmd=None):
+        from dvc.config import Config
+        from dvc.repo import Repo
+        from dvc.exceptions import NotDvcRepoError
+        from dvc.command.daemon import CmdDaemonBase
+
+        if os.getenv("DVC_TEST"):
+            return False
+
+        if isinstance(cmd, CmdDaemonBase):
+            return False
+
+        if cmd is None or not hasattr(cmd, "config"):
+            try:
+                dvc_dir = Repo.find_dvc_dir()
+                config = Config(dvc_dir)
+                assert config is not None
+            except NotDvcRepoError:
+                config = Config(validate=False)
+                assert config is not None
+        else:
+            config = cmd.config
+            assert config is not None
+
+        core = config.config.get(Config.SECTION_CORE, {})
+        enabled = core.get(Config.SECTION_CORE_ANALYTICS, True)
+        logger.debug(
+            "Analytics is {}abled.".format("en" if enabled else "dis")
+        )
+        return enabled
+
+    @staticmethod
+    def send_cmd(cmd, args, ret):
+        """Collect and send analytics for CLI command.
+
+        Args:
+            args (list): parsed args for the CLI command.
+            ret (int): return value of the CLI command.
+        """
+        from dvc.daemon import daemon
+
+        if not Analytics._is_enabled(cmd):
+            return
+
+        analytics = Analytics()
+        analytics.collect_cmd(args, ret)
+        daemon(["analytics", analytics.dump()])
+
+    def send(self):
+        """Collect and send analytics."""
+        import requests
+
+        if not self._is_enabled():
+            return
+
+        self.collect()
+
+        logger.debug("Sending analytics: {}".format(self.info))
+
+        try:
+            requests.post(self.URL, json=self.info, timeout=self.TIMEOUT_POST)
+        except requests.exceptions.RequestException as exc:
+            logger.debug("Failed to send analytics: {}".format(str(exc)))

dvc/cache.py

@@ -1,40 +1,84 @@
+"""Manages cache of a dvc repo."""
+
+from __future__ import unicode_literals
+
 import os
 
 from dvc.config import Config
 
 
 class Cache(object):
-    CACHE_DIR = 'cache'
+    """Class that manages cache locations of a dvc repo.
 
-    def __init__(self, project):
+    Args:
+        repo (dvc.repo.Repo): repo instance that this cache belongs to.
+    """
+
+    CACHE_DIR = "cache"
+
+    def __init__(self, repo):
         from dvc.remote import Remote
 
-        self.project = project
+        self.repo = repo
 
-        config = project.config._config[Config.SECTION_CACHE]
+        config = repo.config.config[Config.SECTION_CACHE]
+        local = config.get(Config.SECTION_CACHE_LOCAL)
 
-        local = config.get(Config.SECTION_CACHE_LOCAL, None)
         if local:
             name = Config.SECTION_REMOTE_FMT.format(local)
-            sect = project.config._config[name]
+            sect = repo.config.config[name]
         else:
-            sect = {}
-            cache_dir = config.get(Config.SECTION_CACHE_DIR, self.CACHE_DIR)
-            if not os.path.isabs(cache_dir):
-                cache_dir = os.path.join(project.dvc_dir, cache_dir)
-                cache_dir = os.path.abspath(os.path.realpath(cache_dir))
-            sect[Config.SECTION_REMOTE_URL] = cache_dir
-            t = config.get(Config.SECTION_CACHE_TYPE, None)
-            if t:
-                sect[Config.SECTION_CACHE_TYPE] = t
+            default_cache_dir = os.path.join(repo.dvc_dir, self.CACHE_DIR)
+            cache_dir = config.get(Config.SECTION_CACHE_DIR, default_cache_dir)
+            cache_type = config.get(Config.SECTION_CACHE_TYPE)
+            protected = config.get(Config.SECTION_CACHE_PROTECTED)
 
-        self.local = Remote(project, sect)
+            sect = {
+                Config.PRIVATE_CWD: config.get(
+                    Config.PRIVATE_CWD, repo.dvc_dir
+                ),
+                Config.SECTION_REMOTE_URL: cache_dir,
+                Config.SECTION_CACHE_TYPE: cache_type,
+                Config.SECTION_CACHE_PROTECTED: protected,
+            }
 
-        self.s3 = self._get_remote(config, Config.SECTION_CACHE_S3)
-        self.gs = self._get_remote(config, Config.SECTION_CACHE_GS)
-        self.ssh = self._get_remote(config, Config.SECTION_CACHE_SSH)
-        self.hdfs = self._get_remote(config, Config.SECTION_CACHE_HDFS)
-        self.azure = self._get_remote(config, Config.SECTION_CACHE_AZURE)
+        self._local = Remote(repo, sect)
+
+        self._s3 = self._get_remote(config, Config.SECTION_CACHE_S3)
+        self._gs = self._get_remote(config, Config.SECTION_CACHE_GS)
+        self._ssh = self._get_remote(config, Config.SECTION_CACHE_SSH)
+        self._hdfs = self._get_remote(config, Config.SECTION_CACHE_HDFS)
+        self._azure = self._get_remote(config, Config.SECTION_CACHE_AZURE)
+
+    @property
+    def local(self):
+        """Remote instance for local cache."""
+        return self._local
+
+    @property
+    def s3(self):
+        """Remote instance for AWS S3 cache."""
+        return self._s3
+
+    @property
+    def gs(self):
+        """Remote instance for Google Cloud Storage cache."""
+        return self._gs
+
+    @property
+    def ssh(self):
+        """Remote instance for SSH cache."""
+        return self._ssh
+
+    @property
+    def hdfs(self):
+        """Remote instance for HDFS cache."""
+        return self._hdfs
+
+    @property
+    def azure(self):
+        """Remote instance for azure cache."""
+        return self._azure
 
     def _get_remote(self, config, name):
         from dvc.remote import Remote
@@ -44,5 +88,5 @@ class Cache(object):
             return None
 
         name = Config.SECTION_REMOTE_FMT.format(remote)
-        sect = self.project.config._config[name]
-        return Remote(self.project, sect)
+        sect = self.repo.config.config[name]
+        return Remote(self.repo, sect)

dvc/cli.py

@@ -1,673 +1,157 @@
+"""Command line interface for dvc."""
+
 from __future__ import print_function
+from __future__ import unicode_literals
 
-import os
 import sys
 import argparse
-from multiprocessing import cpu_count
 
-from dvc.command.init import CmdInit
-from dvc.command.destroy import CmdDestroy
-from dvc.command.remove import CmdRemove
-from dvc.command.move import CmdMove
-from dvc.command.run import CmdRun
-from dvc.command.repro import CmdRepro
-from dvc.command.data_sync import CmdDataPush, CmdDataPull, CmdDataFetch
-from dvc.command.status import CmdDataStatus
-from dvc.command.gc import CmdGC
-from dvc.command.add import CmdAdd
-from dvc.command.imp import CmdImport
-from dvc.command.config import CmdConfig
-from dvc.command.checkout import CmdCheckout
-from dvc.command.remote import CmdRemoteAdd, CmdRemoteRemove
-from dvc.command.remote import CmdRemoteModify, CmdRemoteList
-from dvc.command.metrics import CmdMetricsShow, CmdMetricsAdd
-from dvc.command.metrics import CmdMetricsRemove, CmdMetricsModify
-from dvc.command.install import CmdInstall
-from dvc.command.root import CmdRoot
-from dvc.command.lock import CmdLock, CmdUnlock
-from dvc.command.pipeline import CmdPipelineShow
+import dvc.logger as logger
+from dvc.command.base import fix_subparsers
+import dvc.command.init as init
+import dvc.command.destroy as destroy
+import dvc.command.remove as remove
+import dvc.command.move as move
+import dvc.command.unprotect as unprotect
+import dvc.command.run as run
+import dvc.command.repro as repro
+import dvc.command.data_sync as data_sync
+import dvc.command.gc as gc
+import dvc.command.add as add
+import dvc.command.imp as imp
+import dvc.command.config as config
+import dvc.command.checkout as checkout
+import dvc.command.remote as remote
+import dvc.command.cache as cache
+import dvc.command.metrics as metrics
+import dvc.command.install as install
+import dvc.command.root as root
+import dvc.command.lock as lock
+import dvc.command.pipeline as pipeline
+import dvc.command.daemon as daemon
+import dvc.command.commit as commit
+from dvc.exceptions import DvcParserError
 from dvc import VERSION
 
 
-def _fix_subparsers(subparsers):
-    # NOTE: Workaround for bug in Python 3
-    # More info at:
-    #  https://bugs.python.org/issue16308
-    #  https://github.com/iterative/dvc/issues/769
-    if sys.version_info[0] == 3:
-        subparsers.required = True
-        subparsers.dest = 'cmd'
+COMMANDS = [
+    init,
+    destroy,
+    add,
+    remove,
+    move,
+    unprotect,
+    run,
+    repro,
+    data_sync,
+    gc,
+    add,
+    imp,
+    config,
+    checkout,
+    remote,
+    cache,
+    metrics,
+    install,
+    root,
+    lock,
+    pipeline,
+    daemon,
+    commit,
+]
+
+
+class DvcParser(argparse.ArgumentParser):
+    """Custom parser class for dvc CLI."""
+
+    def error(self, message):
+        """Custom error method.
+        Args:
+            message (str): error message.
+
+        Raises:
+            dvc.exceptions.DvcParser: dvc parser exception.
+        """
+        logger.error(message)
+        self.print_help()
+        raise DvcParserError()
+
+
+class VersionAction(argparse.Action):  # pragma: no cover
+    # pylint: disable=too-few-public-methods
+    """Shows dvc version and exits."""
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        print(VERSION)
+        sys.exit(0)
+
+
+def get_parent_parser():
+    """Create instances of a parser containing common arguments shared among
+    all the commands.
+
+    When overwritting `-q` or `-v`, you need to instantiate a new object
+    in order to prevent some weird behavior.
+    """
+    parent_parser = argparse.ArgumentParser(add_help=False)
+
+    log_level_group = parent_parser.add_mutually_exclusive_group()
+    log_level_group.add_argument(
+        "-q", "--quiet", action="store_true", default=False, help="Be quiet."
+    )
+    log_level_group.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        default=False,
+        help="Be verbose.",
+    )
+
+    return parent_parser
 
 
 def parse_args(argv=None):
-    # Common args
-    parent_parser = argparse.ArgumentParser(add_help=False)
-    parent_parser.add_argument(
-                        '-q',
-                        '--quiet',
-                        action='store_true',
-                        default=False,
-                        help='Be quiet.')
-    parent_parser.add_argument(
-                        '-v',
-                        '--verbose',
-                        action='store_true',
-                        default=False,
-                        help='Be verbose.')
+    """Parses CLI arguments.
+
+    Args:
+        argv: optional list of arguments to parse. sys.argv is used by default.
+
+    Raises:
+        dvc.exceptions.DvcParserError: raised for argument parsing errors.
+    """
+    parent_parser = get_parent_parser()
 
     # Main parser
-    desc = 'Data Version Control'
-    parser = argparse.ArgumentParser(
-                        prog='dvc',
-                        description=desc,
-                        parents=[parent_parser],
-                        formatter_class=argparse.RawTextHelpFormatter)
+    desc = "Data Version Control"
+    parser = DvcParser(
+        prog="dvc",
+        description=desc,
+        parents=[parent_parser],
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
 
-    parser.add_argument('-V',
-                        '--version',
-                        action='version',
-                        version='%(prog)s ' + VERSION,
-                        help='Show program\'s version')
+    # NOTE: On some python versions action='version' prints to stderr
+    # instead of stdout https://bugs.python.org/issue18920
+    parser.add_argument(
+        "-V",
+        "--version",
+        action=VersionAction,
+        nargs=0,
+        help="Show program's version.",
+    )
 
     # Sub commands
     subparsers = parser.add_subparsers(
-                        dest='cmd',
-                        help='Use dvc CMD --help for command-specific help')
+        title="Available Commands",
+        metavar="COMMAND",
+        dest="cmd",
+        help="Use dvc COMMAND --help for command-specific help.",
+    )
 
-    _fix_subparsers(subparsers)
+    fix_subparsers(subparsers)
 
-    # Init
-    init_parser = subparsers.add_parser(
-                        'init',
-                        parents=[parent_parser],
-                        help='Initialize dvc over a directory'
-                             '(should already be a git dir)')
-    init_parser.add_argument(
-                        '--no-scm',
-                        action='store_true',
-                        default=False,
-                        help="Initiate dvc in directory that is "
-                             "not tracked by any scm tool(e.g. git)")
-    init_parser.add_argument(
-                        '-f',
-                        '--force',
-                        action='store_true',
-                        default=False,
-                        help='Force initialization even if \'.dvc\' exists')
-    init_parser.set_defaults(func=CmdInit)
-
-    # Destroy
-    destroy_parser = subparsers.add_parser(
-                        'destroy',
-                        parents=[parent_parser],
-                        help='Destroy dvc')
-    destroy_parser.add_argument(
-                        '-f',
-                        '--force',
-                        action='store_true',
-                        default=False,
-                        help='Force destruction')
-    destroy_parser.set_defaults(func=CmdDestroy)
-
-    # Add
-    add_parser = subparsers.add_parser(
-                        'add',
-                        parents=[parent_parser],
-                        help='Add files/directories to dvc')
-    add_parser.add_argument(
-                        'targets',
-                        nargs='+',
-                        help='Input files/directories')
-    add_parser.set_defaults(func=CmdAdd)
-
-    # Import
-    import_parser = subparsers.add_parser(
-                        'import',
-                        parents=[parent_parser],
-                        help='Import files from URL')
-    import_parser.add_argument(
-                        'url',
-                        help='URL')
-    import_parser.add_argument(
-                        'out',
-                        help='Output')
-    import_parser.set_defaults(func=CmdImport)
-
-    # Checkout
-    checkout_parser = subparsers.add_parser(
-                        'checkout',
-                        parents=[parent_parser],
-                        help='Checkout data files from cache')
-    checkout_parser.add_argument(
-                        'targets',
-                        nargs='*',
-                        help='DVC files')
-    checkout_parser.set_defaults(func=CmdCheckout)
-
-    # Run
-    run_parser = subparsers.add_parser(
-                        'run',
-                        parents=[parent_parser],
-                        help='Generate a stage file from a given '
-                             'command and execute the command')
-    run_parser.add_argument(
-                        '-d',
-                        '--deps',
-                        action='append',
-                        default=[],
-                        help='Declare dependencies for reproducible cmd.')
-    run_parser.add_argument(
-                        '-o',
-                        '--outs',
-                        action='append',
-                        default=[],
-                        help='Declare output data file or data directory.')
-    run_parser.add_argument(
-                        '-O',
-                        '--outs-no-cache',
-                        action='append',
-                        default=[],
-                        help='Declare output regular file or '
-                             'directory (sync to Git, not DVC cache).')
-    run_parser.add_argument(
-                        '-M',
-                        '--metrics-no-cache',
-                        action='append',
-                        default=[],
-                        help='Declare output metric file or '
-                             'directory (not cached by DVC).')
-    run_parser.add_argument(
-                        '-f',
-                        '--file',
-                        help='Specify name of the state file')
-    run_parser.add_argument(
-                        '-c',
-                        '--cwd',
-                        default=os.path.curdir,
-                        help='Directory to run your command and place state '
-                             'file in')
-    run_parser.add_argument(
-                        '--no-exec',
-                        action='store_true',
-                        default=False,
-                        help="Only create stage file without actually "
-                             "running it")
-    run_parser.add_argument(
-                        'command',
-                        nargs=argparse.REMAINDER,
-                        help='Command or command file to execute')
-    run_parser.set_defaults(func=CmdRun)
-
-    # Parent parser used in pull/push/status
-    parent_cache_parser = argparse.ArgumentParser(
-                        add_help=False,
-                        parents=[parent_parser])
-    parent_cache_parser.add_argument(
-                        '-j',
-                        '--jobs',
-                        type=int,
-                        default=8*cpu_count(),
-                        help='Number of jobs to run simultaneously.')
-    parent_cache_parser.add_argument(
-                        '--show-checksums',
-                        action='store_true',
-                        default=False,
-                        help='Show checksums instead of file names.')
-    parent_cache_parser.add_argument(
-                        'targets',
-                        nargs='*',
-                        default=None,
-                        help='DVC files.')
-
-    # Pull
-    pull_parser = subparsers.add_parser(
-                        'pull',
-                        parents=[parent_cache_parser],
-                        help='Pull data files from the cloud')
-    pull_parser.add_argument(
-                        '-r',
-                        '--remote',
-                        help='Remote repository to pull from')
-    pull_parser.add_argument(
-                        '-a',
-                        '--all-branches',
-                        action='store_true',
-                        default=False,
-                        help='Fetch cache for all branches.')
-    pull_parser.set_defaults(func=CmdDataPull)
-
-    # Push
-    push_parser = subparsers.add_parser(
-                        'push',
-                        parents=[parent_cache_parser],
-                        help='Push data files to the cloud')
-    push_parser.add_argument(
-                        '-r',
-                        '--remote',
-                        help='Remote repository to push to')
-    push_parser.add_argument(
-                        '-a',
-                        '--all-branches',
-                        action='store_true',
-                        default=False,
-                        help='Push cache for all branches.')
-    push_parser.set_defaults(func=CmdDataPush)
-
-    # Fetch
-    fetch_parser = subparsers.add_parser(
-                        'fetch',
-                        parents=[parent_cache_parser],
-                        help='Fetch data files from the cloud')
-    fetch_parser.add_argument(
-                        '-r',
-                        '--remote',
-                        help='Remote repository to fetch from')
-    fetch_parser.add_argument(
-                        '-a',
-                        '--all-branches',
-                        action='store_true',
-                        default=False,
-                        help='Fetch cache for all branches.')
-    fetch_parser.set_defaults(func=CmdDataFetch)
-
-    # Status
-    status_parser = subparsers.add_parser(
-                        'status',
-                        parents=[parent_cache_parser],
-                        help='Show the project status')
-    status_parser.add_argument(
-                        '-c',
-                        '--cloud',
-                        action='store_true',
-                        default=False,
-                        help='Show status of a local cache compared to a '
-                             'remote repository')
-    status_parser.add_argument(
-                        '-r',
-                        '--remote',
-                        help='Remote repository to compare local cache to')
-    status_parser.set_defaults(func=CmdDataStatus)
-
-    # Repro
-    repro_parser = subparsers.add_parser(
-                        'repro',
-                        parents=[parent_parser],
-                        help='Reproduce DVC file. Default file name '
-                             '- \'Dvcfile\'')
-    repro_parser.add_argument(
-                        'targets',
-                        nargs='*',
-                        default=['Dvcfile'],
-                        help='DVC file to reproduce.')
-    repro_parser.add_argument(
-                        '-f',
-                        '--force',
-                        action='store_true',
-                        default=False,
-                        help='Reproduce even if dependencies were not '
-                             'changed.')
-    repro_parser.add_argument(
-                        '-s',
-                        '--single-item',
-                        action='store_true',
-                        default=False,
-                        help='Reproduce only single data item without '
-                             'recursive dependencies check.')
-    repro_parser.add_argument(
-                        '-c',
-                        '--cwd',
-                        default=os.path.curdir,
-                        help='Directory to reproduce from.')
-    repro_parser.add_argument(
-                        '-m',
-                        '--metrics',
-                        action='store_true',
-                        default=False,
-                        help='Show metrics after reproduction')
-    repro_parser.set_defaults(func=CmdRepro)
-
-    # Remove
-    remove_parser = subparsers.add_parser(
-                        'remove',
-                        parents=[parent_parser],
-                        help='Remove outputs of DVC file.')
-    remove_parser_group = remove_parser.add_mutually_exclusive_group()
-    remove_parser_group.add_argument(
-                        '-o',
-                        '--outs',
-                        action='store_true',
-                        default=True,
-                        help='Only remove DVC file outputs.')
-    remove_parser_group.add_argument(
-                        '-p',
-                        '--purge',
-                        action='store_true',
-                        default=False,
-                        help='Remove DVC file and all its outputs')
-    remove_parser.add_argument(
-                        'targets',
-                        nargs='+',
-                        help='DVC files.')
-    remove_parser.set_defaults(func=CmdRemove)
-
-    # Move
-    move_parser = subparsers.add_parser(
-                        'move',
-                        parents=[parent_parser],
-                        help='Move output of DVC file.')
-    move_parser.add_argument(
-                        'src',
-                        help='Source')
-    move_parser.add_argument(
-                        'dst',
-                        help='Destination')
-    move_parser.set_defaults(func=CmdMove)
-
-    # Garbage collector
-    gc_parser = subparsers.add_parser(
-                        'gc',
-                        parents=[parent_parser],
-                        help='Collect garbage')
-    gc_parser.add_argument(
-                        '-a',
-                        '--all-branches',
-                        action='store_true',
-                        default=False,
-                        help='Collect garbage for all branches.')
-    gc_parser.add_argument(
-                        '-c',
-                        '--cloud',
-                        action='store_true',
-                        default=False,
-                        help='Collect garbage in remote repository')
-    gc_parser.add_argument(
-                        '-r',
-                        '--remote',
-                        help='Remote repository to collect garbage in')
-    gc_parser.set_defaults(func=CmdGC)
-
-    # Config
-    config_parser = subparsers.add_parser(
-                        'config',
-                        parents=[parent_parser],
-                        help='Get or set config options')
-    config_parser.add_argument(
-                        '-u',
-                        '--unset',
-                        default=False,
-                        action='store_true',
-                        help='Unset option')
-    config_parser.add_argument(
-                        'name',
-                        help='Option name')
-    config_parser.add_argument(
-                        'value',
-                        nargs='?',
-                        default=None,
-                        help='Option value')
-    config_parser.add_argument(
-                        '--local',
-                        action='store_true',
-                        default=False,
-                        help='Use local config')
-    config_parser.set_defaults(func=CmdConfig)
-
-    # Remote
-    remote_parser = subparsers.add_parser(
-                        'remote',
-                        parents=[parent_parser],
-                        help='Manage set of tracked repositories')
-
-    remote_subparsers = remote_parser.add_subparsers(
-                        dest='cmd',
-                        help='Use dvc remote CMD --help for '
-                             'command-specific help')
-
-    _fix_subparsers(remote_subparsers)
-
-    remote_add_parser = remote_subparsers.add_parser(
-                        'add',
-                        parents=[parent_parser],
-                        help='Add remote')
-    remote_add_parser.add_argument(
-                        'name',
-                        help='Name')
-    remote_add_parser.add_argument(
-                        'url',
-                        help='Url')
-    remote_add_parser.add_argument(
-                        '--local',
-                        action='store_true',
-                        default=False,
-                        help='Use local config')
-    remote_add_parser.add_argument(
-                        '-d',
-                        '--default',
-                        action='store_true',
-                        default=False,
-                        help='Set as default remote')
-    remote_add_parser.set_defaults(func=CmdRemoteAdd)
-
-    remote_remove_parser = remote_subparsers.add_parser(
-                        'remove',
-                        parents=[parent_parser],
-                        help='Remove remote')
-    remote_remove_parser.add_argument(
-                        'name',
-                        help='Name')
-    remote_remove_parser.add_argument(
-                        '--local',
-                        action='store_true',
-                        default=False,
-                        help='Use local config')
-    remote_remove_parser.set_defaults(func=CmdRemoteRemove)
-
-    remote_modify_parser = remote_subparsers.add_parser(
-                        'modify',
-                        parents=[parent_parser],
-                        help='Modify remote')
-    remote_modify_parser.add_argument(
-                        'name',
-                        help='Name')
-    remote_modify_parser.add_argument(
-                        'option',
-                        help='Option')
-    remote_modify_parser.add_argument(
-                        'value',
-                        nargs='?',
-                        help='Value')
-    remote_modify_parser.add_argument(
-                        '-u',
-                        '--unset',
-                        default=False,
-                        action='store_true',
-                        help='Unset option')
-    remote_modify_parser.add_argument(
-                        '--local',
-                        action='store_true',
-                        default=False,
-                        help='Use local config')
-    remote_modify_parser.set_defaults(func=CmdRemoteModify)
-
-    remote_list_parser = remote_subparsers.add_parser(
-                        'list',
-                        parents=[parent_parser],
-                        help='List remotes')
-    remote_list_parser.add_argument(
-                        '--local',
-                        action='store_true',
-                        default=False,
-                        help='Use local config')
-    remote_list_parser.set_defaults(func=CmdRemoteList)
-
-    # Metrics
-    metrics_parser = subparsers.add_parser(
-                        'metrics',
-                        parents=[parent_parser],
-                        help='Get metrics from all branches')
-
-    metrics_subparsers = metrics_parser.add_subparsers(
-                        dest='cmd',
-                        help='Use dvc metrics CMD --help for '
-                             'command-specific help')
-
-    _fix_subparsers(metrics_subparsers)
-
-    metrics_show_parser = metrics_subparsers.add_parser(
-                        'show',
-                        parents=[parent_parser],
-                        help='Show metrics')
-    metrics_show_parser.add_argument(
-                        'path',
-                        nargs='?',
-                        help='Path to metrics file')
-    metrics_show_parser.add_argument(
-                        '-t',
-                        '--type',
-                        help='Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV)')
-    metrics_show_parser.add_argument(
-                        '-x',
-                        '--xpath',
-                        help='JSON/TSV/HTSV/CSV/HCSV path')
-    metrics_show_group = metrics_show_parser.add_mutually_exclusive_group()
-    metrics_show_group.add_argument(
-                        '--json-path',
-                        help='JSON path')
-    metrics_show_group.add_argument(
-                        '--tsv-path',
-                        help='TSV path \'row,column\'(e.g. \'1,2\')')
-    metrics_show_group.add_argument(
-                        '--htsv-path',
-                        help='Headed TSV path \'row,column\'(e.g. \'Name,3\'')
-    metrics_show_group.add_argument(
-                        '--csv-path',
-                        help='CSV path \'row,column\'(e.g. \'1,2\')')
-    metrics_show_group.add_argument(
-                        '--hcsv-path',
-                        help='Headed CSV path \'row,column\'(e.g. \'Name,3\'')
-    metrics_show_parser.add_argument(
-                        '-a',
-                        '--all-branches',
-                        action='store_true',
-                        default=False,
-                        help='Show metrics for all branches')
-    metrics_show_parser.set_defaults(func=CmdMetricsShow)
-
-    metrics_add_parser = metrics_subparsers.add_parser(
-                        'add',
-                        parents=[parent_parser],
-                        help='Add metrics')
-    metrics_add_parser.add_argument(
-                        '-t',
-                        '--type',
-                        help='Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV)')
-    metrics_add_parser.add_argument(
-                        '-x',
-                        '--xpath',
-                        help='JSON/TSV/HTSV/CSV/HCSV path')
-    metrics_add_parser.add_argument(
-                        'path',
-                        help='Path to metrics file')
-    metrics_add_parser.set_defaults(func=CmdMetricsAdd)
-
-    metrics_modify_parser = metrics_subparsers.add_parser(
-                        'modify',
-                        parents=[parent_parser],
-                        help='Modify metrics')
-    metrics_modify_parser.add_argument(
-                        '-t',
-                        '--type',
-                        help='Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV)')
-    metrics_modify_parser.add_argument(
-                        '-x',
-                        '--xpath',
-                        help='JSON/TSV/HTSV/CSV/HCSV path')
-    metrics_modify_parser.add_argument(
-                        'path',
-                        help='Metrics file')
-    metrics_modify_parser.set_defaults(func=CmdMetricsModify)
-
-    metrics_remove_parser = metrics_subparsers.add_parser(
-                        'remove',
-                        parents=[parent_parser],
-                        help='Remove metrics')
-    metrics_remove_parser.add_argument(
-                        'path',
-                        help='Path to metrics file')
-    metrics_remove_parser.set_defaults(func=CmdMetricsRemove)
-
-    # Install
-    install_parser = subparsers.add_parser(
-                        'install',
-                        parents=[parent_parser],
-                        help='Install dvc hooks into the repository')
-    install_parser.set_defaults(func=CmdInstall)
-
-    # Root
-    root_parser = subparsers.add_parser(
-                        'root',
-                        parents=[parent_parser],
-                        help='Relative path to project\'s directory')
-    root_parser.set_defaults(func=CmdRoot)
-
-    # Lock
-    lock_parser = subparsers.add_parser(
-                        'lock',
-                        parents=[parent_parser],
-                        help='Lock DVC file')
-    lock_parser.add_argument(
-                        'targets',
-                        nargs='+',
-                        help='DVC files.')
-    lock_parser.set_defaults(func=CmdLock)
-
-    # Unlock
-    unlock_parser = subparsers.add_parser(
-                        'unlock',
-                        parents=[parent_parser],
-                        help='Unlock DVC file')
-    unlock_parser.add_argument(
-                        'targets',
-                        nargs='+',
-                        help='DVC files.')
-    unlock_parser.set_defaults(func=CmdUnlock)
-
-    # Pipeline
-    pipeline_parser = subparsers.add_parser(
-                        'pipeline',
-                        parents=[parent_parser],
-                        help='Manage pipeline')
-
-    pipeline_subparsers = pipeline_parser.add_subparsers(
-                        dest='cmd',
-                        help='Use dvc pipeline CMD --help '
-                             'for command-specific help')
-
-    _fix_subparsers(pipeline_subparsers)
-
-    pipeline_show_parser = pipeline_subparsers.add_parser(
-                        'show',
-                        parents=[parent_parser],
-                        help='Show pipeline')
-    pipeline_show_group = pipeline_show_parser.add_mutually_exclusive_group()
-    pipeline_show_group.add_argument(
-                        '-c',
-                        '--commands',
-                        action='store_true',
-                        default=False,
-                        help='Print commands instead of paths to DVC files.')
-    pipeline_show_group.add_argument(
-                        '-o',
-                        '--outs',
-                        action='store_true',
-                        default=False,
-                        help='Print output files instead of'
-                             'paths to DVC files.')
-    pipeline_show_parser.add_argument(
-                        'targets',
-                        nargs='+',
-                        help='DVC files.')
-    pipeline_show_parser.set_defaults(func=CmdPipelineShow)
+    for cmd in COMMANDS:
+        cmd.add_parser(subparsers, parent_parser)
 
     args = parser.parse_args(argv)
 

dvc/command/add.py

@@ -1,14 +1,59 @@
+from __future__ import unicode_literals
+
+import dvc.logger as logger
 from dvc.exceptions import DvcException
 from dvc.command.base import CmdBase
-from dvc.logger import Logger
 
 
 class CmdAdd(CmdBase):
     def run(self):
-        for target in self.args.targets:
-            try:
-                self.project.add(target)
-            except DvcException as ex:
-                Logger.error('Failed to add \'{}\''.format(target), ex)
-                return 1
+        try:
+            if len(self.args.targets) > 1 and self.args.file:
+                raise DvcException("can't use '--file' with multiple targets")
+
+            for target in self.args.targets:
+                self.repo.add(
+                    target,
+                    recursive=self.args.recursive,
+                    no_commit=self.args.no_commit,
+                    fname=self.args.file,
+                )
+
+        except DvcException:
+            logger.error("failed to add file")
+            return 1
         return 0
+
+
+def add_parser(subparsers, parent_parser):
+    ADD_HELP = "Add files/directories to dvc."
+    add_parser = subparsers.add_parser(
+        "add", parents=[parent_parser], description=ADD_HELP, help=ADD_HELP
+    )
+    add_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="Recursively add each file under the directory.",
+    )
+    add_parser.add_argument(
+        "--no-commit",
+        action="store_true",
+        default=False,
+        help="Don't put files/directories into cache.",
+    )
+    add_parser.add_argument(
+        "-f",
+        "--file",
+        help="Specify name of the stage file. It should be "
+        "either 'Dvcfile' or have a '.dvc' suffix (e.g. "
+        "'prepare.dvc', 'clean.dvc', etc) in order for "
+        "dvc to be able to find it later. By default "
+        "the output basename + .dvc is used as a stage filename. "
+        "(NOTE: It can't be used when specifying multiple targets)",
+    )
+    add_parser.add_argument(
+        "targets", nargs="+", help="Input files/directories."
+    )
+    add_parser.set_defaults(func=CmdAdd)

dvc/command/base.py

@@ -1,45 +1,58 @@
-import os
+from __future__ import unicode_literals
 
-from dvc.logger import Logger
-from dvc.exceptions import DvcException
-from dvc.lock import LockError
+import dvc.logger as logger
+
+
+def fix_subparsers(subparsers):
+    """Workaround for bug in Python 3. See more info at:
+        https://bugs.python.org/issue16308
+        https://github.com/iterative/dvc/issues/769
+
+        Args:
+            subparsers: subparsers to fix.
+    """
+    from dvc.utils.compat import is_py3
+
+    if is_py3:  # pragma: no cover
+        subparsers.required = True
+        subparsers.dest = "cmd"
 
 
 class CmdBase(object):
     def __init__(self, args):
-        from dvc.project import Project
+        from dvc.repo import Repo
 
-        self.project = Project(self._find_root())
+        self.repo = Repo()
+        self.config = self.repo.config
         self.args = args
-        self._set_loglevel(args)
+        self.set_loglevel(args)
+
+    @property
+    def default_targets(self):
+        """Default targets for `dvc repro` and `dvc pipeline`."""
+        from dvc.stage import Stage
+
+        msg = "assuming default target '{}'.".format(Stage.STAGE_FILE)
+        logger.warning(msg)
+        return [Stage.STAGE_FILE]
 
     @staticmethod
-    def _set_loglevel(args):
-        if args.quiet and not args.verbose:
-            Logger.be_quiet()
-        elif not args.quiet and args.verbose:
-            Logger.be_verbose()
+    def set_loglevel(args):
+        """Sets log level from CLI arguments."""
+        if args.quiet:
+            logger.be_quiet()
 
-    def _find_root(self):
-        from dvc.project import Project
-
-        root = os.getcwd()
-        while True:
-            dvc_dir = os.path.join(root, Project.DVC_DIR)
-            if os.path.isdir(dvc_dir):
-                return root
-            if os.path.ismount(root):
-                break
-            root = os.path.dirname(root)
-        msg = "Not a dvc repository (checked up to mount point {})"
-        raise DvcException(msg.format(root))
+        elif args.verbose:
+            logger.be_verbose()
 
     def run_cmd(self):
+        from dvc.lock import LockError
+
         try:
-            with self.project.lock:
+            with self.repo.lock:
                 return self.run()
-        except LockError as ex:
-            Logger.error('Failed to lock before running a command', ex)
+        except LockError:
+            logger.error("failed to lock before running a command")
             return 1
 
     # Abstract methods that have to be implemented by any inheritance class

dvc/command/cache.py

@@ -0,0 +1,62 @@
+from __future__ import unicode_literals
+
+import argparse
+
+from dvc.command.base import fix_subparsers
+from dvc.command.remote import CmdRemoteAdd
+from dvc.command.config import CmdConfig
+
+
+class CmdCacheDir(CmdConfig):
+    def run(self):
+        self.args.name = "cache.dir"
+        self.args.value = CmdRemoteAdd.resolve_path(
+            self.args.value, self.configobj.filename
+        )
+
+        return super(CmdCacheDir, self).run()
+
+
+def add_parser(subparsers, parent_parser):
+    from dvc.command.config import parent_config_parser
+
+    CACHE_HELP = "Manage cache settings."
+    cache_parser = subparsers.add_parser(
+        "cache",
+        parents=[parent_parser],
+        description=CACHE_HELP,
+        help=CACHE_HELP,
+    )
+
+    cache_subparsers = cache_parser.add_subparsers(
+        dest="cmd", help="Use dvc cache CMD --help for command-specific help."
+    )
+
+    fix_subparsers(cache_subparsers)
+
+    parent_cache_config_parser = argparse.ArgumentParser(
+        add_help=False, parents=[parent_config_parser]
+    )
+    CACHE_DIR_HELP = "Configure cache directory location."
+    cache_dir_parser = cache_subparsers.add_parser(
+        "dir",
+        parents=[parent_cache_config_parser],
+        description=CACHE_DIR_HELP,
+        help=CACHE_DIR_HELP,
+    )
+    cache_dir_parser.add_argument(
+        "-u",
+        "--unset",
+        default=False,
+        action="store_true",
+        help="Unset option.",
+    )
+    cache_dir_parser.add_argument(
+        "value",
+        nargs="?",
+        default=None,
+        help="Path to cache directory. Relative paths are resolved relative "
+        "to the current directory and saved to config relative to the "
+        "config file location.",
+    )
+    cache_dir_parser.set_defaults(func=CmdCacheDir)

dvc/command/checkout.py

@@ -1,11 +1,43 @@
+from __future__ import unicode_literals
+
 from dvc.command.base import CmdBase
 
 
 class CmdCheckout(CmdBase):
     def run(self):
         if not self.args.targets:
-            self.project.checkout()
+            self.repo.checkout(force=self.args.force)
         else:
             for target in self.args.targets:
-                self.project.checkout(target=target)
+                self.repo.checkout(
+                    target=target,
+                    with_deps=self.args.with_deps,
+                    force=self.args.force,
+                )
         return 0
+
+
+def add_parser(subparsers, parent_parser):
+    CHECKOUT_HELP = "Checkout data files from cache."
+    checkout_parser = subparsers.add_parser(
+        "checkout",
+        parents=[parent_parser],
+        description=CHECKOUT_HELP,
+        help=CHECKOUT_HELP,
+    )
+    checkout_parser.add_argument(
+        "-d",
+        "--with-deps",
+        action="store_true",
+        default=False,
+        help="Checkout all dependencies of the specified target.",
+    )
+    checkout_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Do not prompt when removing working directory files.",
+    )
+    checkout_parser.add_argument("targets", nargs="*", help="DVC files.")
+    checkout_parser.set_defaults(func=CmdCheckout)

dvc/command/commit.py

@@ -0,0 +1,63 @@
+from __future__ import unicode_literals
+
+import dvc.logger as logger
+from dvc.exceptions import DvcException
+from dvc.command.base import CmdBase
+
+
+class CmdCommit(CmdBase):
+    def run(self):
+        if not self.args.targets:
+            self.args.targets = [None]
+
+        for target in self.args.targets:
+            try:
+                self.repo.commit(
+                    target,
+                    with_deps=self.args.with_deps,
+                    recursive=self.args.recursive,
+                    force=self.args.force,
+                )
+            except DvcException:
+                logger.error(
+                    "failed to commit{}".format(
+                        (" " + target) if target else ""
+                    )
+                )
+                return 1
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    COMMIT_HELP = "Record changes to the repository."
+    commit_parser = subparsers.add_parser(
+        "commit",
+        parents=[parent_parser],
+        description=COMMIT_HELP,
+        help=COMMIT_HELP,
+    )
+    commit_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Commit even if checksums for dependencies/outputs changed.",
+    )
+    commit_parser.add_argument(
+        "-d",
+        "--with-deps",
+        action="store_true",
+        default=False,
+        help="Commit all dependencies of the specified target.",
+    )
+    commit_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="Commit cache for subdirectories of the specified directory.",
+    )
+    commit_parser.add_argument(
+        "targets", nargs="*", default=None, help="DVC files."
+    )
+    commit_parser.set_defaults(func=CmdCommit)

dvc/command/config.py

@@ -1,102 +1,119 @@
-import os
-import configobj
+from __future__ import unicode_literals
 
+import os
+import argparse
+
+import dvc.logger as logger
 from dvc.command.base import CmdBase
-from dvc.logger import Logger
 from dvc.config import Config
+from dvc.exceptions import DvcException
 
 
 class CmdConfig(CmdBase):
     def __init__(self, args):
-        from dvc.project import Project
+        from dvc.repo import Repo, NotDvcRepoError
 
         self.args = args
-        root_dir = self._find_root()
-        if args.local:
-            config = Config.CONFIG_LOCAL
+
+        try:
+            dvc_dir = os.path.join(Repo.find_root(), Repo.DVC_DIR)
+            saved_exc = None
+        except NotDvcRepoError as exc:
+            dvc_dir = None
+            saved_exc = exc
+
+        self.config = Config(dvc_dir, validate=False)
+        if self.args.system:
+            self.configobj = self.config._system_config
+        elif self.args.glob:
+            self.configobj = self.config._global_config
+        elif self.args.local:
+            if dvc_dir is None:
+                raise saved_exc
+            self.configobj = self.config._local_config
         else:
-            config = Config.CONFIG
-        self.config_file = os.path.join(root_dir, Project.DVC_DIR, config)
-        # Using configobj because it doesn't
-        # drop comments like configparser does.
-        self.configobj = configobj.ConfigObj(self.config_file)
+            if dvc_dir is None:
+                raise saved_exc
+            self.configobj = self.config._repo_config
 
     def run_cmd(self):
         return self.run()
 
-    def _get_key(self, d, name, add=False):
-        for k in d.keys():
-            if k.lower() == name.lower():
-                return k
+    def _unset(self, section, opt=None, configobj=None):
+        if configobj is None:
+            configobj = self.configobj
 
-        if add:
-            d[name] = {}
-            return name
-
-        return None
-
-    def save(self):
         try:
-            self.configobj.write()
-        except Exception as exc:
-            msg = "Failed to write config '{}'".format(self.configobj.filename)
-            Logger.error(msg, exc)
+            self.config.unset(configobj, section, opt)
+            self.config.save(configobj)
+        except DvcException:
+            logger.error("failed to unset '{}'".format(self.args.name))
             return 1
         return 0
 
-    def unset(self, section, opt=None):
-        if section not in self.configobj.keys():
-            Logger.error("Section '{}' doesn't exist".format(section))
+    def _show(self, section, opt):
+        try:
+            self.config.show(self.configobj, section, opt)
+        except DvcException:
+            logger.error("failed to show '{}'".format(self.args.name))
             return 1
-
-        if opt in self.configobj[section].keys():
-            del self.configobj[section][opt]
-
-        if len(self.configobj[section]) == 0 or opt is None:
-            del self.configobj[section]
-
-        return self.save()
-
-    def show(self):
-        Logger.info(self.configobj[self.section][self.opt])
         return 0
 
-    def set(self, section, opt, value):
-        if section not in self.configobj.keys():
-            self.configobj[section] = {}
-
-        self.configobj[section][opt] = value
-
-        return self.save()
-
-    def check_opt(self):
-        _section, _opt = self.args.name.strip().split('.', 1)
-        add = (self.args.value is not None and self.args.unset is False)
-
-        section = self._get_key(self.configobj, _section, add)
-
-        if not section:
-            Logger.error('Invalid option name {}'.format(_section))
+    def _set(self, section, opt, value):
+        try:
+            self.config.set(self.configobj, section, opt, value)
+            self.config.save(self.configobj)
+        except DvcException:
+            logger.error(
+                "failed to set '{}.{}' to '{}'".format(section, opt, value)
+            )
             return 1
-
-        opt = self._get_key(self.configobj[section], _opt, add)
-        if not opt:
-            Logger.error('Invalid option value: {}'.format(_opt))
-            return 1
-
-        self.section = section
-        self.opt = opt
-
         return 0
 
     def run(self):
-        if self.check_opt() != 0:
-            return 1
+        section, opt = self.args.name.lower().strip().split(".", 1)
 
         if self.args.unset:
-            return self.unset(self.section, self.opt)
+            return self._unset(section, opt)
+        elif self.args.value is None:
+            return self._show(section, opt)
+        else:
+            return self._set(section, opt, self.args.value)
 
-        if self.args.value is None:
-            return self.show()
 
-        return self.set(self.section, self.opt, self.args.value)
+parent_config_parser = argparse.ArgumentParser(add_help=False)
+parent_config_parser.add_argument(
+    "--global",
+    dest="glob",
+    action="store_true",
+    default=False,
+    help="Use global config.",
+)
+parent_config_parser.add_argument(
+    "--system", action="store_true", default=False, help="Use system config."
+)
+parent_config_parser.add_argument(
+    "--local", action="store_true", default=False, help="Use local config."
+)
+
+
+def add_parser(subparsers, parent_parser):
+    CONFIG_HELP = "Get or set config options."
+    config_parser = subparsers.add_parser(
+        "config",
+        parents=[parent_config_parser, parent_parser],
+        description=CONFIG_HELP,
+        help=CONFIG_HELP,
+    )
+    config_parser.add_argument(
+        "-u",
+        "--unset",
+        default=False,
+        action="store_true",
+        help="Unset option.",
+    )
+    config_parser.add_argument("name", help="Option name.")
+    config_parser.add_argument(
+        "value", nargs="?", default=None, help="Option value."
+    )
+    config_parser.set_defaults(func=CmdConfig)

dvc/command/daemon.py

@@ -0,0 +1,72 @@
+from __future__ import unicode_literals
+
+from dvc.command.base import CmdBase, fix_subparsers
+
+
+class CmdDaemonBase(CmdBase):
+    def __init__(self, args):
+        self.args = args
+        self.config = None
+        self.set_loglevel(args)
+
+    def run_cmd(self):
+        return self.run()
+
+
+class CmdDaemonUpdater(CmdDaemonBase):
+    def run(self):
+        import os
+        from dvc.repo import Repo
+        from dvc.updater import Updater
+
+        root_dir = Repo.find_root()
+        dvc_dir = os.path.join(root_dir, Repo.DVC_DIR)
+        updater = Updater(dvc_dir)
+        updater.fetch(detach=False)
+
+        return 0
+
+
+class CmdDaemonAnalytics(CmdDaemonBase):
+    def run(self):
+        from dvc.analytics import Analytics
+
+        analytics = Analytics.load(self.args.target)
+        analytics.send()
+
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    DAEMON_HELP = "Service daemon."
+    daemon_parser = subparsers.add_parser(
+        "daemon",
+        parents=[parent_parser],
+        description=DAEMON_HELP,
+        help=DAEMON_HELP,
+    )
+
+    daemon_subparsers = daemon_parser.add_subparsers(
+        dest="cmd", help="Use dvc daemon CMD --help for command-specific help."
+    )
+
+    fix_subparsers(daemon_subparsers)
+
+    DAEMON_UPDATER_HELP = "Fetch latest available version."
+    daemon_updater_parser = daemon_subparsers.add_parser(
+        "updater",
+        parents=[parent_parser],
+        description=DAEMON_UPDATER_HELP,
+        help=DAEMON_UPDATER_HELP,
+    )
+    daemon_updater_parser.set_defaults(func=CmdDaemonUpdater)
+
+    DAEMON_ANALYTICS_HELP = "Send dvc usage analytics."
+    daemon_analytics_parser = daemon_subparsers.add_parser(
+        "analytics",
+        parents=[parent_parser],
+        description=DAEMON_ANALYTICS_HELP,
+        help=DAEMON_ANALYTICS_HELP,
+    )
+    daemon_analytics_parser.add_argument("target", help="Analytics file.")
+    daemon_analytics_parser.set_defaults(func=CmdDaemonAnalytics)

dvc/command/data_sync.py

@@ -1,7 +1,14 @@
+from __future__ import unicode_literals
+
+import argparse
+
+import dvc.logger as logger
 from dvc.command.base import CmdBase
 
 
 class CmdDataBase(CmdBase):
+    UP_TO_DATE_MSG = "Everything is up to date."
+
     def do_run(self, target):
         pass
 
@@ -15,46 +22,283 @@ class CmdDataBase(CmdBase):
                 ret = 1
         return ret
 
+    @classmethod
+    def check_up_to_date(cls, processed_files_count):
+        if processed_files_count == 0:
+            logger.info(cls.UP_TO_DATE_MSG)
+
 
 class CmdDataPull(CmdDataBase):
     def do_run(self, target=None):
         try:
-            self.project.pull(target=target,
-                              jobs=self.args.jobs,
-                              remote=self.args.remote,
-                              show_checksums=self.args.show_checksums,
-                              all_branches=self.args.all_branches)
-        except Exception as exc:
-            self.project.logger.error('Failed to pull data from the cloud',
-                                      exc)
+            processed_files_count = self.repo.pull(
+                target=target,
+                jobs=self.args.jobs,
+                remote=self.args.remote,
+                show_checksums=self.args.show_checksums,
+                all_branches=self.args.all_branches,
+                all_tags=self.args.all_tags,
+                with_deps=self.args.with_deps,
+                force=self.args.force,
+                recursive=self.args.recursive,
+            )
+        except Exception:
+            logger.error("failed to pull data from the cloud")
             return 1
+        self.check_up_to_date(processed_files_count)
         return 0
 
 
 class CmdDataPush(CmdDataBase):
     def do_run(self, target=None):
         try:
-            self.project.push(target=target,
-                              jobs=self.args.jobs,
-                              remote=self.args.remote,
-                              show_checksums=self.args.show_checksums,
-                              all_branches=self.args.all_branches)
-        except Exception as exc:
-            self.project.logger.error('Failed to push data to the cloud', exc)
+            processed_files_count = self.repo.push(
+                target=target,
+                jobs=self.args.jobs,
+                remote=self.args.remote,
+                show_checksums=self.args.show_checksums,
+                all_branches=self.args.all_branches,
+                all_tags=self.args.all_tags,
+                with_deps=self.args.with_deps,
+                recursive=self.args.recursive,
+            )
+        except Exception:
+            logger.error("failed to push data to the cloud")
             return 1
+        self.check_up_to_date(processed_files_count)
         return 0
 
 
 class CmdDataFetch(CmdDataBase):
     def do_run(self, target=None):
         try:
-            self.project.fetch(target=target,
-                               jobs=self.args.jobs,
-                               remote=self.args.remote,
-                               show_checksums=self.args.show_checksums,
-                               all_branches=self.args.all_branches)
-        except Exception as exc:
-            self.project.logger.error('Failed to fetch data from the cloud',
-                                      exc)
+            processed_files_count = self.repo.fetch(
+                target=target,
+                jobs=self.args.jobs,
+                remote=self.args.remote,
+                show_checksums=self.args.show_checksums,
+                all_branches=self.args.all_branches,
+                all_tags=self.args.all_tags,
+                with_deps=self.args.with_deps,
+                recursive=self.args.recursive,
+            )
+        except Exception:
+            logger.error("failed to fetch data from the cloud")
             return 1
+        self.check_up_to_date(processed_files_count)
         return 0
+
+
+def shared_parent_parser():
+    from dvc.cli import get_parent_parser
+
+    # Parent parser used in pull/push/status
+    shared_parent_parser = argparse.ArgumentParser(
+        add_help=False, parents=[get_parent_parser()]
+    )
+    shared_parent_parser.add_argument(
+        "-j",
+        "--jobs",
+        type=int,
+        default=None,
+        help="Number of jobs to run simultaneously.",
+    )
+    shared_parent_parser.add_argument(
+        "--show-checksums",
+        action="store_true",
+        default=False,
+        help="Show checksums instead of file names.",
+    )
+    shared_parent_parser.add_argument(
+        "targets", nargs="*", default=None, help="DVC files."
+    )
+
+    return shared_parent_parser
+
+
+def add_parser(subparsers, _parent_parser):
+    from dvc.command.status import CmdDataStatus
+
+    # Pull
+    PULL_HELP = "Pull data files from the cloud."
+    pull_parser = subparsers.add_parser(
+        "pull",
+        parents=[shared_parent_parser()],
+        description=PULL_HELP,
+        help=PULL_HELP,
+    )
+    pull_parser.add_argument(
+        "-r", "--remote", help="Remote repository to pull from."
+    )
+    pull_parser.add_argument(
+        "-a",
+        "--all-branches",
+        action="store_true",
+        default=False,
+        help="Fetch cache for all branches.",
+    )
+    pull_parser.add_argument(
+        "-T",
+        "--all-tags",
+        action="store_true",
+        default=False,
+        help="Fetch cache for all tags.",
+    )
+    pull_parser.add_argument(
+        "-d",
+        "--with-deps",
+        action="store_true",
+        default=False,
+        help="Fetch cache for all dependencies of the specified target.",
+    )
+    pull_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Do not prompt when removing working directory files.",
+    )
+    pull_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="Pull cache for subdirectories of the specified directory.",
+    )
+    pull_parser.set_defaults(func=CmdDataPull)
+
+    # Push
+    PUSH_HELP = "Push data files to the cloud."
+    push_parser = subparsers.add_parser(
+        "push",
+        parents=[shared_parent_parser()],
+        description=PUSH_HELP,
+        help=PUSH_HELP,
+    )
+    push_parser.add_argument(
+        "-r", "--remote", help="Remote repository to push to."
+    )
+    push_parser.add_argument(
+        "-a",
+        "--all-branches",
+        action="store_true",
+        default=False,
+        help="Push cache for all branches.",
+    )
+    push_parser.add_argument(
+        "-T",
+        "--all-tags",
+        action="store_true",
+        default=False,
+        help="Push cache for all tags.",
+    )
+    push_parser.add_argument(
+        "-d",
+        "--with-deps",
+        action="store_true",
+        default=False,
+        help="Push cache for all dependencies of the specified target.",
+    )
+    push_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="Push cache from subdirectories of specified directory.",
+    )
+    push_parser.set_defaults(func=CmdDataPush)
+
+    # Fetch
+    FETCH_HELP = "Fetch data files from the cloud."
+    fetch_parser = subparsers.add_parser(
+        "fetch",
+        parents=[shared_parent_parser()],
+        description=FETCH_HELP,
+        help=FETCH_HELP,
+    )
+    fetch_parser.add_argument(
+        "-r", "--remote", help="Remote repository to fetch from."
+    )
+    fetch_parser.add_argument(
+        "-a",
+        "--all-branches",
+        action="store_true",
+        default=False,
+        help="Fetch cache for all branches.",
+    )
+    fetch_parser.add_argument(
+        "-T",
+        "--all-tags",
+        action="store_true",
+        default=False,
+        help="Fetch cache for all tags.",
+    )
+    fetch_parser.add_argument(
+        "-d",
+        "--with-deps",
+        action="store_true",
+        default=False,
+        help="Fetch cache for all dependencies of the " "specified target.",
+    )
+    fetch_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="Fetch cache for subdirectories of specified directory.",
+    )
+    fetch_parser.set_defaults(func=CmdDataFetch)
+
+    # Status
+    STATUS_HELP = "Show the repo status."
+    status_parser = subparsers.add_parser(
+        "status",
+        parents=[shared_parent_parser()],
+        description=STATUS_HELP,
+        help=STATUS_HELP,
+        conflict_handler="resolve",
+    )
+    status_parser.add_argument(
+        "-q",
+        "--quiet",
+        action="store_true",
+        default=False,
+        help=(
+            "Suppresses all output."
+            " Exit with 0 if pipeline is up to date, otherwise 1."
+        ),
+    )
+    status_parser.add_argument(
+        "-c",
+        "--cloud",
+        action="store_true",
+        default=False,
+        help="Show status of a local cache compared to a remote repository.",
+    )
+    status_parser.add_argument(
+        "-r", "--remote", help="Remote repository to compare local cache to."
+    )
+    status_parser.add_argument(
+        "-a",
+        "--all-branches",
+        action="store_true",
+        default=False,
+        help="Show status of a local cache compared to a remote repository "
+        "for all branches.",
+    )
+    status_parser.add_argument(
+        "-T",
+        "--all-tags",
+        action="store_true",
+        default=False,
+        help="Show status of a local cache compared to a remote repository "
+        "for all tags.",
+    )
+    status_parser.add_argument(
+        "-d",
+        "--with-deps",
+        action="store_true",
+        default=False,
+        help="Show status for all dependencies of the specified target.",
+    )
+    status_parser.set_defaults(func=CmdDataStatus)

dvc/command/destroy.py

@@ -1,22 +1,50 @@
+from __future__ import unicode_literals
+
+import dvc.prompt as prompt
+import dvc.logger as logger
 from dvc.command.base import CmdBase
-from dvc.prompt import prompt
 from dvc.exceptions import DvcException
 
 
 class CmdDestroy(CmdBase):
     def run_cmd(self):
         try:
-            msg = u'This will destroy all information about your pipelines ' \
-                  u'as well as cache in .dvc/cache.\n' \
-                  u'Are you sure you want to continue?'
+            statement = (
+                "This will destroy all information about your pipelines,"
+                " all data files, as well as cache in .dvc/cache."
+                "\n"
+                "Are you sure you want to continue?"
+            )
 
-            if not self.args.force and not prompt(msg, False):
-                msg = u'Cannot destroy without a confirmation from the ' \
-                      u'user. Use \'-f\' to force.'
-                raise DvcException(msg)
+            if not self.args.force and not prompt.confirm(statement):
+                raise DvcException(
+                    "cannot destroy without a confirmation from the user."
+                    " Use '-f' to force."
+                )
 
-            self.project.destroy()
-        except Exception as exc:
-            self.project.logger.error('Failed to destroy DVC', exc)
+            self.repo.destroy()
+        except Exception:
+            logger.error("failed to destroy DVC")
             return 1
         return 0
+
+
+def add_parser(subparsers, parent_parser):
+    DESTROY_HELP = (
+        "Destroy dvc. Will remove all repo's information, "
+        "data files and cache."
+    )
+    destroy_parser = subparsers.add_parser(
+        "destroy",
+        parents=[parent_parser],
+        description=DESTROY_HELP,
+        help=DESTROY_HELP,
+    )
+    destroy_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Force destruction.",
+    )
+    destroy_parser.set_defaults(func=CmdDestroy)

dvc/command/gc.py

@@ -1,9 +1,106 @@
+from __future__ import unicode_literals
+
+import os
+import dvc.prompt as prompt
+import dvc.logger as logger
+
 from dvc.command.base import CmdBase
 
 
 class CmdGC(CmdBase):
     def run(self):
-        self.project.gc(all_branches=self.args.all_branches,
-                        cloud=self.args.cloud,
-                        remote=self.args.remote)
+        msg = "this will remove all cache except the cache that is used in "
+        if not self.args.all_branches and not self.args.all_tags:
+            msg += "the current git branch"
+        elif self.args.all_branches and not self.args.all_tags:
+            msg += "all git branches"
+        elif not self.args.all_branches and self.args.all_tags:
+            msg += "all git tags"
+        else:
+            msg += "all git branches and all git tags"
+
+        if self.args.repos is not None and len(self.args.repos) > 0:
+            msg += " of the current and the following repos:"
+
+            for repo_path in self.args.repos:
+                msg += "\n  - %s" % os.path.abspath(repo_path)
+        else:
+            msg += " of the current repo."
+
+        logger.warning(msg)
+
+        msg = "Are you sure you want to proceed?"
+        if not self.args.force and not prompt.confirm(msg):
+            return 1
+
+        self.repo.gc(
+            all_branches=self.args.all_branches,
+            all_tags=self.args.all_tags,
+            cloud=self.args.cloud,
+            remote=self.args.remote,
+            force=self.args.force,
+            jobs=self.args.jobs,
+            repos=self.args.repos,
+        )
         return 0
+
+
+def add_parser(subparsers, parent_parser):
+    GC_HELP = (
+        "Collect garbage from your DVC cache. \n"
+        "Deletes all files in the cache which are not in use by the "
+        "specified git references (defaults to just HEAD)."
+    )
+    gc_parser = subparsers.add_parser(
+        "gc", parents=[parent_parser], description=GC_HELP, help=GC_HELP
+    )
+    gc_parser.add_argument(
+        "-a",
+        "--all-branches",
+        action="store_true",
+        default=False,
+        help="Keep data files for the tips of all git branches.",
+    )
+    gc_parser.add_argument(
+        "-T",
+        "--all-tags",
+        action="store_true",
+        default=False,
+        help="Keep data files for all git tags.",
+    )
+    gc_parser.add_argument(
+        "-c",
+        "--cloud",
+        action="store_true",
+        default=False,
+        help="Collect garbage in remote repository.",
+    )
+    gc_parser.add_argument(
+        "-r", "--remote", help="Remote repository to collect garbage in."
+    )
+    gc_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Force garbage collection - automatically agree to all prompts.",
+    )
+    gc_parser.add_argument(
+        "-j",
+        "--jobs",
+        type=int,
+        default=None,
+        help="Number of jobs to run simultaneously.",
+    )
+    gc_parser.add_argument(
+        "-p",
+        "--projects",
+        dest="repos",
+        type=str,
+        nargs="*",
+        default=None,
+        help="Keep data files required by these projects "
+        "in addition to the current one. "
+        "Useful if you share a single cache across repos.",
+    )
+    gc_parser.set_defaults(func=CmdGC)

dvc/command/imp.py

@@ -1,13 +1,58 @@
+from __future__ import unicode_literals
+
+import os
+
+import dvc.logger as logger
+from dvc.utils.compat import urlparse
 from dvc.exceptions import DvcException
 from dvc.command.base import CmdBase
-from dvc.logger import Logger
 
 
 class CmdImport(CmdBase):
     def run(self):
         try:
-            self.project.imp(self.args.url, self.args.out)
-        except DvcException as ex:
-            Logger.error('Failed to import {}'.format(self.args.url), ex)
+            default_out = os.path.basename(urlparse(self.args.url).path)
+
+            out = self.args.out or default_out
+
+            self.repo.imp(self.args.url, out, self.args.resume)
+        except DvcException:
+            logger.error(
+                "failed to import {}. You could also try downloading "
+                "it manually and adding it with `dvc add` command.".format(
+                    self.args.url
+                )
+            )
             return 1
         return 0
+
+
+def add_parser(subparsers, parent_parser):
+    IMPORT_HELP = "Import files from URL."
+    import_parser = subparsers.add_parser(
+        "import",
+        parents=[parent_parser],
+        description=IMPORT_HELP,
+        help=IMPORT_HELP,
+    )
+    import_parser.add_argument(
+        "url",
+        help="URL. Supported urls: "
+        "'/path/to/file', "
+        "'C:\\\\path\\to\\file', "
+        "'https://example.com/path/to/file', "
+        "'s3://bucket/path/to/file', "
+        "'gs://bucket/path/to/file', "
+        "'hdfs://example.com/path/to/file', "
+        "'ssh://example.com:/path/to/file', "
+        "'remote://myremote/path/to/file'(see "
+        "`dvc remote` commands). ",
+    )
+    import_parser.add_argument(
+        "--resume",
+        action="store_true",
+        default=False,
+        help="Resume previously started download.",
+    )
+    import_parser.add_argument("out", nargs="?", help="Output.")
+    import_parser.set_defaults(func=CmdImport)

dvc/command/init.py

@@ -1,4 +1,6 @@
-from dvc.logger import Logger
+from __future__ import unicode_literals
+
+import dvc.logger as logger
 
 
 class CmdInit(object):
@@ -6,11 +8,40 @@ class CmdInit(object):
         self.args = args
 
     def run_cmd(self):
-        from dvc.project import Project, InitError
+        from dvc.repo import Repo
+        from dvc.exceptions import InitError
 
         try:
-            Project.init('.', no_scm=self.args.no_scm, force=self.args.force)
-        except InitError as e:
-            Logger.error('Failed to initiate dvc', e)
+            self.repo = Repo.init(
+                ".", no_scm=self.args.no_scm, force=self.args.force
+            )
+            self.config = self.repo.config
+        except InitError:
+            logger.error("failed to initiate dvc")
             return 1
         return 0
+
+
+def add_parser(subparsers, parent_parser):
+    """Setup parser for `dvc init`."""
+    INIT_HELP = (
+        "Initialize dvc over a directory " "(should already be a git dir)."
+    )
+    init_parser = subparsers.add_parser(
+        "init", parents=[parent_parser], description=INIT_HELP, help=INIT_HELP
+    )
+    init_parser.add_argument(
+        "--no-scm",
+        action="store_true",
+        default=False,
+        help="Initiate dvc in directory that is "
+        "not tracked by any scm tool (e.g. git).",
+    )
+    init_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Overwrite '.dvc' if it exists. Will remove all local cache.",
+    )
+    init_parser.set_defaults(func=CmdInit)

dvc/command/install.py

@@ -1,12 +1,25 @@
-from dvc.logger import Logger
+from __future__ import unicode_literals
+
+import dvc.logger as logger
 from dvc.command.base import CmdBase
 
 
 class CmdInstall(CmdBase):
     def run_cmd(self):
         try:
-            self.project.install()
-        except Exception as e:
-            Logger.error('Failed to install dvc hooks', e)
+            self.repo.install()
+        except Exception:
+            logger.error("failed to install dvc hooks")
             return 1
         return 0
+
+
+def add_parser(subparsers, parent_parser):
+    INSTALL_HELP = "Install dvc hooks into the repository."
+    install_parser = subparsers.add_parser(
+        "install",
+        parents=[parent_parser],
+        description=INSTALL_HELP,
+        help=INSTALL_HELP,
+    )
+    install_parser.set_defaults(func=CmdInstall)

dvc/command/lock.py

@@ -1,3 +1,6 @@
+from __future__ import unicode_literals
+
+import dvc.logger as logger
 from dvc.exceptions import DvcException
 from dvc.command.base import CmdBase
 
@@ -6,11 +9,14 @@ class CmdLockBase(CmdBase):
     def run(self, unlock=False):
         for target in self.args.targets:
             try:
-                self.project.lock_stage(target, unlock=unlock)
-            except DvcException as ex:
-                msg = 'Failed to {}lock \'{}\''
-                msg.format('un' if unlock else '', target)
-                self.project.logger.error(msg, ex)
+                self.repo.lock_stage(target, unlock=unlock)
+            except DvcException:
+                logger.error(
+                    "failed to {}lock '{}'".format(
+                        "un" if unlock else "", target
+                    )
+                )
+
                 return 1
         return 0
 
@@ -23,3 +29,22 @@ class CmdLock(CmdLockBase):
 class CmdUnlock(CmdLockBase):
     def run(self):
         return super(CmdUnlock, self).run(True)
+
+
+def add_parser(subparsers, parent_parser):
+    LOCK_HELP = "Lock DVC file."
+    lock_parser = subparsers.add_parser(
+        "lock", parents=[parent_parser], description=LOCK_HELP, help=LOCK_HELP
+    )
+    lock_parser.add_argument("targets", nargs="+", help="DVC files.")
+    lock_parser.set_defaults(func=CmdLock)
+
+    UNLOCK_HELP = "Unlock DVC file."
+    unlock_parser = subparsers.add_parser(
+        "unlock",
+        parents=[parent_parser],
+        description=UNLOCK_HELP,
+        help=UNLOCK_HELP,
+    )
+    unlock_parser.add_argument("targets", nargs="+", help="DVC files.")
+    unlock_parser.set_defaults(func=CmdUnlock)

dvc/command/metrics.py

@@ -1,36 +1,35 @@
-from dvc.command.base import CmdBase
+from __future__ import unicode_literals
+
+import dvc.logger as logger
 from dvc.exceptions import DvcException
+from dvc.command.base import CmdBase, fix_subparsers
 
 
 class CmdMetricsShow(CmdBase):
-    def run(self):
-        try:
-            # backward compatibility
-            if self.args.json_path:
-                typ = 'json'
-                xpath = self.args.json_path
-            elif self.args.tsv_path:
-                typ = 'tsv'
-                xpath = self.args.tsv_path
-            elif self.args.htsv_path:
-                typ = 'htsv'
-                xpath = self.args.htsv_path
-            elif self.args.csv_path:
-                typ = 'csv'
-                xpath = self.args.csv_path
-            elif self.args.hcsv_path:
-                typ = 'hcsv'
-                xpath = self.args.hcsv_path
-            else:
-                typ = self.args.type
-                xpath = self.args.xpath
+    def _show(self, metrics):
+        for branch, val in metrics.items():
+            if self.args.all_branches or self.args.all_tags:
+                logger.info("{}:".format(branch))
 
-            self.project.metrics_show(self.args.path,
-                                      typ=typ,
-                                      xpath=xpath,
-                                      all_branches=self.args.all_branches)
-        except DvcException as exc:
-            self.project.logger.error('Failed to show metrics', exc)
+            for fname, metric in val.items():
+                logger.info("\t{}: {}".format(fname, metric))
+
+    def run(self):
+        typ = self.args.type
+        xpath = self.args.xpath
+        try:
+            metrics = self.repo.metrics.show(
+                self.args.path,
+                typ=typ,
+                xpath=xpath,
+                all_branches=self.args.all_branches,
+                all_tags=self.args.all_tags,
+                recursive=self.args.recursive,
+            )
+
+            self._show(metrics)
+        except DvcException:
+            logger.error("failed to show metrics")
             return 1
 
         return 0
@@ -39,11 +38,11 @@ class CmdMetricsShow(CmdBase):
 class CmdMetricsModify(CmdBase):
     def run(self):
         try:
-            self.project.metrics_modify(self.args.path,
-                                        typ=self.args.type,
-                                        xpath=self.args.xpath)
-        except DvcException as exc:
-            self.project.logger.error('Failed to modify metrics', exc)
+            self.repo.metrics.modify(
+                self.args.path, typ=self.args.type, xpath=self.args.xpath
+            )
+        except DvcException:
+            logger.error("failed to modify metric file settings")
             return 1
 
         return 0
@@ -52,12 +51,12 @@ class CmdMetricsModify(CmdBase):
 class CmdMetricsAdd(CmdBase):
     def run(self):
         try:
-            self.project.metrics_add(self.args.path,
-                                     self.args.type,
-                                     self.args.xpath)
-        except DvcException as exc:
-            msg = 'Failed to add metric file \'{}\''.format(self.args.path)
-            self.project.logger.error(msg, exc)
+            self.repo.metrics.add(
+                self.args.path, self.args.type, self.args.xpath
+            )
+        except DvcException:
+            msg = "failed to add metric file '{}'".format(self.args.path)
+            logger.error(msg)
             return 1
 
         return 0
@@ -66,10 +65,114 @@ class CmdMetricsAdd(CmdBase):
 class CmdMetricsRemove(CmdBase):
     def run(self):
         try:
-            self.project.metrics_remove(self.args.path)
-        except DvcException as exc:
-            msg = 'Failed to remove metric file \'{}\''.format(self.args.path)
-            self.project.logger.error(msg, exc)
+            self.repo.metrics.remove(self.args.path)
+        except DvcException:
+            msg = "failed to remove metric file '{}'".format(self.args.path)
+            logger.error(msg)
             return 1
 
         return 0
+
+
+def add_parser(subparsers, parent_parser):
+    METRICS_HELP = (
+        "A set of commands to add, manage, collect and display project "
+        "metrics."
+    )
+    metrics_parser = subparsers.add_parser(
+        "metrics",
+        parents=[parent_parser],
+        description=METRICS_HELP,
+        help=METRICS_HELP,
+    )
+
+    metrics_subparsers = metrics_parser.add_subparsers(
+        dest="cmd",
+        help="Use dvc metrics CMD --help to display command-specific help.",
+    )
+
+    fix_subparsers(metrics_subparsers)
+
+    METRICS_SHOW_HELP = "Output metric values."
+    metrics_show_parser = metrics_subparsers.add_parser(
+        "show",
+        parents=[parent_parser],
+        description=METRICS_SHOW_HELP,
+        help=METRICS_SHOW_HELP,
+    )
+    metrics_show_parser.add_argument(
+        "path", nargs="?", help="Path to a metric file or a directory."
+    )
+    metrics_show_parser.add_argument(
+        "-t", "--type", help="Type of metrics (raw/json/tsv/htsv/csv/hcsv)."
+    )
+    metrics_show_parser.add_argument(
+        "-x", "--xpath", help="json/tsv/htsv/csv/hcsv path."
+    )
+    metrics_show_parser.add_argument(
+        "-a",
+        "--all-branches",
+        action="store_true",
+        default=False,
+        help="Show metrics for all branches.",
+    )
+    metrics_show_parser.add_argument(
+        "-T",
+        "--all-tags",
+        action="store_true",
+        default=False,
+        help="Show metrics for all tags.",
+    )
+    metrics_show_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help=(
+            "If path is a directory, recursively search and process metric "
+            "files in path."
+        ),
+    )
+    metrics_show_parser.set_defaults(func=CmdMetricsShow)
+
+    METRICS_ADD_HELP = "Tag file as a metric file."
+    metrics_add_parser = metrics_subparsers.add_parser(
+        "add",
+        parents=[parent_parser],
+        description=METRICS_ADD_HELP,
+        help=METRICS_ADD_HELP,
+    )
+    metrics_add_parser.add_argument(
+        "-t", "--type", help="Type of metrics (raw/json/tsv/htsv/csv/hcsv)."
+    )
+    metrics_add_parser.add_argument(
+        "-x", "--xpath", help="json/tsv/htsv/csv/hcsv path."
+    )
+    metrics_add_parser.add_argument("path", help="Path to a metric file.")
+    metrics_add_parser.set_defaults(func=CmdMetricsAdd)
+
+    METRICS_MODIFY_HELP = "Modify metric file options."
+    metrics_modify_parser = metrics_subparsers.add_parser(
+        "modify",
+        parents=[parent_parser],
+        description=METRICS_MODIFY_HELP,
+        help=METRICS_MODIFY_HELP,
+    )
+    metrics_modify_parser.add_argument(
+        "-t", "--type", help="Type of metrics (raw/json/tsv/htsv/csv/hcsv)."
+    )
+    metrics_modify_parser.add_argument(
+        "-x", "--xpath", help="json/tsv/htsv/csv/hcsv path."
+    )
+    metrics_modify_parser.add_argument("path", help="Path to a metric file.")
+    metrics_modify_parser.set_defaults(func=CmdMetricsModify)
+
+    METRICS_REMOVE_HELP = "Remove files's metric tag."
+    metrics_remove_parser = metrics_subparsers.add_parser(
+        "remove",
+        parents=[parent_parser],
+        description=METRICS_REMOVE_HELP,
+        help=METRICS_REMOVE_HELP,
+    )
+    metrics_remove_parser.add_argument("path", help="Path to a metric file.")
+    metrics_remove_parser.set_defaults(func=CmdMetricsRemove)

dvc/command/move.py

@@ -1,3 +1,6 @@
+from __future__ import unicode_literals
+
+import dvc.logger as logger
 from dvc.exceptions import DvcException
 from dvc.command.base import CmdBase
 
@@ -5,10 +8,28 @@ from dvc.command.base import CmdBase
 class CmdMove(CmdBase):
     def run(self):
         try:
-            self.project.move(self.args.src, self.args.dst)
-        except DvcException as ex:
-            msg = 'Failed to move \'{}\' -> \'{}\''.format(self.args.src,
-                                                           self.args.dst)
-            self.project.logger.error(msg, ex)
+            self.repo.move(self.args.src, self.args.dst)
+        except DvcException:
+            msg = "failed to move '{}' -> '{}'".format(
+                self.args.src, self.args.dst
+            )
+            logger.error(msg)
             return 1
         return 0
+
+
+def add_parser(subparsers, parent_parser):
+    description = (
+        "Rename or move a data file or a directory that "
+        "is under DVC control. It renames and modifies "
+        "the corresponding DVC file to reflect the changes."
+    )
+    help = "Rename or move a DVC controlled data file or a directory."
+    move_parser = subparsers.add_parser(
+        "move", parents=[parent_parser], description=description, help=help
+    )
+    move_parser.add_argument(
+        "src", help="Source path to a data file or directory."
+    )
+    move_parser.add_argument("dst", help="Destination path.")
+    move_parser.set_defaults(func=CmdMove)

dvc/command/pipeline.py

@@ -1,35 +1,198 @@
+from __future__ import unicode_literals
+
+from dvc.utils.compat import str
+
 import os
 
+import dvc.logger as logger
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, fix_subparsers
 
 
 class CmdPipelineShow(CmdBase):
     def _show(self, target, commands, outs):
         import networkx
-        from dvc.project import NotDvcFileError
+        from dvc.stage import Stage
 
-        G = self.project.graph()[0]
-        stages = networkx.get_node_attributes(G, 'stage')
-        node = os.path.relpath(os.path.abspath(target), self.project.root_dir)
-        if node not in stages:
-            raise NotDvcFileError(node)
+        stage = Stage.load(self.repo, target)
+        G = self.repo.graph()[0]
+        stages = networkx.get_node_attributes(G, "stage")
+        node = os.path.relpath(stage.path, self.repo.root_dir)
 
         for n in networkx.dfs_postorder_nodes(G, node):
             if commands:
-                self.project.logger.info(stages[n].cmd)
+                logger.info(stages[n].cmd)
             elif outs:
                 for out in stages[n].outs:
-                    self.project.logger.info(out.rel_path)
+                    logger.info(str(out))
             else:
-                self.project.logger.info(n)
+                logger.info(n)
+
+    def __build_graph(self, target, commands, outs):
+        import networkx
+        from dvc.stage import Stage
+
+        stage = Stage.load(self.repo, target)
+        node = os.path.relpath(stage.path, self.repo.root_dir)
+
+        pipelines = list(
+            filter(lambda g: node in g.nodes(), self.repo.pipelines())
+        )
+
+        assert len(pipelines) == 1
+        G = pipelines[0]
+        stages = networkx.get_node_attributes(G, "stage")
+
+        nodes = []
+        for n in G.nodes():
+            stage = stages[n]
+            if commands:
+                if stage.cmd is None:
+                    continue
+                nodes.append(stage.cmd)
+            elif outs:
+                for out in stage.outs:
+                    nodes.append(str(out))
+            else:
+                nodes.append(stage.relpath)
+
+        edges = []
+        for e in G.edges():
+            from_stage = stages[e[0]]
+            to_stage = stages[e[1]]
+            if commands:
+                if to_stage.cmd is None:
+                    continue
+                edges.append((from_stage.cmd, to_stage.cmd))
+            elif outs:
+                for from_out in from_stage.outs:
+                    for to_out in to_stage.outs:
+                        edges.append((str(from_out), str(to_out)))
+            else:
+                edges.append((from_stage.relpath, to_stage.relpath))
+
+        return nodes, edges
+
+    def _show_ascii(self, target, commands, outs):
+        from dvc.dagascii import draw
+
+        nodes, edges = self.__build_graph(target, commands, outs)
+
+        if not nodes:
+            return
+
+        draw(nodes, edges)
+
+    def __write_dot(self, target, commands, outs, filename):
+        import networkx
+        from networkx.drawing.nx_pydot import write_dot
+
+        _, edges = self.__build_graph(target, commands, outs)
+        edges = [edge[::-1] for edge in edges]
+
+        simple_g = networkx.DiGraph()
+        simple_g.add_edges_from(edges)
+        write_dot(simple_g, filename)
 
     def run(self, unlock=False):
+        if not self.args.targets:
+            self.args.targets = self.default_targets
+
         for target in self.args.targets:
             try:
-                self._show(target, self.args.commands, self.args.outs)
-            except DvcException as ex:
-                msg = 'Failed to show pipeline for \'{}\''.format(target)
-                self.project.logger.error(msg, ex)
+                if self.args.ascii:
+                    self._show_ascii(
+                        target, self.args.commands, self.args.outs
+                    )
+                elif self.args.dot:
+                    self.__write_dot(
+                        target,
+                        self.args.commands,
+                        self.args.outs,
+                        self.args.dot,
+                    )
+                else:
+                    self._show(target, self.args.commands, self.args.outs)
+            except DvcException:
+                msg = "failed to show pipeline for '{}'".format(target)
+                logger.error(msg)
                 return 1
         return 0
+
+
+class CmdPipelineList(CmdBase):
+    def run(self):
+        import networkx
+
+        pipelines = self.repo.pipelines()
+        for p in pipelines:
+            stages = networkx.get_node_attributes(p, "stage")
+            for stage in stages:
+                logger.info(stage)
+            if len(stages) != 0:
+                logger.info("=" * 80)
+        logger.info("{} pipeline(s) total".format(len(pipelines)))
+
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    PIPELINE_HELP = "Manage pipeline."
+    pipeline_parser = subparsers.add_parser(
+        "pipeline",
+        parents=[parent_parser],
+        description=PIPELINE_HELP,
+        help=PIPELINE_HELP,
+    )
+
+    pipeline_subparsers = pipeline_parser.add_subparsers(
+        dest="cmd",
+        help="Use dvc pipeline CMD --help for command-specific help.",
+    )
+
+    fix_subparsers(pipeline_subparsers)
+
+    PIPELINE_SHOW_HELP = "Show pipeline."
+    pipeline_show_parser = pipeline_subparsers.add_parser(
+        "show",
+        parents=[parent_parser],
+        description=PIPELINE_SHOW_HELP,
+        help=PIPELINE_SHOW_HELP,
+    )
+    pipeline_show_group = pipeline_show_parser.add_mutually_exclusive_group()
+    pipeline_show_group.add_argument(
+        "-c",
+        "--commands",
+        action="store_true",
+        default=False,
+        help="Print commands instead of paths to DVC files.",
+    )
+    pipeline_show_group.add_argument(
+        "-o",
+        "--outs",
+        action="store_true",
+        default=False,
+        help="Print output files instead of paths to DVC files.",
+    )
+    pipeline_show_parser.add_argument(
+        "--ascii",
+        action="store_true",
+        default=False,
+        help="Output DAG as ASCII.",
+    )
+    pipeline_show_parser.add_argument(
+        "--dot", help="Write DAG in .dot format."
+    )
+    pipeline_show_parser.add_argument(
+        "targets", nargs="*", help="DVC files. 'Dvcfile' by default."
+    )
+    pipeline_show_parser.set_defaults(func=CmdPipelineShow)
+
+    PIPELINE_LIST_HELP = "List pipelines."
+    pipeline_list_parser = pipeline_subparsers.add_parser(
+        "list",
+        parents=[parent_parser],
+        description=PIPELINE_LIST_HELP,
+        help=PIPELINE_LIST_HELP,
+    )
+    pipeline_list_parser.set_defaults(func=CmdPipelineList)

dvc/command/remote.py

@@ -1,47 +1,214 @@
+from __future__ import unicode_literals
+
+import os
 import re
 
+import dvc.logger as logger
+from dvc.command.base import fix_subparsers
 from dvc.config import Config
 from dvc.command.config import CmdConfig
-from dvc.logger import Logger
 
 
 class CmdRemoteAdd(CmdConfig):
+    @staticmethod
+    def resolve_path(path, config_file):
+        """Resolve path relative to config file location.
+
+        Args:
+            path: Path to be resolved.
+            config_file: Path to config file, which `path` is specified
+                relative to.
+
+        Returns:
+            Path relative to the `config_file` location. If `path` is an
+            absolute path then it will be returned without change.
+
+        """
+        if os.path.isabs(path):
+            return path
+        return os.path.relpath(path, os.path.dirname(config_file))
+
     def run(self):
+        from dvc.remote import _get, RemoteLOCAL
+
+        remote = _get({Config.SECTION_REMOTE_URL: self.args.url})
+        if remote == RemoteLOCAL:
+            self.args.url = self.resolve_path(
+                self.args.url, self.configobj.filename
+            )
+
         section = Config.SECTION_REMOTE_FMT.format(self.args.name)
-        ret = self.set(section, Config.SECTION_REMOTE_URL, self.args.url)
+        ret = self._set(section, Config.SECTION_REMOTE_URL, self.args.url)
         if ret != 0:
             return ret
 
         if self.args.default:
-            msg = 'Setting \'{}\' as a default remote.'.format(self.args.name)
-            Logger.info(msg)
-            ret = self.set(Config.SECTION_CORE,
-                           Config.SECTION_CORE_REMOTE,
-                           self.args.name)
+            msg = "Setting '{}' as a default remote.".format(self.args.name)
+            logger.info(msg)
+            ret = self._set(
+                Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, self.args.name
+            )
 
         return ret
 
 
 class CmdRemoteRemove(CmdConfig):
+    def _remove_default(self, config):
+        core = config.get(Config.SECTION_CORE, None)
+        if core is None:
+            return 0
+
+        default = core.get(Config.SECTION_CORE_REMOTE, None)
+        if default is None:
+            return 0
+
+        if default == self.args.name:
+            return self._unset(
+                Config.SECTION_CORE,
+                opt=Config.SECTION_CORE_REMOTE,
+                configobj=config,
+            )
+
     def run(self):
         section = Config.SECTION_REMOTE_FMT.format(self.args.name)
-        return self.unset(section, Config.SECTION_REMOTE_URL)
+        ret = self._unset(section)
+        if ret != 0:
+            return ret
+
+        for configobj in [
+            self.config._local_config,
+            self.config._repo_config,
+            self.config._global_config,
+            self.config._system_config,
+        ]:
+            self._remove_default(configobj)
+            self.config.save(configobj)
+            if configobj == self.configobj:
+                break
+
+        return 0
 
 
 class CmdRemoteModify(CmdConfig):
     def run(self):
         section = Config.SECTION_REMOTE_FMT.format(self.args.name)
-        self.args.name = '{}.{}'.format(section, self.args.option)
+        self.args.name = "{}.{}".format(section, self.args.option)
         return super(CmdRemoteModify, self).run()
 
 
+class CmdRemoteDefault(CmdConfig):
+    def run(self):
+        self.args.value = self.args.name
+        self.args.name = "core.remote"
+        return super(CmdRemoteDefault, self).run()
+
+
 class CmdRemoteList(CmdConfig):
     def run(self):
         for section in self.configobj.keys():
             r = re.match(Config.SECTION_REMOTE_REGEX, section)
             if r:
-                name = r.group('name')
-                url = self.configobj[section].get(Config.SECTION_REMOTE_URL,
-                                                  '')
-                Logger.info('{}\t{}'.format(name, url))
+                name = r.group("name")
+                url = self.configobj[section].get(
+                    Config.SECTION_REMOTE_URL, ""
+                )
+                logger.info("{}\t{}".format(name, url))
         return 0
+
+
+def add_parser(subparsers, parent_parser):
+    from dvc.command.config import parent_config_parser
+
+    REMOTE_HELP = "Manage set of tracked repositories."
+    remote_parser = subparsers.add_parser(
+        "remote",
+        parents=[parent_parser],
+        description=REMOTE_HELP,
+        help=REMOTE_HELP,
+    )
+
+    remote_subparsers = remote_parser.add_subparsers(
+        dest="cmd",
+        help="Use dvc remote CMD --help for " "command-specific help.",
+    )
+
+    fix_subparsers(remote_subparsers)
+
+    REMOTE_ADD_HELP = "Add remote."
+    remote_add_parser = remote_subparsers.add_parser(
+        "add",
+        parents=[parent_config_parser, parent_parser],
+        description=REMOTE_ADD_HELP,
+        help=REMOTE_ADD_HELP,
+    )
+    remote_add_parser.add_argument("name", help="Name.")
+    remote_add_parser.add_argument(
+        "url",
+        help="URL. See full list of supported urls at "
+        "https://dvc.org/doc/commands-reference/remote",
+    )
+    remote_add_parser.add_argument(
+        "-d",
+        "--default",
+        action="store_true",
+        default=False,
+        help="Set as default remote.",
+    )
+    remote_add_parser.set_defaults(func=CmdRemoteAdd)
+
+    REMOTE_DEFAULT_HELP = "Set/unset default remote."
+    remote_default_parser = remote_subparsers.add_parser(
+        "default",
+        parents=[parent_config_parser, parent_parser],
+        description=REMOTE_DEFAULT_HELP,
+        help=REMOTE_DEFAULT_HELP,
+    )
+    remote_default_parser.add_argument(
+        "name", nargs="?", help="Name of the remote."
+    )
+    remote_default_parser.add_argument(
+        "-u",
+        "--unset",
+        action="store_true",
+        default=False,
+        help="Unset default remote.",
+    )
+    remote_default_parser.set_defaults(func=CmdRemoteDefault)
+
+    REMOTE_REMOVE_HELP = "Remove remote."
+    remote_remove_parser = remote_subparsers.add_parser(
+        "remove",
+        parents=[parent_config_parser, parent_parser],
+        description=REMOTE_REMOVE_HELP,
+        help=REMOTE_REMOVE_HELP,
+    )
+    remote_remove_parser.add_argument("name", help="Name")
+    remote_remove_parser.set_defaults(func=CmdRemoteRemove)
+
+    REMOTE_MODIFY_HELP = "Modify remote."
+    remote_modify_parser = remote_subparsers.add_parser(
+        "modify",
+        parents=[parent_config_parser, parent_parser],
+        description=REMOTE_MODIFY_HELP,
+        help=REMOTE_MODIFY_HELP,
+    )
+    remote_modify_parser.add_argument("name", help="Name.")
+    remote_modify_parser.add_argument("option", help="Option.")
+    remote_modify_parser.add_argument("value", nargs="?", help="Value.")
+    remote_modify_parser.add_argument(
+        "-u",
+        "--unset",
+        default=False,
+        action="store_true",
+        help="Unset option.",
+    )
+    remote_modify_parser.set_defaults(func=CmdRemoteModify)
+
+    REMOTE_LIST_HELP = "List remotes."
+    remote_list_parser = remote_subparsers.add_parser(
+        "list",
+        parents=[parent_config_parser, parent_parser],
+        description=REMOTE_LIST_HELP,
+        help=REMOTE_LIST_HELP,
+    )
+    remote_list_parser.set_defaults(func=CmdRemoteList)

dvc/command/remove.py

@@ -1,15 +1,71 @@
+from __future__ import unicode_literals
+
+import dvc.prompt as prompt
+import dvc.logger as logger
 from dvc.exceptions import DvcException
 from dvc.command.base import CmdBase
-from dvc.logger import Logger
 
 
 class CmdRemove(CmdBase):
+    def _is_outs_only(self, target):
+        if not self.args.purge:
+            return True
+
+        if self.args.force:
+            return False
+
+        msg = "Are you sure you want to remove {} with its outputs?".format(
+            target
+        )
+
+        if prompt.confirm(msg):
+            return False
+
+        raise DvcException(
+            "Cannot purge without a confirmation from the user."
+            " Use '-f' to force."
+        )
+
     def run(self):
-        outs_only = not self.args.purge
         for target in self.args.targets:
             try:
-                self.project.remove(target, outs_only=outs_only)
-            except DvcException as ex:
-                Logger.error('Failed to remove {}'.format(target), ex)
+                outs_only = self._is_outs_only(target)
+                self.repo.remove(target, outs_only=outs_only)
+            except DvcException:
+                logger.error("failed to remove {}".format(target))
                 return 1
         return 0
+
+
+def add_parser(subparsers, parent_parser):
+    REMOVE_HELP = "Remove outputs of DVC file."
+    remove_parser = subparsers.add_parser(
+        "remove",
+        parents=[parent_parser],
+        description=REMOVE_HELP,
+        help=REMOVE_HELP,
+    )
+    remove_parser_group = remove_parser.add_mutually_exclusive_group()
+    remove_parser_group.add_argument(
+        "-o",
+        "--outs",
+        action="store_true",
+        default=True,
+        help="Only remove DVC file outputs.(default)",
+    )
+    remove_parser_group.add_argument(
+        "-p",
+        "--purge",
+        action="store_true",
+        default=False,
+        help="Remove DVC file and all its outputs.",
+    )
+    remove_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Force purge.",
+    )
+    remove_parser.add_argument("targets", nargs="+", help="DVC files.")
+    remove_parser.set_defaults(func=CmdRemove)

dvc/command/repro.py

@@ -1,35 +1,134 @@
+from __future__ import unicode_literals
+
 import os
 
+import dvc.logger as logger
 from dvc.command.base import CmdBase
+from dvc.command.status import CmdDataStatus
+from dvc.exceptions import DvcException
 
 
 class CmdRepro(CmdBase):
     def run(self):
-        from dvc.project import ReproductionError
-
         recursive = not self.args.single_item
         saved_dir = os.path.realpath(os.curdir)
         if self.args.cwd:
             os.chdir(self.args.cwd)
 
+        # Dirty hack so the for loop below can at least enter once
+        if self.args.all_pipelines:
+            self.args.targets = [None]
+        elif not self.args.targets:
+            self.args.targets = self.default_targets
+
         ret = 0
         for target in self.args.targets:
             try:
-                self.project.reproduce(target,
-                                       recursive=recursive,
-                                       force=self.args.force)
+                stages = self.repo.reproduce(
+                    target,
+                    recursive=recursive,
+                    force=self.args.force,
+                    dry=self.args.dry,
+                    interactive=self.args.interactive,
+                    pipeline=self.args.pipeline,
+                    all_pipelines=self.args.all_pipelines,
+                    ignore_build_cache=self.args.ignore_build_cache,
+                    no_commit=self.args.no_commit,
+                )
+
+                if len(stages) == 0:
+                    logger.info(CmdDataStatus.UP_TO_DATE_MSG)
+
                 if self.args.metrics:
-                    self.project.metrics_show()
-            except ReproductionError as ex:
-                msg = 'Failed to reproduce \'{}\''.format(target)
-                self.project.logger.error(msg, ex)
-                ret = 1
-                break
-            except Exception as ex:
-                msg = 'Failed to reproduce \'{}\' - unexpected error'
-                self.project.logger.error(msg.format(target), ex)
+                    self.repo.metrics.show()
+            except DvcException:
+                logger.error()
                 ret = 1
                 break
 
         os.chdir(saved_dir)
         return ret
+
+
+def add_parser(subparsers, parent_parser):
+    REPRO_HELP = "Reproduce DVC file. Default file name - 'Dvcfile'."
+    repro_parser = subparsers.add_parser(
+        "repro",
+        parents=[parent_parser],
+        description=REPRO_HELP,
+        help=REPRO_HELP,
+    )
+    repro_parser.add_argument(
+        "targets", nargs="*", help="DVC file to reproduce."
+    )
+    repro_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Reproduce even if dependencies were not changed.",
+    )
+    repro_parser.add_argument(
+        "-s",
+        "--single-item",
+        action="store_true",
+        default=False,
+        help="Reproduce only single data item without recursive dependencies "
+        "check.",
+    )
+    repro_parser.add_argument(
+        "-c",
+        "--cwd",
+        default=os.path.curdir,
+        help="Directory within your repo to reroduce from.",
+    )
+    repro_parser.add_argument(
+        "-m",
+        "--metrics",
+        action="store_true",
+        default=False,
+        help="Show metrics after reproduction.",
+    )
+    repro_parser.add_argument(
+        "--dry",
+        action="store_true",
+        default=False,
+        help="Only print the commands that would be executed without "
+        "actually executing.",
+    )
+    repro_parser.add_argument(
+        "-i",
+        "--interactive",
+        action="store_true",
+        default=False,
+        help="Ask for confirmation before reproducing each stage.",
+    )
+    repro_parser.add_argument(
+        "-p",
+        "--pipeline",
+        action="store_true",
+        default=False,
+        help="Reproduce the whole pipeline that the specified stage file "
+        "belongs to.",
+    )
+    repro_parser.add_argument(
+        "-P",
+        "--all-pipelines",
+        action="store_true",
+        default=False,
+        help="Reproduce all pipelines in the repo.",
+    )
+    repro_parser.add_argument(
+        "--ignore-build-cache",
+        action="store_true",
+        default=False,
+        help="Reproduce all descendants of a changed stage even if their "
+        "direct dependencies didn't change.",
+    )
+    repro_parser.add_argument(
+        "--no-commit",
+        action="store_true",
+        default=False,
+        help="Don't put files/directories into cache.",
+    )
+    repro_parser.set_defaults(func=CmdRepro)

dvc/command/root.py

@@ -1,9 +1,23 @@
+from __future__ import unicode_literals
+
 import os
 
+import dvc.logger as logger
 from dvc.command.base import CmdBase
 
 
 class CmdRoot(CmdBase):
+    def run_cmd(self):
+        return self.run()
+
     def run(self):
-        self.project.logger.info(os.path.relpath(self.project.root_dir))
+        logger.info(os.path.relpath(self.repo.root_dir))
         return 0
+
+
+def add_parser(subparsers, parent_parser):
+    ROOT_HELP = "Relative path to repo's directory."
+    root_parser = subparsers.add_parser(
+        "root", parents=[parent_parser], description=ROOT_HELP, help=ROOT_HELP
+    )
+    root_parser.set_defaults(func=CmdRoot)

dvc/command/run.py

@@ -1,20 +1,180 @@
+from __future__ import unicode_literals
+
+import argparse
+
+import dvc.logger as logger
 from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
 
 
 class CmdRun(CmdBase):
     def run(self):
+        overwrite = self.args.yes or self.args.overwrite_dvcfile
+
+        if not any(
+            [
+                self.args.deps,
+                self.args.outs,
+                self.args.outs_no_cache,
+                self.args.metrics,
+                self.args.metrics_no_cache,
+                self.args.command,
+            ]
+        ):  # pragma: no cover
+            logger.error(
+                "too few arguments. Specify at least one: '-d', "
+                "'-o', '-O', '-m', '-M', 'command'."
+            )
+            return 1
+
         try:
-            self.project.run(cmd=' '.join(self.args.command),
-                             outs=self.args.outs,
-                             outs_no_cache=self.args.outs_no_cache,
-                             metrics_no_cache=self.args.metrics_no_cache,
-                             deps=self.args.deps,
-                             fname=self.args.file,
-                             cwd=self.args.cwd,
-                             no_exec=self.args.no_exec)
-        except DvcException as ex:
-            self.project.logger.error('Failed to run command', ex)
+            self.repo.run(
+                cmd=self._parsed_cmd(),
+                outs=self.args.outs,
+                outs_no_cache=self.args.outs_no_cache,
+                metrics=self.args.metrics,
+                metrics_no_cache=self.args.metrics_no_cache,
+                deps=self.args.deps,
+                fname=self.args.file,
+                cwd=self.args.cwd,
+                wdir=self.args.wdir,
+                no_exec=self.args.no_exec,
+                overwrite=overwrite,
+                ignore_build_cache=self.args.ignore_build_cache,
+                remove_outs=self.args.remove_outs,
+                no_commit=self.args.no_commit,
+            )
+        except DvcException:
+            logger.error("failed to run command")
             return 1
 
         return 0
+
+    def _parsed_cmd(self):
+        """
+        We need to take into account two cases:
+
+        - ['python code.py foo bar']: Used mainly with dvc as a library
+        - ['echo', 'foo bar']: List of arguments received from the CLI
+
+        The second case would need quoting, as it was passed through:
+                dvc run echo "foo bar"
+        """
+        if len(self.args.command) < 2:
+            return " ".join(self.args.command)
+
+        return " ".join(self._quote_argument(arg) for arg in self.args.command)
+
+    def _quote_argument(self, argument):
+        if " " not in argument or '"' in argument:
+            return argument
+
+        return '"{}"'.format(argument)
+
+
+def add_parser(subparsers, parent_parser):
+    RUN_HELP = (
+        "Generate a stage file from a given "
+        "command and execute the command."
+    )
+    run_parser = subparsers.add_parser(
+        "run", parents=[parent_parser], description=RUN_HELP, help=RUN_HELP
+    )
+    run_parser.add_argument(
+        "-d",
+        "--deps",
+        action="append",
+        default=[],
+        help="Declare dependencies for reproducible cmd.",
+    )
+    run_parser.add_argument(
+        "-o",
+        "--outs",
+        action="append",
+        default=[],
+        help="Declare output file or directory.",
+    )
+    run_parser.add_argument(
+        "-O",
+        "--outs-no-cache",
+        action="append",
+        default=[],
+        help="Declare output file or directory "
+        "(do not put into DVC cache).",
+    )
+    run_parser.add_argument(
+        "-m",
+        "--metrics",
+        action="append",
+        default=[],
+        help="Declare output metric file or directory.",
+    )
+    run_parser.add_argument(
+        "-M",
+        "--metrics-no-cache",
+        action="append",
+        default=[],
+        help="Declare output metric file or directory "
+        "(do not put into DVC cache).",
+    )
+    run_parser.add_argument(
+        "-f",
+        "--file",
+        help="Specify name of the stage file. It should be "
+        "either 'Dvcfile' or have a '.dvc' suffix (e.g. "
+        "'prepare.dvc', 'clean.dvc', etc) in order for "
+        "dvc to be able to find it later. By default "
+        "the first output basename + .dvc is used as "
+        "a stage filename.",
+    )
+    run_parser.add_argument(
+        "-c", "--cwd", default=None, help="Deprecated, use -w and -f instead."
+    )
+    run_parser.add_argument(
+        "-w",
+        "--wdir",
+        default=None,
+        help="Directory within your repo to run your command in.",
+    )
+    run_parser.add_argument(
+        "--no-exec",
+        action="store_true",
+        default=False,
+        help="Only create stage file without actually running it.",
+    )
+    run_parser.add_argument(
+        "-y",
+        "--yes",
+        action="store_true",
+        default=False,
+        help="Deprecated, use --overwrite-dvcfile instead",
+    )
+    run_parser.add_argument(
+        "--overwrite-dvcfile",
+        action="store_true",
+        default=False,
+        help="Overwrite existing dvc file without asking for confirmation.",
+    )
+    run_parser.add_argument(
+        "--ignore-build-cache",
+        action="store_true",
+        default=False,
+        help="Run this stage even if it has been already ran with the same "
+        "command/dependencies/outputs/etc before.",
+    )
+    run_parser.add_argument(
+        "--remove-outs",
+        action="store_true",
+        default=False,
+        help="Remove outputs before running the command.",
+    )
+    run_parser.add_argument(
+        "--no-commit",
+        action="store_true",
+        default=False,
+        help="Don't put files/directories into cache.",
+    )
+    run_parser.add_argument(
+        "command", nargs=argparse.REMAINDER, help="Command to execute."
+    )
+    run_parser.set_defaults(func=CmdRun)

dvc/command/status.py

@@ -1,36 +1,63 @@
+from __future__ import unicode_literals
+
+import dvc.logger as logger
 from dvc.command.data_sync import CmdDataBase
+from dvc.utils.compat import str
 
 
 class CmdDataStatus(CmdDataBase):
-    STATUS_LEN = 10
-    STATUS_INDENT = '\t'
+    STATUS_LEN = 20
+    STATUS_INDENT = "\t"
+    UP_TO_DATE_MSG = "Pipeline is up to date. Nothing to reproduce."
 
     def _normalize(self, s):
-        s += ':'
+        s += ":"
         assert len(s) < self.STATUS_LEN
-        return s + (self.STATUS_LEN - len(s))*' '
+        return s + (self.STATUS_LEN - len(s)) * " "
 
     def _show(self, status, indent=0):
         ind = indent * self.STATUS_INDENT
 
+        if isinstance(status, str):
+            logger.info("{}{}".format(ind, status))
+            return
+
+        if isinstance(status, list):
+            for entry in status:
+                self._show(entry, indent)
+            return
+
+        assert isinstance(status, dict)
+
         for key, value in status.items():
-            if isinstance(value, dict):
-                self.project.logger.info('{}{}'.format(ind, key))
-                self._show(value, indent+1)
-            else:
-                msg = '{}{}{}'.format(ind, self._normalize(value), key)
-                self.project.logger.info(msg)
+            if isinstance(value, str):
+                logger.info("{}{}{}".format(ind, self._normalize(value), key))
+            elif value:
+                logger.info("{}{}:".format(ind, key))
+                self._show(value, indent + 1)
 
     def do_run(self, target=None):
         indent = 1 if self.args.cloud else 0
         try:
-            st = self.project.status(target=target,
-                                     jobs=self.args.jobs,
-                                     cloud=self.args.cloud,
-                                     show_checksums=self.args.show_checksums,
-                                     remote=self.args.remote)
-            self._show(st, indent)
-        except Exception as exc:
-            self.project.logger.error('Failed to obtain data status', exc)
+            st = self.repo.status(
+                target=target,
+                jobs=self.args.jobs,
+                cloud=self.args.cloud,
+                show_checksums=self.args.show_checksums,
+                remote=self.args.remote,
+                all_branches=self.args.all_branches,
+                all_tags=self.args.all_tags,
+                with_deps=self.args.with_deps,
+            )
+            if st:
+                if self.args.quiet:
+                    return 1
+                else:
+                    self._show(st, indent)
+            else:
+                logger.info(self.UP_TO_DATE_MSG)
+
+        except Exception:
+            logger.error("failed to obtain data status")
             return 1
         return 0

dvc/command/unprotect.py

@@ -0,0 +1,31 @@
+from __future__ import unicode_literals
+
+import dvc.logger as logger
+from dvc.exceptions import DvcException
+from dvc.command.base import CmdBase
+
+
+class CmdUnprotect(CmdBase):
+    def run(self):
+        for target in self.args.targets:
+            try:
+                self.repo.unprotect(target)
+            except DvcException:
+                msg = "failed to unprotect '{}'".format(target)
+                logger.error(msg)
+                return 1
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    UNPROTECT_HELP = "Unprotect data file/directory."
+    unprotect_parser = subparsers.add_parser(
+        "unprotect",
+        parents=[parent_parser],
+        description=UNPROTECT_HELP,
+        help=UNPROTECT_HELP,
+    )
+    unprotect_parser.add_argument(
+        "targets", nargs="+", help="Data files/directory."
+    )
+    unprotect_parser.set_defaults(func=CmdUnprotect)

dvc/config.py

@@ -1,60 +1,157 @@
-"""
-DVC config objects.
-"""
+"""DVC config objects."""
+
+from __future__ import unicode_literals
+
+from dvc.utils.compat import str, open
+
 import os
+import errno
 import configobj
 from schema import Schema, Optional, And, Use, Regex
 
+import dvc.logger as logger
 from dvc.exceptions import DvcException
 
 
 class ConfigError(DvcException):
-    """ DVC config exception """
-    def __init__(self, ex=None):
-        super(ConfigError, self).__init__('Config file error', ex)
+    """DVC config exception.
 
+    Args:
+        msg (str): error message.
+        ex (Exception): optional exception that has caused this error.
+    """
 
-def supported_url(url):
-    from dvc.remote import supported_url as supported
-    return supported(url)
+    def __init__(self, msg, ex=None):
+        super(ConfigError, self).__init__(
+            "config file error: {}".format(msg), ex
+        )
 
 
 def supported_cache_type(types):
+    """Checks if link type config option has a valid value.
+
+    Args:
+        types (list/string): type(s) of links that dvc should try out.
+    """
     if isinstance(types, str):
-        types = [t.strip() for t in types.split(',')]
-    for t in types:
-        if t not in ['reflink', 'hardlink', 'symlink', 'copy']:
+        types = [typ.strip() for typ in types.split(",")]
+    for typ in types:
+        if typ not in ["reflink", "hardlink", "symlink", "copy"]:
             return False
     return True
 
 
 def supported_loglevel(level):
-    return level in ['info', 'debug', 'warning', 'error']
+    """Checks if log level config option has a valid value.
+
+    Args:
+        level (str): log level name.
+    """
+    return level in ["info", "debug", "warning", "error"]
 
 
 def supported_cloud(cloud):
-    return cloud in ['aws', 'gcp', 'local', '']
+    """Checks if obsoleted cloud option has a valid value.
+
+    Args:
+        cloud (str): cloud type name.
+    """
+    return cloud in ["aws", "gcp", "local", ""]
 
 
-class Config(object):
-    CONFIG = 'config'
-    CONFIG_LOCAL = 'config.local'
+def is_bool(val):
+    """Checks that value is a boolean.
 
-    SECTION_CORE = 'core'
-    SECTION_CORE_LOGLEVEL = 'loglevel'
+    Args:
+        val (str): string value verify.
+
+    Returns:
+        bool: True if value stands for boolean, False otherwise.
+    """
+    return val.lower() in ["true", "false"]
+
+
+def to_bool(val):
+    """Converts value to boolean.
+
+    Args:
+        val (str): string to convert to boolean.
+
+    Returns:
+        bool: True if value.lower() == 'true', False otherwise.
+    """
+    return val.lower() == "true"
+
+
+def is_whole(val):
+    """Checks that value is a whole integer.
+
+    Args:
+        val (str): number string to verify.
+
+    Returns:
+        bool: True if val is a whole number, False otherwise.
+    """
+    return int(val) >= 0
+
+
+def is_percent(val):
+    """Checks that value is a percent.
+
+    Args:
+        val (str): number string to verify.
+
+    Returns:
+        bool: True if 0<=value<=100, False otherwise.
+    """
+    return int(val) >= 0 and int(val) <= 100
+
+
+class Config(object):  # pylint: disable=too-many-instance-attributes
+    """Class that manages configuration files for a dvc repo.
+
+    Args:
+        dvc_dir (str): optional path to `.dvc` directory, that is used to
+            access repo-specific configs like .dvc/config and
+            .dvc/config.local.
+        validate (bool): optional flag to tell dvc if it should validate the
+            config or just load it as is. 'True' by default.
+
+
+    Raises:
+        ConfigError: thrown when config has an invalid format.
+    """
+
+    APPNAME = "dvc"
+    APPAUTHOR = "iterative"
+
+    # NOTE: used internally in RemoteLOCAL to know config
+    # location, that url should resolved relative to.
+    PRIVATE_CWD = "_cwd"
+
+    CONFIG = "config"
+    CONFIG_LOCAL = "config.local"
+
+    SECTION_CORE = "core"
+    SECTION_CORE_LOGLEVEL = "loglevel"
     SECTION_CORE_LOGLEVEL_SCHEMA = And(Use(str.lower), supported_loglevel)
-    SECTION_CORE_REMOTE = 'remote'
+    SECTION_CORE_REMOTE = "remote"
+    SECTION_CORE_INTERACTIVE_SCHEMA = And(str, is_bool, Use(to_bool))
+    SECTION_CORE_INTERACTIVE = "interactive"
+    SECTION_CORE_ANALYTICS = "analytics"
+    SECTION_CORE_ANALYTICS_SCHEMA = And(str, is_bool, Use(to_bool))
 
-    SECTION_CACHE = 'cache'
-    SECTION_CACHE_DIR = 'dir'
-    SECTION_CACHE_TYPE = 'type'
+    SECTION_CACHE = "cache"
+    SECTION_CACHE_DIR = "dir"
+    SECTION_CACHE_TYPE = "type"
     SECTION_CACHE_TYPE_SCHEMA = supported_cache_type
-    SECTION_CACHE_LOCAL = 'local'
-    SECTION_CACHE_S3 = 's3'
-    SECTION_CACHE_GS = 'gs'
-    SECTION_CACHE_SSH = 'ssh'
-    SECTION_CACHE_HDFS = 'hdfs'
-    SECTION_CACHE_AZURE = 'azure'
+    SECTION_CACHE_PROTECTED = "protected"
+    SECTION_CACHE_LOCAL = "local"
+    SECTION_CACHE_S3 = "s3"
+    SECTION_CACHE_GS = "gs"
+    SECTION_CACHE_SSH = "ssh"
+    SECTION_CACHE_HDFS = "hdfs"
+    SECTION_CACHE_AZURE = "azure"
     SECTION_CACHE_SCHEMA = {
         Optional(SECTION_CACHE_LOCAL): str,
         Optional(SECTION_CACHE_S3): str,
@@ -62,121 +159,405 @@ class Config(object):
         Optional(SECTION_CACHE_HDFS): str,
         Optional(SECTION_CACHE_SSH): str,
         Optional(SECTION_CACHE_AZURE): str,
-
-        # backward compatibility
-        Optional(SECTION_CACHE_DIR, default='cache'): str,
+        Optional(SECTION_CACHE_DIR): str,
         Optional(SECTION_CACHE_TYPE, default=None): SECTION_CACHE_TYPE_SCHEMA,
+        Optional(SECTION_CACHE_PROTECTED, default=False): And(
+            str, is_bool, Use(to_bool)
+        ),
+        Optional(PRIVATE_CWD): str,
     }
 
     # backward compatibility
-    SECTION_CORE_CLOUD = 'cloud'
+    SECTION_CORE_CLOUD = "cloud"
     SECTION_CORE_CLOUD_SCHEMA = And(Use(str.lower), supported_cloud)
-    SECTION_CORE_STORAGEPATH = 'storagepath'
+    SECTION_CORE_STORAGEPATH = "storagepath"
 
     SECTION_CORE_SCHEMA = {
-        Optional(SECTION_CORE_LOGLEVEL,
-                 default='info'): And(str, Use(str.lower),
-                                      SECTION_CORE_LOGLEVEL_SCHEMA),
-        Optional(SECTION_CORE_REMOTE, default=''): And(str, Use(str.lower)),
-
+        Optional(SECTION_CORE_LOGLEVEL, default="info"): And(
+            str, Use(str.lower), SECTION_CORE_LOGLEVEL_SCHEMA
+        ),
+        Optional(SECTION_CORE_REMOTE, default=""): And(str, Use(str.lower)),
+        Optional(
+            SECTION_CORE_INTERACTIVE, default=False
+        ): SECTION_CORE_INTERACTIVE_SCHEMA,
+        Optional(
+            SECTION_CORE_ANALYTICS, default=True
+        ): SECTION_CORE_ANALYTICS_SCHEMA,
         # backward compatibility
-        Optional(SECTION_CORE_CLOUD, default=''): SECTION_CORE_CLOUD_SCHEMA,
-        Optional(SECTION_CORE_STORAGEPATH, default=''): str,
+        Optional(SECTION_CORE_CLOUD, default=""): SECTION_CORE_CLOUD_SCHEMA,
+        Optional(SECTION_CORE_STORAGEPATH, default=""): str,
     }
 
     # backward compatibility
-    SECTION_AWS = 'aws'
-    SECTION_AWS_STORAGEPATH = 'storagepath'
-    SECTION_AWS_CREDENTIALPATH = 'credentialpath'
-    SECTION_AWS_ENDPOINT_URL = 'endpointurl'
-    SECTION_AWS_REGION = 'region'
-    SECTION_AWS_PROFILE = 'profile'
+    SECTION_AWS = "aws"
+    SECTION_AWS_STORAGEPATH = "storagepath"
+    SECTION_AWS_CREDENTIALPATH = "credentialpath"
+    SECTION_AWS_ENDPOINT_URL = "endpointurl"
+    SECTION_AWS_LIST_OBJECTS = "listobjects"
+    SECTION_AWS_REGION = "region"
+    SECTION_AWS_PROFILE = "profile"
+    SECTION_AWS_USE_SSL = "use_ssl"
     SECTION_AWS_SCHEMA = {
         SECTION_AWS_STORAGEPATH: str,
         Optional(SECTION_AWS_REGION): str,
-        Optional(SECTION_AWS_PROFILE, default='default'): str,
-        Optional(SECTION_AWS_CREDENTIALPATH, default=''): str,
-        Optional(SECTION_AWS_ENDPOINT_URL, default=None): str,
+        Optional(SECTION_AWS_PROFILE): str,
+        Optional(SECTION_AWS_CREDENTIALPATH): str,
+        Optional(SECTION_AWS_ENDPOINT_URL): str,
+        Optional(SECTION_AWS_LIST_OBJECTS, default=False): And(
+            str, is_bool, Use(to_bool)
+        ),
+        Optional(SECTION_AWS_USE_SSL, default=True): And(
+            str, is_bool, Use(to_bool)
+        ),
     }
 
     # backward compatibility
-    SECTION_GCP = 'gcp'
+    SECTION_GCP = "gcp"
     SECTION_GCP_STORAGEPATH = SECTION_AWS_STORAGEPATH
-    SECTION_GCP_PROJECTNAME = 'projectname'
+    SECTION_GCP_CREDENTIALPATH = SECTION_AWS_CREDENTIALPATH
+    SECTION_GCP_PROJECTNAME = "projectname"
     SECTION_GCP_SCHEMA = {
         SECTION_GCP_STORAGEPATH: str,
         Optional(SECTION_GCP_PROJECTNAME): str,
     }
 
     # backward compatibility
-    SECTION_LOCAL = 'local'
+    SECTION_LOCAL = "local"
     SECTION_LOCAL_STORAGEPATH = SECTION_AWS_STORAGEPATH
-    SECTION_LOCAL_SCHEMA = {
-        SECTION_LOCAL_STORAGEPATH: str,
-    }
+    SECTION_LOCAL_SCHEMA = {SECTION_LOCAL_STORAGEPATH: str}
+
+    SECTION_AZURE_CONNECTION_STRING = "connection_string"
 
     SECTION_REMOTE_REGEX = r'^\s*remote\s*"(?P<name>.*)"\s*$'
     SECTION_REMOTE_FMT = 'remote "{}"'
-    SECTION_REMOTE_URL = 'url'
-    SECTION_REMOTE_USER = 'user'
+    SECTION_REMOTE_URL = "url"
+    SECTION_REMOTE_USER = "user"
+    SECTION_REMOTE_PORT = "port"
+    SECTION_REMOTE_KEY_FILE = "keyfile"
+    SECTION_REMOTE_TIMEOUT = "timeout"
+    SECTION_REMOTE_PASSWORD = "password"
+    SECTION_REMOTE_ASK_PASSWORD = "ask_password"
     SECTION_REMOTE_SCHEMA = {
-        SECTION_REMOTE_URL: supported_url,
+        SECTION_REMOTE_URL: str,
         Optional(SECTION_AWS_REGION): str,
-        Optional(SECTION_AWS_PROFILE, default='default'): str,
-        Optional(SECTION_AWS_CREDENTIALPATH, default=''): str,
-        Optional(SECTION_AWS_ENDPOINT_URL, default=None): str,
+        Optional(SECTION_AWS_PROFILE): str,
+        Optional(SECTION_AWS_CREDENTIALPATH): str,
+        Optional(SECTION_AWS_ENDPOINT_URL): str,
+        Optional(SECTION_AWS_LIST_OBJECTS, default=False): And(
+            str, is_bool, Use(to_bool)
+        ),
+        Optional(SECTION_AWS_USE_SSL, default=True): And(
+            str, is_bool, Use(to_bool)
+        ),
         Optional(SECTION_GCP_PROJECTNAME): str,
         Optional(SECTION_CACHE_TYPE): SECTION_CACHE_TYPE_SCHEMA,
+        Optional(SECTION_CACHE_PROTECTED, default=False): And(
+            str, is_bool, Use(to_bool)
+        ),
         Optional(SECTION_REMOTE_USER): str,
+        Optional(SECTION_REMOTE_PORT): Use(int),
+        Optional(SECTION_REMOTE_KEY_FILE): str,
+        Optional(SECTION_REMOTE_TIMEOUT): Use(int),
+        Optional(SECTION_REMOTE_PASSWORD): str,
+        Optional(SECTION_REMOTE_ASK_PASSWORD): And(str, is_bool, Use(to_bool)),
+        Optional(SECTION_AZURE_CONNECTION_STRING): str,
+        Optional(PRIVATE_CWD): str,
+    }
+
+    SECTION_STATE = "state"
+    SECTION_STATE_ROW_LIMIT = "row_limit"
+    SECTION_STATE_ROW_CLEANUP_QUOTA = "row_cleanup_quota"
+    SECTION_STATE_SCHEMA = {
+        Optional(SECTION_STATE_ROW_LIMIT): And(Use(int), is_whole),
+        Optional(SECTION_STATE_ROW_CLEANUP_QUOTA): And(Use(int), is_percent),
     }
 
     SCHEMA = {
         Optional(SECTION_CORE, default={}): SECTION_CORE_SCHEMA,
         Optional(Regex(SECTION_REMOTE_REGEX)): SECTION_REMOTE_SCHEMA,
         Optional(SECTION_CACHE, default={}): SECTION_CACHE_SCHEMA,
-
+        Optional(SECTION_STATE, default={}): SECTION_STATE_SCHEMA,
         # backward compatibility
         Optional(SECTION_AWS, default={}): SECTION_AWS_SCHEMA,
         Optional(SECTION_GCP, default={}): SECTION_GCP_SCHEMA,
         Optional(SECTION_LOCAL, default={}): SECTION_LOCAL_SCHEMA,
     }
 
-    def __init__(self, dvc_dir):
-        self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
-        self.config_file = os.path.join(dvc_dir, self.CONFIG)
-        self.config_local_file = os.path.join(dvc_dir, self.CONFIG_LOCAL)
+    def __init__(self, dvc_dir=None, validate=True):
+        self.system_config_file = os.path.join(
+            self.get_system_config_dir(), self.CONFIG
+        )
+        self.global_config_file = os.path.join(
+            self.get_global_config_dir(), self.CONFIG
+        )
 
+        if dvc_dir is not None:
+            self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
+            self.config_file = os.path.join(dvc_dir, self.CONFIG)
+            self.config_local_file = os.path.join(dvc_dir, self.CONFIG_LOCAL)
+        else:
+            self.dvc_dir = None
+            self.config_file = None
+            self.config_local_file = None
+
+        self._system_config = None
+        self._global_config = None
+        self._repo_config = None
+        self._local_config = None
+
+        self.config = None
+
+        self.load(validate=validate)
+
+    @staticmethod
+    def get_global_config_dir():
+        """Returns global config location. E.g. ~/.config/dvc/config.
+
+        Returns:
+            str: path to the global config directory.
+        """
+        from appdirs import user_config_dir
+
+        return user_config_dir(
+            appname=Config.APPNAME, appauthor=Config.APPAUTHOR
+        )
+
+    @staticmethod
+    def get_system_config_dir():
+        """Returns system config location. E.g. /etc/dvc.conf.
+
+        Returns:
+            str: path to the system config directory.
+        """
+        from appdirs import site_config_dir
+
+        return site_config_dir(
+            appname=Config.APPNAME, appauthor=Config.APPAUTHOR
+        )
+
+    @staticmethod
+    def init(dvc_dir):
+        """Initializes dvc config.
+
+        Args:
+            dvc_dir (str): path to .dvc directory.
+
+        Returns:
+            dvc.config.Config: config object.
+        """
+        config_file = os.path.join(dvc_dir, Config.CONFIG)
+        open(config_file, "w+").close()
+        return Config(dvc_dir)
+
+    def _load(self):
+        self._system_config = configobj.ConfigObj(self.system_config_file)
+        self._global_config = configobj.ConfigObj(self.global_config_file)
+
+        if self.config_file is not None:
+            self._repo_config = configobj.ConfigObj(self.config_file)
+        else:
+            self._repo_config = configobj.ConfigObj()
+
+        if self.config_local_file is not None:
+            self._local_config = configobj.ConfigObj(self.config_local_file)
+        else:
+            self._local_config = configobj.ConfigObj()
+
+        self.config = None
+
+    def _load_config(self, path):
+        config = configobj.ConfigObj(path)
+        config = self._lower(config)
+        self._resolve_paths(config, path)
+        return config
+
+    @staticmethod
+    def _resolve_path(path, config_file):
+        assert os.path.isabs(config_file)
+        config_dir = os.path.dirname(config_file)
+        return os.path.abspath(os.path.join(config_dir, path))
+
+    def _resolve_cache_path(self, config, fname):
+        cache = config.get(self.SECTION_CACHE)
+        if cache is None:
+            return
+
+        cache_dir = cache.get(self.SECTION_CACHE_DIR)
+        if cache_dir is None:
+            return
+
+        cache[self.PRIVATE_CWD] = os.path.dirname(fname)
+
+    def _resolve_paths(self, config, fname):
+        if fname is None:
+            return
+
+        self._resolve_cache_path(config, fname)
+        for section in config.values():
+            if self.SECTION_REMOTE_URL not in section.keys():
+                continue
+
+            section[self.PRIVATE_CWD] = os.path.dirname(fname)
+
+    def load(self, validate=True):
+        """Loads config from all the config files.
+
+        Args:
+            validate (bool): optional flag to tell dvc if it should validate
+                the config or just load it as is. 'True' by default.
+
+
+        Raises:
+            dvc.config.ConfigError: thrown if config has invalid format.
+        """
+        self._load()
         try:
-            self._config = configobj.ConfigObj(self.config_file)
-            local = configobj.ConfigObj(self.config_local_file)
+            self.config = self._load_config(self.system_config_file)
+            user = self._load_config(self.global_config_file)
+            config = self._load_config(self.config_file)
+            local = self._load_config(self.config_local_file)
 
             # NOTE: schema doesn't support ConfigObj.Section validation, so we
             # need to convert our config to dict before passing it to
-            self._config = self._lower(self._config)
-            local = self._lower(local)
-            self._config.update(local)
+            for conf in [user, config, local]:
+                self.config = self._merge(self.config, conf)
 
-            self._config = Schema(self.SCHEMA).validate(self._config)
+            if validate:
+                self.config = Schema(self.SCHEMA).validate(self.config)
 
             # NOTE: now converting back to ConfigObj
-            self._config = configobj.ConfigObj(self._config,
-                                               write_empty_values=True)
-            self._config.filename = self.config_file
+            self.config = configobj.ConfigObj(
+                self.config, write_empty_values=True
+            )
+            self.config.filename = self.config_file
+            self._resolve_paths(self.config, self.config_file)
         except Exception as ex:
             raise ConfigError(ex)
 
+    @staticmethod
+    def _get_key(conf, name, add=False):
+        for k in conf.keys():
+            if k.lower() == name.lower():
+                return k
+
+        if add:
+            conf[name] = {}
+            return name
+
+        return None
+
+    def save(self, config=None):
+        """Saves config to config files.
+
+        Args:
+            config (configobj.ConfigObj): optional config object to save.
+
+        Raises:
+            dvc.config.ConfigError: thrown if failed to write config file.
+        """
+        if config is not None:
+            clist = [config]
+        else:
+            clist = [
+                self._system_config,
+                self._global_config,
+                self._repo_config,
+                self._local_config,
+            ]
+
+        for conf in clist:
+            if conf.filename is None:
+                continue
+
+            try:
+                logger.debug("Writing '{}'.".format(conf.filename))
+                dname = os.path.dirname(os.path.abspath(conf.filename))
+                try:
+                    os.makedirs(dname)
+                except OSError as exc:
+                    if exc.errno != errno.EEXIST:
+                        raise
+                conf.write()
+            except Exception as exc:
+                msg = "failed to write config '{}'".format(conf.filename)
+                raise ConfigError(msg, exc)
+
+    @staticmethod
+    def unset(config, section, opt=None):
+        """Unsets specified option and/or section in the config.
+
+        Args:
+            config (configobj.ConfigObj): config to work on.
+            section (str): section name.
+            opt (str): optional option name.
+        """
+        if section not in config.keys():
+            raise ConfigError("section '{}' doesn't exist".format(section))
+
+        if opt is None:
+            del config[section]
+            return
+
+        if opt not in config[section].keys():
+            raise ConfigError(
+                "option '{}.{}' doesn't exist".format(section, opt)
+            )
+        del config[section][opt]
+
+        if not config[section]:
+            del config[section]
+
+    @staticmethod
+    def set(config, section, opt, value):
+        """Sets specified option in the config.
+
+        Args:
+            config (configobj.ConfigObj): config to work on.
+            section (str): section name.
+            opt (str): option name.
+            value: value to set option to.
+        """
+        if section not in config.keys():
+            config[section] = {}
+
+        config[section][opt] = value
+
+    @staticmethod
+    def show(config, section, opt):
+        """Prints option value from the config.
+
+        Args:
+            config (configobj.ConfigObj): config to work on.
+            section (str): section name.
+            opt (str): option name.
+        """
+        if section not in config.keys():
+            raise ConfigError("section '{}' doesn't exist".format(section))
+
+        if opt not in config[section].keys():
+            raise ConfigError(
+                "option '{}.{}' doesn't exist".format(section, opt)
+            )
+
+        logger.info(config[section][opt])
+
+    @staticmethod
+    def _merge(first, second):
+        res = {}
+        sections = list(first.keys()) + list(second.keys())
+        for section in sections:
+            first_copy = first.get(section, {}).copy()
+            second_copy = second.get(section, {}).copy()
+            first_copy.update(second_copy)
+            res[section] = first_copy
+        return res
+
     @staticmethod
     def _lower(config):
         new_config = {}
         for s_key, s_value in config.items():
             new_s = {}
             for key, value in s_value.items():
-                new_s[key.lower()] = value
+                new_s[key.lower()] = str(value)
             new_config[s_key.lower()] = new_s
         return new_config
-
-    @staticmethod
-    def init(dvc_dir):
-        config_file = os.path.join(dvc_dir, Config.CONFIG)
-        open(config_file, 'w+').close()
-        return Config(dvc_dir)

dvc/daemon.py

@@ -0,0 +1,95 @@
+"""Launch `dvc daemon` command in a separate detached process."""
+
+from __future__ import unicode_literals
+
+import os
+import sys
+import inspect
+from subprocess import Popen
+
+import dvc.logger as logger
+from dvc.utils import is_binary, fix_env
+from dvc.utils.compat import cast_bytes_py2
+
+
+CREATE_NEW_PROCESS_GROUP = 0x00000200
+DETACHED_PROCESS = 0x00000008
+
+
+def _spawn_windows(cmd, env):
+    from subprocess import STARTUPINFO, STARTF_USESHOWWINDOW
+
+    creationflags = CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS
+
+    startupinfo = STARTUPINFO()
+    startupinfo.dwFlags |= STARTF_USESHOWWINDOW
+
+    Popen(
+        cmd,
+        env=env,
+        close_fds=True,
+        shell=False,
+        creationflags=creationflags,
+        startupinfo=startupinfo,
+    ).communicate()
+
+
+def _spawn_posix(cmd, env):
+    # NOTE: using os._exit instead of sys.exit, because dvc built
+    # with PyInstaller has trouble with SystemExit exeption and throws
+    # errors such as "[26338] Failed to execute script __main__"
+    try:
+        pid = os.fork()
+        if pid > 0:
+            return
+    except OSError:
+        logger.error("failed at first fork")
+        os._exit(1)  # pylint: disable=protected-access
+
+    os.setsid()
+    os.umask(0)
+
+    try:
+        pid = os.fork()
+        if pid > 0:
+            os._exit(0)  # pylint: disable=protected-access
+    except OSError:
+        logger.error("failed at second fork")
+        os._exit(1)  # pylint: disable=protected-access
+
+    sys.stdin.close()
+    sys.stdout.close()
+    sys.stderr.close()
+
+    Popen(cmd, env=env, close_fds=True, shell=False).communicate()
+
+    os._exit(0)  # pylint: disable=protected-access
+
+
+def daemon(args):
+    """Launch a `dvc daemon` command in a detached process.
+
+    Args:
+        args (list): list of arguments to append to `dvc daemon` command.
+    """
+    cmd = [sys.executable]
+    if not is_binary():
+        cmd += ["-m", "dvc"]
+    cmd += ["daemon", "-q"] + args
+
+    env = fix_env()
+    file_path = os.path.abspath(inspect.stack()[0][1])
+    env[cast_bytes_py2("PYTHONPATH")] = cast_bytes_py2(
+        os.path.dirname(os.path.dirname(file_path))
+    )
+
+    logger.debug("Trying to spawn '{}' with env '{}'".format(cmd, env))
+
+    if os.name == "nt":
+        _spawn_windows(cmd, env)
+    elif os.name == "posix":
+        _spawn_posix(cmd, env)
+    else:
+        raise NotImplementedError
+
+    logger.debug("Spawned '{}'".format(cmd))

dvc/dagascii.py

@@ -0,0 +1,370 @@
+"""Draws DAG in ASCII."""
+
+from __future__ import unicode_literals
+from __future__ import print_function
+
+import sys
+import math
+
+from grandalf.graphs import Vertex, Edge, Graph
+from grandalf.layouts import SugiyamaLayout
+from grandalf.routing import route_with_lines, EdgeViewer
+
+
+class VertexViewer(object):
+    """Class to define vertex box boundaries that will be accounted for during
+    graph building by grandalf.
+
+    Args:
+        name (str): name of the vertex.
+    """
+
+    HEIGHT = 3  # top and bottom box edges + text
+
+    def __init__(self, name):
+        # pylint: disable=invalid-name
+        self._h = self.HEIGHT  # top and bottom box edges + text
+        self._w = len(name) + 2  # right and left bottom edges + text
+
+    @property
+    def h(self):  # pylint: disable=invalid-name
+        """Height of the box."""
+        return self._h
+
+    @property
+    def w(self):  # pylint: disable=invalid-name
+        """Width of the box."""
+        return self._w
+
+
+class AsciiCanvas(object):
+    """Class for drawing in ASCII.
+
+    Args:
+        cols (int): number of columns in the canvas. Should be > 1.
+        lines (int): number of lines in the canvas. Should be > 1.
+    """
+
+    TIMEOUT = 10
+
+    def __init__(self, cols, lines):
+        assert cols > 1
+        assert lines > 1
+
+        self.cols = cols
+        self.lines = lines
+
+        self.canvas = [[" "] * cols for l in range(lines)]
+
+    def draw(self):
+        """Draws ASCII canvas on the screen."""
+        if sys.stdout.isatty():  # pragma: no cover
+            from asciimatics.screen import Screen
+
+            Screen.wrapper(self._do_draw)
+        else:
+            for line in self.canvas:
+                print("".join(line))
+
+    def _do_draw(self, screen):  # pragma: no cover
+        # pylint: disable=too-many-locals
+        # pylint: disable=too-many-branches, too-many-statements
+        from dvc.system import System
+        from asciimatics.event import KeyboardEvent
+
+        offset_x = 0
+        offset_y = 0
+        smaxrow, smaxcol = screen.dimensions
+        assert smaxrow > 1
+        assert smaxcol > 1
+        smaxrow -= 1
+        smaxcol -= 1
+
+        if self.lines + 1 > smaxrow:
+            max_y = self.lines + 1 - smaxrow
+        else:
+            max_y = 0
+
+        if self.cols + 1 > smaxcol:
+            max_x = self.cols + 1 - smaxcol
+        else:
+            max_x = 0
+
+        while True:
+            for y in range(smaxrow + 1):
+                y_index = offset_y + y
+                line = []
+                for x in range(smaxcol + 1):
+                    x_index = offset_x + x
+                    if (
+                        len(self.canvas) > y_index
+                        and len(self.canvas[y_index]) > x_index
+                    ):
+                        line.append(self.canvas[y_index][x_index])
+                    else:
+                        line.append(" ")
+                assert len(line) == (smaxcol + 1)
+                screen.print_at("".join(line), 0, y)
+
+            screen.refresh()
+
+            # NOTE: get_event() doesn't block by itself,
+            # so we have to do the blocking ourselves.
+            #
+            # NOTE: using this workaround while waiting for PR [1]
+            # to get merged and released. After that need to adjust
+            # asciimatics version requirements.
+            #
+            # [1] https://github.com/peterbrittain/asciimatics/pull/188
+            System.wait_for_input(self.TIMEOUT)
+
+            event = screen.get_event()
+            if not isinstance(event, KeyboardEvent):
+                continue
+
+            k = event.key_code
+            if k == screen.KEY_DOWN or k == ord("s"):
+                offset_y += 1
+            elif k == screen.KEY_PAGE_DOWN or k == ord("S"):
+                offset_y += smaxrow
+            elif k == screen.KEY_UP or k == ord("w"):
+                offset_y -= 1
+            elif k == screen.KEY_PAGE_UP or k == ord("W"):
+                offset_y -= smaxrow
+            elif k == screen.KEY_RIGHT or k == ord("d"):
+                offset_x += 1
+            elif k == ord("D"):
+                offset_x += smaxcol
+            elif k == screen.KEY_LEFT or k == ord("a"):
+                offset_x -= 1
+            elif k == ord("A"):
+                offset_x -= smaxcol
+            elif k == ord("q") or k == ord("Q"):
+                break
+
+            if offset_y > max_y:
+                offset_y = max_y
+            elif offset_y < 0:
+                offset_y = 0
+
+            if offset_x > max_x:
+                offset_x = max_x
+            elif offset_x < 0:
+                offset_x = 0
+
+    def point(self, x, y, char):
+        """Create a point on ASCII canvas.
+
+        Args:
+            x (int): x coordinate. Should be >= 0 and < number of columns in
+                the canvas.
+            y (int): y coordinate. Should be >= 0 an < number of lines in the
+                canvas.
+            char (str): character to place in the specified point on the
+                canvas.
+        """
+        assert len(char) == 1
+        assert x >= 0
+        assert x < self.cols
+        assert y >= 0
+        assert y < self.lines
+
+        self.canvas[y][x] = char
+
+    def line(self, x0, y0, x1, y1, char):
+        """Create a line on ASCII canvas.
+
+        Args:
+            x0 (int): x coordinate where the line should start.
+            y0 (int): y coordinate where the line should start.
+            x1 (int): x coordinate where the line should end.
+            y1 (int): y coordinate where the line should end.
+            char (str): character to draw the line with.
+        """
+        # pylint: disable=too-many-arguments, too-many-branches
+        if x0 > x1:
+            x1, x0 = x0, x1
+            y1, y0 = y0, y1
+
+        dx = x1 - x0
+        dy = y1 - y0
+
+        if dx == 0 and dy == 0:
+            self.point(x0, y0, char)
+        elif abs(dx) >= abs(dy):
+            for x in range(x0, x1 + 1):
+                if dx == 0:
+                    y = y0
+                else:
+                    y = y0 + int(round((x - x0) * dy / float((dx))))
+                self.point(x, y, char)
+        elif y0 < y1:
+            for y in range(y0, y1 + 1):
+                if dy == 0:
+                    x = x0
+                else:
+                    x = x0 + int(round((y - y0) * dx / float((dy))))
+                self.point(x, y, char)
+        else:
+            for y in range(y1, y0 + 1):
+                if dy == 0:
+                    x = x0
+                else:
+                    x = x1 + int(round((y - y1) * dx / float((dy))))
+                self.point(x, y, char)
+
+    def text(self, x, y, text):
+        """Print a text on ASCII canvas.
+
+        Args:
+            x (int): x coordinate where the text should start.
+            y (int): y coordinate where the text should start.
+            text (str): string that should be printed.
+        """
+        for i, char in enumerate(text):
+            self.point(x + i, y, char)
+
+    def box(self, x0, y0, width, height):
+        """Create a box on ASCII canvas.
+
+        Args:
+            x0 (int): x coordinate of the box corner.
+            y0 (int): y coordinate of the box corner.
+            width (int): box width.
+            height (int): box height.
+        """
+        assert width > 1
+        assert height > 1
+
+        width -= 1
+        height -= 1
+
+        for x in range(x0, x0 + width):
+            self.point(x, y0, "-")
+            self.point(x, y0 + height, "-")
+
+        for y in range(y0, y0 + height):
+            self.point(x0, y, "|")
+            self.point(x0 + width, y, "|")
+
+        self.point(x0, y0, "+")
+        self.point(x0 + width, y0, "+")
+        self.point(x0, y0 + height, "+")
+        self.point(x0 + width, y0 + height, "+")
+
+
+def _build_sugiyama_layout(vertexes, edges):
+    #
+    # Just a reminder about naming conventions:
+    # +------------X
+    # |
+    # |
+    # |
+    # |
+    # Y
+    #
+
+    vertexes = {v: Vertex(" {} ".format(v)) for v in vertexes}
+    # NOTE: reverting edges to correctly orientate the graph
+    edges = [Edge(vertexes[e], vertexes[s]) for s, e in edges]
+    vertexes = vertexes.values()
+    graph = Graph(vertexes, edges)
+
+    for vertex in vertexes:
+        vertex.view = VertexViewer(vertex.data)
+
+    # NOTE: determine min box length to create the best layout
+    minw = min([v.view.w for v in vertexes])
+
+    for edge in edges:
+        edge.view = EdgeViewer()
+
+    sug = SugiyamaLayout(graph.C[0])
+    graph = graph.C[0]
+    roots = list(filter(lambda x: len(x.e_in()) == 0, graph.sV))
+
+    sug.init_all(roots=roots, optimize=True)
+
+    sug.yspace = VertexViewer.HEIGHT
+    sug.xspace = minw
+    sug.route_edge = route_with_lines
+
+    sug.draw()
+
+    return sug
+
+
+def draw(vertexes, edges):
+    """Build a DAG and draw it in ASCII.
+
+    Args:
+        vertexes (list): list of graph vertexes.
+        edges (list): list of graph edges.
+    """
+    # pylint: disable=too-many-locals
+    # NOTE: coordinates might me negative, so we need to shift
+    # everything to the positive plane before we actually draw it.
+    Xs = []  # pylint: disable=invalid-name
+    Ys = []  # pylint: disable=invalid-name
+
+    sug = _build_sugiyama_layout(vertexes, edges)
+
+    for vertex in sug.g.sV:
+        # NOTE: moving boxes w/2 to the left
+        Xs.append(vertex.view.xy[0] - vertex.view.w / 2.0)
+        Xs.append(vertex.view.xy[0] + vertex.view.w / 2.0)
+        Ys.append(vertex.view.xy[1])
+        Ys.append(vertex.view.xy[1] + vertex.view.h)
+
+    for edge in sug.g.sE:
+        for x, y in edge.view._pts:  # pylint: disable=protected-access
+            Xs.append(x)
+            Ys.append(y)
+
+    minx = min(Xs)
+    miny = min(Ys)
+    maxx = max(Xs)
+    maxy = max(Ys)
+
+    canvas_cols = int(math.ceil(math.ceil(maxx) - math.floor(minx))) + 1
+    canvas_lines = int(round(maxy - miny))
+
+    canvas = AsciiCanvas(canvas_cols, canvas_lines)
+
+    # NOTE: first draw edges so that node boxes could overwrite them
+    for edge in sug.g.sE:
+        # pylint: disable=protected-access
+        assert len(edge.view._pts) > 1
+        for index in range(1, len(edge.view._pts)):
+            start = edge.view._pts[index - 1]
+            end = edge.view._pts[index]
+
+            start_x = int(round(start[0] - minx))
+            start_y = int(round(start[1] - miny))
+            end_x = int(round(end[0] - minx))
+            end_y = int(round(end[1] - miny))
+
+            assert start_x >= 0
+            assert start_y >= 0
+            assert end_x >= 0
+            assert end_y >= 0
+
+            canvas.line(start_x, start_y, end_x, end_y, "*")
+
+    for vertex in sug.g.sV:
+        # NOTE: moving boxes w/2 to the left
+        x = vertex.view.xy[0] - vertex.view.w / 2.0
+        y = vertex.view.xy[1]
+
+        canvas.box(
+            int(round(x - minx)),
+            int(round(y - miny)),
+            vertex.view.w,
+            vertex.view.h,
+        )
+
+        canvas.text(
+            int(round(x - minx)) + 1, int(round(y - miny)) + 1, vertex.data
+        )
+
+    canvas.draw()

dvc/data_cloud.py

@@ -1,122 +1,165 @@
-from dvc.logger import Logger
+"""Manages dvc remotes that user can use with push/pull/status comamnds."""
+
+from __future__ import unicode_literals
+
+import dvc.logger as logger
 from dvc.config import Config, ConfigError
 
+from dvc.remote import Remote
 from dvc.remote.s3 import RemoteS3
 from dvc.remote.gs import RemoteGS
 from dvc.remote.azure import RemoteAzure
 from dvc.remote.ssh import RemoteSSH
 from dvc.remote.hdfs import RemoteHDFS
 from dvc.remote.local import RemoteLOCAL
+from dvc.remote.http import RemoteHTTP
 
 
 class DataCloud(object):
+    """Class that manages dvc remotes.
+
+    Args:
+        repo (dvc.repo.Repo): repo instance that belongs to the repo that
+            we are working on.
+        config (dict): config of the repo.
+
+    Raises:
+        config.ConfigError: thrown when config has invalid format.
+    """
+
     CLOUD_MAP = {
-        'aws': RemoteS3,
-        'gcp': RemoteGS,
-        'azure': RemoteAzure,
-        'ssh': RemoteSSH,
-        'hdfs': RemoteHDFS,
-        'local': RemoteLOCAL,
+        "aws": RemoteS3,
+        "gcp": RemoteGS,
+        "azure": RemoteAzure,
+        "ssh": RemoteSSH,
+        "hdfs": RemoteHDFS,
+        "local": RemoteLOCAL,
+        "http": RemoteHTTP,
+        "https": RemoteHTTP,
     }
 
-    def __init__(self, project, config=None):
-        self.project = project
+    def __init__(self, repo, config=None):
+        self.repo = repo
         self._config = config
         self._core = self._config[Config.SECTION_CORE]
 
-        remote = self._core.get(Config.SECTION_CORE_REMOTE, '')
-        if remote == '':
-            if self._core.get(Config.SECTION_CORE_CLOUD, None):
-                # backward compatibility
-                msg = 'Using obsoleted config format. Consider updating.'
-                Logger.warn(msg)
-                self._cloud = self.__init__compat()
-            else:
-                self._cloud = None
-            return
+    @property
+    def _cloud(self):
+        remote = self._core.get(Config.SECTION_CORE_REMOTE, "")
+        if remote != "":
+            return self._init_remote(remote)
 
-        self._cloud = self._init_remote(remote)
+        if self._core.get(Config.SECTION_CORE_CLOUD, None):
+            # backward compatibility
+            msg = "using obsoleted config format. Consider updating."
+            logger.warning(msg)
+            return self._init_compat()
 
-    @staticmethod
-    def supported(config):
-        for cloud in DataCloud.CLOUD_MAP.values():
-            if cloud.supported(config):
-                return cloud
         return None
 
     def _init_remote(self, remote):
-        section = Config.SECTION_REMOTE_FMT.format(remote)
+        section = Config.SECTION_REMOTE_FMT.format(remote).lower()
         cloud_config = self._config.get(section, None)
         if not cloud_config:
-            msg = "Can't find remote section '{}' in config"
+            msg = "can't find remote section '{}' in config"
             raise ConfigError(msg.format(section))
 
-        cloud_type = self.supported(cloud_config)
-        if not cloud_type:
-            raise ConfigError("Unsupported cloud '{}'".format(cloud_config))
+        return Remote(self.repo, cloud_config)
 
-        return self._init_cloud(cloud_config, cloud_type)
-
-    def __init__compat(self):
-        name = self._core.get(Config.SECTION_CORE_CLOUD, '').strip().lower()
-        if name == '':
+    def _init_compat(self):
+        name = self._core.get(Config.SECTION_CORE_CLOUD, "").strip().lower()
+        if name == "":
             self._cloud = None
-            return
+            return None
 
         cloud_type = self.CLOUD_MAP.get(name, None)
         if not cloud_type:
-            msg = "Wrong cloud type {} specified"
-            raise ConfigError(msg.format(name))
+            msg = "wrong cloud type '{}' specified".format(name)
+            raise ConfigError(msg)
 
         cloud_config = self._config.get(name, None)
         if not cloud_config:
-            msg = "Can't find cloud section '{}' in config"
-            raise ConfigError(msg.format(name))
+            msg = "can't find cloud section '{}' in config".format(name)
+            raise ConfigError(msg)
+
+        # NOTE: check if the class itself has everything needed for operation.
+        # E.g. all the imported packages.
+        if not cloud_type.supported(cloud_type.compat_config(cloud_config)):
+            raise ConfigError("unsupported cloud '{}'".format(name))
 
         return self._init_cloud(cloud_config, cloud_type)
 
     def _init_cloud(self, cloud_config, cloud_type):
         global_storage_path = self._core.get(Config.SECTION_CORE_STORAGEPATH)
         if global_storage_path:
-            Logger.warn('Using obsoleted config format. Consider updating.')
+            logger.warning("using obsoleted config format. Consider updating.")
 
-        cloud = cloud_type(self.project, cloud_config)
+        cloud = cloud_type(self.repo, cloud_config)
         return cloud
 
-    def _get_cloud(self, remote):
+    def _get_cloud(self, remote, cmd):
         if remote:
             return self._init_remote(remote)
 
         if self._cloud:
             return self._cloud
 
-        msg = "No remote repository specified. Setup default repository " \
-              "with 'dvc config core.remote <name>' or use '-r <name>'."
-        raise ConfigError(msg)
+        raise ConfigError(
+            "No remote repository specified. Setup default repository with\n"
+            "    dvc config core.remote <name>\n"
+            "or use:\n"
+            "    dvc {} -r <name>\n".format(cmd)
+        )
 
-    def push(self, targets, jobs=1, remote=None, show_checksums=False):
-        """
-        Push data items in a cloud-agnostic way.
-        """
-        return self.project.cache.local.push(targets,
-                                             jobs=jobs,
-                                             remote=self._get_cloud(remote),
-                                             show_checksums=show_checksums)
+    def push(self, targets, jobs=None, remote=None, show_checksums=False):
+        """Push data items in a cloud-agnostic way.
 
-    def pull(self, targets, jobs=1, remote=None, show_checksums=False):
+        Args:
+            targets (list): list of targets to push to the cloud.
+            jobs (int): number of jobs that can be running simultaneously.
+            remote (dvc.remote.base.RemoteBase): optional remote to push to.
+                By default remote from core.remote config option is used.
+            show_checksums (bool): show checksums instead of file names in
+                information messages.
         """
-        Pull data items in a cloud-agnostic way.
-        """
-        return self.project.cache.local.pull(targets,
-                                             jobs=jobs,
-                                             remote=self._get_cloud(remote),
-                                             show_checksums=show_checksums)
+        return self.repo.cache.local.push(
+            targets,
+            jobs=jobs,
+            remote=self._get_cloud(remote, "push"),
+            show_checksums=show_checksums,
+        )
 
-    def status(self, targets, jobs=1, remote=None, show_checksums=False):
+    def pull(self, targets, jobs=None, remote=None, show_checksums=False):
+        """Pull data items in a cloud-agnostic way.
+
+        Args:
+            targets (list): list of targets to pull from the cloud.
+            jobs (int): number of jobs that can be running simultaneously.
+            remote (dvc.remote.base.RemoteBase): optional remote to pull from.
+                By default remote from core.remote config option is used.
+            show_checksums (bool): show checksums instead of file names in
+                information messages.
         """
-        Check status of data items in a cloud-agnostic way.
+        return self.repo.cache.local.pull(
+            targets,
+            jobs=jobs,
+            remote=self._get_cloud(remote, "pull"),
+            show_checksums=show_checksums,
+        )
+
+    def status(self, targets, jobs=None, remote=None, show_checksums=False):
+        """Check status of data items in a cloud-agnostic way.
+
+        Args:
+            targets (list): list of targets to check status for.
+            jobs (int): number of jobs that can be running simultaneously.
+            remote (dvc.remote.base.RemoteBase): optional remote to compare
+                targets to. By default remote from core.remote config option
+                is used.
+            show_checksums (bool): show checksums instead of file names in
+                information messages.
         """
-        return self.project.cache.local.status(targets,
-                                               jobs=jobs,
-                                               remote=self._get_cloud(remote),
-                                               show_checksums=show_checksums)
+        cloud = self._get_cloud(remote, "status")
+        return self.repo.cache.local.status(
+            targets, jobs=jobs, remote=cloud, show_checksums=show_checksums
+        )

dvc/dependency/__init__.py

@@ -1,63 +1,68 @@
+from __future__ import unicode_literals
+
 import schema
 
-try:
-    from urlparse import urlparse
-except ImportError:
-    from urllib.parse import urlparse
-
-from dvc.exceptions import DvcException
 from dvc.config import Config
+from dvc.utils.compat import urlparse
 
-from dvc.dependency.base import DependencyBase
+import dvc.output as output
+from dvc.output.base import OutputBase
 from dvc.dependency.s3 import DependencyS3
 from dvc.dependency.gs import DependencyGS
 from dvc.dependency.local import DependencyLOCAL
 from dvc.dependency.hdfs import DependencyHDFS
 from dvc.dependency.ssh import DependencySSH
+from dvc.dependency.http import DependencyHTTP
 
 from dvc.remote import Remote
-from dvc.remote.local import RemoteLOCAL
-from dvc.remote.s3 import RemoteS3
-from dvc.remote.hdfs import RemoteHDFS
 
-DEPS = [DependencyHDFS,
-        DependencyS3,
-        DependencyGS,
-        DependencySSH,
-        DependencyLOCAL]
+DEPS = [
+    DependencyGS,
+    DependencyHDFS,
+    DependencyHTTP,
+    DependencyS3,
+    DependencySSH,
+    # NOTE: DependencyLOCAL is the default choice
+]
 
-DEP_MAP = {'': DependencyLOCAL,
-           'ssh': DependencySSH,
-           's3': DependencyS3,
-           'gs': DependencyGS,
-           'hdfs': DependencyHDFS}
-
-SCHEMA = {
-    DependencyBase.PARAM_PATH: str,
-    schema.Optional(RemoteLOCAL.PARAM_MD5): schema.Or(str, None),
-    schema.Optional(RemoteS3.PARAM_ETAG): schema.Or(str, None),
-    schema.Optional(RemoteHDFS.PARAM_CHECKSUM): schema.Or(str, None),
+DEP_MAP = {
+    "local": DependencyLOCAL,
+    "ssh": DependencySSH,
+    "s3": DependencyS3,
+    "gs": DependencyGS,
+    "hdfs": DependencyHDFS,
+    "http": DependencyHTTP,
+    "https": DependencyHTTP,
 }
 
 
+# NOTE: schema for dependencies is basically the same as for outputs, but
+# without output-specific entries like 'cache' (whether or not output is
+# cached, see -o and -O flags for `dvc run`) and 'metric' (whether or not
+# output is a metric file and how to parse it, see `-M` flag for `dvc run`).
+SCHEMA = output.SCHEMA.copy()
+del SCHEMA[schema.Optional(OutputBase.PARAM_CACHE)]
+del SCHEMA[schema.Optional(OutputBase.PARAM_METRIC)]
+
+
 def _get(stage, p, info):
     parsed = urlparse(p)
-    if parsed.scheme == 'remote':
+    if parsed.scheme == "remote":
         name = Config.SECTION_REMOTE_FMT.format(parsed.netloc)
-        sect = stage.project.config._config[name]
-        remote = Remote(stage.project, sect)
+        sect = stage.repo.config.config[name]
+        remote = Remote(stage.repo, sect)
         return DEP_MAP[remote.scheme](stage, p, info, remote=remote)
 
     for d in DEPS:
         if d.supported(p):
             return d(stage, p, info)
-    raise DvcException('Dependency \'{}\' is not supported'.format(p))
+    return DependencyLOCAL(stage, p, info)
 
 
 def loadd_from(stage, d_list):
     ret = []
     for d in d_list:
-        p = d.pop(DependencyBase.PARAM_PATH)
+        p = d.pop(OutputBase.PARAM_PATH)
         ret.append(_get(stage, p, d))
     return ret
 

dvc/dependency/base.py

@@ -1,72 +1,22 @@
-import re
+from __future__ import unicode_literals
 
 from dvc.exceptions import DvcException
 
 
-class DependencyError(DvcException):
-    def __init__(self, path, msg):
-        msg = 'Dependency \'{}\' error: {}'
-        super(DependencyError, self).__init__(msg.format(path, msg))
-
-
-class DependencyDoesNotExistError(DependencyError):
+class DependencyDoesNotExistError(DvcException):
     def __init__(self, path):
-        msg = 'does not exist'
-        super(DependencyDoesNotExistError, self).__init__(path, msg)
+        msg = "dependency '{}' does not exist".format(path)
+        super(DependencyDoesNotExistError, self).__init__(msg)
 
 
-class DependencyIsNotFileOrDirError(DependencyError):
+class DependencyIsNotFileOrDirError(DvcException):
     def __init__(self, path):
-        msg = 'not a file or directory'
-        super(DependencyIsNotFileOrDirError, self).__init__(path, msg)
+        msg = "dependency '{}' is not a file or directory".format(path)
+        super(DependencyIsNotFileOrDirError, self).__init__(msg)
 
 
 class DependencyBase(object):
-    REGEX = None
+    IS_DEPENDENCY = True
 
-    PARAM_PATH = 'path'
-
-    def __init__(self, stage, path):
-        self.stage = stage
-        self.project = stage.project
-        self.path = path
-
-    @classmethod
-    def match(cls, url):
-        return re.match(cls.REGEX, url)
-
-    def group(self, name):
-        match = self.match(self.path)
-        if not match:
-            return None
-        return match.group(name)
-
-    @classmethod
-    def supported(cls, url):
-        return cls.match(url) is not None
-
-    @property
-    def sep(self):
-        return '/'
-
-    @property
-    def exists(self):
-        return self.remote.exists([self.path_info])
-
-    def changed(self):
-        raise NotImplementedError
-
-    def status(self):
-        if self.changed():
-            # FIXME better msgs
-            return {self.rel_path: 'changed'}
-        return {}
-
-    def save(self):
-        raise NotImplementedError
-
-    def dumpd(self):
-        return {self.PARAM_PATH: self.path}
-
-    def download(self, to_info):
-        self.remote.download([self.path_info], [to_info])
+    DoesNotExistError = DependencyDoesNotExistError
+    IsNotFileOrDirError = DependencyIsNotFileOrDirError

dvc/dependency/gs.py

@@ -1,25 +1,8 @@
-import posixpath
+from __future__ import unicode_literals
 
-try:
-    from urlparse import urlparse
-except ImportError:
-    from urllib.parse import urlparse
-
-from dvc.dependency.s3 import DependencyS3
-from dvc.remote.gs import RemoteGS
+from dvc.output.gs import OutputGS
+from dvc.dependency.base import DependencyBase
 
 
-class DependencyGS(DependencyS3):
-    REGEX = RemoteGS.REGEX
-
-    def __init__(self, stage, path, info=None, remote=None):
-        super(DependencyGS, self).__init__(stage, path)
-        self.info = info
-        self.remote = remote if remote else RemoteGS(stage.project, {})
-        bucket = remote.bucket if remote else urlparse(path).netloc
-        key = urlparse(path).path.lstrip('/')
-        if remote:
-            key = posixpath.join(remote.prefix, key)
-        self.path_info = {'scheme': 'gs',
-                          'bucket': bucket,
-                          'key': key}
+class DependencyGS(DependencyBase, OutputGS):
+    pass

dvc/dependency/hdfs.py

@@ -1,35 +1,8 @@
-import posixpath
-
-try:
-    from urlparse import urlparse
-except ImportError:
-    from urllib.parse import urlparse
+from __future__ import unicode_literals
 
+from dvc.output.hdfs import OutputHDFS
 from dvc.dependency.base import DependencyBase
-from dvc.remote.hdfs import RemoteHDFS
 
 
-class DependencyHDFS(DependencyBase):
-    REGEX = RemoteHDFS.REGEX
-
-    def __init__(self, stage, path, info=None, remote=None):
-        super(DependencyHDFS, self).__init__(stage, path)
-        self.info = info
-        self.remote = remote if remote else RemoteHDFS(stage.project, {})
-        if remote:
-            path = posixpath.join(remote.url, urlparse(path).path.lstrip('/'))
-        user = remote.user if remote else self.group('user')
-        self.path_info = {'scheme': 'hdfs',
-                          'user': user,
-                          'url': path}
-
-    def changed(self):
-        return self.info != self.remote.save_info(self.path_info)
-
-    def save(self):
-        self.info = self.remote.save_info(self.path_info)
-
-    def dumpd(self):
-        ret = self.info
-        ret[self.PARAM_PATH] = self.path
-        return ret
+class DependencyHDFS(DependencyBase, OutputHDFS):
+    pass

dvc/dependency/http.py

@@ -0,0 +1,19 @@
+from __future__ import unicode_literals
+
+from dvc.utils.compat import urlparse, urljoin
+from dvc.output.base import OutputBase
+from dvc.remote.http import RemoteHTTP
+from dvc.dependency.base import DependencyBase
+
+
+class DependencyHTTP(DependencyBase, OutputBase):
+    REMOTE = RemoteHTTP
+
+    def __init__(self, stage, path, info=None, remote=None):
+        super(DependencyHTTP, self).__init__(
+            stage, path, info=info, remote=remote
+        )
+        if path.startswith("remote"):
+            path = urljoin(self.remote.cache_dir, urlparse(path).path)
+
+        self.path_info = {"scheme": urlparse(path).scheme, "path": path}

dvc/dependency/local.py

@@ -1,79 +1,8 @@
-import os
-
-try:
-    from urlparse import urlparse
-except ImportError:
-    from urllib.parse import urlparse
+from __future__ import unicode_literals
 
 from dvc.dependency.base import DependencyBase
-from dvc.dependency.base import DependencyDoesNotExistError
-from dvc.dependency.base import DependencyIsNotFileOrDirError
-from dvc.logger import Logger
-from dvc.remote.local import RemoteLOCAL
+from dvc.output.local import OutputLOCAL
 
 
-class DependencyLOCAL(DependencyBase):
-    REGEX = r'^(?P<path>(/+|.:\\+)?.*)$'
-
-    DoesNotExistError = DependencyDoesNotExistError
-    IsNotFileOrDirError = DependencyIsNotFileOrDirError
-
-    def __init__(self, stage, path, info=None, remote=None):
-        self.stage = stage
-        self.project = stage.project
-        self.info = info
-        if remote is not None:
-            self.remote = remote
-        else:
-            self.remote = RemoteLOCAL(stage.project, {})
-
-        if remote:
-            path = os.path.join(remote.prefix, urlparse(path).path.lstrip('/'))
-
-        if not os.path.isabs(path):
-            path = self.remote.ospath(path)
-            path = os.path.join(stage.cwd, path)
-        self.path = os.path.abspath(os.path.normpath(path))
-
-        self.path_info = {'scheme': 'local',
-                          'path': self.path}
-
-    @property
-    def sep(self):
-        return os.sep
-
-    @property
-    def rel_path(self):
-        return os.path.relpath(self.path)
-
-    def changed(self):
-        if not self.exists:
-            return True
-
-        info = self.remote.save_info(self.path_info)
-
-        return self.info != info
-
-    def save(self):
-        if not self.exists:
-            raise self.DoesNotExistError(self.rel_path)
-
-        if not os.path.isfile(self.path) and not os.path.isdir(self.path):
-            raise self.IsNotFileOrDirError(self.rel_path)
-
-        if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \
-           (os.path.isdir(self.path) and len(os.listdir(self.path)) == 0):
-            Logger.warn("File/directory '{}' is empty.".format(self.rel_path))
-
-        self.info = self.remote.save_info(self.path_info)
-
-    def dumpd(self):
-        if self.path.startswith(self.stage.project.root_dir):
-            path = self.remote.unixpath(os.path.relpath(self.path,
-                                                        self.stage.cwd))
-        else:
-            path = self.path
-
-        info = self.info.copy()
-        info[self.PARAM_PATH] = path
-        return info
+class DependencyLOCAL(DependencyBase, OutputLOCAL):
+    pass

dvc/dependency/s3.py

@@ -1,37 +1,8 @@
-import posixpath
-
-try:
-    from urlparse import urlparse
-except ImportError:
-    from urllib.parse import urlparse
+from __future__ import unicode_literals
 
+from dvc.output.s3 import OutputS3
 from dvc.dependency.base import DependencyBase
-from dvc.remote.s3 import RemoteS3
 
 
-class DependencyS3(DependencyBase):
-    REGEX = RemoteS3.REGEX
-
-    def __init__(self, stage, path, info=None, remote=None):
-        super(DependencyS3, self).__init__(stage, path)
-        self.info = info
-        self.remote = remote if remote else RemoteS3(stage.project, {})
-
-        bucket = remote.bucket if remote else urlparse(path).netloc
-        key = urlparse(path).path.lstrip('/')
-        if remote:
-            key = posixpath.join(remote.prefix, key)
-        self.path_info = {'scheme': 's3',
-                          'bucket': bucket,
-                          'key': key}
-
-    def changed(self):
-        return self.info != self.remote.save_info(self.path_info)
-
-    def save(self):
-        self.info = self.remote.save_info(self.path_info)
-
-    def dumpd(self):
-        ret = self.info
-        ret[self.PARAM_PATH] = self.path
-        return ret
+class DependencyS3(DependencyBase, OutputS3):
+    pass

dvc/dependency/ssh.py

@@ -1,42 +1,8 @@
-import posixpath
-
-try:
-    from urlparse import urlparse
-except ImportError:
-    from urllib.parse import urlparse
+from __future__ import unicode_literals
 
+from dvc.output.ssh import OutputSSH
 from dvc.dependency.base import DependencyBase
-from dvc.remote.ssh import RemoteSSH
 
 
-class DependencySSH(DependencyBase):
-    REGEX = RemoteSSH.REGEX
-
-    def __init__(self, stage, path, info=None, remote=None):
-        super(DependencySSH, self).__init__(stage, path)
-        self.info = info
-        self.remote = remote if remote else RemoteSSH(stage.project, {})
-
-        host = remote.host if remote else self.group('host')
-        user = remote.user if remote else self.group('user')
-        if remote:
-            path = posixpath.join(remote.prefix,
-                                  urlparse(path).path.lstrip('/'))
-        else:
-            path = self.match(self.path).group('path')
-
-        self.path_info = {'scheme': 'ssh',
-                          'host': host,
-                          'user': user,
-                          'path': path}
-
-    def changed(self):
-        return self.info != self.remote.save_info(self.path_info)
-
-    def save(self):
-        self.info = self.remote.save_info(self.path_info)
-
-    def dumpd(self):
-        ret = self.info
-        ret[self.PARAM_PATH] = self.path
-        return ret
+class DependencySSH(DependencyBase, OutputSSH):
+    pass

dvc/exceptions.py

@@ -1,29 +1,226 @@
+"""Exceptions raised by the dvc."""
+
+from __future__ import unicode_literals
+
+from dvc.utils.compat import str, builtin_str
+
+import os
 import traceback
 
 
 class DvcException(Exception):
+    """Base class for all dvc exceptions.
+
+    Args:
+        msg (unicode): message for this exception.
+        cause (Exception): optional cause exception.
+    """
+
     def __init__(self, msg, cause=None):
+        # NOTE: unlike python 3, python 2 doesn't have built-in support
+        # for chained exceptions, so we are using our own implementation.
         self.cause = cause
-        self.case_tb = None
+        self.cause_tb = None
         if cause:
             try:
                 self.cause_tb = traceback.format_exc()
-            except Exception:
+            except AttributeError:  # pragma: no cover
                 pass
         super(DvcException, self).__init__(msg)
 
 
-class UnsupportedRemoteError(DvcException):
-    def __init__(self, config):
-        msg = "Remote '{}' is not supported.".format(config)
-        super(UnsupportedRemoteError, self).__init__(msg)
-
-
 class OutputDuplicationError(DvcException):
+    """Thrown if a file/directory is specified as an output in more than one
+    stage.
+
+    Args:
+        output (unicode): path to the file/directory.
+        stages (list): list of paths to stages.
+    """
+
     def __init__(self, output, stages):
-        assert isinstance(output, str)
+        assert isinstance(output, str) or isinstance(output, builtin_str)
         assert isinstance(stages, list)
-        assert all(isinstance(stage, str) for stage in stages)
-        msg = "File '{}' is specified as an output in more than one stage: {}"
-        super(OutputDuplicationError, self).__init__(msg.format(output,
-                                                                str(stages)))
+        assert all(
+            isinstance(stage, str) or isinstance(stage, builtin_str)
+            for stage in stages
+        )
+        msg = (
+            "file/directory '{}' is specified as an output in more than one"
+            "stage: {}"
+        ).format(output, "\n    ".join(stages))
+        super(OutputDuplicationError, self).__init__(msg)
+
+
+class OutputNotFoundError(DvcException):
+    """Thrown if a file/directory not found in repository pipelines.
+
+    Args:
+        output (unicode): path to the file/directory.
+    """
+
+    def __init__(self, output):
+        super(OutputNotFoundError, self).__init__(
+            "unable to find stage file with output '{path}'".format(
+                path=os.path.relpath(output)
+            )
+        )
+
+
+class StagePathAsOutputError(DvcException):
+    """Thrown if directory that stage is going to be saved in is specified as
+    an output of another stage.
+
+    Args:
+        cwd (str): path to the directory.
+        fname (str): path to the stage file that has cwd specified as an
+            output.
+    """
+
+    def __init__(self, wdir, fname):
+        assert isinstance(wdir, str) or isinstance(wdir, builtin_str)
+        assert isinstance(fname, str) or isinstance(fname, builtin_str)
+        msg = (
+            "current working directory '{cwd}' is specified as an output in"
+            " '{fname}'. Use another CWD to prevent any data removal.".format(
+                cwd=wdir, fname=fname
+            )
+        )
+        super(StagePathAsOutputError, self).__init__(msg)
+
+
+class CircularDependencyError(DvcException):
+    """Thrown if a file/directory specified both as an output and as a
+    dependency.
+
+    Args:
+        dependency (str): path to the dependency.
+    """
+
+    def __init__(self, dependency):
+        assert isinstance(dependency, str) or isinstance(
+            dependency, builtin_str
+        )
+
+        msg = (
+            "file/directory '{}' is specified as an output and as a "
+            "dependency."
+        )
+        super(CircularDependencyError, self).__init__(msg.format(dependency))
+
+
+class ArgumentDuplicationError(DvcException):
+    """Thrown if a file/directory is specified as a dependency/output more
+    than once.
+
+    Args:
+        path (str): path to the file/directory.
+    """
+
+    def __init__(self, path):
+        assert isinstance(path, str) or isinstance(path, builtin_str)
+        msg = "file '{}' is specified more than once."
+        super(ArgumentDuplicationError, self).__init__(msg.format(path))
+
+
+class MoveNotDataSourceError(DvcException):
+    """Thrown if attempted to move a file/directory that is not an output
+    in a data source stage.
+
+    Args:
+        path (str): path to the file/directory.
+    """
+
+    def __init__(self, path):
+        msg = (
+            "move is not permitted for stages that are not data sources. "
+            "You need to either move '{path}' to a new location and edit "
+            "it by hand, or remove '{path}' and create a new one at the "
+            "desired location."
+        )
+        super(MoveNotDataSourceError, self).__init__(msg.format(path=path))
+
+
+class NotDvcRepoError(DvcException):
+    """Thrown if a directory is not a dvc repo.
+
+    Args:
+        root (str): path to the directory.
+    """
+
+    def __init__(self, root):
+        msg = "not a dvc repository (checked up to mount point '{}')"
+        super(NotDvcRepoError, self).__init__(msg.format(root))
+
+
+class DvcParserError(DvcException):
+    """Base class for CLI parser errors."""
+
+    def __init__(self):
+        super(DvcParserError, self).__init__("parser error")
+
+
+class CyclicGraphError(DvcException):
+    def __init__(self, stages):
+        assert isinstance(stages, list)
+        stages = "\n".join("\t- {}".format(stage) for stage in stages)
+        msg = (
+            "you've introduced a cycle in your pipeline that involves"
+            " the following stages:"
+            "\n"
+            "{stages}".format(stages=stages)
+        )
+        super(CyclicGraphError, self).__init__(msg)
+
+
+class ConfirmRemoveError(DvcException):
+    def __init__(self, path):
+        super(ConfirmRemoveError, self).__init__(
+            "unable to remove '{}' without a confirmation from the user. Use "
+            "'-f' to force.".format(path)
+        )
+
+
+class InitError(DvcException):
+    def __init__(self, msg):
+        super(InitError, self).__init__(msg)
+
+
+class ReproductionError(DvcException):
+    def __init__(self, dvc_file_name, ex):
+        self.path = dvc_file_name
+        msg = "failed to reproduce '{}'".format(dvc_file_name)
+        super(ReproductionError, self).__init__(msg, cause=ex)
+
+
+class BadMetricError(DvcException):
+    def __init__(self, path):
+        super(BadMetricError, self).__init__(
+            "'{}' does not exist, not a metric or is malformed".format(
+                os.path.relpath(path)
+            )
+        )
+
+
+class NoMetricsError(DvcException):
+    def __init__(self):
+        super(NoMetricsError, self).__init__(
+            "no metric files in this repository. "
+            "Use 'dvc metrics add' to add a metric file to track."
+        )
+
+
+class StageFileCorruptedError(DvcException):
+    def __init__(self, path):
+        path = os.path.relpath(path)
+        super(StageFileCorruptedError, self).__init__(
+            "unable to read stage file: {} "
+            "YAML file structure is corrupted".format(path)
+        )
+
+
+class RecursiveAddingWhileUsingFilename(DvcException):
+    def __init__(self):
+        super(RecursiveAddingWhileUsingFilename, self).__init__(
+            "using fname with recursive is not allowed."
+        )

dvc/istextfile.py

@@ -1,19 +1,24 @@
+"""Use heuristics to guess if it is a text file or a binary file."""
+
+from __future__ import unicode_literals
+
+from dvc.utils.compat import is_py3, open
+
 # Based on https://eli.thegreenplace.net/2011/10/19/
 # perls-guess-if-file-is-text-or-binary-implemented-in-python
 
-import sys
-PY3 = sys.version_info[0] == 3
 
 # A function that takes an integer in the 8-bit range and returns
 # a single-character byte object in py3 / a single-character string
 # in py2.
 #
-int2byte = (lambda x: bytes((x,))) if PY3 else chr
+def _int2byte(i):
+    if is_py3:
+        return bytes((i,))
+    return chr(i)
 
 
-_text_characters = (
-        b''.join(int2byte(i) for i in range(32, 127)) +
-        b'\n\r\t\f\b')
+TEXT_CHARS = b"".join(_int2byte(i) for i in range(32, 127)) + b"\n\r\t\f\b"
 
 
 def istextfile(fname, blocksize=512):
@@ -22,17 +27,18 @@ def istextfile(fname, blocksize=512):
         If more than 30% of the chars in the block are non-text, or there
         are NUL ('\x00') bytes in the block, assume this is a binary file.
     """
-    with open(fname, 'rb') as fd:
-        block = fd.read(blocksize)
+    with open(fname, "rb") as fobj:
+        block = fobj.read(blocksize)
 
-    if b'\x00' in block:
-        # Files with null bytes are binary
-        return False
-    elif not block:
+    if not block:
         # An empty file is considered a valid text file
         return True
 
+    if b"\x00" in block:
+        # Files with null bytes are binary
+        return False
+
     # Use translate's 'deletechars' argument to efficiently remove all
-    # occurrences of _text_characters from the block
-    nontext = block.translate(None, _text_characters)
+    # occurrences of TEXT_CHARS from the block
+    nontext = block.translate(None, TEXT_CHARS)
     return float(len(nontext)) / len(block) <= 0.30

dvc/lock.py

@@ -1,3 +1,7 @@
+"""Manages dvc lock file."""
+
+from __future__ import unicode_literals
+
 import os
 import time
 import zc.lockfile
@@ -6,29 +10,36 @@ from dvc.exceptions import DvcException
 
 
 class LockError(DvcException):
-    pass
+    """Thrown when unable to acquire the lock for dvc repo."""
 
 
 class Lock(object):
-    LOCK_FILE = 'lock'
+    """Class for dvc repo lock.
+
+    Args:
+        dvc_dir (str): path to the directory that the lock should be created
+            in.
+        name (str): name of the lock file.
+    """
+
+    LOCK_FILE = "lock"
     TIMEOUT = 5
 
     def __init__(self, dvc_dir, name=LOCK_FILE):
         self.lock_file = os.path.join(dvc_dir, name)
         self._lock = None
 
-    @staticmethod
-    def init(dvc_dir):
-        return Lock(dvc_dir)
-
     def _do_lock(self):
         try:
             self._lock = zc.lockfile.LockFile(self.lock_file)
         except zc.lockfile.LockError:
-            raise LockError('Cannot perform the cmd since DVC is busy and '
-                            'locked. Please retry the cmd later.')
+            raise LockError(
+                "cannot perform the cmd since DVC is busy and "
+                "locked. Please retry the cmd later."
+            )
 
     def lock(self):
+        """Acquire lock for dvc repo."""
         try:
             self._do_lock()
             return
@@ -38,11 +49,12 @@ class Lock(object):
         self._do_lock()
 
     def unlock(self):
+        """Release lock for dvc repo."""
         self._lock.close()
         self._lock = None
 
     def __enter__(self):
         self.lock()
 
-    def __exit__(self, type, value, tb):
+    def __exit__(self, typ, value, tbck):
         self.unlock()

dvc/logger.py

@@ -1,103 +1,312 @@
+"""Manages logger for dvc repo."""
+
+from __future__ import unicode_literals
+
+from dvc.exceptions import DvcException
+from dvc.utils.compat import str
+from dvc.progress import progress_aware
+
+import re
 import sys
 import logging
-import colorama
 import traceback
 
+from contextlib import contextmanager
 
-colorama.init()
+import colorama
 
 
-class Logger(object):
-    FMT = '%(message)s'
-    DEFAULT_LEVEL = logging.INFO
+@progress_aware
+def info(message):
+    """Prints an info message."""
+    logger.info(message)
 
-    LEVEL_MAP = {
-        'debug': logging.DEBUG,
-        'info': logging.INFO,
-        'warn': logging.WARNING,
-        'error': logging.ERROR
+
+def debug(message):
+    """Prints a debug message."""
+    prefix = colorize("Debug", color="blue")
+
+    out = "{prefix}: {message}".format(prefix=prefix, message=message)
+
+    logger.debug(out)
+
+
+@progress_aware
+def warning(message, parse_exception=False):
+    """Prints a warning message."""
+    prefix = colorize("Warning", color="yellow")
+
+    exception, stack_trace = None, ""
+    if parse_exception:
+        exception, stack_trace = _parse_exc()
+
+    out = "{prefix}: {description}".format(
+        prefix=prefix, description=_description(message, exception)
+    )
+
+    if stack_trace:
+        out += "\n{stack_trace}".format(stack_trace=stack_trace)
+
+    logger.warning(out)
+
+
+@progress_aware
+def error(message=None):
+    """Prints an error message."""
+    prefix = colorize("Error", color="red")
+
+    exception, stack_trace = _parse_exc()
+
+    out = (
+        "{prefix}: {description}"
+        "\n"
+        "{stack_trace}"
+        "\n"
+        "{footer}".format(
+            prefix=prefix,
+            description=_description(message, exception),
+            stack_trace=stack_trace,
+            footer=_footer(),
+        )
+    )
+
+    logger.error(out)
+
+
+def box(message, border_color=None):
+    """Prints a message in a box.
+
+    Args:
+        message (unicode): message to print.
+        border_color (unicode): name of a color to outline the box with.
+    """
+    lines = message.split("\n")
+    max_width = max(_visual_width(line) for line in lines)
+
+    padding_horizontal = 5
+    padding_vertical = 1
+
+    box_size_horizontal = max_width + (padding_horizontal * 2)
+
+    chars = {"corner": "+", "horizontal": "-", "vertical": "|", "empty": " "}
+
+    margin = "{corner}{line}{corner}\n".format(
+        corner=chars["corner"], line=chars["horizontal"] * box_size_horizontal
+    )
+
+    padding_lines = [
+        "{border}{space}{border}\n".format(
+            border=colorize(chars["vertical"], color=border_color),
+            space=chars["empty"] * box_size_horizontal,
+        )
+        * padding_vertical
+    ]
+
+    content_lines = [
+        "{border}{space}{content}{space}{border}\n".format(
+            border=colorize(chars["vertical"], color=border_color),
+            space=chars["empty"] * padding_horizontal,
+            content=_visual_center(line, max_width),
+        )
+        for line in lines
+    ]
+
+    box_str = "{margin}{padding}{content}{padding}{margin}".format(
+        margin=colorize(margin, color=border_color),
+        padding="".join(padding_lines),
+        content="".join(content_lines),
+    )
+
+    logger.info(box_str)
+
+
+def level():
+    """Returns current log level."""
+    return logger.getEffectiveLevel()
+
+
+def set_level(level_name):
+    """Sets log level.
+
+    Args:
+        level_name (str): log level name. E.g. info, debug, warning, error,
+            critical.
+    """
+    if not level_name:
+        return
+
+    levels = {
+        "info": logging.INFO,
+        "debug": logging.DEBUG,
+        "warning": logging.WARNING,
+        "error": logging.ERROR,
+        "critical": logging.CRITICAL,
     }
 
-    COLOR_MAP = {
-        'debug': colorama.Fore.BLUE,
-        'warn': colorama.Fore.YELLOW,
-        'error': colorama.Fore.RED
+    logger.setLevel(levels.get(level_name))
+
+
+def be_quiet():
+    """Disables all messages except critical ones."""
+    logger.setLevel(logging.CRITICAL)
+
+
+def be_verbose():
+    """Enables all messages."""
+    logger.setLevel(logging.DEBUG)
+
+
+@contextmanager
+def verbose():
+    """Enables verbose mode for the context."""
+    previous_level = level()
+    be_verbose()
+    yield
+    logger.setLevel(previous_level)
+
+
+@contextmanager
+def quiet():
+    """Enables quiet mode for the context."""
+    previous_level = level()
+    be_quiet()
+    yield
+    logger.setLevel(previous_level)
+
+
+def is_quiet():
+    """Returns whether or not all messages except critical ones are
+    disabled.
+    """
+    return level() == logging.CRITICAL
+
+
+def is_verbose():
+    """Returns whether or not all messages are enabled."""
+    return level() == logging.DEBUG
+
+
+def colorize(message, color=None):
+    """Returns a message in a specified color."""
+    if not color:
+        return message
+
+    colors = {
+        "green": colorama.Fore.GREEN,
+        "yellow": colorama.Fore.YELLOW,
+        "blue": colorama.Fore.BLUE,
+        "red": colorama.Fore.RED,
     }
 
-    def __init__(self, loglevel=None):
-        if loglevel:
-            Logger.set_level(loglevel)
+    return "{color}{message}{nc}".format(
+        color=colors.get(color, ""), message=message, nc=colorama.Fore.RESET
+    )
 
-    @staticmethod
-    def init():
-        sh = logging.StreamHandler(sys.stdout)
-        sh.setFormatter(logging.Formatter(Logger.FMT))
-        sh.setLevel(logging.DEBUG)
 
-        Logger.logger().addHandler(sh)
-        Logger.set_level()
+def _init_colorama():
+    colorama.init()
 
-    @staticmethod
-    def logger():
-        return logging.getLogger('dvc')
 
-    @staticmethod
-    def set_level(level=None):
-        if not level:
-            lvl = Logger.DEFAULT_LEVEL
-        else:
-            lvl = Logger.LEVEL_MAP.get(level.lower(), Logger.DEFAULT_LEVEL)
-        Logger.logger().setLevel(lvl)
+def set_default_level():
+    """Sets default log level."""
+    logger.setLevel(logging.INFO)
 
-    @staticmethod
-    def be_quiet():
-        Logger.logger().setLevel(logging.CRITICAL)
 
-    @staticmethod
-    def be_verbose():
-        Logger.logger().setLevel(logging.DEBUG)
+def _add_handlers():
+    formatter = "%(message)s"
 
-    @staticmethod
-    def colorize(msg, typ):
-        header = ''
-        footer = ''
+    class _LogLevelFilter(logging.Filter):
+        # pylint: disable=too-few-public-methods
+        def filter(self, record):
+            return record.levelno <= logging.WARNING
 
-        if sys.stdout.isatty():
-            header = Logger.COLOR_MAP.get(typ.lower(), '')
-            footer = colorama.Style.RESET_ALL
+    sh_out = logging.StreamHandler(sys.stdout)
+    sh_out.setFormatter(logging.Formatter(formatter))
+    sh_out.setLevel(logging.DEBUG)
+    sh_out.addFilter(_LogLevelFilter())
 
-        return u'{}{}{}'.format(header, msg, footer)
+    sh_err = logging.StreamHandler(sys.stderr)
+    sh_err.setFormatter(logging.Formatter(formatter))
+    sh_err.setLevel(logging.ERROR)
 
-    @staticmethod
-    def parse_exc(exc, tb=None):
-        str_tb = tb if tb else None
-        str_exc = ': {}'.format(str(exc)) if exc else ""
+    logger.addHandler(sh_out)
+    logger.addHandler(sh_err)
 
-        if exc and hasattr(exc, 'cause') and exc.cause:
-            cause_tb = exc.cause_tb if hasattr(exc, 'cause_tb') else None
-            cause_str_exc, cause_str_tb = Logger.parse_exc(exc.cause, cause_tb)
 
-            str_tb = cause_str_tb
-            str_exc = '{}{}'.format(str_exc, cause_str_exc)
+def _walk_exc(exc):
+    exc_list = [str(exc)]
+    tb_list = [traceback.format_exc()]
 
-        return (str_exc, str_tb)
+    # NOTE: parsing chained exceptions. See dvc/exceptions.py for more info.
+    while hasattr(exc, "cause") and exc.cause is not None:
+        exc_list.append(str(exc.cause))
+        if hasattr(exc, "cause_tb") and exc.cause_tb is not None:
+            tb_list.insert(0, str(exc.cause_tb))
+        exc = exc.cause
 
-    @staticmethod
-    def error(msg, exc=None):
-        str_exc, str_tb = Logger.parse_exc(exc)
-        if Logger.logger().getEffectiveLevel() == logging.DEBUG and exc:
-            str_tb = str_tb if str_tb else traceback.format_exc()
-            Logger.logger().error(str_tb)
-        return Logger.logger().error(Logger.colorize(msg + str_exc, 'error'))
+    return exc_list, tb_list
 
-    @staticmethod
-    def warn(msg):
-        return Logger.logger().warn(Logger.colorize(msg, 'warn'))
 
-    @staticmethod
-    def debug(msg):
-        return Logger.logger().debug(Logger.colorize(msg, 'debug'))
+def _parse_exc():
+    exc = sys.exc_info()[1]
+    if not exc:
+        return (None, "")
 
-    @staticmethod
-    def info(msg):
-        return Logger.logger().info(Logger.colorize(msg, 'info'))
+    exc_list, tb_list = _walk_exc(exc)
+
+    exception = ": ".join(exc_list)
+
+    if is_verbose():
+        stack_trace = "{line}\n{stack_trace}{line}\n".format(
+            line=colorize("-" * 60, color="red"),
+            stack_trace="\n".join(tb_list),
+        )
+    else:
+        stack_trace = ""
+
+    return (exception, stack_trace)
+
+
+def _description(message, exception):
+    if exception and message:
+        description = "{message} - {exception}"
+    elif exception:
+        description = "{exception}"
+    elif message:
+        description = "{message}"
+    else:
+        raise DvcException(
+            "Unexpected error - either exception or message must be provided"
+        )
+
+    return description.format(message=message, exception=exception)
+
+
+def _footer():
+    return "{phrase} Hit us up at {url}, we are always happy to help!".format(
+        phrase=colorize("Having any troubles?", "yellow"),
+        url=colorize("https://dvc.org/support", "blue"),
+    )
+
+
+def _visual_width(line):
+    """Get the the number of columns required to display a string"""
+
+    return len(re.sub(colorama.ansitowin32.AnsiToWin32.ANSI_CSI_RE, "", line))
+
+
+def _visual_center(line, width):
+    """Center align string according to it's visual width"""
+
+    spaces = max(width - _visual_width(line), 0)
+    left_padding = int(spaces / 2)
+    right_padding = spaces - left_padding
+
+    return (left_padding * " ") + line + (right_padding * " ")
+
+
+logger = logging.getLogger("dvc")  # pylint: disable=invalid-name
+
+set_default_level()
+_add_handlers()
+_init_colorama()

dvc/main.py

@@ -1,27 +1,47 @@
-from dvc.logger import Logger
+"""Main entry point for dvc CLI."""
+
+from __future__ import unicode_literals
+
+import dvc.logger as logger
 from dvc.cli import parse_args
 from dvc.command.base import CmdBase
+from dvc.analytics import Analytics
+from dvc.exceptions import NotDvcRepoError, DvcParserError
 
 
 def main(argv=None):
-    Logger.init()
+    """Run dvc CLI command.
 
-    args = parse_args(argv)
-
-    # Init loglevel early in case we'll run
-    # into errors before setting it properly
-    CmdBase._set_loglevel(args)
+    Args:
+        argv: optional list of arguments to parse. sys.argv is used by default.
 
+    Returns:
+        int: command's return code.
+    """
+    args = None
+    cmd = None
     try:
+        args = parse_args(argv)
+
+        # Init loglevel early in case we'll run
+        # into errors before setting it properly
+        CmdBase.set_loglevel(args)
+
         cmd = args.func(args)
-    except Exception as ex:
-        Logger.error('Initialization error', ex)
-        return 255
 
-    try:
         ret = cmd.run_cmd()
-    except Exception as ex:
-        Logger.error('Unexpected error', ex)
-        return 254
+    except KeyboardInterrupt:
+        logger.error("interrupted by the user")
+        ret = 252
+    except NotDvcRepoError:
+        logger.error()
+        ret = 253
+    except DvcParserError:
+        ret = 254
+    except Exception:  # pylint: disable=broad-except
+        logger.error("unexpected error")
+        ret = 255
+
+    Analytics().send_cmd(cmd, args, ret)
 
     return ret

dvc/output/__init__.py

@@ -1,10 +1,11 @@
+from __future__ import unicode_literals
+
 import schema
 
-from dvc.exceptions import DvcException
 from dvc.config import Config
+from dvc.utils.compat import urlparse, str
 
-from dvc.dependency import SCHEMA, urlparse
-from dvc.dependency.base import DependencyBase
+from dvc.output.base import OutputBase
 from dvc.output.s3 import OutputS3
 from dvc.output.gs import OutputGS
 from dvc.output.local import OutputLOCAL
@@ -12,45 +13,66 @@ from dvc.output.hdfs import OutputHDFS
 from dvc.output.ssh import OutputSSH
 
 from dvc.remote import Remote
+from dvc.remote.s3 import RemoteS3
+from dvc.remote.hdfs import RemoteHDFS
+from dvc.remote.local import RemoteLOCAL
 
+OUTS = [
+    OutputHDFS,
+    OutputS3,
+    OutputGS,
+    OutputSSH,
+    # NOTE: OutputLOCAL is the default choice
+]
 
-OUTS = [OutputHDFS, OutputS3, OutputGS, OutputSSH, OutputLOCAL]
+OUTS_MAP = {
+    "hdfs": OutputHDFS,
+    "s3": OutputS3,
+    "gs": OutputGS,
+    "ssh": OutputSSH,
+    "local": OutputLOCAL,
+}
 
-OUTS_MAP = {'hdfs': OutputHDFS,
-            's3': OutputS3,
-            'gs': OutputGS,
-            'ssh': OutputSSH,
-            '': OutputLOCAL}
-
-SCHEMA[schema.Optional(OutputLOCAL.PARAM_CACHE)] = bool
-SCHEMA[schema.Optional(OutputLOCAL.PARAM_METRIC)] = OutputLOCAL.METRIC_SCHEMA
+SCHEMA = {
+    OutputBase.PARAM_PATH: str,
+    # NOTE: currently there are only 3 possible checksum names:
+    #
+    #    1) md5 (LOCAL, SSH, GS);
+    #    2) etag (S3);
+    #    3) checksum (HDFS);
+    #
+    # so when a few types of outputs share the same name, we only need
+    # specify it once.
+    schema.Optional(RemoteLOCAL.PARAM_CHECKSUM): schema.Or(str, None),
+    schema.Optional(RemoteS3.PARAM_CHECKSUM): schema.Or(str, None),
+    schema.Optional(RemoteHDFS.PARAM_CHECKSUM): schema.Or(str, None),
+    schema.Optional(OutputBase.PARAM_CACHE): bool,
+    schema.Optional(OutputBase.PARAM_METRIC): OutputBase.METRIC_SCHEMA,
+}
 
 
 def _get(stage, p, info, cache, metric):
     parsed = urlparse(p)
-    if parsed.scheme == 'remote':
+    if parsed.scheme == "remote":
         name = Config.SECTION_REMOTE_FMT.format(parsed.netloc)
-        sect = stage.project.config._config[name]
-        remote = Remote(stage.project, sect)
-        return OUTS_MAP[remote.scheme](stage,
-                                       p,
-                                       info,
-                                       cache=cache,
-                                       remote=remote,
-                                       metric=metric)
+        sect = stage.repo.config.config[name]
+        remote = Remote(stage.repo, sect)
+        return OUTS_MAP[remote.scheme](
+            stage, p, info, cache=cache, remote=remote, metric=metric
+        )
 
     for o in OUTS:
         if o.supported(p):
             return o(stage, p, info, cache=cache, remote=None, metric=metric)
-    raise DvcException('Output \'{}\' is not supported'.format(p))
+    return OutputLOCAL(stage, p, info, cache=cache, remote=None, metric=metric)
 
 
 def loadd_from(stage, d_list):
     ret = []
     for d in d_list:
-        p = d.pop(DependencyBase.PARAM_PATH)
-        cache = d.pop(OutputLOCAL.PARAM_CACHE, True)
-        metric = d.pop(OutputLOCAL.PARAM_METRIC, False)
+        p = d.pop(OutputBase.PARAM_PATH)
+        cache = d.pop(OutputBase.PARAM_CACHE, True)
+        metric = d.pop(OutputBase.PARAM_METRIC, False)
         ret.append(_get(stage, p, info=d, cache=cache, metric=metric))
     return ret
 

dvc/output/base.py

@@ -1,24 +1,222 @@
+from __future__ import unicode_literals
+
+import re
+from schema import Or, Optional
+
 from dvc.exceptions import DvcException
+from dvc.utils.compat import str
+from dvc.remote.base import RemoteBase
 
 
-class OutputError(DvcException):
-    def __init__(self, path, msg):
-        msg = 'Output \'{}\' error: {}'.format(path, msg)
-        super(OutputError, self).__init__(msg)
-
-
-class OutputDoesNotExistError(OutputError):
+class OutputDoesNotExistError(DvcException):
     def __init__(self, path):
-        super(OutputDoesNotExistError, self).__init__(path, 'does not exist')
+        msg = "output '{}' does not exist".format(path)
+        super(OutputDoesNotExistError, self).__init__(msg)
 
 
-class OutputIsNotFileOrDirError(OutputError):
+class OutputIsNotFileOrDirError(DvcException):
     def __init__(self, path):
-        msg = 'not a file or directory'
-        super(OutputIsNotFileOrDirError, self).__init__(path, msg)
+        msg = "output '{}' is not a file or directory".format(path)
+        super(OutputIsNotFileOrDirError, self).__init__(msg)
 
 
-class OutputAlreadyTrackedError(OutputError):
+class OutputAlreadyTrackedError(DvcException):
     def __init__(self, path):
-        msg = 'already tracked by scm(e.g. git)'
-        super(OutputAlreadyTrackedError, self).__init__(path, msg)
+        msg = "output '{}' is already tracked by scm (e.g. git)".format(path)
+        super(OutputAlreadyTrackedError, self).__init__(msg)
+
+
+class OutputBase(object):
+    IS_DEPENDENCY = False
+
+    REMOTE = RemoteBase
+
+    PARAM_PATH = "path"
+    PARAM_CACHE = "cache"
+    PARAM_METRIC = "metric"
+    PARAM_METRIC_TYPE = "type"
+    PARAM_METRIC_XPATH = "xpath"
+
+    METRIC_SCHEMA = Or(
+        None,
+        bool,
+        {
+            Optional(PARAM_METRIC_TYPE): Or(str, None),
+            Optional(PARAM_METRIC_XPATH): Or(str, None),
+        },
+    )
+
+    DoesNotExistError = OutputDoesNotExistError
+    IsNotFileOrDirError = OutputIsNotFileOrDirError
+
+    def __init__(
+        self, stage, path, info=None, remote=None, cache=True, metric=False
+    ):
+        self.stage = stage
+        self.repo = stage.repo
+        self.url = path
+        self.info = info
+        self.remote = remote or self.REMOTE(self.repo, {})
+        self.use_cache = False if self.IS_DEPENDENCY else cache
+        self.metric = False if self.IS_DEPENDENCY else metric
+
+        if (
+            self.use_cache
+            and getattr(self.repo.cache, self.REMOTE.scheme) is None
+        ):
+            raise DvcException(
+                "no cache location setup for '{}' outputs.".format(
+                    self.REMOTE.scheme
+                )
+            )
+
+    def __repr__(self):
+        return "{class_name}: '{url}'".format(
+            class_name=type(self).__name__, url=(self.url or "No url")
+        )
+
+    def __str__(self):
+        return self.url
+
+    @property
+    def is_local(self):
+        raise DvcException(
+            "is local is not supported for {}".format(self.scheme)
+        )
+
+    @classmethod
+    def match(cls, url):
+        return re.match(cls.REMOTE.REGEX, url)
+
+    def group(self, name):
+        match = self.match(self.url)
+        if not match:
+            return None
+        return match.group(name)
+
+    @classmethod
+    def supported(cls, url):
+        return cls.match(url) is not None
+
+    @property
+    def scheme(self):
+        return self.REMOTE.scheme
+
+    @property
+    def path(self):
+        return self.path_info["path"]
+
+    @property
+    def sep(self):
+        return "/"
+
+    @property
+    def checksum(self):
+        return self.info.get(self.remote.PARAM_CHECKSUM)
+
+    @property
+    def exists(self):
+        return self.remote.exists(self.path_info)
+
+    def changed_checksum(self):
+        return (
+            self.checksum
+            != self.remote.save_info(self.path_info)[
+                self.remote.PARAM_CHECKSUM
+            ]
+        )
+
+    def changed_cache(self):
+        if not self.use_cache or not self.checksum:
+            return True
+
+        cache = self.repo.cache.__getattribute__(self.scheme)
+
+        return cache.changed_cache(self.checksum)
+
+    def status(self):
+        if self.checksum and self.use_cache and self.changed_cache():
+            return {str(self): "not in cache"}
+
+        if not self.exists:
+            return {str(self): "deleted"}
+
+        if self.changed_checksum():
+            return {str(self): "modified"}
+
+        if not self.checksum:
+            return {str(self): "new"}
+
+        return {}
+
+    def changed(self):
+        return bool(self.status())
+
+    def save(self):
+        if not self.exists:
+            raise self.DoesNotExistError(self)
+
+        self.info = self.remote.save_info(self.path_info)
+
+    def commit(self):
+        if self.use_cache:
+            getattr(self.repo.cache, self.scheme).save(
+                self.path_info, self.info
+            )
+
+    def dumpd(self):
+        ret = self.info.copy()
+        ret[self.PARAM_PATH] = self.url
+
+        if self.IS_DEPENDENCY:
+            return ret
+
+        ret[self.PARAM_CACHE] = self.use_cache
+
+        if isinstance(self.metric, dict):
+            if (
+                self.PARAM_METRIC_XPATH in self.metric
+                and not self.metric[self.PARAM_METRIC_XPATH]
+            ):
+                del self.metric[self.PARAM_METRIC_XPATH]
+
+        ret[self.PARAM_METRIC] = self.metric
+
+        return ret
+
+    def verify_metric(self):
+        raise DvcException(
+            "verify metric is not supported for {}".format(self.scheme)
+        )
+
+    def download(self, to_info, resume=False):
+        self.remote.download([self.path_info], [to_info], resume=resume)
+
+    def checkout(self, force=False):
+        if not self.use_cache:
+            return
+
+        getattr(self.repo.cache, self.scheme).checkout(
+            self.path_info, self.info, force=force
+        )
+
+    def remove(self, ignore_remove=False):
+        self.remote.remove(self.path_info)
+        if self.scheme != "local":
+            return
+
+        if ignore_remove and self.use_cache and self.is_local:
+            self.repo.scm.ignore_remove(self.path)
+
+    def move(self, out):
+        if self.scheme == "local" and self.use_cache and self.is_local:
+            self.repo.scm.ignore_remove(self.path)
+
+        self.remote.move(self.path_info, out.path_info)
+        self.url = out.url
+        self.path_info = out.path_info
+        self.save()
+        self.commit()
+
+        if self.scheme == "local" and self.use_cache and self.is_local:
+            self.repo.scm.ignore(self.path)

dvc/output/gs.py

@@ -1,53 +1,8 @@
-from dvc.dependency.gs import DependencyGS
-from dvc.exceptions import DvcException
+from __future__ import unicode_literals
+
+from dvc.remote.gs import RemoteGS
+from dvc.output.s3 import OutputS3
 
 
-class OutputGS(DependencyGS):
-    PARAM_CACHE = 'cache'
-    PARAM_METRIC = 'metric'
-
-    def __init__(self,
-                 stage,
-                 path,
-                 info=None,
-                 remote=None,
-                 cache=True,
-                 metric=False):
-        super(OutputGS, self).__init__(stage, path, info, remote=remote)
-        self.use_cache = cache
-        self.metric = metric
-        if cache and self.project.cache.gs is None:
-            raise DvcException("No cache location setup for \'gs\' outputs.")
-
-    def dumpd(self):
-        ret = super(OutputGS, self).dumpd()
-        ret[self.PARAM_CACHE] = self.use_cache
-        ret[self.PARAM_METRIC] = self.metric
-        return ret
-
-    def changed(self):
-        if super(OutputGS, self).changed():
-            return True
-
-        if self.use_cache \
-           and self.info != self.project.cache.gs.save_info(self.path_info):
-            return True
-
-        return False
-
-    def checkout(self):
-        if not self.use_cache:
-            return
-
-        self.project.cache.gs.checkout(self.path_info, self.info)
-
-    def save(self):
-        super(OutputGS, self).save()
-
-        if not self.use_cache:
-            return
-
-        self.info = self.project.cache.gs.save(self.path_info)
-
-    def remove(self, ignore_remove=False):
-        self.remote.remove(self.path_info)
+class OutputGS(OutputS3):
+    REMOTE = RemoteGS

dvc/output/hdfs.py

@@ -1,53 +1,22 @@
-from dvc.dependency.hdfs import DependencyHDFS
-from dvc.exceptions import DvcException
+from __future__ import unicode_literals
+
+import posixpath
+
+from dvc.utils.compat import urlparse
+from dvc.output.base import OutputBase
+from dvc.remote.hdfs import RemoteHDFS
 
 
-class OutputHDFS(DependencyHDFS):
-    PARAM_CACHE = 'cache'
-    PARAM_METRIC = 'metric'
+class OutputHDFS(OutputBase):
+    REMOTE = RemoteHDFS
 
-    def __init__(self,
-                 stage,
-                 path,
-                 info=None,
-                 remote=None,
-                 cache=True,
-                 metric=False):
-        super(OutputHDFS, self).__init__(stage, path, info, remote=remote)
-        self.use_cache = cache
-        self.metric = metric
-        if cache and self.project.cache.hdfs is None:
-            raise DvcException("No cache location setup for \'hdfs\' outputs.")
-
-    def dumpd(self):
-        ret = super(OutputHDFS, self).dumpd()
-        ret[self.PARAM_CACHE] = self.use_cache
-        ret[self.PARAM_METRIC] = self.metric
-        return ret
-
-    def changed(self):
-        if super(OutputHDFS, self).changed():
-            return True
-
-        if self.use_cache \
-           and self.info != self.project.cache.hdfs.save_info(self.path_info):
-            return True
-
-        return False
-
-    def checkout(self):
-        if not self.use_cache:
-            return
-
-        self.project.cache.hdfs.checkout(self.path_info, self.info)
-
-    def save(self):
-        super(OutputHDFS, self).save()
-
-        if not self.use_cache:
-            return
-
-        self.info = self.project.cache.hdfs.save(self.path_info)
-
-    def remove(self, ignore_remove=False):
-        self.remote.remove(self.path_info)
+    def __init__(
+        self, stage, path, info=None, remote=None, cache=True, metric=False
+    ):
+        super(OutputHDFS, self).__init__(
+            stage, path, info=info, remote=remote, cache=cache, metric=metric
+        )
+        if remote:
+            path = posixpath.join(remote.url, urlparse(path).path.lstrip("/"))
+        user = remote.user if remote else self.group("user")
+        self.path_info = {"scheme": "hdfs", "user": user, "path": path}

dvc/output/local.py

@@ -1,138 +1,120 @@
+from __future__ import unicode_literals
+
 import os
-from schema import Optional, Or
 
-from dvc.output.base import OutputDoesNotExistError, OutputIsNotFileOrDirError
-from dvc.output.base import OutputAlreadyTrackedError
-from dvc.dependency.local import DependencyLOCAL
-from dvc.exceptions import DvcException
+import dvc.logger as logger
+from dvc.utils.compat import urlparse
 from dvc.istextfile import istextfile
+from dvc.exceptions import DvcException
+from dvc.remote.local import RemoteLOCAL
+from dvc.output.base import OutputBase, OutputAlreadyTrackedError
 
 
-class OutputLOCAL(DependencyLOCAL):
-    PARAM_CACHE = 'cache'
-    PARAM_METRIC = 'metric'
-    PARAM_METRIC_TYPE = 'type'
-    PARAM_METRIC_XPATH = 'xpath'
+class OutputLOCAL(OutputBase):
+    REMOTE = RemoteLOCAL
 
-    METRIC_SCHEMA = Or(None, bool,
-                       {Optional(PARAM_METRIC_TYPE): Or(str, None),
-                        Optional(PARAM_METRIC_XPATH): Or(str, None)})
+    def __init__(
+        self, stage, path, info=None, remote=None, cache=True, metric=False
+    ):
+        super(OutputLOCAL, self).__init__(
+            stage, path, info, remote=remote, cache=cache, metric=metric
+        )
+        if remote:
+            p = os.path.join(
+                remote.prefix, urlparse(self.url).path.lstrip("/")
+            )
+        else:
+            p = path
 
-    DoesNotExistError = OutputDoesNotExistError
-    IsNotFileOrDirError = OutputIsNotFileOrDirError
+        if not os.path.isabs(p):
+            p = self.remote.to_ospath(p)
+            p = os.path.join(stage.wdir, p)
+        p = os.path.abspath(os.path.normpath(p))
 
-    def __init__(self,
-                 stage,
-                 path,
-                 info=None,
-                 remote=None,
-                 cache=True,
-                 metric=False):
-        super(OutputLOCAL, self).__init__(stage, path, info, remote=remote)
-        self.use_cache = cache
-        self.metric = metric
+        self.path_info = {"scheme": "local", "path": p}
 
-    @property
-    def md5(self):
-        return self.info.get(self.project.cache.local.PARAM_MD5, None)
-
-    @property
-    def cache(self):
-        return self.project.cache.local.get(self.md5)
+    def __str__(self):
+        return self.rel_path
 
     @property
     def is_local(self):
-        return self.path.startswith(self.project.root_dir)
+        return urlparse(self.url).scheme != "remote" and not os.path.isabs(
+            self.url
+        )
+
+    @property
+    def sep(self):
+        return os.sep
+
+    @property
+    def rel_path(self):
+        return os.path.relpath(self.path)
+
+    @property
+    def cache(self):
+        return self.repo.cache.local.get(self.checksum)
 
     def dumpd(self):
         ret = super(OutputLOCAL, self).dumpd()
-        ret[self.PARAM_CACHE] = self.use_cache
+        if self.is_local:
+            path = self.remote.unixpath(
+                os.path.relpath(self.path, self.stage.wdir)
+            )
+        else:
+            path = self.url
 
-        if isinstance(self.metric, dict):
-            if self.PARAM_METRIC_XPATH in self.metric and \
-               not self.metric[self.PARAM_METRIC_XPATH]:
-                del self.metric[self.PARAM_METRIC_XPATH]
-
-        if self.metric:
-            ret[self.PARAM_METRIC] = self.metric
+        ret[self.PARAM_PATH] = path
 
         return ret
 
-    def changed(self):
-        if not self.use_cache:
-            return super(OutputLOCAL, self).changed()
-
-        return self.info != self.project.cache.local.save_info(self.path_info)
-
-    def checkout(self):
-        if not self.use_cache:
-            return
-
-        if not self.changed():
-            msg = u'Data file \'{}\' didn\'t change, skipping checkout.'
-            self.project.logger.debug(msg.format(self.rel_path))
-            return
-
-        self.project.cache.local.checkout(self.path_info, self.info)
-
-    def _verify_metric(self):
+    def verify_metric(self):
         if not self.metric:
             return
 
+        if not os.path.exists(self.path):
+            return
+
         if os.path.isdir(self.path):
-            msg = 'Directory \'{}\' cannot be used as metrics.'
+            msg = "directory '{}' cannot be used as metrics."
             raise DvcException(msg.format(self.rel_path))
 
         if not istextfile(self.path):
-            msg = 'Binary file \'{}\' cannot be used as metrics.'
+            msg = "binary file '{}' cannot be used as metrics."
             raise DvcException(msg.format(self.rel_path))
 
     def save(self):
-        if not self.use_cache:
-            super(OutputLOCAL, self).save()
-            self._verify_metric()
-            msg = 'Output \'{}\' doesn\'t use cache. Skipping saving.'
-            self.project.logger.debug(msg.format(self.rel_path))
-            return
-
         if not os.path.exists(self.path):
             raise self.DoesNotExistError(self.rel_path)
 
         if not os.path.isfile(self.path) and not os.path.isdir(self.path):
             raise self.IsNotFileOrDirError(self.rel_path)
 
-        if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \
-           (os.path.isdir(self.path) and len(os.listdir(self.path)) == 0):
-            msg = "File/directory '{}' is empty.".format(self.rel_path)
-            self.project.logger.warn(msg)
+        if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or (
+            os.path.isdir(self.path) and len(os.listdir(self.path)) == 0
+        ):
+            msg = "file/directory '{}' is empty.".format(self.rel_path)
+            logger.warning(msg)
+
+        if not self.use_cache:
+            self.info = self.remote.save_info(self.path_info)
+            self.verify_metric()
+            if not self.IS_DEPENDENCY:
+                msg = "Output '{}' doesn't use cache. Skipping saving."
+                logger.info(msg.format(self.rel_path))
+            return
+
+        assert not self.IS_DEPENDENCY
 
         if not self.changed():
-            msg = 'Output \'{}\' didn\'t change. Skipping saving.'
-            self.project.logger.debug(msg.format(self.rel_path))
+            msg = "Output '{}' didn't change. Skipping saving."
+            logger.info(msg.format(self.rel_path))
             return
 
         if self.is_local:
-            if self.project.scm.is_tracked(self.path):
+            if self.repo.scm.is_tracked(self.path):
                 raise OutputAlreadyTrackedError(self.rel_path)
 
             if self.use_cache:
-                self.project.scm.ignore(self.path)
+                self.repo.scm.ignore(self.path)
 
-        self.info = self.project.cache.local.save(self.path_info)
-
-    def remove(self, ignore_remove=False):
-        self.remote.remove(self.path_info)
-        if ignore_remove and self.use_cache and self.is_local:
-            self.project.scm.ignore_remove(self.path)
-
-    def move(self, out):
-        if self.use_cache and self.is_local:
-            self.project.scm.ignore_remove(self.path)
-
-        self.remote.move(self.path_info, out.path_info)
-        self.path = out.path
-        self.path_info = out.path_info
-        self.save()
-
-        if self.use_cache and self.is_local:
-            self.project.scm.ignore(self.path)
+        self.info = self.remote.save_info(self.path_info)

dvc/output/s3.py

@@ -1,53 +1,27 @@
-from dvc.dependency.s3 import DependencyS3
-from dvc.exceptions import DvcException
+from __future__ import unicode_literals
+
+import posixpath
+
+from dvc.remote.s3 import RemoteS3
+from dvc.utils.compat import urlparse
+from dvc.output.base import OutputBase
 
 
-class OutputS3(DependencyS3):
-    PARAM_CACHE = 'cache'
-    PARAM_METRIC = 'metric'
+class OutputS3(OutputBase):
+    REMOTE = RemoteS3
 
-    def __init__(self,
-                 stage,
-                 path,
-                 info=None,
-                 remote=None,
-                 cache=True,
-                 metric=False):
-        super(OutputS3, self).__init__(stage, path, info, remote=remote)
-        self.use_cache = cache
-        self.metric = metric
-        if cache and self.project.cache.s3 is None:
-            raise DvcException("No cache location setup for \'s3\' outputs.")
-
-    def dumpd(self):
-        ret = super(OutputS3, self).dumpd()
-        ret[self.PARAM_CACHE] = self.use_cache
-        ret[self.PARAM_METRIC] = self.metric
-        return ret
-
-    def changed(self):
-        if super(OutputS3, self).changed():
-            return True
-
-        if self.use_cache \
-           and self.info != self.project.cache.s3.save_info(self.path_info):
-            return True
-
-        return False
-
-    def checkout(self):
-        if not self.use_cache:
-            return
-
-        self.project.cache.s3.checkout(self.path_info, self.info)
-
-    def save(self):
-        super(OutputS3, self).save()
-
-        if not self.use_cache:
-            return
-
-        self.info = self.project.cache.s3.save(self.path_info)
-
-    def remove(self, ignore_remove=False):
-        self.remote.remove(self.path_info)
+    def __init__(
+        self, stage, path, info=None, remote=None, cache=True, metric=False
+    ):
+        super(OutputS3, self).__init__(
+            stage, path, info=info, remote=remote, cache=cache, metric=metric
+        )
+        bucket = remote.bucket if remote else urlparse(path).netloc
+        path = urlparse(path).path.lstrip("/")
+        if remote:
+            path = posixpath.join(remote.prefix, path)
+        self.path_info = {
+            "scheme": self.scheme,
+            "bucket": bucket,
+            "path": path,
+        }

dvc/output/ssh.py

@@ -1,53 +1,42 @@
-from dvc.dependency.ssh import DependencySSH
-from dvc.exceptions import DvcException
+from __future__ import unicode_literals
+
+import getpass
+import posixpath
+
+from dvc.utils.compat import urlparse
+from dvc.output.base import OutputBase
+from dvc.remote.ssh import RemoteSSH
 
 
-class OutputSSH(DependencySSH):
-    PARAM_CACHE = 'cache'
-    PARAM_METRIC = 'metric'
+class OutputSSH(OutputBase):
+    REMOTE = RemoteSSH
 
-    def __init__(self,
-                 stage,
-                 path,
-                 info=None,
-                 remote=None,
-                 cache=True,
-                 metric=False):
-        super(OutputSSH, self).__init__(stage, path, info, remote=remote)
-        self.use_cache = cache
-        self.metric = metric
-        if cache and self.project.cache.ssh is None:
-            raise DvcException("No cache location setup for \'ssh\' outputs.")
+    def __init__(
+        self, stage, path, info=None, remote=None, cache=True, metric=False
+    ):
+        super(OutputSSH, self).__init__(
+            stage, path, info=info, remote=remote, cache=cache, metric=metric
+        )
+        parsed = urlparse(path)
+        host = remote.host if remote else parsed.hostname
+        port = (
+            remote.port if remote else (parsed.port or RemoteSSH.DEFAULT_PORT)
+        )
+        user = (
+            remote.user if remote else (parsed.username or getpass.getuser())
+        )
 
-    def dumpd(self):
-        ret = super(OutputSSH, self).dumpd()
-        ret[self.PARAM_CACHE] = self.use_cache
-        ret[self.PARAM_METRIC] = self.metric
-        return ret
+        if remote:
+            path = posixpath.join(
+                remote.prefix, urlparse(path).path.lstrip("/")
+            )
+        else:
+            path = parsed.path
 
-    def changed(self):
-        if super(OutputSSH, self).changed():
-            return True
-
-        if self.use_cache \
-           and self.info != self.project.cache.ssh.save_info(self.path_info):
-            return True
-
-        return False
-
-    def checkout(self):
-        if not self.use_cache:
-            return
-
-        self.project.cache.ssh.checkout(self.path_info, self.info)
-
-    def save(self):
-        super(OutputSSH, self).save()
-
-        if not self.use_cache:
-            return
-
-        self.info = self.project.cache.ssh.save(self.path_info)
-
-    def remove(self, ignore_remove=False):
-        self.remote.remove(self.path_info)
+        self.path_info = {
+            "scheme": "ssh",
+            "host": host,
+            "port": port,
+            "user": user,
+            "path": path,
+        }

dvc/progress.py

@@ -1,4 +1,10 @@
+"""Manages progress bars for dvc repo."""
+
 from __future__ import print_function
+from __future__ import unicode_literals
+
+from dvc.utils.compat import str
+
 import sys
 import threading
 
@@ -7,46 +13,63 @@ class Progress(object):
     """
     Simple multi-target progress bar.
     """
+
     def __init__(self):
         self._n_total = 0
         self._n_finished = 0
         self._lock = threading.Lock()
+        self._line = None
 
     def set_n_total(self, total):
+        """Sets total number of targets."""
         self._n_total = total
         self._n_finished = 0
 
+    @property
+    def is_finished(self):
+        """Returns if all targets have finished."""
+        return self._n_total == self._n_finished
+
     def _clearln(self):
-        print('\r\x1b[K', end='')
+        self._print("\r\x1b[K", end="")
 
     def _writeln(self, line):
         self._clearln()
-        print(line, end='')
+        self._print(line, end="")
         sys.stdout.flush()
 
-    def update_target(self, name, current, total):
+    def refresh(self, line=None):
+        """Refreshes progress bar."""
         # Just go away if it is locked. Will update next time
         if not self._lock.acquire(False):
             return
 
-        bar = self._bar(name, current, total)
+        if line is None:
+            line = self._line
 
-        if sys.stdout.isatty():
-            self._writeln(bar)
+        if sys.stdout.isatty() and line is not None:
+            self._writeln(line)
+            self._line = line
 
         self._lock.release()
 
+    def update_target(self, name, current, total):
+        """Updates progress bar for a specified target."""
+        self.refresh(self._bar(name, current, total))
+
     def finish_target(self, name):
+        """Finishes progress bar for a specified target."""
         # We have to write a msg about finished target
         with self._lock:
-            bar = self._bar(name, 100, 100)
+            pbar = self._bar(name, 100, 100)
 
             if sys.stdout.isatty():
                 self._clearln()
 
-            print(bar)
+            self._print(pbar)
 
             self._n_finished += 1
+            self._line = None
 
     def _bar(self, target_name, current, total):
         """
@@ -56,23 +79,56 @@ class Progress(object):
         bar_len = 30
 
         if total is None:
-            progress = 0
+            state = 0
             percent = "?% "
         else:
             total = int(total)
-            progress = int((100 * current)/total) if current < total else 100
-            percent = str(progress) + "% "
+            state = int((100 * current) / total) if current < total else 100
+            percent = str(state) + "% "
 
-        if self._n_total:
+        if self._n_total > 1:
             num = "({}/{}): ".format(self._n_finished + 1, self._n_total)
         else:
             num = ""
 
-        n_sh = int((progress * bar_len)/100)
+        n_sh = int((state * bar_len) / 100)
         n_sp = bar_len - n_sh
-        bar = "[" + '#'*n_sh + ' '*n_sp + "] "
+        pbar = "[" + "#" * n_sh + " " * n_sp + "] "
 
-        return num + bar + percent + target_name
+        return num + pbar + percent + target_name
+
+    @staticmethod
+    def _print(*args, **kwargs):
+        import dvc.logger as logger
+
+        if logger.is_quiet():
+            return
+
+        print(*args, **kwargs)
+
+    def __enter__(self):
+        self._lock.acquire(True)
+        if self._line is not None:
+            self._clearln()
+
+    def __exit__(self, typ, value, tbck):
+        if self._line is not None:
+            self.refresh()
+        self._lock.release()
 
 
-progress = Progress()
+def progress_aware(f):
+    """ Decorator to add a new line if progress bar hasn't finished  """
+    from functools import wraps
+
+    @wraps(f)
+    def wrapper(*args, **kwargs):
+        if not progress.is_finished:
+            progress._print()
+
+        return f(*args, **kwargs)
+
+    return wrapper
+
+
+progress = Progress()  # pylint: disable=invalid-name

dvc/project.py

@@ -1,707 +0,0 @@
-import os
-
-from dvc.exceptions import DvcException
-from dvc.stage import Stage
-
-
-class InitError(DvcException):
-    def __init__(self, msg):
-        super(InitError, self).__init__(msg)
-
-
-class NotDvcFileError(DvcException):
-    def __init__(self, path):
-        msg = '\'{}\' is not a DVC file'.format(path)
-        p = path + Stage.STAGE_FILE_SUFFIX
-        if Stage.is_stage_file(p):
-            msg += '. Maybe you meant \'{}\'?'.format(p)
-        super(NotDvcFileError, self).__init__(msg)
-
-
-class ReproductionError(DvcException):
-    def __init__(self, dvc_file_name, ex):
-        self.path = dvc_file_name
-        msg = 'Failed to reproduce \'{}\''.format(dvc_file_name)
-        super(ReproductionError, self).__init__(msg, cause=ex)
-
-
-class Project(object):
-    DVC_DIR = '.dvc'
-
-    def __init__(self, root_dir):
-        from dvc.logger import Logger
-        from dvc.config import Config
-        from dvc.state import LinkState, State
-        from dvc.lock import Lock
-        from dvc.scm import SCM
-        from dvc.cache import Cache
-        from dvc.data_cloud import DataCloud
-        from dvc.updater import Updater
-
-        self.root_dir = os.path.abspath(os.path.realpath(root_dir))
-        self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR)
-
-        self.config = Config(self.dvc_dir)
-        self.scm = SCM(self.root_dir)
-        self.lock = Lock(self.dvc_dir)
-        # NOTE: storing state and link_state in the repository itself to avoid
-        # any possible state corruption in 'shared cache dir' scenario.
-        self.state = State(self)
-        self.link_state = LinkState(self)
-
-        core = self.config._config[Config.SECTION_CORE]
-        self.logger = Logger(core.get(Config.SECTION_CORE_LOGLEVEL, None))
-
-        self.cache = Cache(self)
-        self.cloud = DataCloud(self, config=self.config._config)
-        self.updater = Updater(self.dvc_dir)
-
-        self._ignore()
-
-        self.updater.check()
-
-    @staticmethod
-    def init(root_dir=os.curdir, no_scm=False, force=False):
-        """
-        Initiate dvc project in directory.
-
-        Args:
-            root_dir: Path to project's root directory.
-
-        Returns:
-            Project instance.
-
-        Raises:
-            KeyError: Raises an exception.
-        """
-        import shutil
-        from dvc.scm import SCM, Base
-        from dvc.config import Config
-
-        root_dir = os.path.abspath(root_dir)
-        dvc_dir = os.path.join(root_dir, Project.DVC_DIR)
-
-        scm = SCM(root_dir)
-        if type(scm) == Base and not no_scm:
-            msg = "{} is not tracked by any supported scm tool(e.g. git)."
-            raise InitError(msg.format(root_dir))
-
-        if os.path.isdir(dvc_dir):
-            if not force:
-                msg = "'{}' exists. Use '-f' to force."
-                raise InitError(msg.format(os.path.relpath(dvc_dir)))
-            shutil.rmtree(dvc_dir)
-
-        os.mkdir(dvc_dir)
-
-        config = Config.init(dvc_dir)
-        proj = Project(root_dir)
-
-        scm.add([config.config_file])
-        if scm.ignore_file():
-            scm.add([os.path.join(dvc_dir, scm.ignore_file())])
-
-        return proj
-
-    def destroy(self):
-        import shutil
-
-        for stage in self.stages():
-            stage.remove()
-
-        shutil.rmtree(self.dvc_dir)
-
-    def _ignore(self):
-        flist = [self.state.state_file,
-                 self.state._lock_file.lock_file,
-                 self.link_state.state_file,
-                 self.link_state._lock_file.lock_file,
-                 self.lock.lock_file,
-                 self.config.config_local_file,
-                 self.updater.updater_file]
-
-        if self.cache.local.cache_dir.startswith(self.root_dir):
-            flist += [self.cache.local.cache_dir]
-
-        self.scm.ignore_list(flist)
-
-    def install(self):
-        self.scm.install()
-
-    def to_dvc_path(self, path):
-        return os.path.relpath(path, self.root_dir)
-
-    def add(self, fname):
-        stage = Stage.loads(project=self,
-                            outs=[fname],
-                            add=True)
-
-        stage.save()
-        stage.dump()
-        return stage
-
-    def remove(self, target, outs_only=False):
-        if not Stage.is_stage_file(target):
-            raise NotDvcFileError(target)
-
-        stage = Stage.load(self, target)
-        if outs_only:
-            stage.remove_outs()
-        else:
-            stage.remove()
-
-        return stage
-
-    def lock_stage(self, target, unlock=False):
-        if not Stage.is_stage_file(target):
-            raise NotDvcFileError(target)
-
-        stage = Stage.load(self, target)
-        stage.locked = False if unlock else True
-        stage.dump()
-
-        return stage
-
-    def move(self, from_path, to_path):
-        import dvc.output as Output
-
-        from_out = Output.loads_from(Stage(self, cwd=os.curdir),
-                                     [from_path])[0]
-
-        found = False
-        for stage in self.stages():
-            for out in stage.outs:
-                if out.path != from_out.path:
-                    continue
-
-                if not stage.is_data_source:
-                    msg = 'Dvcfile \'{}\' is not a data source.'
-                    raise DvcException(msg.format(stage.rel_path))
-
-                found = True
-                to_out = Output.loads_from(out.stage,
-                                           [to_path],
-                                           out.cache,
-                                           out.metric)[0]
-                out.move(to_out)
-
-                stage_base = os.path.basename(stage.path)
-                stage_base = stage_base.rstrip(Stage.STAGE_FILE_SUFFIX)
-
-                stage_dir = os.path.dirname(stage.path)
-                from_base = os.path.basename(from_path)
-                to_base = os.path.basename(to_path)
-                if stage_base == from_base:
-                    os.unlink(stage.path)
-                    path = to_base + Stage.STAGE_FILE_SUFFIX
-                    stage.path = os.path.join(stage_dir, path)
-
-            stage.dump()
-
-        if not found:
-            msg = 'Unable to find dvcfile with output \'{}\''
-            raise DvcException(msg.format(from_path))
-
-    def run(self,
-            cmd=None,
-            deps=[],
-            outs=[],
-            outs_no_cache=[],
-            metrics_no_cache=[],
-            fname=Stage.STAGE_FILE,
-            cwd=os.curdir,
-            no_exec=False):
-        stage = Stage.loads(project=self,
-                            fname=fname,
-                            cmd=cmd,
-                            cwd=cwd,
-                            outs=outs,
-                            outs_no_cache=outs_no_cache,
-                            metrics_no_cache=metrics_no_cache,
-                            deps=deps)
-        if not no_exec:
-            stage.run()
-        stage.dump()
-        return stage
-
-    def imp(self, url, out):
-        stage = Stage.loads(project=self,
-                            cmd=None,
-                            deps=[url],
-                            outs=[out])
-
-        stage.run()
-        stage.dump()
-        return stage
-
-    def _reproduce_stage(self, stages, node, force):
-        stage = stages[node]
-
-        if stage.locked:
-            msg = 'DVC file \'{}\' is locked. Its dependecies are not ' \
-                  'going to be reproduced.'
-            self.logger.warn(msg.format(stage.relpath))
-
-        stage = stage.reproduce(force=force)
-        if not stage:
-            return []
-        stage.dump()
-        return [stage]
-
-    def reproduce(self, target, recursive=True, force=False):
-        import networkx as nx
-
-        G = self.graph()[1]
-        stages = nx.get_node_attributes(G, 'stage')
-        node = os.path.relpath(os.path.abspath(target), self.root_dir)
-        if node not in stages:
-            raise NotDvcFileError(target)
-
-        if recursive:
-            return self._reproduce_stages(G, stages, node, force)
-
-        return self._reproduce_stage(stages, node, force)
-
-    def _reproduce_stages(self, G, stages, node, force):
-        import networkx as nx
-
-        result = []
-        for n in nx.dfs_postorder_nodes(G, node):
-            try:
-                result += self._reproduce_stage(stages, n, force)
-            except Exception as ex:
-                raise ReproductionError(stages[n].relpath, ex)
-        return result
-
-    def _cleanup_unused_links(self, all_stages):
-        used = []
-        for stage in all_stages:
-            for out in stage.outs:
-                used.append(out.path)
-        self.link_state.remove_unused(used)
-
-    def checkout(self, target=None):
-        all_stages = self.active_stages()
-
-        if target:
-            if not Stage.is_stage_file(target):
-                raise NotDvcFileError(target)
-            stages = [Stage.load(self, target)]
-        else:
-            stages = all_stages
-
-        self._cleanup_unused_links(all_stages)
-
-        for stage in stages:
-            if stage.locked:
-                msg = 'DVC file \'{}\' is locked. Its dependecies are not ' \
-                      'going to be checked out.'
-                self.logger.warn(msg.format(stage.relpath))
-
-            stage.checkout()
-
-    def _used_cache(self, target=None, all_branches=False, active=True):
-        cache = {}
-        cache['local'] = []
-        cache['s3'] = []
-        cache['gs'] = []
-        cache['hdfs'] = []
-        cache['ssh'] = []
-
-        for branch in self.scm.brancher(all_branches=all_branches):
-            if target:
-                stages = [Stage.load(self, target)]
-            elif active:
-                stages = self.active_stages()
-            else:
-                stages = self.stages()
-
-            for stage in stages:
-                if active and not target and stage.locked:
-                    msg = 'DVC file \'{}\' is locked. Its dependecies are ' \
-                          'not going to be pushed/pulled/fetched.'
-                    self.logger.warn(msg.format(stage.relpath))
-
-                for out in stage.outs:
-                    if not out.use_cache or not out.info:
-                        continue
-
-                    info = out.dumpd()
-                    info['branch'] = branch
-
-                    cache[out.path_info['scheme']] += [info]
-
-        return cache
-
-    def gc(self, all_branches=False, cloud=False, remote=None):
-        clist = self._used_cache(target=None,
-                                 all_branches=all_branches,
-                                 active=False)
-        self.cache.local.gc(clist)
-
-        if self.cache.s3:
-            self.cache.s3.gc(clist)
-
-        if self.cache.gs:
-            self.cache.gs.gc(clist)
-
-        if self.cache.ssh:
-            self.cache.ssh.gc(clist)
-
-        if self.cache.hdfs:
-            self.cache.hdfs.gc(clist)
-
-        if self.cache.azure:
-            self.cache.azure.gc(clist)
-
-        if cloud:
-            self.cloud._get_cloud(remote).gc(clist)
-
-    def push(self,
-             target=None,
-             jobs=1,
-             remote=None,
-             all_branches=False,
-             show_checksums=False):
-        self.cloud.push(self._used_cache(target, all_branches)['local'],
-                        jobs,
-                        remote=remote,
-                        show_checksums=show_checksums)
-
-    def fetch(self,
-              target=None,
-              jobs=1,
-              remote=None,
-              all_branches=False,
-              show_checksums=False):
-        self.cloud.pull(self._used_cache(target, all_branches)['local'],
-                        jobs,
-                        remote=remote,
-                        show_checksums=show_checksums)
-
-    def pull(self,
-             target=None,
-             jobs=1,
-             remote=None,
-             all_branches=False,
-             show_checksums=False):
-        self.fetch(target,
-                   jobs,
-                   remote=remote,
-                   all_branches=all_branches,
-                   show_checksums=show_checksums)
-        self.checkout(target=target)
-
-    def _local_status(self, target=None):
-        status = {}
-
-        if target:
-            stages = [Stage.load(self, target)]
-        else:
-            stages = self.active_stages()
-
-        for stage in stages:
-            if stage.locked:
-                msg = 'DVC file \'{}\' is locked. Its dependecies are not ' \
-                      'going to be shown in status output.'
-                self.logger.warn(msg.format(stage.relpath))
-
-            status.update(stage.status())
-
-        return status
-
-    def _cloud_status(self,
-                      target=None,
-                      jobs=1,
-                      remote=None,
-                      show_checksums=False):
-        import dvc.remote.base as cloud
-
-        status = {}
-        for md5, ret in self.cloud.status(self._used_cache(target)['local'],
-                                          jobs,
-                                          remote=remote,
-                                          show_checksums=show_checksums):
-            if ret == cloud.STATUS_OK:
-                continue
-
-            prefix_map = {
-                cloud.STATUS_DELETED: 'deleted',
-                cloud.STATUS_NEW: 'new',
-            }
-
-            status[md5] = prefix_map[ret]
-
-        return status
-
-    def status(self,
-               target=None,
-               jobs=1,
-               cloud=False,
-               remote=None,
-               show_checksums=False):
-        if cloud:
-            return self._cloud_status(target,
-                                      jobs,
-                                      remote=remote,
-                                      show_checksums=show_checksums)
-        return self._local_status(target)
-
-    def _read_metric_json(self, fd, json_path):
-        import json
-        from jsonpath_rw import parse
-
-        parser = parse(json_path)
-        return [x.value for x in parser.find(json.load(fd))]
-
-    def _do_read_metric_xsv(self, reader, row, col):
-        if col is not None and row is not None:
-            return [reader[row][col]]
-        elif col is not None:
-            return [r[col] for r in reader]
-        elif row is not None:
-            return reader[row]
-        return None
-
-    def _read_metric_hxsv(self, fd, hxsv_path, delimiter):
-        import csv
-
-        col, row = hxsv_path.split(',')
-        row = int(row)
-        reader = list(csv.DictReader(fd, delimiter=delimiter))
-        return self._do_read_metric_xsv(reader, row, col)
-
-    def _read_metric_xsv(self, fd, xsv_path, delimiter):
-        import csv
-
-        col, row = xsv_path.split(',')
-        row = int(row)
-        col = int(col)
-        reader = list(csv.reader(fd, delimiter=delimiter))
-        return self._do_read_metric_xsv(reader, row, col)
-
-    def _read_metric(self, path, typ=None, xpath=None):
-        ret = None
-
-        if not os.path.exists(path):
-            return ret
-
-        try:
-            with open(path, 'r') as fd:
-                if typ == 'json':
-                    ret = self._read_metric_json(fd, xpath)
-                elif typ == 'csv':
-                    ret = self._read_metric_xsv(fd, xpath, ',')
-                elif typ == 'tsv':
-                    ret = self._read_metric_xsv(fd, xpath, '\t')
-                elif typ == 'hcsv':
-                    ret = self._read_metric_hxsv(fd, xpath, ',')
-                elif typ == 'htsv':
-                    ret = self._read_metric_hxsv(fd, xpath, '\t')
-                else:
-                    ret = fd.read()
-        except Exception as exc:
-            self.logger.error('Unable to read metric in \'{}\''.format(path),
-                              exc)
-
-        return ret
-
-    def _find_output_by_path(self, path, outs=None):
-        from dvc.exceptions import OutputDuplicationError
-
-        if not outs:
-            astages = self.active_stages()
-            outs = [out for stage in astages for out in stage.outs]
-
-        abs_path = os.path.abspath(path)
-        matched = [out for out in outs if out.path == abs_path]
-        stages = [out.stage.path for out in matched]
-        if len(stages) > 1:
-            raise OutputDuplicationError(path, stages)
-
-        return matched[0] if matched else None
-
-    def metrics_show(self,
-                     path=None,
-                     typ=None,
-                     xpath=None,
-                     all_branches=False):
-        res = {}
-        for branch in self.scm.brancher(all_branches=all_branches):
-            astages = self.active_stages()
-            outs = [out for stage in astages for out in stage.outs]
-
-            if path:
-                out = self._find_output_by_path(path, outs=outs)
-                stage = out.stage.path if out else None
-                if out and all([out.metric,
-                                not typ,
-                                isinstance(out.metric, dict)]):
-                    entries = [(path,
-                                out.metric.get(out.PARAM_METRIC_TYPE, None),
-                                out.metric.get(out.PARAM_METRIC_XPATH, None))]
-                else:
-                    entries = [(path, typ, xpath)]
-            else:
-                metrics = filter(lambda o: o.metric, outs)
-                stage = None
-                entries = []
-                for o in metrics:
-                    if not typ and isinstance(o.metric, dict):
-                        t = o.metric.get(o.PARAM_METRIC_TYPE, None)
-                        x = o.metric.get(o.PARAM_METRIC_XPATH, None)
-                    else:
-                        t = typ
-                        x = xpath
-                    entries.append((o.path, t, x))
-
-            for fname, t, x in entries:
-                if stage:
-                    self.checkout(stage)
-
-                rel = os.path.relpath(fname)
-                metric = self._read_metric(fname,
-                                           typ=t,
-                                           xpath=x)
-                if not metric:
-                    continue
-
-                if branch not in res:
-                    res[branch] = {}
-
-                res[branch][rel] = metric
-
-        for branch, val in res.items():
-            if all_branches:
-                self.logger.info('{}:'.format(branch))
-            for fname, metric in val.items():
-                self.logger.info('\t{}: {}'.format(fname, metric))
-
-        if res:
-            return res
-
-        if path:
-            msg = 'File \'{}\' does not exist'.format(path)
-        else:
-            msg = 'No metric files in this repository. ' \
-                  'Use \'dvc metrics add\' to add a metric file to track.'
-        raise DvcException(msg)
-
-    def _metrics_modify(self, path, typ=None, xpath=None, delete=False):
-        out = self._find_output_by_path(path)
-        if not out:
-            msg = 'Unable to find file \'{}\' in the pipeline'.format(path)
-            raise DvcException(msg)
-
-        if out.path_info['scheme'] != 'local':
-            msg = 'Output \'{}\' scheme \'{}\' is not supported for metrics'
-            raise DvcException(msg.format(out.path, out.path_info['scheme']))
-
-        if out.use_cache:
-            msg = 'Cached output \'{}\' is not supported for metrics'
-            raise DvcException(msg.format(out.rel_path))
-
-        if typ:
-            if not isinstance(out.metric, dict):
-                out.metric = {}
-            out.metric[out.PARAM_METRIC_TYPE] = typ
-
-        if xpath:
-            if not isinstance(out.metric, dict):
-                out.metric = {}
-            out.metric[out.PARAM_METRIC_XPATH] = xpath
-
-        if delete:
-            out.metric = None
-
-        out._verify_metric()
-
-        out.stage.dump()
-
-    def metrics_modify(self, path=None, typ=None, xpath=None):
-        self._metrics_modify(path, typ, xpath)
-
-    def metrics_add(self, path, typ=None, xpath=None):
-        if not typ:
-            typ = 'raw'
-        self._metrics_modify(path, typ, xpath)
-
-    def metrics_remove(self, path):
-        self._metrics_modify(path, delete=True)
-
-    def graph(self):
-        import networkx as nx
-
-        G = nx.DiGraph()
-        G_active = nx.DiGraph()
-        stages = self.stages()
-
-        outs = []
-        for stage in stages:
-            outs += stage.outs
-
-        # collect the whole DAG
-        for stage in stages:
-            node = os.path.relpath(stage.path, self.root_dir)
-
-            G.add_node(node, stage=stage)
-            G_active.add_node(node, stage=stage)
-
-            for dep in stage.deps:
-                for out in outs:
-                    if out.path != dep.path \
-                       and not dep.path.startswith(out.path + out.sep):
-                        continue
-
-                    dep_stage = out.stage
-                    dep_node = os.path.relpath(dep_stage.path, self.root_dir)
-                    G.add_node(dep_node, stage=dep_stage)
-                    G.add_edge(node, dep_node)
-                    if not stage.locked:
-                        G_active.add_node(dep_node, stage=dep_stage)
-                        G_active.add_edge(node, dep_node)
-
-        return G, G_active
-
-    def pipelines(self):
-        import networkx as nx
-
-        G, G_active = self.graph()
-
-        if len(G.nodes()) == 0:
-            return []
-
-        # find pipeline ends aka "output stages"
-        ends = [node for node, in_degree in G.in_degree() if in_degree == 0]
-
-        # filter out subgraphs that didn't exist in original G
-        pipelines = []
-        for c in nx.weakly_connected_components(G_active):
-            H = G_active.subgraph(c)
-            found = False
-            for node in ends:
-                if node in H:
-                    found = True
-                    break
-            if found:
-                pipelines.append(H)
-
-        return pipelines
-
-    def stages(self):
-        stages = []
-        for root, dirs, files in os.walk(self.root_dir):
-            for fname in files:
-                path = os.path.join(root, fname)
-                if not Stage.is_stage_file(path):
-                    continue
-                stages.append(Stage.load(self, path))
-        return stages
-
-    def active_stages(self):
-        import networkx as nx
-
-        stages = []
-        for G in self.pipelines():
-            stages.extend(list(nx.get_node_attributes(G, 'stage').values()))
-        return stages

dvc/prompt.py

@@ -1,18 +1,61 @@
+"""Manages user prompts."""
+
+from __future__ import unicode_literals
+from __future__ import print_function
+
 import sys
+from getpass import getpass
 
-try:
-    # NOTE: in Python3 raw_input() was renamed to input()
-    input = raw_input
-except NameError:
-    pass
+from dvc.progress import progress_aware
+from dvc.utils.compat import input
 
 
-def prompt(msg, default=False):
+def _ask(prompt, limited_to=None):
     if not sys.stdout.isatty():
-        return default
+        return None
 
-    answer = input(msg + u'\n').lower()
-    while answer not in ['yes', 'no', 'y', 'n']:
-        answer = input('Enter \'yes\' or \'no\'.\n').lower()
+    while True:
+        try:
+            answer = input(prompt + " ").lower()
+        except EOFError:
+            return None
 
-    return answer[0] == "y"
+        if not limited_to:
+            return answer
+
+        if answer in limited_to:
+            return answer
+
+        print(
+            "Your response must be one of: {options}. "
+            "Please try again.".format(options=limited_to)
+        )
+
+
+@progress_aware
+def confirm(statement):
+    """Ask the user for confirmation about the specified statement.
+
+    Args:
+        statement (unicode): statement to ask the user confirmation about.
+
+    Returns:
+        bool: whether or not specified statement was confirmed.
+    """
+    prompt = "{statement} [y/n]".format(statement=statement)
+    answer = _ask(prompt, limited_to=["yes", "no", "y", "n"])
+    return answer and answer.startswith("y")
+
+
+@progress_aware
+def password(statement):
+    """Ask the user for a password.
+
+    Args:
+        statement (str): string to prompt the user with.
+
+    Returns:
+        str: password entered by the user.
+    """
+    prompt = "{statement}: ".format(statement=statement)
+    return getpass(prompt)

dvc/remote/__init__.py

@@ -1,27 +1,31 @@
-from dvc.remote.local import RemoteLOCAL
-from dvc.remote.s3 import RemoteS3
+from __future__ import unicode_literals
+
+from dvc.remote.azure import RemoteAzure
 from dvc.remote.gs import RemoteGS
 from dvc.remote.hdfs import RemoteHDFS
+from dvc.remote.local import RemoteLOCAL
+from dvc.remote.s3 import RemoteS3
 from dvc.remote.ssh import RemoteSSH
-from dvc.remote.azure import RemoteAzure
-
-from dvc.config import Config
-from dvc.exceptions import UnsupportedRemoteError
+from dvc.remote.http import RemoteHTTP
 
 
-REMOTES = [RemoteHDFS, RemoteSSH, RemoteS3, RemoteGS, RemoteAzure, RemoteLOCAL]
+REMOTES = [
+    RemoteAzure,
+    RemoteGS,
+    RemoteHDFS,
+    RemoteHTTP,
+    RemoteS3,
+    RemoteSSH,
+    # NOTE: RemoteLOCAL is the default
+]
 
 
-def supported_url(url):
-    config = {Config.SECTION_REMOTE_URL: url}
-    for r in REMOTES:
-        if r.supported(config):
-            return True
-    return False
+def _get(config):
+    for remote in REMOTES:
+        if remote.supported(config):
+            return remote
+    return RemoteLOCAL
 
 
-def Remote(project, config):
-    for r in REMOTES:
-        if r.supported(config):
-            return r(project, config)
-    raise UnsupportedRemoteError(str(config))
+def Remote(repo, config):
+    return _get(config)(repo, config)

dvc/remote/azure.py

@@ -1,4 +1,6 @@
 from __future__ import absolute_import
+from __future__ import unicode_literals
+
 import os
 import re
 
@@ -7,7 +9,9 @@ try:
 except ImportError:
     BlockBlobService = None
 
-from dvc.logger import Logger
+import dvc.logger as logger
+from dvc.utils import tmp_fname, move
+from dvc.utils.compat import urlparse, makedirs
 from dvc.progress import progress
 from dvc.config import Config
 from dvc.remote.base import RemoteBase
@@ -22,216 +26,168 @@ class Callback(object):
 
 
 class RemoteAzure(RemoteBase):
-    scheme = 'azure'
-    REGEX = (r'^azure://'
-             r'(ContainerName=(?P<container_name>[^;]+);?)?'
-             r'(?P<connection_string>.+)?$')
-    REQUIRES = {'azure-storage-blob': BlockBlobService}
-    PARAM_ETAG = 'etag'
+    scheme = "azure"
+    REGEX = (
+        r"azure://((?P<path>[^=;]*)?|("
+        # backward compatibility
+        r"(ContainerName=(?P<container_name>[^;]+);?)?"
+        r"(?P<connection_string>.+)?)?)$"
+    )
+    REQUIRES = {"azure-storage-blob": BlockBlobService}
+    PARAM_CHECKSUM = "etag"
     COPY_POLL_SECONDS = 5
 
-    def __init__(self, project, config):
-        super(RemoteAzure, self).__init__(project, config)
-        self.project = project
+    def __init__(self, repo, config):
+        super(RemoteAzure, self).__init__(repo, config)
 
-        url = config.get(Config.SECTION_REMOTE_URL)
-        match = re.match(self.REGEX, url)
+        self.url = config.get(Config.SECTION_REMOTE_URL, "azure://")
+        match = re.match(self.REGEX, self.url)  # backward compatibility
 
+        path = match.group("path")
         self.bucket = (
-            match.group('container_name')
-            or os.getenv('AZURE_STORAGE_CONTAINER_NAME'))
+            urlparse(self.url if path else "").netloc
+            or match.group("container_name")  # backward compatibility
+            or os.getenv("AZURE_STORAGE_CONTAINER_NAME")
+        )
+
+        self.prefix = urlparse(self.url).path.lstrip("/") if path else ""
 
         self.connection_string = (
-            match.group('connection_string')
-            or os.getenv('AZURE_STORAGE_CONNECTION_STRING'))
+            config.get(Config.SECTION_AZURE_CONNECTION_STRING)
+            or match.group("connection_string")  # backward compatibility
+            or os.getenv("AZURE_STORAGE_CONNECTION_STRING")
+        )
 
         if not self.bucket:
-            raise ValueError('Azure Storage container name missing')
+            raise ValueError("azure storage container name missing")
 
         if not self.connection_string:
-            raise ValueError('Azure Storage connection string missing')
+            raise ValueError("azure storage connection string missing")
 
         self.__blob_service = None
 
+        self.path_info = {"scheme": self.scheme, "bucket": self.bucket}
+
     @property
     def blob_service(self):
         if self.__blob_service is None:
+            logger.debug("URL {}".format(self.url))
+            logger.debug("Connection string {}".format(self.connection_string))
             self.__blob_service = BlockBlobService(
-                connection_string=self.connection_string)
+                connection_string=self.connection_string
+            )
+            logger.debug("Container name {}".format(self.bucket))
             self.__blob_service.create_container(self.bucket)
         return self.__blob_service
 
-# FIXME: temporarily disabled because of the lack of test for external azure
-# dependencies/outputs/cache.
-#
-#    def _get_etag(self, bucket, key):
-#        try:
-#            blob = self.blob_service.get_blob_properties(bucket, key)
-#            return blob.properties.etag
-#        except Exception:
-#            return None
-#
-#    def save_info(self, path_info):
-#        if path_info['scheme'] != self.scheme:
-#            raise NotImplementedError
-#
-#        return {self.PARAM_ETAG: self._get_etag(
-#            path_info['bucket'], path_info['key'])}
-#
-#    def save(self, path_info):
-#        if path_info['scheme'] != self.scheme:
-#            raise NotImplementedError
-#
-#        etag = self._get_etag(path_info['bucket'], path_info['key'])
-#        dest_key = '{}/{}'.format(etag[0:2], etag[2:])
-#
-#        self._copy_blob(to_bucket=self.bucket,
-#                        to_key=dest_key,
-#                        from_bucket=path_info['bucket'],
-#                        from_key=path_info['key'])
-#
-#        return {self.PARAM_ETAG: etag}
-#
-#    def _copy_blob(self, to_bucket, to_key, from_bucket, from_key):
-#        source = self.blob_service.make_blob_url(from_bucket, from_key)
-#
-#        copy = self.blob_service.copy_blob(to_bucket, to_key, source)
-#
-#        if self.COPY_POLL_SECONDS <= 0:
-#            return
-#
-#        while copy.status != 'success':
-#            time.sleep(self.COPY_POLL_SECONDS)
-#            copy = self.blob_service.get_blob_properties(
-#                to_bucket, to_key).properties.copy
-#
-#    def checkout(self, path_info, checksum_info):
-#        if path_info['scheme'] != self.scheme:
-#            raise NotImplementedError
-#
-#        etag = checksum_info.get(self.PARAM_ETAG, None)
-#        if not etag:
-#            return
-#
-#        self._copy_blob(to_bucket=path_info['bucket'],
-#                        to_key=path_info['key'],
-#                        from_bucket=self.bucket,
-#                        from_key='{}/{}'.format(etag[0:2], etag[2:]))
-#
-#    def remove(self, path_info):
-#        if path_info['scheme'] != self.scheme:
-#            raise NotImplementedError
-#
-#        Logger.debug('Removing azure://{}/{}'.format(path_info['bucket'],
-#                                                     path_info['key']))
-#
-#        self.blob_service.delete_blob(path_info['bucket'], path_info['key'])
+    def remove(self, path_info):
+        if path_info["scheme"] != self.scheme:
+            raise NotImplementedError
 
-    def md5s_to_path_infos(self, md5s):
-        return [{
-            'scheme': self.scheme,
-            'bucket': self.bucket,
-            'key': '{}/{}'.format(md5[0:2], md5[2:])
-        } for md5 in md5s]
+        logger.debug(
+            "Removing azure://{}/{}".format(
+                path_info["bucket"], path_info["path"]
+            )
+        )
 
-    def exists(self, path_infos):
-        keys = {blob.name
-                for blob in self.blob_service.list_blobs(self.bucket)}
+        self.blob_service.delete_blob(path_info["bucket"], path_info["path"])
 
-        ret = []
-        for path_info in path_infos:
-            if path_info['scheme'] != self.scheme:
-                raise NotImplementedError
+    def _list_paths(self, bucket, prefix):
+        blob_service = self.blob_service
+        next_marker = None
+        while True:
+            blobs = blob_service.list_blobs(
+                bucket, prefix=prefix, marker=next_marker
+            )
 
-            key = path_info['key']
-            ret.append(key in keys)
+            for blob in blobs:
+                yield blob.name
 
-        return ret
+            if not blobs.next_marker:
+                break
+
+            next_marker = blobs.next_marker
+
+    def list_cache_paths(self):
+        return self._list_paths(self.bucket, self.prefix)
 
     def upload(self, from_infos, to_infos, names=None):
         names = self._verify_path_args(to_infos, from_infos, names)
 
         for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info['scheme'] != self.scheme:
+            if to_info["scheme"] != self.scheme:
                 raise NotImplementedError
 
-            if from_info['scheme'] != 'local':
+            if from_info["scheme"] != "local":
                 raise NotImplementedError
 
-            bucket = to_info['bucket']
-            key = to_info['key']
+            bucket = to_info["bucket"]
+            path = to_info["path"]
 
-            Logger.debug("Uploading '{}' to '{}/{}'".format(
-                from_info['path'], bucket, key))
+            logger.debug(
+                "Uploading '{}' to '{}/{}'".format(
+                    from_info["path"], bucket, path
+                )
+            )
 
             if not name:
-                name = os.path.basename(from_info['path'])
+                name = os.path.basename(from_info["path"])
 
             cb = Callback(name)
 
             try:
                 self.blob_service.create_blob_from_path(
-                    bucket, key, from_info['path'], progress_callback=cb)
-            except Exception as ex:
-                Logger.error("Failed to upload '{}'".format(from_info['path']),
-                             ex)
+                    bucket, path, from_info["path"], progress_callback=cb
+                )
+            except Exception:
+                msg = "failed to upload '{}'".format(from_info["path"])
+                logger.warning(msg)
             else:
                 progress.finish_target(name)
 
-    def download(self,
-                 from_infos,
-                 to_infos,
-                 no_progress_bar=False,
-                 names=None):
+    def download(
+        self,
+        from_infos,
+        to_infos,
+        no_progress_bar=False,
+        names=None,
+        resume=False,
+    ):
         names = self._verify_path_args(from_infos, to_infos, names)
 
         for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info['scheme'] != self.scheme:
+            if from_info["scheme"] != self.scheme:
                 raise NotImplementedError
 
-            if to_info['scheme'] != 'local':
+            if to_info["scheme"] != "local":
                 raise NotImplementedError
 
-            bucket = from_info['bucket']
-            key = from_info['key']
+            bucket = from_info["bucket"]
+            path = from_info["path"]
 
-            Logger.debug("Downloading '{}/{}' to '{}'".format(
-                bucket, key, to_info['path']))
+            logger.debug(
+                "Downloading '{}/{}' to '{}'".format(
+                    bucket, path, to_info["path"]
+                )
+            )
 
-            tmp_file = self.tmp_file(to_info['path'])
+            tmp_file = tmp_fname(to_info["path"])
             if not name:
-                name = os.path.basename(to_info['path'])
+                name = os.path.basename(to_info["path"])
 
             cb = None if no_progress_bar else Callback(name)
 
-            self._makedirs(to_info['path'])
+            makedirs(os.path.dirname(to_info["path"]), exist_ok=True)
 
             try:
                 self.blob_service.get_blob_to_path(
-                    bucket, key, tmp_file, progress_callback=cb)
-            except Exception as exc:
-                Logger.error("Failed to download '{}/{}'".format(
-                    bucket, key), exc)
+                    bucket, path, tmp_file, progress_callback=cb
+                )
+            except Exception:
+                msg = "failed to download '{}/{}'".format(bucket, path)
+                logger.warning(msg)
             else:
-                os.rename(tmp_file, to_info['path'])
+                move(tmp_file, to_info["path"])
 
                 if not no_progress_bar:
                     progress.finish_target(name)
-
-# FIXME: temporarily disabled because of the lack of test for external azure
-# dependencies/outputs/cache.
-#
-#    def gc(self, cinfos):
-#        used = [info[self.PARAM_ETAG] for info in cinfos['azure']]
-#        used += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
-#
-#        all_blobs = self.blob_service.list_blobs(self.bucket)
-#
-#        for blob in all_blobs:
-#            etag = blob.properties.etag
-#            if etag in used:
-#                continue
-#            path_info = {'scheme': self.scheme,
-#                         'key': blob.name,
-#                         'bucket': self.bucket}
-#            self.remove(path_info)

dvc/remote/base.py

@@ -1,10 +1,15 @@
-import os
-import re
-import errno
+from __future__ import unicode_literals
 
+from dvc.utils.compat import str
+
+import re
+import posixpath
+from multiprocessing import cpu_count
+
+import dvc.prompt as prompt
+import dvc.logger as logger
 from dvc.config import Config
-from dvc.logger import Logger
-from dvc.exceptions import DvcException
+from dvc.exceptions import DvcException, ConfirmRemoveError
 
 
 STATUS_OK = 1
@@ -23,28 +28,75 @@ STATUS_MAP = {
 
 class DataCloudError(DvcException):
     """ Data Cloud exception """
+
     def __init__(self, msg):
-        super(DataCloudError, self).__init__('Data sync error: {}'.format(msg))
+        super(DataCloudError, self).__init__("Data sync error: {}".format(msg))
+
+
+class RemoteCmdError(DvcException):
+    def __init__(self, remote, cmd, ret, err):
+        super(RemoteCmdError, self).__init__(
+            "{remote} command '{cmd}' finished with non-zero return code"
+            " {ret}': {err}".format(remote=remote, cmd=cmd, ret=ret, err=err)
+        )
+
+
+class RemoteActionNotImplemented(DvcException):
+    def __init__(self, action, scheme):
+        m = "{} is not supported by {} remote".format(action, scheme)
+        super(RemoteActionNotImplemented, self).__init__(m)
+
+
+class RemoteMissingDepsError(DvcException):
+    pass
 
 
 class RemoteBase(object):
+    scheme = None
     REGEX = None
     REQUIRES = {}
+    JOBS = 4 * cpu_count()
 
-    def __init__(self, project, config):
-        pass
+    def __init__(self, repo, config):
+        self.repo = repo
+        deps_ok = all(self.REQUIRES.values())
+        if not deps_ok:
+            missing = [k for k, v in self.REQUIRES.items() if v is None]
+            url = config.get(
+                Config.SECTION_REMOTE_URL, "{}://".format(self.scheme)
+            )
+            msg = (
+                "URL '{}' is supported but requires these missing "
+                "dependencies: {}. If you have installed dvc using pip, "
+                "choose one of these options to proceed: \n"
+                "\n"
+                "    1) Install specific missing dependencies:\n"
+                "        pip install {}\n"
+                "    2) Install dvc package that includes those missing "
+                "dependencies: \n"
+                "        pip install dvc[{}]\n"
+                "    3) Install dvc package with all possible "
+                "dependencies included: \n"
+                "        pip install dvc[all]\n"
+                "\n"
+                "If you have installed dvc from a binary package and you "
+                "are still seeing this message, please report it to us "
+                "using https://github.com/iterative/dvc/issues. Thank you!"
+            ).format(url, missing, " ".join(missing), self.scheme)
+            raise RemoteMissingDepsError(msg)
+
+    def __repr__(self):
+        return "{class_name}: '{url}'".format(
+            class_name=type(self).__name__, url=(self.url or "No url")
+        )
+
+    def compat_config(config):
+        return config.copy()
 
     @classmethod
     def supported(cls, config):
         url = config[Config.SECTION_REMOTE_URL]
-        url_ok = cls.match(url)
-        deps_ok = all(cls.REQUIRES.values())
-        if url_ok and not deps_ok:
-            missing = [k for k, v in cls.REQUIRES.items() if v is None]
-            msg = "URL \'{}\' is supported but requires " \
-                  "these missing dependencies: {}"
-            Logger.warn(msg.format(url, str(missing)))
-        return url_ok and deps_ok
+        return cls.match(url) is not None
 
     @classmethod
     def match(cls, url):
@@ -56,44 +108,97 @@ class RemoteBase(object):
             return None
         return m.group(name)
 
-    @staticmethod
-    def tmp_file(fname):
-        """ Temporary name for a partial download """
-        # FIXME probably better use uuid()
-        return fname + '.part'
-
     def save_info(self, path_info):
         raise NotImplementedError
 
-    def save(self, path_info):
-        raise NotImplementedError
+    def changed(self, path_info, checksum_info):
+        """Checks if data has changed.
 
-    def checkout(self, path_info, checksum_info):
-        raise NotImplementedError
+        A file is considered changed if:
+            - It doesn't exist on the working directory (was unlinked)
+            - Checksum is not computed (saving a new file)
+            - The checkusm stored in the State is different from the given one
+            - There's no file in the cache
 
-    def download(self, from_infos, to_infos, no_progress_bar=False, name=None):
-        raise NotImplementedError
+        Args:
+            path_info: dict with path information.
+            checksum: expected checksum for this data.
 
-    def upload(self, from_infos, to_infos, path_info, name=None):
-        raise NotImplementedError
+        Returns:
+            bool: True if data has changed, False otherwise.
+        """
+
+        logger.debug(
+            "checking if '{}'('{}') has changed.".format(
+                path_info, checksum_info
+            )
+        )
+
+        if not self.exists(path_info):
+            logger.debug("'{}' doesn't exist.".format(path_info))
+            return True
+
+        checksum = checksum_info.get(self.PARAM_CHECKSUM)
+        if checksum is None:
+            logger.debug("checksum for '{}' is missing.".format(path_info))
+            return True
+
+        if self.changed_cache(checksum):
+            logger.debug(
+                "cache for '{}'('{}') has changed.".format(path_info, checksum)
+            )
+            return True
+
+        actual = self.save_info(path_info)[self.PARAM_CHECKSUM]
+        if checksum != actual:
+            logger.debug(
+                "checksum '{}'(actual '{}') for '{}' has changed.".format(
+                    checksum, actual, path_info
+                )
+            )
+            return True
+
+        logger.debug("'{}' hasn't changed.".format(path_info))
+        return False
+
+    def save(self, path_info, checksum_info):
+        if path_info["scheme"] != self.scheme:
+            raise RemoteActionNotImplemented(
+                "save {} -> {}".format(path_info["scheme"], self.scheme),
+                self.scheme,
+            )
+
+        checksum = checksum_info[self.PARAM_CHECKSUM]
+        if not self.changed_cache(checksum):
+            return
+
+        to_info = self.checksum_to_path_info(checksum)
+
+        logger.info("Saving '{}' to '{}'.".format(path_info, to_info))
+
+        self.copy(path_info, to_info)
+
+    def download(
+        self,
+        from_infos,
+        to_infos,
+        no_progress_bar=False,
+        name=None,
+        resume=False,
+    ):
+        raise RemoteActionNotImplemented("download", self.scheme)
+
+    def upload(self, from_infos, to_infos, names=None):
+        raise RemoteActionNotImplemented("upload", self.scheme)
 
     def remove(self, path_info):
-        raise NotImplementedError
+        raise RemoteActionNotImplemented("remove", self.scheme)
 
     def move(self, path_info):
-        raise NotImplementedError
+        raise RemoteActionNotImplemented("move", self.scheme)
 
-    def _makedirs(self, fname):
-        dname = os.path.dirname(fname)
-
-        try:
-            os.makedirs(dname)
-        except OSError as e:
-            if e.errno != errno.EEXIST:
-                raise
-
-    def md5s_to_path_infos(self, md5s):
-        raise NotImplementedError
+    def copy(self, from_info, to_info):
+        raise RemoteActionNotImplemented("copy", self.scheme)
 
     def exists(self, path_infos):
         raise NotImplementedError
@@ -111,3 +216,143 @@ class RemoteBase(object):
             assert len(names) == len(to_infos)
 
         return names
+
+    @property
+    def ospath(self):
+        return posixpath
+
+    def checksum_to_path(self, checksum):
+        return self.ospath.join(self.prefix, checksum[0:2], checksum[2:])
+
+    def path_to_checksum(self, path):
+        relpath = self.ospath.relpath(path, self.prefix)
+        return self.ospath.dirname(relpath) + self.ospath.basename(relpath)
+
+    def checksum_to_path_info(self, checksum):
+        path_info = self.path_info.copy()
+        path_info["path"] = self.checksum_to_path(checksum)
+        return path_info
+
+    def md5s_to_path_infos(self, md5s):
+        return [self.checksum_to_path_info(md5) for md5 in md5s]
+
+    def list_cache_paths(self):
+        raise NotImplementedError
+
+    def all(self):
+        # NOTE: The list might be way too big(e.g. 100M entries, md5 for each
+        # is 32 bytes, so ~3200Mb list) and we don't really need all of it at
+        # the same time, so it makes sense to use a generator to gradually
+        # iterate over it, without keeping all of it in memory.
+        return (
+            self.path_to_checksum(path) for path in self.list_cache_paths()
+        )
+
+    def gc(self, cinfos):
+        from dvc.remote.local import RemoteLOCAL
+
+        used = [info[RemoteLOCAL.PARAM_CHECKSUM] for info in cinfos["local"]]
+
+        if self.scheme != "":
+            used += [info[self.PARAM_CHECKSUM] for info in cinfos[self.scheme]]
+
+        removed = False
+        for checksum in self.all():
+            if checksum in used:
+                continue
+            path_info = self.checksum_to_path_info(checksum)
+            self.remove(path_info)
+            removed = True
+        return removed
+
+    def changed_cache(self, checksum):
+        cache = self.checksum_to_path_info(checksum)
+        expected = {self.PARAM_CHECKSUM: checksum}
+
+        if not self.exists(cache):
+            return True
+
+        actual = self.save_info(cache)
+
+        logger.debug(
+            "Cache '{}' actual '{}'.".format(str(expected), str(actual))
+        )
+
+        if expected != actual:
+            if self.exists(cache):
+                msg = "corrupted cache file {}"
+                logger.warning(msg.format(str(cache)))
+                self.remove(cache)
+            return True
+
+        return False
+
+    def cache_exists(self, checksums):
+        # NOTE: The reason for such an odd logic is that most of the remotes
+        # take much shorter time to just retrieve everything they have under
+        # a certain prefix(e.g. s3, gs, ssh, hdfs). Other remotes that can
+        # check if particular file exists much quicker, use their own
+        # implementation of cache_exists(see http, local).
+        #
+        # Result of all() might be way too big, so we should walk through
+        # it in one pass.
+        return list(filter(lambda checksum: checksum in checksums, self.all()))
+
+    def already_cached(self, path_info):
+        current = self.save_info(path_info)[self.PARAM_CHECKSUM]
+
+        if not current:
+            return False
+
+        return not self.changed_cache(current)
+
+    def safe_remove(self, path_info, force=False):
+        if not self.exists(path_info):
+            return
+
+        if not force and not self.already_cached(path_info):
+            msg = (
+                "file '{}' is going to be removed."
+                " Are you sure you want to proceed?".format(str(path_info))
+            )
+
+            if not prompt.confirm(msg):
+                raise ConfirmRemoveError(str(path_info))
+
+        self.remove(path_info)
+
+    def do_checkout(self, path_info, checksum, force=False):
+        if self.exists(path_info):
+            msg = "data '{}' exists. Removing before checkout."
+            logger.warning(msg.format(str(path_info)))
+            self.safe_remove(path_info, force=force)
+
+        from_info = self.checksum_to_path_info(checksum)
+        self.copy(from_info, path_info)
+
+    def checkout(self, path_info, checksum_info, force=False):
+        scheme = path_info["scheme"]
+        if scheme not in ["", "local"] and scheme != self.scheme:
+            raise NotImplementedError
+
+        checksum = checksum_info.get(self.PARAM_CHECKSUM)
+        if not checksum:
+            msg = "No checksum info for '{}'."
+            logger.info(msg.format(str(path_info)))
+            return
+
+        if not self.changed(path_info, checksum_info):
+            msg = "Data '{}' didn't change."
+            logger.info(msg.format(str(path_info)))
+            return
+
+        if self.changed_cache(checksum):
+            msg = "Cache '{}' not found. File '{}' won't be created."
+            logger.warning(msg.format(checksum, str(path_info)))
+            self.safe_remove(path_info, force=force)
+            return
+
+        msg = "Checking out '{}' with cache '{}'."
+        logger.info(msg.format(str(path_info), checksum))
+
+        self.do_checkout(path_info, checksum, force=force)

dvc/remote/gs.py

@@ -1,134 +1,121 @@
+from __future__ import unicode_literals
+
 import os
-import posixpath
 
 try:
     from google.cloud import storage
 except ImportError:
     storage = None
 
-try:
-    from urlparse import urlparse
-except ImportError:
-    from urllib.parse import urlparse
-
-from dvc.logger import Logger
+import dvc.logger as logger
+from dvc.utils import tmp_fname, move
+from dvc.utils.compat import urlparse, makedirs
 from dvc.remote.base import RemoteBase
-from dvc.remote.local import RemoteLOCAL
 from dvc.config import Config
 from dvc.progress import progress
 from dvc.exceptions import DvcException
 
 
 class RemoteGS(RemoteBase):
-    scheme = 'gs'
-    REGEX = r'^gs://(?P<path>.*)$'
-    REQUIRES = {'google.cloud.storage': storage}
-    PARAM_ETAG = 'etag'
+    scheme = "gs"
+    REGEX = r"^gs://(?P<path>.*)$"
+    REQUIRES = {"google.cloud.storage": storage}
+    PARAM_CHECKSUM = "md5"
 
-    def __init__(self, project, config):
-        self.project = project
-        storagepath = 'gs://'
-        storagepath += config.get(Config.SECTION_AWS_STORAGEPATH, '/')
-        storagepath.lstrip('/')
+    def __init__(self, repo, config):
+        super(RemoteGS, self).__init__(repo, config)
+        storagepath = "gs://"
+        storagepath += config.get(Config.SECTION_AWS_STORAGEPATH, "/")
+        storagepath.lstrip("/")
         self.url = config.get(Config.SECTION_REMOTE_URL, storagepath)
         self.projectname = config.get(Config.SECTION_GCP_PROJECTNAME, None)
+        self.credentialpath = config.get(Config.SECTION_GCP_CREDENTIALPATH)
 
-    @property
-    def bucket(self):
-        return urlparse(self.url).netloc
+        parsed = urlparse(self.url)
+        self.bucket = parsed.netloc
+        self.prefix = parsed.path.lstrip("/")
 
-    @property
-    def prefix(self):
-        return urlparse(self.url).path.lstrip('/')
+        self.path_info = {"scheme": "gs", "bucket": self.bucket}
+
+    @staticmethod
+    def compat_config(config):
+        ret = config.copy()
+        url = "gs://" + ret.pop(Config.SECTION_AWS_STORAGEPATH, "").lstrip("/")
+        ret[Config.SECTION_REMOTE_URL] = url
+        return ret
 
     @property
     def gs(self):
-        return storage.Client()
+        return (
+            storage.Client.from_service_account_json(self.credentialpath)
+            if self.credentialpath
+            else storage.Client(self.projectname)
+        )
 
-    def get_etag(self, bucket, key):
-        blob = self.gs.bucket(bucket).get_blob(key)
+    def get_md5(self, bucket, path):
+        import base64
+        import codecs
+
+        blob = self.gs.bucket(bucket).get_blob(path)
         if not blob:
             return None
 
-        return blob.etag
+        b64_md5 = blob.md5_hash
+        md5 = base64.b64decode(b64_md5)
+        return codecs.getencoder("hex")(md5)[0].decode("utf-8")
 
     def save_info(self, path_info):
-        if path_info['scheme'] != 'gs':
+        if path_info["scheme"] != "gs":
             raise NotImplementedError
 
-        return {self.PARAM_ETAG: self.get_etag(path_info['bucket'],
-                                               path_info['key'])}
+        return {
+            self.PARAM_CHECKSUM: self.get_md5(
+                path_info["bucket"], path_info["path"]
+            )
+        }
 
-    def _copy(self, from_info, to_info, gs=None):
+    def copy(self, from_info, to_info, gs=None):
         gs = gs if gs else self.gs
 
-        blob = gs.bucket(from_info['bucket']).get_blob(from_info['key'])
+        blob = gs.bucket(from_info["bucket"]).get_blob(from_info["path"])
         if not blob:
-            msg = '{} doesn\'t exist in the cloud'
-            raise DvcException(msg.format(from_info['key']))
+            msg = "'{}' doesn't exist in the cloud".format(from_info["path"])
+            raise DvcException(msg)
 
-        bucket = self.gs.bucket(to_info['bucket'])
-        bucket.copy_blob(blob,
-                         self.gs.bucket(to_info['bucket']),
-                         new_name=to_info['key'])
-
-    def save(self, path_info):
-        if path_info['scheme'] != 'gs':
-            raise NotImplementedError
-
-        etag = self.get_etag(path_info['bucket'], path_info['key'])
-        key = posixpath.join(self.prefix, etag[0:2], etag[2:])
-        to_info = {'scheme': 'gs', 'bucket': self.bucket, 'key': key}
-
-        self._copy(path_info, to_info)
-
-        return {self.PARAM_ETAG: etag}
-
-    def checkout(self, path_info, checksum_info):
-        if path_info['scheme'] != 'gs':
-            raise NotImplementedError
-
-        etag = checksum_info.get(self.PARAM_ETAG, None)
-        if not etag:
-            return
-
-        key = posixpath.join(self.prefix, etag[0:2], etag[2:])
-        from_info = {'scheme': 'gs', 'bucket': self.bucket, 'key': key}
-
-        self._copy(from_info, path_info)
+        bucket = self.gs.bucket(to_info["bucket"])
+        bucket.copy_blob(
+            blob, self.gs.bucket(to_info["bucket"]), new_name=to_info["path"]
+        )
 
     def remove(self, path_info):
-        if path_info['scheme'] != 'gs':
+        if path_info["scheme"] != "gs":
             raise NotImplementedError
 
-        Logger.debug("Removing gs://{}/{}".format(path_info['bucket'],
-                                                  path_info['key']))
+        logger.debug(
+            "Removing gs://{}/{}".format(
+                path_info["bucket"], path_info["path"]
+            )
+        )
 
-        blob = self.gs.bucket(path_info['bucket']).get_blob(path_info['key'])
+        blob = self.gs.bucket(path_info["bucket"]).get_blob(path_info["path"])
         if not blob:
             return
 
         blob.delete()
 
-    def md5s_to_path_infos(self, md5s):
-        return [{'scheme': 'gs',
-                 'bucket': self.bucket,
-                 'key': posixpath.join(self.prefix,
-                                       md5[0:2], md5[2:])} for md5 in md5s]
+    def _list_paths(self, bucket, prefix):
+        for blob in self.gs.bucket(bucket).list_blobs(prefix=prefix):
+            yield blob.name
 
-    def exists(self, path_infos):
-        ret = []
-        gs = self.gs
+    def list_cache_paths(self):
+        return self._list_paths(self.bucket, self.prefix)
 
-        bucket = gs.bucket(self.bucket)
-        keys = [blob.name for blob in bucket.list_blobs(prefix=self.prefix)]
+    def exists(self, path_info):
+        assert not isinstance(path_info, list)
+        assert path_info["scheme"] == "gs"
 
-        for path_info in path_infos:
-            exists = False
-            if path_info['key'] in keys:
-                exists = True
-            ret.append(exists)
-        return ret
+        paths = self._list_paths(path_info["bucket"], path_info["path"])
+        return any(path_info["path"] == path for path in paths)
 
     def upload(self, from_infos, to_infos, names=None):
         names = self._verify_path_args(to_infos, from_infos, names)
@@ -136,104 +123,91 @@ class RemoteGS(RemoteBase):
         gs = self.gs
 
         for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info['scheme'] != 'gs':
+            if to_info["scheme"] != "gs":
                 raise NotImplementedError
 
-            if from_info['scheme'] != 'local':
+            if from_info["scheme"] != "local":
                 raise NotImplementedError
 
-            Logger.debug("Uploading '{}' to '{}/{}'".format(from_info['path'],
-                                                            to_info['bucket'],
-                                                            to_info['key']))
+            logger.debug(
+                "Uploading '{}' to '{}/{}'".format(
+                    from_info["path"], to_info["bucket"], to_info["path"]
+                )
+            )
 
             if not name:
-                name = os.path.basename(from_info['path'])
+                name = os.path.basename(from_info["path"])
 
             progress.update_target(name, 0, None)
 
             try:
-                bucket = gs.bucket(to_info['bucket'])
-                blob = bucket.blob(to_info['key'])
-                blob.upload_from_filename(from_info['path'])
-            except Exception as exc:
-                msg = "Failed to upload '{}' to '{}/{}'"
-                Logger.error(msg.format(from_info['path'],
-                                        to_info['bucket'],
-                                        to_info['key']), exc)
+                bucket = gs.bucket(to_info["bucket"])
+                blob = bucket.blob(to_info["path"])
+                blob.upload_from_filename(from_info["path"])
+            except Exception:
+                msg = "failed to upload '{}' to '{}/{}'"
+                logger.error(
+                    msg.format(
+                        from_info["path"], to_info["bucket"], to_info["path"]
+                    )
+                )
                 continue
 
             progress.finish_target(name)
 
-    def download(self,
-                 from_infos,
-                 to_infos,
-                 no_progress_bar=False,
-                 names=None):
+    def download(
+        self,
+        from_infos,
+        to_infos,
+        no_progress_bar=False,
+        names=None,
+        resume=False,
+    ):
         names = self._verify_path_args(from_infos, to_infos, names)
 
         gs = self.gs
 
         for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info['scheme'] != 'gs':
+            if from_info["scheme"] != "gs":
                 raise NotImplementedError
 
-            if to_info['scheme'] == 'gs':
-                self._copy(from_info, to_info, gs=gs)
+            if to_info["scheme"] == "gs":
+                self.copy(from_info, to_info, gs=gs)
                 continue
 
-            if to_info['scheme'] != 'local':
+            if to_info["scheme"] != "local":
                 raise NotImplementedError
 
-            msg = "Downloading '{}/{}' to '{}'".format(from_info['bucket'],
-                                                       from_info['key'],
-                                                       to_info['path'])
-            Logger.debug(msg)
+            msg = "Downloading '{}/{}' to '{}'".format(
+                from_info["bucket"], from_info["path"], to_info["path"]
+            )
+            logger.debug(msg)
 
-            tmp_file = self.tmp_file(to_info['path'])
+            tmp_file = tmp_fname(to_info["path"])
             if not name:
-                name = os.path.basename(to_info['path'])
+                name = os.path.basename(to_info["path"])
 
             if not no_progress_bar:
                 # percent_cb is not available for download_to_filename, so
-                # lets at least update progress at keypoints(start, finish)
+                # lets at least update progress at pathpoints(start, finish)
                 progress.update_target(name, 0, None)
 
-            self._makedirs(to_info['path'])
+            makedirs(os.path.dirname(to_info["path"]), exist_ok=True)
 
             try:
-                bucket = gs.bucket(from_info['bucket'])
-                blob = bucket.get_blob(from_info['key'])
+                bucket = gs.bucket(from_info["bucket"])
+                blob = bucket.get_blob(from_info["path"])
                 blob.download_to_filename(tmp_file)
-            except Exception as exc:
-                msg = "Failed to download '{}/{}' to '{}'"
-                Logger.error(msg.format(from_info['bucket'],
-                                        from_info['key'],
-                                        to_info['path']), exc)
+            except Exception:
+                msg = "failed to download '{}/{}' to '{}'"
+                logger.error(
+                    msg.format(
+                        from_info["bucket"], from_info["path"], to_info["path"]
+                    )
+                )
                 continue
 
-            os.rename(tmp_file, to_info['path'])
+            move(tmp_file, to_info["path"])
 
             if not no_progress_bar:
                 progress.finish_target(name)
-
-    def _path_to_etag(self, path):
-        relpath = posixpath.relpath(path, self.prefix)
-        return posixpath.dirname(relpath) + posixpath.basename(relpath)
-
-    def _all_etags(self):
-        blobs = self.gs.bucket(self.bucket).list_blobs(prefix=self.prefix)
-        blobs = list(blobs)
-        return [self._path_to_etag(blob.name) for blob in blobs]
-
-    def gc(self, cinfos):
-        used = [info[self.PARAM_ETAG] for info in cinfos['gs']]
-        used += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
-
-        for etag in self._all_etags():
-            if etag in used:
-                continue
-            path_info = {'scheme': 'gs',
-                         'bucket': self.bucket,
-                         'key': posixpath.join(self.prefix,
-                                               etag[0:2], etag[2:])}
-            self.remove(path_info)

dvc/remote/hdfs.py

@@ -1,44 +1,58 @@
+from __future__ import unicode_literals
+
 import os
 import re
 import getpass
 import posixpath
 from subprocess import Popen, PIPE
 
+import dvc.logger as logger
 from dvc.config import Config
-from dvc.remote.base import RemoteBase
-from dvc.remote.local import RemoteLOCAL
-from dvc.exceptions import DvcException
-from dvc.logger import Logger
+from dvc.remote.base import RemoteBase, RemoteCmdError
+from dvc.utils import fix_env, tmp_fname
 
 
 class RemoteHDFS(RemoteBase):
-    scheme = 'hdfs'
-    REGEX = r'^hdfs://((?P<user>.*)@)?.*$'
-    PARAM_CHECKSUM = 'checksum'
+    scheme = "hdfs"
+    REGEX = r"^hdfs://((?P<user>.*)@)?.*$"
+    PARAM_CHECKSUM = "checksum"
 
-    def __init__(self, project, config):
-        self.project = project
-        self.url = config.get(Config.SECTION_REMOTE_URL, '/')
-        self.user = self.group('user')
+    def __init__(self, repo, config):
+        super(RemoteHDFS, self).__init__(repo, config)
+        self.url = config.get(Config.SECTION_REMOTE_URL, "/")
+        self.prefix = self.url
+        self.user = self.group("user")
         if not self.user:
-            self.user = config.get(Config.SECTION_REMOTE_USER,
-                                   getpass.getuser())
+            self.user = config.get(
+                Config.SECTION_REMOTE_USER, getpass.getuser()
+            )
+
+        self.path_info = {"scheme": "hdfs", "user": self.user}
 
     def hadoop_fs(self, cmd, user=None):
-        cmd = 'hadoop fs -' + cmd
+        cmd = "hadoop fs -" + cmd
         if user:
-            cmd = 'HADOOP_USER_NAME={} '.format(user) + cmd
-        p = Popen(cmd,
-                  shell=True,
-                  close_fds=True,
-                  executable=os.getenv('SHELL'),
-                  stdin=PIPE,
-                  stdout=PIPE,
-                  stderr=PIPE)
+            cmd = "HADOOP_USER_NAME={} ".format(user) + cmd
+
+        # NOTE: close_fds doesn't work with redirected stdin/stdout/stderr.
+        # See https://github.com/iterative/dvc/issues/1197.
+        close_fds = os.name != "nt"
+
+        executable = os.getenv("SHELL") if os.name != "nt" else None
+        p = Popen(
+            cmd,
+            shell=True,
+            close_fds=close_fds,
+            executable=executable,
+            env=fix_env(os.environ),
+            stdin=PIPE,
+            stdout=PIPE,
+            stderr=PIPE,
+        )
         out, err = p.communicate()
         if p.returncode != 0:
-            raise DvcException('HDFS command failed: {}: {}'.format(cmd, err))
-        return out.decode('utf-8')
+            raise RemoteCmdError(self.scheme, cmd, p.returncode, err)
+        return out.decode("utf-8")
 
     @staticmethod
     def _group(regex, s, gname):
@@ -47,161 +61,125 @@ class RemoteHDFS(RemoteBase):
         return match.group(gname)
 
     def checksum(self, path_info):
-        regex = r'.*\t.*\t(?P<checksum>.*)'
-        stdout = self.hadoop_fs('checksum {}'.format(path_info['url']),
-                                user=path_info['user'])
-        return self._group(regex, stdout, 'checksum')
+        regex = r".*\t.*\t(?P<checksum>.*)"
+        stdout = self.hadoop_fs(
+            "checksum {}".format(path_info["path"]), user=path_info["user"]
+        )
+        return self._group(regex, stdout, "checksum")
 
-    def cp(self, from_info, to_info):
-        self.hadoop_fs('mkdir -p {}'.format(posixpath.dirname(to_info['url'])),
-                       user=to_info['user'])
-        self.hadoop_fs('cp -f {} {}'.format(from_info['url'], to_info['url']),
-                       user=to_info['user'])
+    def copy(self, from_info, to_info):
+        dname = posixpath.dirname(to_info["path"])
+        self.hadoop_fs("mkdir -p {}".format(dname), user=to_info["user"])
+        self.hadoop_fs(
+            "cp -f {} {}".format(from_info["path"], to_info["path"]),
+            user=to_info["user"],
+        )
 
     def rm(self, path_info):
-        self.hadoop_fs('rm {}'.format(path_info['url']),
-                       user=path_info['user'])
+        self.hadoop_fs(
+            "rm {}".format(path_info["path"]), user=path_info["user"]
+        )
 
     def save_info(self, path_info):
-        if path_info['scheme'] != 'hdfs':
+        if path_info["scheme"] != "hdfs":
             raise NotImplementedError
 
-        assert path_info.get('url')
+        assert path_info.get("path")
 
         return {self.PARAM_CHECKSUM: self.checksum(path_info)}
 
-    def save(self, path_info):
-        if path_info['scheme'] != 'hdfs':
-            raise NotImplementedError
-
-        assert path_info.get('url')
-
-        checksum = self.checksum(path_info)
-        dest = path_info.copy()
-        dest['url'] = posixpath.join(self.url, checksum[0:2], checksum[2:])
-
-        self.cp(path_info, dest)
-
-        return {self.PARAM_CHECKSUM: checksum}
-
-    def checkout(self, path_info, checksum_info):
-        if path_info['scheme'] != 'hdfs':
-            raise NotImplementedError
-
-        assert path_info.get('url')
-
-        checksum = checksum_info.get(self.PARAM_CHECKSUM, None)
-        if not checksum:
-            return
-
-        src = path_info.copy()
-        src['url'] = posixpath.join(self.url, checksum[0:2], checksum[2:])
-
-        self.cp(src, path_info)
-
     def remove(self, path_info):
-        if path_info['scheme'] != 'hdfs':
+        if path_info["scheme"] != "hdfs":
             raise NotImplementedError
 
-        assert path_info.get('url')
+        assert path_info.get("path")
 
-        Logger.debug('Removing {}'.format(path_info['url']))
+        logger.debug("Removing {}".format(path_info["path"]))
 
         self.rm(path_info)
 
-    def md5s_to_path_infos(self, md5s):
-        return [{'scheme': 'hdfs',
-                 'user': self.user,
-                 'url': posixpath.join(self.url,
-                                       md5[0:2], md5[2:])} for md5 in md5s]
+    def exists(self, path_info):
+        assert not isinstance(path_info, list)
+        assert path_info["scheme"] == "hdfs"
 
-    def exists(self, path_infos):
         try:
-            stdout = self.hadoop_fs('ls -R {}'.format(self.url))
-        except DvcException:
-            return len(path_infos) * [False]
-
-        lines = stdout.split('\n')
-        lurl = []
-        for line in lines:
-            if not line.startswith('-'):
-                continue
-            lurl.append(line.split()[-1])
-
-        ret = []
-        for path_info in path_infos:
-            exists = False
-            if path_info['url'] in lurl:
-                exists = True
-            ret.append(exists)
-
-        return ret
+            self.hadoop_fs("test -e {}".format(path_info["path"]))
+            return True
+        except RemoteCmdError:
+            return False
 
     def upload(self, from_infos, to_infos, names=None):
         names = self._verify_path_args(to_infos, from_infos, names)
 
         for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info['scheme'] != 'hdfs':
+            if to_info["scheme"] != "hdfs":
                 raise NotImplementedError
 
-            if from_info['scheme'] != 'local':
+            if from_info["scheme"] != "local":
                 raise NotImplementedError
 
-            cmd = 'mkdir -p {}'.format(posixpath.dirname(to_info['url']))
-            self.hadoop_fs(cmd, user=to_info['user'])
+            self.hadoop_fs(
+                "mkdir -p {}".format(posixpath.dirname(to_info["path"])),
+                user=to_info["user"],
+            )
 
-            cmd = 'copyFromLocal {} {}'.format(from_info['path'],
-                                               to_info['url'])
-            self.hadoop_fs(cmd, user=to_info['user'])
+            tmp_file = tmp_fname(to_info["path"])
 
-    def download(self,
-                 from_infos,
-                 to_infos,
-                 no_progress_bar=False,
-                 names=None):
+            self.hadoop_fs(
+                "copyFromLocal {} {}".format(from_info["path"], tmp_file),
+                user=to_info["user"],
+            )
+
+            self.hadoop_fs(
+                "mv {} {}".format(tmp_file, to_info["path"]),
+                user=to_info["user"],
+            )
+
+    def download(
+        self,
+        from_infos,
+        to_infos,
+        no_progress_bar=False,
+        names=None,
+        resume=False,
+    ):
         names = self._verify_path_args(from_infos, to_infos, names)
 
         for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info['scheme'] != 'hdfs':
+            if from_info["scheme"] != "hdfs":
                 raise NotImplementedError
 
-            if to_info['scheme'] == 'hdfs':
-                self.cp(from_info, to_info)
+            if to_info["scheme"] == "hdfs":
+                self.copy(from_info, to_info)
                 continue
 
-            if to_info['scheme'] != 'local':
+            if to_info["scheme"] != "local":
                 raise NotImplementedError
 
-            dname = os.path.dirname(to_info['path'])
+            dname = os.path.dirname(to_info["path"])
             if not os.path.exists(dname):
                 os.makedirs(dname)
 
-            cmd = 'copyToLocal {} {}'.format(from_info['url'], to_info['path'])
-            self.hadoop_fs(cmd, user=from_info['user'])
+            tmp_file = tmp_fname(to_info["path"])
 
-    def _path_to_checksum(self, path):
-        relpath = posixpath.relpath(path, self.url)
-        return posixpath.dirname(relpath) + posixpath.basename(relpath)
+            self.hadoop_fs(
+                "copyToLocal {} {}".format(from_info["path"], tmp_file),
+                user=from_info["user"],
+            )
 
-    def _all_checksums(self):
-        stdout = self.hadoop_fs('ls -R {}'.format(self.url))
-        lines = stdout.split('\n')
+            os.rename(tmp_file, to_info["path"])
+
+    def list_cache_paths(self):
+        try:
+            self.hadoop_fs("test -e {}".format(self.prefix))
+        except RemoteCmdError:
+            return []
+
+        stdout = self.hadoop_fs("ls -R {}".format(self.prefix))
+        lines = stdout.split("\n")
         flist = []
         for line in lines:
-            if not line.startswith('-'):
+            if not line.startswith("-"):
                 continue
             flist.append(line.split()[-1])
-        return [self._path_to_checksum(path) for path in flist]
-
-    def gc(self, cinfos):
-        used = [info[self.PARAM_CHECKSUM] for info in cinfos['hdfs']]
-        used += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
-
-        for checksum in self._all_checksums():
-            if checksum in used:
-                continue
-            path_info = {'scheme': 'hdfs',
-                         'user': self.user,
-                         'url': posixpath.join(self.url,
-                                               checksum[0:2], checksum[2:])}
-            self.remove(path_info)
+        return flist

dvc/remote/http.py

@@ -0,0 +1,196 @@
+from __future__ import unicode_literals
+from dvc.utils.compat import open, makedirs
+
+import os
+import threading
+import requests
+
+import dvc.logger as logger
+from dvc.progress import progress
+from dvc.exceptions import DvcException
+from dvc.config import Config
+from dvc.remote.base import RemoteBase
+from dvc.utils import move
+
+
+class ProgressBarCallback(object):
+    def __init__(self, name, total):
+        self.name = name
+        self.total = total
+        self.current = 0
+        self.lock = threading.Lock()
+
+    def __call__(self, byts):
+        with self.lock:
+            self.current += byts
+            progress.update_target(self.name, self.current, self.total)
+
+
+class RemoteHTTP(RemoteBase):
+    scheme = "http"
+    REGEX = r"^https?://.*$"
+    REQUEST_TIMEOUT = 10
+    CHUNK_SIZE = 2 ** 16
+    PARAM_CHECKSUM = "etag"
+
+    def __init__(self, repo, config):
+        super(RemoteHTTP, self).__init__(repo, config)
+        self.cache_dir = config.get(Config.SECTION_REMOTE_URL)
+        self.url = self.cache_dir
+        self.path_info = {"scheme": "http"}
+
+    @property
+    def prefix(self):
+        return self.cache_dir
+
+    def download(
+        self,
+        from_infos,
+        to_infos,
+        no_progress_bar=False,
+        names=None,
+        resume=False,
+    ):
+        names = self._verify_path_args(to_infos, from_infos, names)
+
+        for to_info, from_info, name in zip(to_infos, from_infos, names):
+            if from_info["scheme"] not in ["http", "https"]:
+                raise NotImplementedError
+
+            if to_info["scheme"] != "local":
+                raise NotImplementedError
+
+            msg = "Downloading '{}' to '{}'".format(
+                from_info["path"], to_info["path"]
+            )
+            logger.debug(msg)
+
+            if not name:
+                name = os.path.basename(to_info["path"])
+
+            makedirs(os.path.dirname(to_info["path"]), exist_ok=True)
+
+            total = self._content_length(from_info["path"])
+
+            if no_progress_bar or not total:
+                cb = None
+            else:
+                cb = ProgressBarCallback(name, total)
+
+            try:
+                self._download_to(
+                    from_info["path"],
+                    to_info["path"],
+                    callback=cb,
+                    resume=resume,
+                )
+
+            except Exception:
+                msg = "failed to download '{}'".format(from_info["path"])
+                logger.error(msg)
+                continue
+
+            if not no_progress_bar:
+                progress.finish_target(name)
+
+    def exists(self, path_info):
+        assert not isinstance(path_info, list)
+        assert path_info["scheme"] in ["http", "https"]
+        return bool(self._request("HEAD", path_info.get("path")))
+
+    def cache_exists(self, md5s):
+        assert isinstance(md5s, list)
+
+        def func(md5):
+            return bool(self._request("HEAD", self.checksum_to_path(md5)))
+
+        return list(filter(func, md5s))
+
+    def save_info(self, path_info):
+        if path_info["scheme"] not in ["http", "https"]:
+            raise NotImplementedError
+
+        return {self.PARAM_CHECKSUM: self._etag(path_info["path"])}
+
+    def _content_length(self, url):
+        return self._request("HEAD", url).headers.get("Content-Length")
+
+    def _etag(self, url):
+        etag = self._request("HEAD", url).headers.get("ETag") or self._request(
+            "HEAD", url
+        ).headers.get("Content-MD5")
+
+        if not etag:
+            raise DvcException(
+                "could not find an ETag or "
+                "Content-MD5 header for '{url}'".format(url=url)
+            )
+
+        if etag.startswith("W/"):
+            raise DvcException(
+                "Weak ETags are not supported."
+                " (Etag: '{etag}', URL: '{url}')".format(etag=etag, url=url)
+            )
+
+        return etag
+
+    def _download_to(self, url, target_file, callback=None, resume=False):
+        request = self._request("GET", url, stream=True)
+        partial_file = target_file + ".part"
+
+        mode, transferred_bytes = self._determine_mode_get_transferred_bytes(
+            partial_file, resume
+        )
+
+        self._validate_existing_file_size(transferred_bytes, partial_file)
+
+        self._write_request_content(
+            mode, partial_file, request, transferred_bytes, callback
+        )
+
+        move(partial_file, target_file)
+
+    def _write_request_content(
+        self, mode, partial_file, request, transferred_bytes, callback=None
+    ):
+        with open(partial_file, mode) as fd:
+
+            for index, chunk in enumerate(
+                request.iter_content(chunk_size=self.CHUNK_SIZE)
+            ):
+                chunk_number = index + 1
+                if chunk_number * self.CHUNK_SIZE > transferred_bytes:
+                    fd.write(chunk)
+                    fd.flush()
+                    transferred_bytes += len(chunk)
+
+                if callback:
+                    callback(transferred_bytes)
+
+    def _validate_existing_file_size(self, bytes_transferred, partial_file):
+        if bytes_transferred % self.CHUNK_SIZE != 0:
+            raise DvcException(
+                "File {}, might be corrupted, please remove "
+                "it and retry importing".format(partial_file)
+            )
+
+    def _determine_mode_get_transferred_bytes(self, partial_file, resume):
+        if os.path.exists(partial_file) and resume:
+            mode = "ab"
+            bytes_transfered = os.path.getsize(partial_file)
+        else:
+            mode = "wb"
+            bytes_transfered = 0
+        return mode, bytes_transfered
+
+    def _request(self, method, url, **kwargs):
+        kwargs.setdefault("allow_redirects", True)
+        kwargs.setdefault("timeout", self.REQUEST_TIMEOUT)
+
+        try:
+            return requests.request(method, url, **kwargs)
+        except requests.exceptions.RequestException:
+            raise DvcException("could not perform a {} request".format(method))
+
+    def gc(self):
+        raise NotImplementedError

dvc/remote/local.py

@@ -1,4 +1,9 @@
+from __future__ import unicode_literals
+
+from dvc.utils.compat import str, makedirs
+
 import os
+import stat
 import uuid
 import json
 import ntpath
@@ -6,43 +11,50 @@ import shutil
 import posixpath
 from operator import itemgetter
 
+import dvc.logger as logger
 from dvc.system import System
-from dvc.remote.base import RemoteBase, STATUS_MAP
-from dvc.logger import Logger
-from dvc.utils import remove, move, copyfile, file_md5, dict_md5, to_chunks
+from dvc.remote.base import RemoteBase, STATUS_MAP, STATUS_NEW, STATUS_DELETED
+from dvc.utils import remove, move, copyfile, dict_md5, to_chunks, tmp_fname
+from dvc.utils import LARGE_DIR_SIZE
 from dvc.config import Config
 from dvc.exceptions import DvcException
 from dvc.progress import progress
 from concurrent.futures import ThreadPoolExecutor
 
+from dvc.utils.fs import get_mtime_and_size, get_inode
+
 
 class RemoteLOCAL(RemoteBase):
-    scheme = ''
-    REGEX = r'^(?P<path>(/+|.:\\+).*)$'
-    PARAM_MD5 = 'md5'
-    PARAM_PATH = 'path'
-    PARAM_RELPATH = 'relpath'
-    MD5_DIR_SUFFIX = '.dir'
+    scheme = "local"
+    REGEX = r"^(?P<path>.*)$"
+    PARAM_CHECKSUM = "md5"
+    PARAM_PATH = "path"
+    PARAM_RELPATH = "relpath"
+    MD5_DIR_SUFFIX = ".dir"
 
-    CACHE_TYPES = ['reflink', 'hardlink', 'symlink', 'copy']
+    CACHE_TYPES = ["reflink", "hardlink", "symlink", "copy"]
     CACHE_TYPE_MAP = {
-        'copy': shutil.copyfile,
-        'symlink': System.symlink,
-        'hardlink': System.hardlink,
-        'reflink': System.reflink,
+        "copy": shutil.copyfile,
+        "symlink": System.symlink,
+        "hardlink": System.hardlink,
+        "reflink": System.reflink,
     }
 
-    def __init__(self, project, config):
-        self.project = project
-        self.state = self.project.state
-        self.link_state = project.link_state
+    def __init__(self, repo, config):
+        super(RemoteLOCAL, self).__init__(repo, config)
+        self.state = self.repo.state if self.repo else None
+        self.protected = config.get(Config.SECTION_CACHE_PROTECTED, False)
         storagepath = config.get(Config.SECTION_AWS_STORAGEPATH, None)
         self.cache_dir = config.get(Config.SECTION_REMOTE_URL, storagepath)
 
+        if self.cache_dir is not None and not os.path.isabs(self.cache_dir):
+            cwd = config[Config.PRIVATE_CWD]
+            self.cache_dir = os.path.abspath(os.path.join(cwd, self.cache_dir))
+
         types = config.get(Config.SECTION_CACHE_TYPE, None)
         if types:
             if isinstance(types, str):
-                types = [t.strip() for t in types.split(',')]
+                types = [t.strip() for t in types.split(",")]
             self.cache_types = types
         else:
             self.cache_types = self.CACHE_TYPES
@@ -50,11 +62,24 @@ class RemoteLOCAL(RemoteBase):
         if self.cache_dir is not None and not os.path.exists(self.cache_dir):
             os.mkdir(self.cache_dir)
 
+        self.path_info = {"scheme": "local"}
+
+    @staticmethod
+    def compat_config(config):
+        ret = config.copy()
+        url = ret.pop(Config.SECTION_AWS_STORAGEPATH, "")
+        ret[Config.SECTION_REMOTE_URL] = url
+        return ret
+
+    @property
+    def url(self):
+        return self.cache_dir
+
     @property
     def prefix(self):
         return self.cache_dir
 
-    def all(self):
+    def list_cache_paths(self):
         clist = []
         for entry in os.listdir(self.cache_dir):
             subdir = os.path.join(self.cache_dir, entry)
@@ -62,8 +87,7 @@ class RemoteLOCAL(RemoteBase):
                 continue
 
             for cache in os.listdir(subdir):
-                path = os.path.join(subdir, cache)
-                clist.append(self.path_to_md5(path))
+                clist.append(os.path.join(subdir, cache))
 
         return clist
 
@@ -73,18 +97,38 @@ class RemoteLOCAL(RemoteBase):
 
         return os.path.join(self.cache_dir, md5[0:2], md5[2:])
 
-    def path_to_md5(self, path):
-        relpath = os.path.relpath(path, self.cache_dir)
-        return os.path.dirname(relpath) + os.path.basename(relpath)
-
-    def changed(self, md5):
+    def changed_cache_file(self, md5):
         cache = self.get(md5)
         if self.state.changed(cache, md5=md5):
             if os.path.exists(cache):
-                msg = 'Corrupted cache file {}'
-                Logger.warn(msg.format(os.path.relpath(cache)))
+                msg = "Corrupted cache file {}."
+                logger.warning(msg.format(os.path.relpath(cache)))
                 remove(cache)
             return True
+        return False
+
+    def exists(self, path_info):
+        assert not isinstance(path_info, list)
+        assert path_info["scheme"] == "local"
+        return os.path.exists(path_info["path"])
+
+    def changed_cache(self, md5):
+        cache = self.get(md5)
+        clist = [(cache, md5)]
+
+        while True:
+            if len(clist) == 0:
+                break
+
+            cache, md5 = clist.pop()
+            if self.changed_cache_file(md5):
+                return True
+
+            if self.is_dir_cache(cache) and self._cache_metadata_changed():
+                for entry in self.load_dir_cache(md5):
+                    md5 = entry[self.PARAM_CHECKSUM]
+                    cache = self.get(md5)
+                    clist.append((cache, md5))
 
         return False
 
@@ -95,22 +139,49 @@ class RemoteLOCAL(RemoteBase):
         if not os.path.exists(dname):
             os.makedirs(dname)
 
+        # NOTE: just create an empty file for an empty cache
+        if os.path.getsize(cache) == 0:
+            open(path, "w+").close()
+
+            msg = "Created empty file: {} -> {}".format(cache, path)
+            logger.debug(msg)
+            return
+
         i = len(self.cache_types)
         while i > 0:
             try:
                 self.CACHE_TYPE_MAP[self.cache_types[0]](cache, path)
+
+                if self.protected:
+                    os.chmod(path, stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH)
+
+                msg = "Created {}'{}': {} -> {}".format(
+                    "protected " if self.protected else "",
+                    self.cache_types[0],
+                    cache,
+                    path,
+                )
+
+                logger.debug(msg)
                 return
-            except Exception as exc:
-                msg = 'Cache type \'{}\' is not supported: {}'
-                Logger.debug(msg.format(self.cache_types[0], str(exc)))
+
+            except DvcException as exc:
+                msg = "Cache type '{}' is not supported: {}"
+                logger.debug(msg.format(self.cache_types[0], str(exc)))
                 del self.cache_types[0]
                 i -= 1
 
-        raise DvcException('No possible cache types left to try out.')
+        raise DvcException("no possible cache types left to try out.")
+
+    @property
+    def ospath(self):
+        if os.name == "nt":
+            return ntpath
+        return posixpath
 
     @classmethod
-    def ospath(cls, path):
-        if os.name == 'nt':
+    def to_ospath(cls, path):
+        if os.name == "nt":
             return cls.ntpath(path)
         return cls.unixpath(path)
 
@@ -118,36 +189,53 @@ class RemoteLOCAL(RemoteBase):
     def unixpath(path):
         assert not ntpath.isabs(path)
         assert not posixpath.isabs(path)
-        return path.replace('\\', '/')
+        return path.replace("\\", "/")
 
     @staticmethod
     def ntpath(path):
         assert not ntpath.isabs(path)
         assert not posixpath.isabs(path)
-        return path.replace('/', '\\')
+        return path.replace("/", "\\")
 
     def collect_dir_cache(self, dname):
         dir_info = []
 
-        for root, dirs, files in os.walk(dname):
+        for root, dirs, files in os.walk(str(dname)):
+            bar = False
+
+            if len(files) > LARGE_DIR_SIZE:
+                msg = (
+                    "Computing md5 for a large directory {}. "
+                    "This is only done once."
+                )
+                logger.info(msg.format(os.path.relpath(root)))
+                bar = True
+                title = os.path.relpath(root)
+                processed = 0
+                total = len(files)
+                progress.update_target(title, 0, total)
+
             for fname in files:
                 path = os.path.join(root, fname)
                 relpath = self.unixpath(os.path.relpath(path, dname))
 
-                # FIXME: we could've used md5 = state.update(path, dump=False)
-                # here, but it is around twice as slow(on ssd, don't know about
-                # hdd) for a directory with small files. What we could do here
-                # is introduce some kind of a limit for file size, after which
-                # we would actually register it in our state file.
-                md5 = file_md5(path)[0]
-                dir_info.append({self.PARAM_RELPATH: relpath,
-                                 self.PARAM_MD5: md5})
+                if bar:
+                    progress.update_target(title, processed, total)
+                    processed += 1
+
+                md5 = self.state.update(path)
+                dir_info.append(
+                    {self.PARAM_RELPATH: relpath, self.PARAM_CHECKSUM: md5}
+                )
+
+            if bar:
+                progress.finish_target(title)
 
         # NOTE: sorting the list by path to ensure reproducibility
         dir_info = sorted(dir_info, key=itemgetter(self.PARAM_RELPATH))
 
         md5 = dict_md5(dir_info) + self.MD5_DIR_SUFFIX
-        if self.changed(md5):
+        if self.changed_cache_file(md5):
             self.dump_dir_cache(md5, dir_info)
 
         return (md5, dir_info)
@@ -158,20 +246,20 @@ class RemoteLOCAL(RemoteBase):
         assert self.is_dir_cache(path)
 
         try:
-            with open(path, 'r') as fd:
+            with open(path, "r") as fd:
                 d = json.load(fd)
-        except Exception as exc:
-            msg = u'Failed to load dir cache \'{}\''
-            Logger.error(msg.format(os.path.relpath(path)), exc)
+        except Exception:
+            msg = "Failed to load dir cache '{}'"
+            logger.error(msg.format(os.path.relpath(path)))
             return []
 
         if not isinstance(d, list):
-            msg = u'Dir cache file format error \'{}\': skipping the file'
-            Logger.error(msg.format(os.path.relpath(path)))
+            msg = "dir cache file format error '{}' [skipping the file]"
+            logger.error(msg.format(os.path.relpath(path)))
             return []
 
         for info in d:
-            info['relpath'] = self.ospath(info['relpath'])
+            info["relpath"] = self.to_ospath(info["relpath"])
 
         return d
 
@@ -187,8 +275,9 @@ class RemoteLOCAL(RemoteBase):
 
         # NOTE: Writing first and renaming after that
         # to make sure that the operation is atomic.
-        tmp = '{}.{}'.format(path, str(uuid.uuid4()))
-        with open(tmp, 'w+') as fd:
+        tmp = "{}.{}".format(path, str(uuid.uuid4()))
+
+        with open(tmp, "w+") as fd:
             json.dump(dir_info, fd, sort_keys=True)
         move(tmp, path)
 
@@ -196,33 +285,17 @@ class RemoteLOCAL(RemoteBase):
     def is_dir_cache(cls, cache):
         return cache.endswith(cls.MD5_DIR_SUFFIX)
 
-    def checkout(self, path_info, checksum_info):
-        path = path_info['path']
-        md5 = checksum_info.get(self.PARAM_MD5, None)
+    def do_checkout(self, path_info, checksum, force=False):
+        path = path_info["path"]
+        md5 = checksum
         cache = self.get(md5)
 
-        if not cache:
-            msg = 'No cache info for \'{}\'. Skipping checkout.'
-            Logger.warn(msg.format(os.path.relpath(path)))
-            return
-
-        if self.changed(md5):
-            msg = u'Cache \'{}\' not found. File \'{}\' won\'t be created.'
-            Logger.warn(msg.format(md5, os.path.relpath(path)))
-            remove(path)
-            return
-
-        if os.path.exists(path):
-            msg = u'Data \'{}\' exists. Removing before checkout'
-            Logger.debug(msg.format(os.path.relpath(path)))
-            remove(path)
-
-        msg = u'Checking out \'{}\' with cache \'{}\''
-        Logger.debug(msg.format(os.path.relpath(path), md5))
-
         if not self.is_dir_cache(cache):
+            if os.path.exists(path):
+                self.safe_remove(path_info, force=force)
+
             self.link(cache, path)
-            self.link_state.update(path)
+            self.state.update_link(path)
             return
 
         # Create dir separately so that dir is created
@@ -230,330 +303,410 @@ class RemoteLOCAL(RemoteBase):
         if not os.path.exists(path):
             os.makedirs(path)
 
-        for entry in self.load_dir_cache(md5):
-            md5 = entry[self.PARAM_MD5]
-            c = self.get(md5)
+        dir_info = self.load_dir_cache(md5)
+        dir_relpath = os.path.relpath(path)
+        dir_size = len(dir_info)
+        bar = dir_size > LARGE_DIR_SIZE
+
+        logger.info("Linking directory '{}'.".format(dir_relpath))
+
+        for processed, entry in enumerate(dir_info):
             relpath = entry[self.PARAM_RELPATH]
+            m = entry[self.PARAM_CHECKSUM]
             p = os.path.join(path, relpath)
-            self.link(c, p)
-        self.link_state.update(path)
+            c = self.get(m)
+
+            entry_info = {"scheme": path_info["scheme"], self.PARAM_PATH: p}
+
+            entry_checksum_info = {self.PARAM_CHECKSUM: m}
+
+            if self.changed(entry_info, entry_checksum_info):
+                if os.path.exists(p):
+                    self.safe_remove(entry_info, force=force)
+
+                self.link(c, p)
+
+            if bar:
+                progress.update_target(dir_relpath, processed, dir_size)
+
+        self._discard_working_directory_changes(path, dir_info, force=force)
+
+        self.state.update_link(path)
+
+        if bar:
+            progress.finish_target(dir_relpath)
+
+    def already_cached(self, path_info):
+        assert path_info["scheme"] in ["", "local"]
+
+        current_md5 = self.state.update(path_info["path"])
+
+        if not current_md5:
+            return False
+
+        return not self.changed_cache(current_md5)
+
+    def _discard_working_directory_changes(self, path, dir_info, force=False):
+        working_dir_files = set(
+            os.path.join(root, file)
+            for root, _, files in os.walk(str(path))
+            for file in files
+        )
+
+        cached_files = set(
+            os.path.join(path, file["relpath"]) for file in dir_info
+        )
+
+        delta = working_dir_files - cached_files
+
+        for file in delta:
+            self.safe_remove({"scheme": "local", "path": file}, force=force)
 
     def _move(self, inp, outp):
-        # moving in two stages to make last the move atomic in
-        # case inp and outp are in different filesystems
-        tmp = '{}.{}'.format(outp, str(uuid.uuid4()))
+        # moving in two stages to make the whole operation atomic in
+        # case inp and outp are in different filesystems and actual
+        # physical copying of data is happening
+        tmp = "{}.{}".format(outp, str(uuid.uuid4()))
         move(inp, tmp)
         move(tmp, outp)
 
-    def _save_file(self, path_info):
-        path = path_info['path']
-        md5 = self.state.update(path)
+    def _save_file(self, path, md5):
         assert md5 is not None
 
         cache = self.get(md5)
 
-        if self.changed(md5):
+        if self.changed_cache(md5):
             self._move(path, cache)
         else:
             remove(path)
 
         self.link(cache, path)
-        self.link_state.update(path)
+        self.state.update_link(path)
 
-        return {self.PARAM_MD5: md5}
+        # we need to update path and cache, since in case of reflink,
+        # or copy cache type moving original file results in updates on
+        # next executed command, which causes md5 recalculation
+        self.state.update(path, md5)
+        self.state.update(cache, md5)
 
-    def _save_dir(self, path_info):
-        path = path_info['path']
-        md5, dir_info = self.state.update_info(path)
+        return {self.PARAM_CHECKSUM: md5}
 
-        for entry in dir_info:
+    def _save_dir(self, path, md5):
+        dir_info = self.load_dir_cache(md5)
+        dir_relpath = os.path.relpath(path)
+        dir_size = len(dir_info)
+        bar = dir_size > LARGE_DIR_SIZE
+
+        logger.info("Linking directory '{}'.".format(dir_relpath))
+
+        for processed, entry in enumerate(dir_info):
             relpath = entry[self.PARAM_RELPATH]
-            m = entry[self.PARAM_MD5]
+            m = entry[self.PARAM_CHECKSUM]
             p = os.path.join(path, relpath)
             c = self.get(m)
 
-            if self.changed(m):
+            if self.changed_cache(m):
                 self._move(p, c)
             else:
                 remove(p)
 
             self.link(c, p)
 
-        self.link_state.update(path)
+            self.state.update(p, m)
+            self.state.update(c, m)
 
-        return {self.PARAM_MD5: md5}
+            if bar:
+                progress.update_target(dir_relpath, processed, dir_size)
 
-    def save(self, path_info):
-        if path_info['scheme'] != 'local':
+        self.state.update_link(path)
+
+        cache = self.get(md5)
+        self.state.update(cache)
+        self.state.update(path, md5)
+
+        if bar:
+            progress.finish_target(dir_relpath)
+
+    def save(self, path_info, checksum_info):
+        if path_info["scheme"] != "local":
             raise NotImplementedError
 
-        path = path_info['path']
+        path = path_info["path"]
 
+        msg = "Saving '{}' to cache '{}'."
+        logger.info(
+            msg.format(os.path.relpath(path), os.path.relpath(self.cache_dir))
+        )
+
+        md5 = checksum_info[self.PARAM_CHECKSUM]
         if os.path.isdir(path):
-            return self._save_dir(path_info)
+            self._save_dir(path, md5)
         else:
-            return self._save_file(path_info)
+            self._save_file(path, md5)
 
     def save_info(self, path_info):
-        if path_info['scheme'] != 'local':
+        if path_info["scheme"] != "local":
             raise NotImplementedError
 
-        return {self.PARAM_MD5: self.state.update(path_info['path'])}
+        return {self.PARAM_CHECKSUM: self.state.update(path_info["path"])}
 
     def remove(self, path_info):
-        if path_info['scheme'] != 'local':
+        if path_info["scheme"] != "local":
             raise NotImplementedError
 
-        remove(path_info['path'])
+        remove(path_info["path"])
 
     def move(self, from_info, to_info):
-        if from_info['scheme'] != 'local' or to_info['scheme'] != 'local':
+        if from_info["scheme"] != "local" or to_info["scheme"] != "local":
             raise NotImplementedError
 
-        move(from_info['path'], to_info['path'])
+        move(from_info["path"], to_info["path"])
 
-    def md5s_to_path_infos(self, md5s):
-        return [{'scheme': 'local',
-                 'path': os.path.join(self.prefix,
-                                      md5[0:2], md5[2:])} for md5 in md5s]
-
-    def exists(self, path_infos):
-        ret = []
-        for path_info in path_infos:
-            ret.append(os.path.exists(path_info['path']))
-        return ret
+    def cache_exists(self, md5s):
+        assert isinstance(md5s, list)
+        return list(filter(lambda md5: not self.changed_cache_file(md5), md5s))
 
     def upload(self, from_infos, to_infos, names=None):
         names = self._verify_path_args(to_infos, from_infos, names)
 
         for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info['scheme'] != 'local':
+            if to_info["scheme"] != "local":
                 raise NotImplementedError
 
-            if from_info['scheme'] != 'local':
+            if from_info["scheme"] != "local":
                 raise NotImplementedError
 
-            Logger.debug("Uploading '{}' to '{}'".format(from_info['path'],
-                                                         to_info['path']))
+            logger.debug(
+                "Uploading '{}' to '{}'".format(
+                    from_info["path"], to_info["path"]
+                )
+            )
 
             if not name:
-                name = os.path.basename(from_info['path'])
+                name = os.path.basename(from_info["path"])
 
-            self._makedirs(to_info['path'])
+            makedirs(os.path.dirname(to_info["path"]), exist_ok=True)
+            tmp_file = tmp_fname(to_info["path"])
 
             try:
-                copyfile(from_info['path'], to_info['path'], name=name)
-            except Exception as exc:
-                msg = "Failed to upload '{}' tp '{}'"
-                Logger.error(msg.format(from_info['path'],
-                                        to_info['path']), exc)
+                copyfile(from_info["path"], tmp_file, name=name)
+                os.rename(tmp_file, to_info["path"])
+            except Exception:
+                logger.error(
+                    "failed to upload '{}' to '{}'".format(
+                        from_info["path"], to_info["path"]
+                    )
+                )
 
-    def download(self,
-                 from_infos,
-                 to_infos,
-                 no_progress_bar=False,
-                 names=None):
+    def download(
+        self,
+        from_infos,
+        to_infos,
+        no_progress_bar=False,
+        names=None,
+        resume=False,
+    ):
         names = self._verify_path_args(from_infos, to_infos, names)
 
         for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info['scheme'] != 'local':
+            if from_info["scheme"] != "local":
                 raise NotImplementedError
 
-            if to_info['scheme'] != 'local':
+            if to_info["scheme"] != "local":
                 raise NotImplementedError
 
-            Logger.debug("Downloading '{}' to '{}'".format(from_info['path'],
-                                                           to_info['path']))
+            logger.debug(
+                "Downloading '{}' to '{}'".format(
+                    from_info["path"], to_info["path"]
+                )
+            )
 
             if not name:
-                name = os.path.basename(to_info['path'])
+                name = os.path.basename(to_info["path"])
 
-            self._makedirs(to_info['path'])
-            tmp_file = self.tmp_file(to_info['path'])
+            makedirs(os.path.dirname(to_info["path"]), exist_ok=True)
+            tmp_file = tmp_fname(to_info["path"])
             try:
-                copyfile(from_info['path'],
-                         tmp_file,
-                         no_progress_bar=no_progress_bar,
-                         name=name)
-            except Exception as exc:
-                msg = "Failed to download '{}' to '{}'"
-                Logger.error(msg.format(from_info['path'],
-                                        to_info['path']), exc)
+                copyfile(
+                    from_info["path"],
+                    tmp_file,
+                    no_progress_bar=no_progress_bar,
+                    name=name,
+                )
+
+                move(tmp_file, to_info["path"])
+            except Exception:
+                logger.error(
+                    "failed to download '{}' to '{}'".format(
+                        from_info["path"], to_info["path"]
+                    )
+                )
+
                 continue
 
-            os.rename(tmp_file, to_info['path'])
-
-    def _collect(self, checksum_infos):
-        missing = []
-        collected = []
-        for info in checksum_infos:
-            md5 = info[self.PARAM_MD5]
-            cache = self.get(md5)
-
-            if not self.is_dir_cache(info[self.PARAM_MD5]):
-                continue
-
-            if not os.path.exists(cache):
-                missing.append(info)
-                continue
-
-            for i in self.load_dir_cache(md5):
-                if info.get('branch'):
-                    i['branch'] = info['branch']
-                i[self.PARAM_PATH] = posixpath.join(info[self.PARAM_PATH],
-                                                    i[self.PARAM_RELPATH])
-                collected.append(i.copy())
-
-        collected.extend(checksum_infos)
-        return collected, missing
-
     def _group(self, checksum_infos, show_checksums=False):
         by_md5 = {}
 
         for info in checksum_infos:
-            md5 = info[self.PARAM_MD5]
+            md5 = info[self.PARAM_CHECKSUM]
 
             if show_checksums:
-                by_md5[md5] = md5
+                by_md5[md5] = {"name": md5}
                 continue
 
             name = info[self.PARAM_PATH]
-            branch = info.get('branch')
+            branch = info.get("branch")
             if branch:
-                name += '({})'.format(branch)
+                name += "({})".format(branch)
 
             if md5 not in by_md5.keys():
-                by_md5[md5] = ''
+                by_md5[md5] = {"name": name}
             else:
-                by_md5[md5] += ' '
+                by_md5[md5]["name"] += " " + name
 
-            by_md5[md5] += name
+        return by_md5
 
-        return list(by_md5.keys()), list(by_md5.values())
+    def status(self, checksum_infos, remote, jobs=None, show_checksums=False):
+        logger.info("Preparing to collect status from {}".format(remote.url))
+        title = "Collecting information"
 
-    def gc(self, checksum_infos):
-        checksum_infos = self._collect(checksum_infos['local'])[0]
-        used_md5s = [info[self.PARAM_MD5] for info in checksum_infos]
+        ret = {}
 
-        for md5 in self.all():
-            if md5 in used_md5s:
-                continue
-            remove(self.get(md5))
+        progress.set_n_total(1)
+        progress.update_target(title, 0, 100)
 
-    def status(self, checksum_infos, remote, jobs=1, show_checksums=False):
-        checksum_infos, missing = self._collect(checksum_infos)
-        checksum_infos += missing
-        md5s, names = self._group(checksum_infos,
-                                  show_checksums=show_checksums)
-        path_infos = remote.md5s_to_path_infos(md5s)
-        remote_exists = remote.exists(path_infos)
-        local_exists = [not self.changed(md5) for md5 in md5s]
+        progress.update_target(title, 10, 100)
 
-        return [(name, STATUS_MAP[l, r]) for name, l, r in zip(names,
-                                                               local_exists,
-                                                               remote_exists)]
+        ret = self._group(checksum_infos, show_checksums=show_checksums)
+        md5s = list(ret.keys())
 
-    def _do_pull(self,
-                 checksum_infos,
-                 remote,
-                 jobs=1,
-                 no_progress_bar=False,
-                 show_checksums=False):
-        md5s = []
+        progress.update_target(title, 30, 100)
+
+        remote_exists = list(remote.cache_exists(md5s))
+
+        progress.update_target(title, 90, 100)
+
+        local_exists = self.cache_exists(md5s)
+
+        progress.finish_target(title)
+
+        for md5, info in ret.items():
+            info["status"] = STATUS_MAP[
+                (md5 in local_exists, md5 in remote_exists)
+            ]
+
+        return ret
+
+    def _get_chunks(self, download, remote, status_info, status, jobs):
+        title = "Analysing status."
+
+        progress.set_n_total(1)
+        total = len(status_info)
+        current = 0
+
+        cache = []
+        path_infos = []
         names = []
-        # NOTE: filter files that are not corrupted
-        for md5, name in zip(*self._group(checksum_infos,
-                                          show_checksums=show_checksums)):
-            if self.changed(md5):
-                md5s.append(md5)
-                names.append(name)
+        for md5, info in status_info.items():
+            if info["status"] == status:
+                cache.append(self.checksum_to_path_info(md5))
+                path_infos.append(remote.checksum_to_path_info(md5))
+                names.append(info["name"])
+            current += 1
+            progress.update_target(title, current, total)
 
-        cache = [{'scheme': 'local', 'path': self.get(md5)} for md5 in md5s]
-        path_infos = remote.md5s_to_path_infos(md5s)
-
-        assert len(path_infos) == len(cache) == len(md5s) == len(names)
-
-        chunks = list(zip(to_chunks(path_infos, jobs),
-                          to_chunks(cache, jobs),
-                          to_chunks(names, jobs)))
+        progress.finish_target(title)
 
         progress.set_n_total(len(names))
 
+        if download:
+            to_infos = cache
+            from_infos = path_infos
+        else:
+            to_infos = path_infos
+            from_infos = cache
+
+        return list(
+            zip(
+                to_chunks(from_infos, jobs),
+                to_chunks(to_infos, jobs),
+                to_chunks(names, jobs),
+            )
+        )
+
+    def _process(
+        self,
+        checksum_infos,
+        remote,
+        jobs=None,
+        show_checksums=False,
+        download=False,
+    ):
+        msg = "Preparing to {} data {} '{}'"
+        logger.info(
+            msg.format(
+                "download" if download else "upload",
+                "from" if download else "to",
+                remote.url,
+            )
+        )
+
+        if download:
+            func = remote.download
+            status = STATUS_DELETED
+        else:
+            func = remote.upload
+            status = STATUS_NEW
+
+        if jobs is None:
+            jobs = remote.JOBS
+
+        status_info = self.status(
+            checksum_infos, remote, jobs=jobs, show_checksums=show_checksums
+        )
+
+        chunks = self._get_chunks(download, remote, status_info, status, jobs)
+
         if len(chunks) == 0:
-            return
+            return 0
 
         futures = []
-        with ThreadPoolExecutor(max_workers=len(chunks)) as executor:
+        with ThreadPoolExecutor(max_workers=jobs) as executor:
             for from_infos, to_infos, names in chunks:
-                res = executor.submit(remote.download,
-                                      from_infos,
-                                      to_infos,
-                                      names=names,
-                                      no_progress_bar=no_progress_bar)
+                res = executor.submit(func, from_infos, to_infos, names=names)
                 futures.append(res)
 
         for f in futures:
             f.result()
 
-    def pull(self, checksum_infos, remote, jobs=1, show_checksums=False):
-        # NOTE: try fetching missing dir info
-        checksum_infos, missing = self._collect(checksum_infos)
-        if len(missing) > 0:
-            self._do_pull(missing,
-                          remote,
-                          jobs,
-                          no_progress_bar=True,
-                          show_checksums=show_checksums)
-            checksum_infos += self._collect(missing)[0]
+        return len(chunks)
 
-        self._do_pull(checksum_infos,
-                      remote,
-                      jobs,
-                      show_checksums=show_checksums)
+    def push(self, checksum_infos, remote, jobs=None, show_checksums=False):
+        return self._process(
+            checksum_infos,
+            remote,
+            jobs=jobs,
+            show_checksums=show_checksums,
+            download=False,
+        )
 
-    def push(self, checksum_infos, remote, jobs=1, show_checksums=False):
-        checksum_infos = self._collect(checksum_infos)[0]
+    def pull(self, checksum_infos, remote, jobs=None, show_checksums=False):
+        return self._process(
+            checksum_infos,
+            remote,
+            jobs=jobs,
+            show_checksums=show_checksums,
+            download=True,
+        )
 
-        # NOTE: verifying that our cache is not corrupted
-        def func(info):
-            return not self.changed(info[self.PARAM_MD5])
-        checksum_infos = list(filter(func, checksum_infos))
+    def _cache_metadata_changed(self):
+        mtime, size = get_mtime_and_size(self.cache_dir)
+        inode = get_inode(self.cache_dir)
 
-        # NOTE: filter files that are already uploaded
-        md5s = [i[self.PARAM_MD5] for i in checksum_infos]
-        exists = remote.exists(remote.md5s_to_path_infos(md5s))
+        existing_record = self.state.get_state_record_for_inode(inode)
 
-        def func(entry):
-            return not entry[0]
+        if existing_record:
+            cached_mtime, cached_size, _, _ = existing_record
+            return not (mtime == cached_mtime and size == cached_size)
 
-        assert len(exists) == len(checksum_infos)
-        infos_exist = list(filter(func, zip(exists, checksum_infos)))
-        checksum_infos = [i for e, i in infos_exist]
-
-        md5s, names = self._group(checksum_infos,
-                                  show_checksums=show_checksums)
-        cache = [{'scheme': 'local', 'path': self.get(md5)} for md5 in md5s]
-        path_infos = remote.md5s_to_path_infos(md5s)
-
-        assert len(path_infos) == len(cache) == len(md5s) == len(names)
-
-        chunks = list(zip(to_chunks(path_infos, jobs),
-                          to_chunks(cache, jobs),
-                          to_chunks(names, jobs)))
-
-        progress.set_n_total(len(names))
-
-        if len(chunks) == 0:
-            return
-
-        futures = []
-        with ThreadPoolExecutor(max_workers=len(chunks)) as executor:
-            for to_infos, from_infos, names in chunks:
-                res = executor.submit(remote.upload,
-                                      from_infos,
-                                      to_infos,
-                                      names=names)
-                futures.append(res)
-
-        for f in futures:
-            f.result()
+        return True

dvc/remote/s3.py

@@ -1,22 +1,19 @@
+from __future__ import unicode_literals
+
 import os
 import threading
-import posixpath
 
 try:
     import boto3
 except ImportError:
     boto3 = None
 
-try:
-    from urlparse import urlparse
-except ImportError:
-    from urllib.parse import urlparse
-
-from dvc.logger import Logger
+import dvc.logger as logger
+from dvc.utils import tmp_fname, move
+from dvc.utils.compat import urlparse, makedirs
 from dvc.progress import progress
 from dvc.config import Config
 from dvc.remote.base import RemoteBase
-from dvc.remote.local import RemoteLOCAL
 from dvc.exceptions import DvcException
 
 
@@ -34,152 +31,126 @@ class Callback(object):
 
 
 class RemoteS3(RemoteBase):
-    scheme = 's3'
-    REGEX = r'^s3://(?P<path>.*)$'
-    REQUIRES = {'boto3': boto3}
-    PARAM_ETAG = 'etag'
+    scheme = "s3"
+    REGEX = r"^s3://(?P<path>.*)$"
+    REQUIRES = {"boto3": boto3}
+    PARAM_CHECKSUM = "etag"
 
-    def __init__(self, project, config):
-        import configobj
+    def __init__(self, repo, config):
+        super(RemoteS3, self).__init__(repo, config)
+
+        storagepath = "s3://{}".format(
+            config.get(Config.SECTION_AWS_STORAGEPATH, "").lstrip("/")
+        )
 
-        self.project = project
-        storagepath = 's3://' + config.get(Config.SECTION_AWS_STORAGEPATH,
-                                           '').lstrip('/')
         self.url = config.get(Config.SECTION_REMOTE_URL, storagepath)
-        self.region = config.get(Config.SECTION_AWS_REGION, None)
-        self.profile = config.get(Config.SECTION_AWS_PROFILE, 'default')
-        self.endpoint_url = config.get(Config.SECTION_AWS_ENDPOINT_URL, None)
 
-        credentialpath = config.get(Config.SECTION_AWS_CREDENTIALPATH, None)
-        if credentialpath:
-            creds_conf = configobj.ConfigObj(credentialpath)
-            creds = creds_conf.get(self.profile, {})
-        else:
-            creds = {}
+        self.region = os.environ.get("AWS_DEFAULT_REGION") or config.get(
+            Config.SECTION_AWS_REGION
+        )
 
-        self.region = creds.get('region', self.region)
-        self.aws_access_key_id = creds.get('aws_access_key_id', None)
-        self.aws_secret_access_key = creds.get('aws_secret_access_key', None)
+        self.profile = os.environ.get("AWS_PROFILE") or config.get(
+            Config.SECTION_AWS_PROFILE
+        )
 
-    @property
-    def bucket(self):
-        return urlparse(self.url).netloc
+        self.endpoint_url = config.get(Config.SECTION_AWS_ENDPOINT_URL)
 
-    @property
-    def prefix(self):
-        return urlparse(self.url).path.lstrip('/')
+        self.list_objects = config.get(Config.SECTION_AWS_LIST_OBJECTS)
+
+        self.use_ssl = config.get(Config.SECTION_AWS_USE_SSL, True)
+
+        shared_creds = config.get(Config.SECTION_AWS_CREDENTIALPATH)
+        if shared_creds:
+            os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", shared_creds)
+
+        parsed = urlparse(self.url)
+        self.bucket = parsed.netloc
+        self.prefix = parsed.path.lstrip("/")
+
+        self.path_info = {"scheme": self.scheme, "bucket": self.bucket}
+
+    @staticmethod
+    def compat_config(config):
+        ret = config.copy()
+        url = "s3://" + ret.pop(Config.SECTION_AWS_STORAGEPATH, "").lstrip("/")
+        ret[Config.SECTION_REMOTE_URL] = url
+        return ret
 
     @property
     def s3(self):
-        session = boto3.session.Session(profile_name=self.profile)
-        return session.client('s3',
-                              aws_access_key_id=self.aws_access_key_id,
-                              aws_secret_access_key=self.aws_secret_access_key,
-                              region_name=self.region,
-                              endpoint_url=self.endpoint_url)
+        session = boto3.session.Session(
+            profile_name=self.profile, region_name=self.region
+        )
 
-    def get_etag(self, bucket, key):
+        return session.client(
+            "s3", endpoint_url=self.endpoint_url, use_ssl=self.use_ssl
+        )
+
+    def get_etag(self, bucket, path):
         try:
-            obj = self.s3.head_object(Bucket=bucket, Key=key)
+            obj = self.s3.head_object(Bucket=bucket, Key=path)
         except Exception:
-            raise DvcException('s3://{}/{} does not exist'.format(bucket, key))
+            raise DvcException(
+                "s3://{}/{} does not exist".format(bucket, path)
+            )
 
-        return obj['ETag'].strip('"')
+        return obj["ETag"].strip('"')
 
     def save_info(self, path_info):
-        if path_info['scheme'] != 's3':
+        if path_info["scheme"] != "s3":
             raise NotImplementedError
 
-        return {self.PARAM_ETAG: self.get_etag(path_info['bucket'],
-                                               path_info['key'])}
+        return {
+            self.PARAM_CHECKSUM: self.get_etag(
+                path_info["bucket"], path_info["path"]
+            )
+        }
 
-    def _copy(self, from_info, to_info, s3=None):
+    def copy(self, from_info, to_info, s3=None):
         s3 = s3 if s3 else self.s3
 
-        source = {'Bucket': from_info['bucket'],
-                  'Key': from_info['key']}
-        self.s3.copy(source, to_info['bucket'], to_info['key'])
-
-    def save(self, path_info):
-        if path_info['scheme'] != 's3':
-            raise NotImplementedError
-
-        etag = self.get_etag(path_info['bucket'], path_info['key'])
-        key = posixpath.join(self.prefix, etag[0:2], etag[2:])
-        to_info = {'scheme': 's3', 'bucket': self.bucket, 'key': key}
-
-        self._copy(path_info, to_info)
-
-        return {self.PARAM_ETAG: etag}
-
-    def checkout(self, path_info, checksum_info):
-        if path_info['scheme'] != 's3':
-            raise NotImplementedError
-
-        etag = checksum_info.get(self.PARAM_ETAG, None)
-        if not etag:
-            return
-
-        key = posixpath.join(self.prefix, etag[0:2], etag[2:])
-        from_info = {'scheme': 's3', 'bucket': self.bucket, 'key': key}
-
-        self._copy(from_info, path_info)
+        source = {"Bucket": from_info["bucket"], "Key": from_info["path"]}
+        self.s3.copy(source, to_info["bucket"], to_info["path"])
 
     def remove(self, path_info):
-        if path_info['scheme'] != 's3':
+        if path_info["scheme"] != "s3":
             raise NotImplementedError
 
-        Logger.debug('Removing s3://{}/{}'.format(path_info['bucket'],
-                                                  path_info['key']))
+        logger.debug(
+            "Removing s3://{}/{}".format(
+                path_info["bucket"], path_info["path"]
+            )
+        )
 
-        self.s3.delete_object(Bucket=path_info['bucket'],
-                              Key=path_info['key'])
+        self.s3.delete_object(
+            Bucket=path_info["bucket"], Key=path_info["path"]
+        )
 
-    def md5s_to_path_infos(self, md5s):
-        return [{'scheme': self.scheme,
-                 'bucket': self.bucket,
-                 'key': posixpath.join(self.prefix,
-                                       md5[0:2], md5[2:])} for md5 in md5s]
-
-    def _all_keys(self):
+    def _list_paths(self, bucket, prefix):
+        """ Read config for list object api, paginate through list objects."""
         s3 = self.s3
-
-        keys = []
-        kwargs = {'Bucket': self.bucket,
-                  'Prefix': self.prefix}
-        while True:
-            resp = s3.list_objects_v2(**kwargs)
-            contents = resp.get('Contents', None)
+        kwargs = {"Bucket": bucket, "Prefix": prefix}
+        if self.list_objects:
+            list_objects_api = "list_objects"
+        else:
+            list_objects_api = "list_objects_v2"
+        paginator = s3.get_paginator(list_objects_api)
+        for page in paginator.paginate(**kwargs):
+            contents = page.get("Contents", None)
             if not contents:
-                break
+                continue
+            for item in contents:
+                yield item["Key"]
 
-            for obj in contents:
-                keys.append(obj['Key'])
+    def list_cache_paths(self):
+        return self._list_paths(self.bucket, self.prefix)
 
-            token = resp.get('NextContinuationToken', None)
-            if not token:
-                break
+    def exists(self, path_info):
+        assert not isinstance(path_info, list)
+        assert path_info["scheme"] == "s3"
 
-            kwargs['ContinuationToken'] = token
-
-        return keys
-
-    def exists(self, path_infos):
-        # NOTE: We mostly use exists() method when filtering a bulk of cache
-        # files to decide if we need to download/upload them and in s3
-        # list_objects_v2() is much-much faster than trying to check keys
-        # one-by-one.
-        ret = []
-
-        keys = self._all_keys()
-
-        for path_info in path_infos:
-            exists = False
-            if path_info['key'] in keys:
-                exists = True
-            ret.append(exists)
-
-        return ret
+        paths = self._list_paths(path_info["bucket"], path_info["path"])
+        return any(path_info["path"] == path for path in paths)
 
     def upload(self, from_infos, to_infos, names=None):
         names = self._verify_path_args(to_infos, from_infos, names)
@@ -187,105 +158,95 @@ class RemoteS3(RemoteBase):
         s3 = self.s3
 
         for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info['scheme'] != 's3':
+            if to_info["scheme"] != "s3":
                 raise NotImplementedError
 
-            if from_info['scheme'] != 'local':
+            if from_info["scheme"] != "local":
                 raise NotImplementedError
 
-            Logger.debug("Uploading '{}' to '{}/{}'".format(from_info['path'],
-                                                            to_info['bucket'],
-                                                            to_info['key']))
+            logger.debug(
+                "Uploading '{}' to '{}/{}'".format(
+                    from_info["path"], to_info["bucket"], to_info["path"]
+                )
+            )
 
             if not name:
-                name = os.path.basename(from_info['path'])
+                name = os.path.basename(from_info["path"])
 
-            total = os.path.getsize(from_info['path'])
+            total = os.path.getsize(from_info["path"])
             cb = Callback(name, total)
 
             try:
-                s3.upload_file(from_info['path'],
-                               to_info['bucket'],
-                               to_info['key'],
-                               Callback=cb)
-            except Exception as exc:
-                msg = "Failed to upload '{}'".format(from_info['path'])
-                Logger.error(msg, exc)
+                s3.upload_file(
+                    from_info["path"],
+                    to_info["bucket"],
+                    to_info["path"],
+                    Callback=cb,
+                )
+            except Exception:
+                msg = "failed to upload '{}'".format(from_info["path"])
+                logger.error(msg)
                 continue
 
             progress.finish_target(name)
 
-    def download(self,
-                 from_infos,
-                 to_infos,
-                 no_progress_bar=False,
-                 names=None):
+    def download(
+        self,
+        from_infos,
+        to_infos,
+        no_progress_bar=False,
+        names=None,
+        resume=False,
+    ):
         names = self._verify_path_args(from_infos, to_infos, names)
 
         s3 = self.s3
 
         for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info['scheme'] != 's3':
+            if from_info["scheme"] != "s3":
                 raise NotImplementedError
 
-            if to_info['scheme'] == 's3':
-                self._copy(from_info, to_info, s3=s3)
+            if to_info["scheme"] == "s3":
+                self.copy(from_info, to_info, s3=s3)
                 continue
 
-            if to_info['scheme'] != 'local':
+            if to_info["scheme"] != "local":
                 raise NotImplementedError
 
-            msg = "Downloading '{}/{}' to '{}'".format(from_info['bucket'],
-                                                       from_info['key'],
-                                                       to_info['path'])
-            Logger.debug(msg)
+            msg = "Downloading '{}/{}' to '{}'".format(
+                from_info["bucket"], from_info["path"], to_info["path"]
+            )
+            logger.debug(msg)
 
-            tmp_file = self.tmp_file(to_info['path'])
+            tmp_file = tmp_fname(to_info["path"])
             if not name:
-                name = os.path.basename(to_info['path'])
+                name = os.path.basename(to_info["path"])
 
-            if no_progress_bar:
-                cb = None
-            else:
-                total = s3.head_object(Bucket=from_info['bucket'],
-                                       Key=from_info['key'])['ContentLength']
-                cb = Callback(name, total)
-
-            self._makedirs(to_info['path'])
+            makedirs(os.path.dirname(to_info["path"]), exist_ok=True)
 
             try:
-                s3.download_file(from_info['bucket'],
-                                 from_info['key'],
-                                 tmp_file,
-                                 Callback=cb)
-            except Exception as exc:
-                msg = "Failed to download '{}/{}'".format(from_info['bucket'],
-                                                          from_info['key'])
-                Logger.error(msg, exc)
-                return
+                if no_progress_bar:
+                    cb = None
+                else:
+                    total = s3.head_object(
+                        Bucket=from_info["bucket"], Key=from_info["path"]
+                    )["ContentLength"]
+                    cb = Callback(name, total)
 
-            os.rename(tmp_file, to_info['path'])
+                s3.download_file(
+                    from_info["bucket"],
+                    from_info["path"],
+                    tmp_file,
+                    Callback=cb,
+                )
+            except Exception:
+                msg = "failed to download '{}/{}'".format(
+                    from_info["bucket"], from_info["path"]
+                )
+                logger.error(msg)
+                continue
+
+            move(tmp_file, to_info["path"])
 
             if not no_progress_bar:
                 progress.finish_target(name)
-
-    def _path_to_etag(self, path):
-        relpath = posixpath.relpath(path, self.prefix)
-        return posixpath.dirname(relpath) + posixpath.basename(relpath)
-
-    def _all(self):
-        keys = self._all_keys()
-        return [self._path_to_etag(key) for key in keys]
-
-    def gc(self, cinfos):
-        used_etags = [info[self.PARAM_ETAG] for info in cinfos['s3']]
-        used_etags += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
-
-        for etag in self._all():
-            if etag in used_etags:
-                continue
-            path_info = {'scheme': 's3',
-                         'key': posixpath.join(self.prefix,
-                                               etag[0:2], etag[2:]),
-                         'bucket': self.bucket}
-            self.remove(path_info)

dvc/remote/ssh.py

@@ -1,300 +0,0 @@
-import os
-import getpass
-import posixpath
-
-try:
-    import paramiko
-except ImportError:
-    paramiko = None
-
-from dvc.logger import Logger
-from dvc.progress import progress
-from dvc.remote.base import RemoteBase
-from dvc.remote.local import RemoteLOCAL
-from dvc.config import Config
-from dvc.exceptions import DvcException
-
-
-def sizeof_fmt(num, suffix='B'):
-    """ Convert number of bytes to human-readable string """
-    for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
-        if abs(num) < 1024.0:
-            return "%3.1f%s%s" % (num, unit, suffix)
-        num /= 1024.0
-    return "%.1f%s%s" % (num, 'Y', suffix)
-
-
-def percent_cb(name, complete, total):
-    """ Callback for updating target progress """
-    Logger.debug('{}: {} transferred out of {}'.format(name,
-                                                       sizeof_fmt(complete),
-                                                       sizeof_fmt(total)))
-    progress.update_target(name, complete, total)
-
-
-def create_cb(name):
-    """ Create callback function for multipart object """
-    return (lambda cur, tot: percent_cb(name, cur, tot))
-
-
-class RemoteSSH(RemoteBase):
-    scheme = 'ssh'
-
-    # NOTE: temporarily only absolute paths are allowed
-    REGEX = r'^ssh://((?P<user>.*)@)?(?P<host>[^/]*):(?P<path>/.*)$'
-
-    REQUIRES = {'paramiko': paramiko}
-    PARAM_MD5 = 'md5'
-
-    def __init__(self, project, config):
-        self.project = project
-        self.url = config.get(Config.SECTION_REMOTE_URL, '/')
-        self.host = self.group('host')
-        self.user = self.group('user')
-        if not self.user:
-            self.user = config.get(Config.SECTION_REMOTE_USER,
-                                   getpass.getuser())
-        self.prefix = self.group('path')
-
-    def md5s_to_path_infos(self, md5s):
-        return [{'scheme': 'ssh',
-                 'host': self.host,
-                 'user': self.user,
-                 'path': posixpath.join(self.prefix,
-                                        md5[0:2], md5[2:])} for md5 in md5s]
-
-    def ssh(self, host=None, user=None):
-        msg = "Establishing ssh connection with '{}' as user '{}'"
-        Logger.debug(msg.format(host, user))
-
-        ssh = paramiko.SSHClient()
-
-        ssh.load_system_host_keys()
-        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-
-        ssh.connect(host, username=user)
-
-        return ssh
-
-    def exists(self, path_infos):
-        ret = []
-        ssh = self.ssh(host=self.host, user=self.user)
-        cmd = 'find {} -type f -follow -print'.format(self.prefix)
-        stdout = self._exec(ssh, cmd)
-        plist = stdout.split()
-        ssh.close()
-
-        for path_info in path_infos:
-            exists = False
-            if path_info['path'] in plist:
-                exists = True
-            ret.append(exists)
-
-        return ret
-
-    def _exec(self, ssh, cmd):
-        stdin, stdout, stderr = ssh.exec_command(cmd)
-        if stdout.channel.recv_exit_status() != 0:
-            DvcException('SSH command \'{}\' failed: {}'.format(cmd,
-                                                                stderr.read()))
-        return stdout.read().decode('utf-8')
-
-    def md5(self, path_info):
-        if path_info['scheme'] != 'ssh':
-            raise NotImplementedError
-
-        ssh = self.ssh(host=path_info['host'], user=path_info['user'])
-
-        # Use different md5 commands depending on os
-        stdout = self._exec(ssh, 'uname').strip()
-        if stdout == 'Darwin':
-            md5cmd = 'md5'
-            index = -1
-        elif stdout == 'Linux':
-            md5cmd = 'md5sum'
-            index = 0
-        else:
-            msg = '\'{}\' is not supported as a remote'.format(stdout)
-            raise DvcException(msg)
-
-        stdout = self._exec(ssh, '{} {}'.format(md5cmd, path_info['path']))
-        md5 = stdout.split()[index]
-        ssh.close()
-
-        assert len(md5) == 32
-
-        return md5
-
-    def cp(self, from_info, to_info, ssh=None):
-        if from_info['scheme'] != 'ssh' or to_info['scheme'] != 'ssh':
-            raise NotImplementedError
-
-        assert from_info['host'] == to_info['host']
-        assert from_info['user'] == to_info['user']
-
-        s = ssh if ssh else self.ssh(host=from_info['host'],
-                                     user=from_info['user'])
-
-        dname = posixpath.dirname(to_info['path'])
-        self._exec(s, 'mkdir -p {}'.format(dname))
-        self._exec(s, 'cp {} {}'.format(from_info['path'], to_info['path']))
-
-        if not ssh:
-            s.close()
-
-    def save_info(self, path_info):
-        if path_info['scheme'] != 'ssh':
-            raise NotImplementedError
-
-        return {self.PARAM_MD5: self.md5(path_info)}
-
-    def save(self, path_info):
-        if path_info['scheme'] != 'ssh':
-            raise NotImplementedError
-
-        md5 = self.md5(path_info)
-        dest = path_info.copy()
-        dest['path'] = posixpath.join(self.prefix, md5[0:2], md5[2:])
-
-        self.cp(path_info, dest)
-
-        return {self.PARAM_MD5: md5}
-
-    def checkout(self, path_info, checksum_info):
-        if path_info['scheme'] != 'ssh':
-            raise NotImplementedError
-
-        md5 = checksum_info.get(self.PARAM_MD5, None)
-        if not md5:
-            return
-
-        src = path_info.copy()
-        src['path'] = posixpath.join(self.prefix, md5[0:2], md5[2:])
-
-        self.cp(src, path_info)
-
-    def remove(self, path_info):
-        if path_info['scheme'] != 'ssh':
-            raise NotImplementedError
-
-        Logger.debug('Removing ssh://{}@{}/{}'.format(path_info['user'],
-                                                      path_info['host'],
-                                                      path_info['path']))
-
-        ssh = self.ssh(host=path_info['host'], user=path_info['user'])
-        ssh.open_sftp().remove(path_info['path'])
-        ssh.close()
-
-    def download(self,
-                 from_infos,
-                 to_infos,
-                 no_progress_bar=False,
-                 names=None):
-        names = self._verify_path_args(from_infos, to_infos, names)
-
-        ssh = self.ssh(host=from_infos[0]['host'], user=from_infos[0]['user'])
-
-        for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info['scheme'] != 'ssh':
-                raise NotImplementedError
-
-            if to_info['scheme'] == 'ssh':
-                assert from_info['host'] == to_info['host']
-                assert from_info['user'] == to_info['user']
-                self.cp(from_info, to_info, ssh=ssh)
-                continue
-
-            if to_info['scheme'] != 'local':
-                raise NotImplementedError
-
-            msg = "Downloading '{}/{}' to '{}'".format(from_info['host'],
-                                                       from_info['path'],
-                                                       to_info['path'])
-            Logger.debug(msg)
-
-            if not name:
-                name = os.path.basename(to_info['path'])
-
-            self._makedirs(to_info['path'])
-            tmp_file = self.tmp_file(to_info['path'])
-            try:
-                ssh.open_sftp().get(from_info['path'],
-                                    tmp_file,
-                                    callback=create_cb(name))
-            except Exception as exc:
-                msg = "Failed to download '{}/{}' to '{}'"
-                Logger.error(msg.format(from_info['host'],
-                                        from_info['path'],
-                                        to_info['path']), exc)
-                continue
-
-            os.rename(tmp_file, to_info['path'])
-            progress.finish_target(name)
-
-        ssh.close()
-
-    def upload(self, from_infos, to_infos, names=None):
-        names = self._verify_path_args(to_infos, from_infos, names)
-
-        ssh = self.ssh(host=to_infos[0]['host'], user=to_infos[0]['user'])
-        sftp = ssh.open_sftp()
-
-        for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info['scheme'] != 'ssh':
-                raise NotImplementedError
-
-            if from_info['scheme'] != 'local':
-                raise NotImplementedError
-
-            Logger.debug("Uploading '{}' to '{}/{}'".format(from_info['path'],
-                                                            to_info['host'],
-                                                            to_info['path']))
-
-            if not name:
-                name = os.path.basename(from_info['path'])
-
-            dname = posixpath.dirname(to_info['path'])
-            self._exec(ssh, 'mkdir -p {}'.format(dname))
-
-            try:
-                sftp.put(from_info['path'],
-                         to_info['path'],
-                         callback=create_cb(name))
-            except Exception as exc:
-                msg = "Failed to upload '{}' to '{}/{}'"
-                Logger.error(msg.format(from_info['path'],
-                                        to_info['host'],
-                                        to_info['path'], exc))
-                continue
-
-            progress.finish_target(name)
-
-        sftp.close()
-        ssh.close()
-
-    def _path_to_md5(self, path):
-        relpath = posixpath.relpath(path, self.prefix)
-        return posixpath.dirname(relpath) + posixpath.basename(relpath)
-
-    def _all_md5s(self):
-        ssh = self.ssh(host=self.host, user=self.user)
-        cmd = 'find {} -type f -follow -print'.format(self.prefix)
-        stdout = self._exec(ssh, cmd)
-        flist = stdout.split()
-        ssh.close()
-
-        return [self._path_to_md5(path) for path in flist]
-
-    def gc(self, cinfos):
-        used = [info[self.PARAM_MD5] for info in cinfos['ssh']]
-        used += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
-
-        for md5 in self._all_md5s():
-            if md5 in used:
-                continue
-            path_info = {'scheme': 'ssh',
-                         'user': self.user,
-                         'host': self.host,
-                         'path': posixpath.join(self.prefix,
-                                                md5[0:2], md5[2:])}
-            self.remove(path_info)

dvc/remote/ssh/__init__.py

@@ -0,0 +1,198 @@
+from __future__ import unicode_literals
+
+import getpass
+
+try:
+    import paramiko
+except ImportError:
+    paramiko = None
+
+import dvc.prompt as prompt
+import dvc.logger as logger
+from dvc.remote.ssh.connection import SSHConnection
+from dvc.config import Config
+from dvc.utils.compat import urlparse
+from dvc.remote.base import RemoteBase
+
+
+class RemoteSSH(RemoteBase):
+    scheme = "ssh"
+
+    # NOTE: we support both URL-like (ssh://[user@]host.xz[:port]/path) and
+    # SCP-like (ssh://[user@]host.xz:/absolute/path) urls.
+    REGEX = r"^ssh://.*$"
+
+    REQUIRES = {"paramiko": paramiko}
+
+    JOBS = 4
+    PARAM_CHECKSUM = "md5"
+    DEFAULT_PORT = 22
+    TIMEOUT = 1800
+
+    def __init__(self, repo, config):
+        super(RemoteSSH, self).__init__(repo, config)
+        self.url = config.get(Config.SECTION_REMOTE_URL, "ssh://")
+
+        parsed = urlparse(self.url)
+        self.host = parsed.hostname
+        self.user = (
+            config.get(Config.SECTION_REMOTE_USER)
+            or parsed.username
+            or getpass.getuser()
+        )
+        self.prefix = parsed.path or "/"
+        self.port = (
+            config.get(Config.SECTION_REMOTE_PORT)
+            or parsed.port
+            or self.DEFAULT_PORT
+        )
+        self.keyfile = config.get(Config.SECTION_REMOTE_KEY_FILE, None)
+        self.timeout = config.get(Config.SECTION_REMOTE_TIMEOUT, self.TIMEOUT)
+        self.password = config.get(Config.SECTION_REMOTE_PASSWORD, None)
+        self.ask_password = config.get(
+            Config.SECTION_REMOTE_ASK_PASSWORD, False
+        )
+
+        self.path_info = {
+            "scheme": "ssh",
+            "host": self.host,
+            "user": self.user,
+            "port": self.port,
+        }
+
+    def ssh(self, host=None, user=None, port=None, **kwargs):
+        logger.debug(
+            "Establishing ssh connection with '{host}' "
+            "through port '{port}' as user '{user}'".format(
+                host=host, user=user, port=port
+            )
+        )
+
+        if self.ask_password and not self.password:
+            self.password = prompt.password(
+                "Enter a private key passphrase or a password for "
+                "host '{host}' port '{port}' user '{user}'".format(
+                    host=host, port=port, user=user
+                )
+            )
+
+        return SSHConnection(
+            host,
+            username=user,
+            port=port,
+            key_filename=self.keyfile,
+            timeout=self.timeout,
+            password=self.password,
+        )
+
+    def exists(self, path_info):
+        assert not isinstance(path_info, list)
+        assert path_info["scheme"] == self.scheme
+
+        with self.ssh(**path_info) as ssh:
+            return ssh.file_exists(path_info["path"])
+
+    def md5(self, path_info):
+        if path_info["scheme"] != self.scheme:
+            raise NotImplementedError
+
+        with self.ssh(**path_info) as ssh:
+            return ssh.md5(path_info["path"])
+
+    def copy(self, from_info, to_info):
+        if (
+            from_info["scheme"] != self.scheme
+            or to_info["scheme"] != self.scheme
+        ):
+            raise NotImplementedError
+
+        with self.ssh(**from_info) as ssh:
+            ssh.cp(from_info["path"], to_info["path"])
+
+    def save_info(self, path_info):
+        if path_info["scheme"] != self.scheme:
+            raise NotImplementedError
+
+        return {self.PARAM_CHECKSUM: self.md5(path_info)}
+
+    def remove(self, path_info):
+        if path_info["scheme"] != self.scheme:
+            raise NotImplementedError
+
+        with self.ssh(**path_info) as ssh:
+            ssh.remove(path_info["path"])
+
+    def download(
+        self,
+        from_infos,
+        to_infos,
+        no_progress_bar=False,
+        names=None,
+        resume=False,
+    ):
+        names = self._verify_path_args(from_infos, to_infos, names)
+        ssh = self.ssh(**from_infos[0])
+
+        for to_info, from_info, name in zip(to_infos, from_infos, names):
+            if from_info["scheme"] != self.scheme:
+                raise NotImplementedError
+
+            if to_info["scheme"] == self.scheme:
+                ssh.cp(from_info["path"], to_info["path"])
+                continue
+
+            if to_info["scheme"] != "local":
+                raise NotImplementedError
+
+            logger.debug(
+                "Downloading '{host}/{path}' to '{dest}'".format(
+                    host=from_info["host"],
+                    path=from_info["path"],
+                    dest=to_info["path"],
+                )
+            )
+
+            try:
+                ssh.download(
+                    from_info["path"], to_info["path"], progress_title=name
+                )
+            except Exception:
+                logger.error(
+                    "failed to download '{host}/{path}' to '{dest}'".format(
+                        host=from_info["host"],
+                        path=from_info["path"],
+                        dest=to_info["path"],
+                    )
+                )
+                continue
+
+        ssh.close()
+
+    def upload(self, from_infos, to_infos, names=None):
+        names = self._verify_path_args(to_infos, from_infos, names)
+
+        with self.ssh(**to_infos[0]) as ssh:
+            for from_info, to_info, name in zip(from_infos, to_infos, names):
+                if to_info["scheme"] != self.scheme:
+                    raise NotImplementedError
+
+                if from_info["scheme"] != "local":
+                    raise NotImplementedError
+
+                try:
+                    ssh.upload(
+                        from_info["path"], to_info["path"], progress_title=name
+                    )
+                except Exception:
+                    logger.error(
+                        "failed to upload '{host}/{path}' to '{dest}'".format(
+                            host=from_info["host"],
+                            path=from_info["path"],
+                            dest=to_info["path"],
+                        )
+                    )
+                    pass
+
+    def list_cache_paths(self):
+        with self.ssh(**self.path_info) as ssh:
+            return list(ssh.walk_files(self.prefix))

dvc/remote/ssh/connection.py

@@ -0,0 +1,262 @@
+import os
+import posixpath
+
+try:
+    import paramiko
+except ImportError:
+    paramiko = None
+
+import dvc.logger as logger
+from dvc.utils import tmp_fname
+from dvc.utils.compat import makedirs
+from dvc.progress import progress
+from dvc.exceptions import DvcException
+from dvc.remote.base import RemoteCmdError
+
+
+def sizeof_fmt(num, suffix="B"):
+    """ Convert number of bytes to human-readable string """
+    for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]:
+        if abs(num) < 1024.0:
+            return "%3.1f%s%s" % (num, unit, suffix)
+        num /= 1024.0
+    return "%.1f%s%s" % (num, "Y", suffix)
+
+
+def percent_cb(name, complete, total):
+    """ Callback for updating target progress """
+    logger.debug(
+        "{}: {} transferred out of {}".format(
+            name, sizeof_fmt(complete), sizeof_fmt(total)
+        )
+    )
+    progress.update_target(name, complete, total)
+
+
+def create_cb(name):
+    """ Create callback function for multipart object """
+    return lambda cur, tot: percent_cb(name, cur, tot)
+
+
+class SSHConnection:
+    def __init__(self, *args, **kwargs):
+        self.timeout = kwargs.get("timeout", 1800)
+
+        self._ssh = paramiko.SSHClient()
+        self._ssh.load_system_host_keys()
+        self._ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+        self._ssh.connect(*args, **kwargs)
+        self._sftp = None
+        self._sftp_alive = False
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exception_type, exception_value, traceback):
+        self.close()
+
+    def _sftp_connect(self):
+        if not self._sftp or not self._sftp_alive:
+            self._sftp = self._ssh.open_sftp()
+            self._sftp_alive = True
+
+    def close(self):
+        if self._sftp:
+            self._sftp.close()
+            self._sftp_alive = False
+
+        self._ssh.close()
+
+    def file_exists(self, path):
+        self._sftp_connect()
+        try:
+            return self._sftp.stat(path)
+        except IOError:
+            return False
+        pass
+
+    def isdir(self, path):
+        from stat import S_ISDIR
+
+        self._sftp_connect()
+        try:
+            return S_ISDIR(self._sftp.stat(path).st_mode)
+        except IOError:
+            return False
+
+    def isfile(self, path):
+        from stat import S_ISREG
+
+        self._sftp_connect()
+        try:
+            return S_ISREG(self._sftp.stat(path).st_mode)
+        except IOError:
+            return False
+
+    def islink(self, path):
+        from stat import S_ISLNK
+
+        self._sftp_connect()
+        try:
+            return S_ISLNK(self._sftp.stat(path).st_mode)
+        except IOError:
+            return False
+
+    def makedirs(self, path):
+        self._sftp_connect()
+
+        if self.isdir(path):
+            return
+
+        if self.isfile(path) or self.islink(path):
+            raise DvcException(
+                "a file with the same name '{}' already exists".format(path)
+            )
+
+        head, tail = os.path.split(path)
+
+        if head:
+            self.makedirs(head)
+
+        if tail:
+            self._sftp.mkdir(path)
+
+    def walk_files(self, directory):
+        from stat import S_ISLNK, S_ISDIR, S_ISREG
+
+        self._sftp_connect()
+
+        for entry in self._sftp.listdir_attr(directory):
+            path = posixpath.join(directory, entry.filename)
+
+            if S_ISLNK(entry.st_mode):
+                path = self._sftp.readlink(directory)
+                entry = self._sftp.stat(path)
+
+            if S_ISDIR(entry.st_mode):
+                for inner_path in self.walk_files(path):
+                    yield inner_path
+
+            elif S_ISREG(entry.st_mode):
+                yield path
+
+    def remove(self, path):
+        self._sftp_connect()
+        self._sftp.remove(path)
+
+    def download(self, src, dest, no_progress_bar=False, progress_title=None):
+        self._sftp_connect()
+
+        makedirs(posixpath.dirname(dest), exist_ok=True)
+        tmp_file = tmp_fname(dest)
+
+        if no_progress_bar:
+            self._sftp.get(src, tmp_file)
+        else:
+            if not progress_title:
+                progress_title = os.path.basename(dest)
+
+            self._sftp.get(src, tmp_file, callback=create_cb(progress_title))
+            progress.finish_target(progress_title)
+
+        if os.path.exists(dest):
+            os.remove(dest)
+
+        os.rename(tmp_file, dest)
+
+    def upload(self, src, dest, no_progress_bar=False, progress_title=None):
+        self._sftp_connect()
+
+        self.makedirs(posixpath.dirname(dest))
+        tmp_file = tmp_fname(dest)
+
+        if no_progress_bar:
+            self._sftp.put(src, tmp_file)
+        else:
+            if not progress_title:
+                progress_title = os.path.basename(dest)
+
+            self._sftp.put(src, tmp_file, callback=create_cb(progress_title))
+            progress.finish_target(progress_title)
+
+        self._sftp.rename(tmp_file, dest)
+
+    def execute(self, cmd):
+        stdin, stdout, stderr = self._ssh.exec_command(cmd)
+        channel = stdout.channel
+
+        stdin.close()
+        channel.shutdown_write()
+
+        stdout_chunks = []
+        stderr_chunks = []
+        while (
+            not channel.closed
+            or channel.recv_ready()
+            or channel.recv_stderr_ready()
+        ):
+            import select
+
+            got_chunk = False
+            readq, _, _ = select.select([stdout.channel], [], [], self.timeout)
+            for c in readq:
+                if c.recv_ready():
+                    stdout_chunks.append(stdout.channel.recv(len(c.in_buffer)))
+                    got_chunk = True
+
+                if c.recv_stderr_ready():
+                    stderr_len = len(c.in_stderr_buffer)
+                    s = stderr.channel.recv_stderr(stderr_len)
+                    stderr_chunks.append(s)
+                    got_chunk = True
+
+            if (
+                not got_chunk
+                and stdout.channel.exit_status_ready()
+                and not stderr.channel.recv_stderr_ready()
+                and not stdout.channel.recv_ready()
+            ):
+                stdout.channel.shutdown_read()
+                stdout.channel.close()
+                break
+
+        stdout.close()
+        stderr.close()
+
+        ret = stdout.channel.recv_exit_status()
+        if ret != 0:
+            err = b"".join(stderr_chunks).decode("utf-8")
+            raise RemoteCmdError("ssh", cmd, ret, err)
+
+        return b"".join(stdout_chunks).decode("utf-8")
+
+    def md5(self, path):
+        """
+        Use different md5 commands depending on the OS:
+
+         - Darwin's `md5` returns BSD-style checksums by default
+         - Linux's `md5sum` needs the `--tag` flag for a similar output
+
+         Example:
+              MD5 (foo.txt) = f3d220a856b52aabbf294351e8a24300
+        """
+        uname = self.execute("uname").strip()
+
+        command = {
+            "Darwin": "md5 {}".format(path),
+            "Linux": "md5sum --tag {}".format(path),
+        }.get(uname)
+
+        if not command:
+            raise DvcException(
+                "'{uname}' is not supported as a remote".format(uname=uname)
+            )
+
+        md5 = self.execute(command).split()[-1]
+        assert len(md5) == 32
+        return md5
+
+    def cp(self, src, dest):
+        self.makedirs(posixpath.dirname(dest))
+        self.execute("cp {} {}".format(src, dest))

dvc/repo/__init__.py

@@ -0,0 +1,417 @@
+from __future__ import unicode_literals
+
+from dvc.utils.compat import str
+
+import os
+import dvc.prompt as prompt
+import dvc.logger as logger
+
+from dvc.exceptions import DvcException, NotDvcRepoError, OutputNotFoundError
+
+
+class Repo(object):
+    DVC_DIR = ".dvc"
+
+    from dvc.repo.destroy import destroy
+    from dvc.repo.install import install
+    from dvc.repo.add import add
+    from dvc.repo.remove import remove
+    from dvc.repo.lock import lock as lock_stage
+    from dvc.repo.move import move
+    from dvc.repo.run import run
+    from dvc.repo.imp import imp
+    from dvc.repo.reproduce import reproduce
+    from dvc.repo.checkout import checkout
+    from dvc.repo.push import push
+    from dvc.repo.fetch import fetch
+    from dvc.repo.pull import pull
+    from dvc.repo.status import status
+    from dvc.repo.gc import gc
+    from dvc.repo.commit import commit
+
+    def __init__(self, root_dir=None):
+        from dvc.config import Config
+        from dvc.state import State
+        from dvc.lock import Lock
+        from dvc.scm import SCM
+        from dvc.cache import Cache
+        from dvc.data_cloud import DataCloud
+        from dvc.updater import Updater
+        from dvc.repo.metrics import Metrics
+
+        root_dir = self.find_root(root_dir)
+
+        self.root_dir = os.path.abspath(os.path.realpath(root_dir))
+        self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR)
+
+        self.config = Config(self.dvc_dir)
+
+        self.scm = SCM(self.root_dir, repo=self)
+        self.lock = Lock(self.dvc_dir)
+        # NOTE: storing state and link_state in the repository itself to avoid
+        # any possible state corruption in 'shared cache dir' scenario.
+        self.state = State(self, self.config.config)
+
+        core = self.config.config[Config.SECTION_CORE]
+
+        logger.set_level(core.get(Config.SECTION_CORE_LOGLEVEL))
+
+        self.cache = Cache(self)
+        self.cloud = DataCloud(self, config=self.config.config)
+        self.updater = Updater(self.dvc_dir)
+
+        self.metrics = Metrics(self)
+
+        self._ignore()
+
+        self.updater.check()
+
+    def __repr__(self):
+        return "Repo: '{root_dir}'".format(root_dir=self.root_dir)
+
+    @staticmethod
+    def find_root(root=None):
+        if root is None:
+            root = os.getcwd()
+        else:
+            root = os.path.abspath(os.path.realpath(root))
+
+        while True:
+            dvc_dir = os.path.join(root, Repo.DVC_DIR)
+            if os.path.isdir(dvc_dir):
+                return root
+            if os.path.ismount(root):
+                break
+            root = os.path.dirname(root)
+        raise NotDvcRepoError(root)
+
+    @staticmethod
+    def find_dvc_dir(root=None):
+        root_dir = Repo.find_root(root)
+        return os.path.join(root_dir, Repo.DVC_DIR)
+
+    @staticmethod
+    def init(root_dir=os.curdir, no_scm=False, force=False):
+        from dvc.repo.init import init
+
+        init(root_dir=root_dir, no_scm=no_scm, force=force)
+        return Repo(root_dir)
+
+    @staticmethod
+    def unprotect(target):
+        from dvc.repo.unprotect import unprotect
+
+        return unprotect(target)
+
+    def _ignore(self):
+        flist = [
+            self.state.state_file,
+            self.lock.lock_file,
+            self.config.config_local_file,
+            self.updater.updater_file,
+            self.updater.lock.lock_file,
+        ] + self.state.temp_files
+
+        if self.cache.local.cache_dir.startswith(self.root_dir):
+            flist += [self.cache.local.cache_dir]
+
+        self.scm.ignore_list(flist)
+
+    def check_dag(self, stages):
+        """Generate graph including the new stage to check for errors"""
+        self.graph(stages=stages)
+
+    @staticmethod
+    def _check_cyclic_graph(graph):
+        import networkx as nx
+        from dvc.exceptions import CyclicGraphError
+
+        cycles = list(nx.simple_cycles(graph))
+
+        if cycles:
+            raise CyclicGraphError(cycles[0])
+
+    def _get_pipeline(self, node):
+        pipelines = list(filter(lambda g: node in g.nodes(), self.pipelines()))
+        assert len(pipelines) == 1
+        return pipelines[0]
+
+    def collect(self, target, with_deps=False):
+        import networkx as nx
+        from dvc.stage import Stage
+
+        stage = Stage.load(self, target)
+        if not with_deps:
+            return [stage]
+
+        node = os.path.relpath(stage.path, self.root_dir)
+        G = self._get_pipeline(node)
+        stages = nx.get_node_attributes(G, "stage")
+
+        ret = [stage]
+        for n in nx.dfs_postorder_nodes(G, node):
+            ret.append(stages[n])
+
+        return ret
+
+    def _collect_dir_cache(
+        self, out, branch=None, remote=None, force=False, jobs=None
+    ):
+        info = out.dumpd()
+        ret = [info]
+        r = out.remote
+        md5 = info[r.PARAM_CHECKSUM]
+
+        if self.cache.local.changed_cache_file(md5):
+            try:
+                self.cloud.pull(
+                    ret, jobs=jobs, remote=remote, show_checksums=False
+                )
+            except DvcException as exc:
+                msg = "Failed to pull cache for '{}': {}"
+                logger.debug(msg.format(out, exc))
+
+        if self.cache.local.changed_cache_file(md5):
+            msg = (
+                "Missing cache for directory '{}'. "
+                "Cache for files inside will be lost. "
+                "Would you like to continue? Use '-f' to force. "
+            )
+            if not force and not prompt.confirm(msg):
+                raise DvcException(
+                    "unable to fully collect used cache"
+                    " without cache for directory '{}'".format(out)
+                )
+            else:
+                return ret
+
+        for i in self.cache.local.load_dir_cache(md5):
+            i["branch"] = branch
+            i[r.PARAM_PATH] = os.path.join(
+                info[r.PARAM_PATH], i[r.PARAM_RELPATH]
+            )
+            ret.append(i)
+
+        return ret
+
+    def _collect_used_cache(
+        self, out, branch=None, remote=None, force=False, jobs=None
+    ):
+        if not out.use_cache or not out.info:
+            if not out.info:
+                logger.warning(
+                    "Output '{}'({}) is missing version "
+                    "info. Cache for it will not be collected. "
+                    "Use dvc repro to get your pipeline up to "
+                    "date.".format(out, out.stage)
+                )
+            return []
+
+        info = out.dumpd()
+        info["branch"] = branch
+        ret = [info]
+
+        if out.scheme != "local":
+            return ret
+
+        md5 = info[out.remote.PARAM_CHECKSUM]
+        cache = self.cache.local.get(md5)
+        if not out.remote.is_dir_cache(cache):
+            return ret
+
+        return self._collect_dir_cache(
+            out, branch=branch, remote=remote, force=force, jobs=jobs
+        )
+
+    def used_cache(
+        self,
+        target=None,
+        all_branches=False,
+        active=True,
+        with_deps=False,
+        all_tags=False,
+        remote=None,
+        force=False,
+        jobs=None,
+        recursive=False,
+    ):
+        cache = {}
+        cache["local"] = []
+        cache["s3"] = []
+        cache["gs"] = []
+        cache["hdfs"] = []
+        cache["ssh"] = []
+        cache["azure"] = []
+
+        for branch in self.scm.brancher(
+            all_branches=all_branches, all_tags=all_tags
+        ):
+            if target:
+                if recursive:
+                    stages = self.stages(target)
+                else:
+                    stages = self.collect(target, with_deps=with_deps)
+            elif active:
+                stages = self.active_stages()
+            else:
+                stages = self.stages()
+
+            for stage in stages:
+                if active and not target and stage.locked:
+                    logger.warning(
+                        "DVC file '{path}' is locked. Its dependencies are"
+                        " not going to be pushed/pulled/fetched.".format(
+                            path=stage.relpath
+                        )
+                    )
+
+                for out in stage.outs:
+                    scheme = out.path_info["scheme"]
+                    cache[scheme] += self._collect_used_cache(
+                        out,
+                        branch=branch,
+                        remote=remote,
+                        force=force,
+                        jobs=jobs,
+                    )
+
+        return cache
+
+    def graph(self, stages=None, from_directory=None):
+        import networkx as nx
+        from dvc.exceptions import (
+            OutputDuplicationError,
+            StagePathAsOutputError,
+        )
+
+        G = nx.DiGraph()
+        G_active = nx.DiGraph()
+        stages = stages or self.stages(from_directory, check_dag=False)
+        stages = [stage for stage in stages if stage]
+        outs = []
+
+        for stage in stages:
+            for out in stage.outs:
+                existing = [o.stage for o in outs if o.path == out.path]
+
+                if existing:
+                    stages = [stage.relpath, existing[0].relpath]
+                    raise OutputDuplicationError(out.path, stages)
+
+                outs.append(out)
+
+        for stage in stages:
+            path_dir = os.path.dirname(stage.path) + os.sep
+            for out in outs:
+                if path_dir.startswith(out.path + os.sep):
+                    raise StagePathAsOutputError(stage.wdir, stage.relpath)
+
+        # collect the whole DAG
+        for stage in stages:
+            node = os.path.relpath(stage.path, self.root_dir)
+
+            G.add_node(node, stage=stage)
+            G_active.add_node(node, stage=stage)
+
+            for dep in stage.deps:
+                for out in outs:
+                    if (
+                        out.path != dep.path
+                        and not dep.path.startswith(out.path + out.sep)
+                        and not out.path.startswith(dep.path + dep.sep)
+                    ):
+                        continue
+
+                    dep_stage = out.stage
+                    dep_node = os.path.relpath(dep_stage.path, self.root_dir)
+                    G.add_node(dep_node, stage=dep_stage)
+                    G.add_edge(node, dep_node)
+                    if not stage.locked:
+                        G_active.add_node(dep_node, stage=dep_stage)
+                        G_active.add_edge(node, dep_node)
+
+        self._check_cyclic_graph(G)
+
+        return G, G_active
+
+    def pipelines(self, from_directory=None):
+        import networkx as nx
+
+        G, G_active = self.graph(from_directory=from_directory)
+
+        return [
+            G.subgraph(c).copy() for c in nx.weakly_connected_components(G)
+        ]
+
+    def stages(self, from_directory=None, check_dag=True):
+        """
+        Walks down the root directory looking for Dvcfiles,
+        skipping the directories that are related with
+        any SCM (e.g. `.git`), DVC itself (`.dvc`), or directories
+        tracked by DVC (e.g. `dvc add data` would skip `data/`)
+
+        NOTE: For large repos, this could be an expensive
+              operation. Consider using some memoization.
+        """
+        from dvc.stage import Stage
+
+        if not from_directory:
+            from_directory = self.root_dir
+
+        stages = []
+        outs = []
+        for root, dirs, files in os.walk(str(from_directory)):
+            for fname in files:
+                path = os.path.join(root, fname)
+                if not Stage.is_stage_file(path):
+                    continue
+                stage = Stage.load(self, path)
+                for out in stage.outs:
+                    outs.append(out.path + out.sep)
+                stages.append(stage)
+
+            def filter_dirs(dname, root=root):
+                path = os.path.join(root, dname)
+                if path in (self.dvc_dir, self.scm.dir):
+                    return False
+                for out in outs:
+                    if path == os.path.normpath(out) or path.startswith(out):
+                        return False
+                return True
+
+            dirs[:] = list(filter(filter_dirs, dirs))
+
+        if check_dag:
+            self.check_dag(stages)
+
+        return stages
+
+    def active_stages(self, from_directory=None):
+        import networkx as nx
+
+        stages = []
+        for G in self.pipelines(from_directory):
+            stages.extend(list(nx.get_node_attributes(G, "stage").values()))
+        return stages
+
+    def find_outs_by_path(self, path, outs=None, recursive=False):
+        if not outs:
+            outs = [out for stage in self.stages() for out in stage.outs]
+
+        abs_path = os.path.abspath(path)
+        is_dir = os.path.isdir(abs_path)
+
+        def func(out):
+            if out.path == abs_path:
+                return True
+
+            if is_dir and recursive and out.path.startswith(abs_path + os.sep):
+                return True
+
+            return False
+
+        matched = list(filter(func, outs))
+        if not matched:
+            raise OutputNotFoundError(path)
+
+        return matched

dvc/repo/add.py

@@ -0,0 +1,51 @@
+import os
+
+from dvc.scm import scm_context
+from dvc.stage import Stage
+from dvc.utils import walk_files
+from dvc.exceptions import RecursiveAddingWhileUsingFilename
+
+
+@scm_context
+def add(repo, target, recursive=False, no_commit=False, fname=None):
+    if recursive and fname:
+        raise RecursiveAddingWhileUsingFilename()
+
+    targets = [target]
+
+    if os.path.isdir(target) and recursive:
+        targets = [
+            file
+            for file in walk_files(target)
+            if not Stage.is_stage_file(file)
+            if os.path.basename(file) != repo.scm.ignore_file
+            if not repo.scm.is_tracked(file)
+        ]
+
+    stages = _create_stages(repo, targets, fname, no_commit)
+
+    repo.check_dag(repo.stages() + stages)
+
+    for stage in stages:
+        stage.dump()
+    return stages
+
+
+def _create_stages(repo, targets, fname, no_commit):
+    stages = []
+
+    with repo.state:
+        for out in targets:
+            stage = Stage.create(repo=repo, outs=[out], add=True, fname=fname)
+
+            if not stage:
+                continue
+
+            stage.save()
+
+            if not no_commit:
+                stage.commit()
+
+            stages.append(stage)
+
+    return stages

dvc/repo/checkout.py

@@ -0,0 +1,40 @@
+from __future__ import unicode_literals
+
+import dvc.logger as logger
+from dvc.exceptions import DvcException
+
+
+def _cleanup_unused_links(self, all_stages):
+    used = []
+    for stage in all_stages:
+        for out in stage.outs:
+            used.append(out.path)
+    self.state.remove_unused_links(used)
+
+
+def checkout(self, target=None, with_deps=False, force=False, recursive=False):
+    from dvc.stage import StageFileDoesNotExistError, StageFileBadNameError
+
+    if target and not recursive:
+        all_stages = self.active_stages()
+        try:
+            stages = self.collect(target, with_deps=with_deps)
+        except (StageFileDoesNotExistError, StageFileBadNameError) as exc:
+            raise DvcException(
+                str(exc) + " Did you mean 'git checkout {}'?".format(target)
+            )
+    else:
+        all_stages = self.active_stages(target)
+        stages = all_stages
+
+    with self.state:
+        _cleanup_unused_links(self, all_stages)
+
+        for stage in stages:
+            if stage.locked:
+                logger.warning(
+                    "DVC file '{path}' is locked. Its dependencies are"
+                    " not going to be checked out.".format(path=stage.relpath)
+                )
+
+            stage.checkout(force=force)

dvc/repo/commit.py

@@ -0,0 +1,11 @@
+def commit(self, target, with_deps=False, recursive=False, force=False):
+    if target and not recursive:
+        stages = self.collect(target, with_deps=with_deps)
+    else:
+        stages = self.active_stages(target)
+
+    with self.state:
+        for stage in stages:
+            stage.check_can_commit(force=force)
+            stage.commit()
+            stage.dump()

dvc/repo/destroy.py

@@ -0,0 +1,8 @@
+import shutil
+
+
+def destroy(self):
+    for stage in self.stages():
+        stage.remove()
+
+    shutil.rmtree(self.dvc_dir)

dvc/repo/fetch.py

@@ -0,0 +1,28 @@
+from __future__ import unicode_literals
+
+
+def fetch(
+    self,
+    target=None,
+    jobs=1,
+    remote=None,
+    all_branches=False,
+    show_checksums=False,
+    with_deps=False,
+    all_tags=False,
+    recursive=False,
+):
+    with self.state:
+        used = self.used_cache(
+            target,
+            all_branches=all_branches,
+            all_tags=all_tags,
+            with_deps=with_deps,
+            force=True,
+            remote=remote,
+            jobs=jobs,
+            recursive=recursive,
+        )["local"]
+        return self.cloud.pull(
+            used, jobs, remote=remote, show_checksums=show_checksums
+        )

dvc/repo/gc.py

@@ -0,0 +1,111 @@
+from __future__ import unicode_literals
+
+import collections
+
+import dvc.logger as logger
+
+
+def _merge_cache_lists(clists):
+    merged_cache = collections.defaultdict(list)
+
+    for cache_list in clists:
+        for scheme, cache in cache_list.items():
+            for item in cache:
+                if item not in merged_cache[scheme]:
+                    merged_cache[scheme].append(item)
+
+    return merged_cache
+
+
+def _load_all_used_cache(
+    repos,
+    target=None,
+    all_branches=False,
+    active=True,
+    with_deps=False,
+    all_tags=False,
+    remote=None,
+    force=False,
+    jobs=None,
+):
+    clists = []
+
+    for repo in repos:
+        with repo.state:
+            repo_clist = repo.used_cache(
+                target=None,
+                all_branches=all_branches,
+                active=False,
+                with_deps=with_deps,
+                all_tags=all_tags,
+                remote=remote,
+                force=force,
+                jobs=jobs,
+            )
+
+            clists.append(repo_clist)
+
+    return clists
+
+
+def _do_gc(typ, func, clist):
+    removed = func(clist)
+    if not removed:
+        logger.info("No unused {} cache to remove.".format(typ))
+
+
+def gc(
+    self,
+    all_branches=False,
+    cloud=False,
+    remote=None,
+    with_deps=False,
+    all_tags=False,
+    force=False,
+    jobs=None,
+    repos=None,
+):
+    from dvc.repo import Repo
+
+    all_repos = [self]
+
+    if repos:
+        all_repos.extend(Repo(path) for path in repos)
+
+    all_clists = _load_all_used_cache(
+        all_repos,
+        target=None,
+        all_branches=all_branches,
+        active=False,
+        with_deps=with_deps,
+        all_tags=all_tags,
+        remote=remote,
+        force=force,
+        jobs=jobs,
+    )
+
+    if len(all_clists) > 1:
+        clist = _merge_cache_lists(all_clists)
+    else:
+        clist = all_clists[0]
+
+    with self.state:
+        _do_gc("local", self.cache.local.gc, clist)
+
+        if self.cache.s3:
+            _do_gc("s3", self.cache.s3.gc, clist)
+
+        if self.cache.gs:
+            _do_gc("gs", self.cache.gs.gc, clist)
+
+        if self.cache.ssh:
+            _do_gc("ssh", self.cache.ssh.gc, clist)
+
+        if self.cache.hdfs:
+            _do_gc("hdfs", self.cache.hdfs.gc, clist)
+
+        if self.cache.azure:
+            _do_gc("azure", self.cache.azure.gc, clist)
+
+        if cloud:
+            _do_gc("remote", self.cloud._get_cloud(remote, "gc -c").gc, clist)

dvc/repo/imp.py

@@ -0,0 +1,21 @@
+from dvc.scm import scm_context
+
+
+@scm_context
+def imp(self, url, out, resume=False):
+    from dvc.stage import Stage
+
+    with self.state:
+        stage = Stage.create(repo=self, cmd=None, deps=[url], outs=[out])
+
+    if stage is None:
+        return None
+
+    self.check_dag(self.stages() + [stage])
+
+    with self.state:
+        stage.run(resume=resume)
+
+    stage.dump()
+
+    return stage

dvc/repo/init.py

@@ -0,0 +1,90 @@
+import os
+import shutil
+import colorama
+
+import dvc.logger as logger
+from dvc.repo import Repo
+from dvc.scm import SCM, Base
+from dvc.config import Config
+from dvc.exceptions import InitError
+
+
+def _welcome_message():
+    logger.box(
+        "DVC has enabled anonymous aggregate usage analytics.\n"
+        "Read the analytics documentation (and how to opt-out) here:\n"
+        "{blue}https://dvc.org/doc/user-guide/analytics{nc}".format(
+            blue=colorama.Fore.BLUE, nc=colorama.Fore.RESET
+        ),
+        border_color="red",
+    )
+
+    msg = (
+        "{yellow}What's next?{nc}\n"
+        "{yellow}------------{nc}\n"
+        "- Check out the documentation: {blue}https://dvc.org/doc{nc}\n"
+        "- Get help and share ideas: {blue}https://dvc.org/chat{nc}\n"
+        "- Star us on GitHub: {blue}https://github.com/iterative/dvc{nc}"
+    ).format(
+        yellow=colorama.Fore.YELLOW,
+        blue=colorama.Fore.BLUE,
+        nc=colorama.Fore.RESET,
+    )
+
+    logger.info(msg)
+
+
+def init(root_dir=os.curdir, no_scm=False, force=False):
+    """
+    Creates an empty repo on the given directory -- basically a
+    `.dvc` directory with subdirectories for configuration and cache.
+
+    It should be tracked by a SCM or use the `--no-scm` flag.
+
+    If the given directory is not empty, you must use the `--force`
+    flag to override it.
+
+    Args:
+        root_dir: Path to repo's root directory.
+
+    Returns:
+        Repo instance.
+
+    Raises:
+        KeyError: Raises an exception.
+    """
+    root_dir = os.path.abspath(root_dir)
+    dvc_dir = os.path.join(root_dir, Repo.DVC_DIR)
+    scm = SCM(root_dir)
+    if type(scm) == Base and not no_scm:
+        raise InitError(
+            "{repo} is not tracked by any supported scm tool (e.g. git). "
+            "Use '--no-scm' if you don't want to use any scm.".format(
+                repo=root_dir
+            )
+        )
+
+    if os.path.isdir(dvc_dir):
+        if not force:
+            raise InitError(
+                "'{repo}' exists. Use '-f' to force.".format(
+                    repo=os.path.relpath(dvc_dir)
+                )
+            )
+
+        shutil.rmtree(dvc_dir)
+
+    os.mkdir(dvc_dir)
+
+    config = Config.init(dvc_dir)
+    proj = Repo(root_dir)
+
+    scm.add([config.config_file])
+
+    if scm.ignore_file:
+        scm.add([os.path.join(dvc_dir, scm.ignore_file)])
+        logger.info("\nYou can now commit the changes to git.\n")
+
+    _welcome_message()
+
+    return proj

dvc/repo/install.py

@@ -0,0 +1,2 @@
+def install(self):
+    self.scm.install()

dvc/repo/lock.py

@@ -0,0 +1,8 @@
+def lock(self, target, unlock=False):
+    from dvc.stage import Stage
+
+    stage = Stage.load(self, target)
+    stage.locked = False if unlock else True
+    stage.dump()
+
+    return stage

dvc/repo/metrics/__init__.py

@@ -0,0 +1,23 @@
+class Metrics(object):
+    def __init__(self, repo):
+        self.repo = repo
+
+    def add(self, *args, **kwargs):
+        from dvc.repo.metrics.add import add
+
+        return add(self.repo, *args, **kwargs)
+
+    def modify(self, *args, **kwargs):
+        from dvc.repo.metrics.modify import modify
+
+        return modify(self.repo, *args, **kwargs)
+
+    def show(self, *args, **kwargs):
+        from dvc.repo.metrics.show import show
+
+        return show(self.repo, *args, **kwargs)
+
+    def remove(self, *args, **kwargs):
+        from dvc.repo.metrics.remove import remove
+
+        return remove(self.repo, *args, **kwargs)

dvc/repo/metrics/add.py

@@ -0,0 +1,7 @@
+from dvc.repo.metrics.modify import modify
+
+
+def add(repo, path, typ=None, xpath=None):
+    if not typ:
+        typ = "raw"
+    modify(repo, path, typ, xpath)

dvc/repo/metrics/modify.py

@@ -0,0 +1,40 @@
+from __future__ import unicode_literals
+
+from dvc.exceptions import DvcException
+
+
+def modify(repo, path, typ=None, xpath=None, delete=False):
+    supported_types = ["raw", "json", "csv", "tsv", "hcsv", "htsv"]
+    outs = repo.find_outs_by_path(path)
+    assert len(outs) == 1
+    out = outs[0]
+
+    if out.scheme != "local":
+        msg = "output '{}' scheme '{}' is not supported for metrics"
+        raise DvcException(msg.format(out.path, out.path_info["scheme"]))
+
+    if typ is not None:
+        typ = typ.lower().strip()
+        if typ not in ["raw", "json", "csv", "tsv", "hcsv", "htsv"]:
+            msg = (
+                "metric type '{typ}' is not supported, "
+                "must be one of [{types}]"
+            )
+            raise DvcException(
+                msg.format(typ=typ, types=", ".join(supported_types))
+            )
+        if not isinstance(out.metric, dict):
+            out.metric = {}
+        out.metric[out.PARAM_METRIC_TYPE] = typ
+
+    if xpath is not None:
+        if not isinstance(out.metric, dict):
+            out.metric = {}
+        out.metric[out.PARAM_METRIC_XPATH] = xpath
+
+    if delete:
+        out.metric = None
+
+    out.verify_metric()
+
+    out.stage.dump()

dvc/repo/metrics/remove.py

@@ -0,0 +1,5 @@
+from dvc.repo.metrics.modify import modify
+
+
+def remove(repo, path):
+    modify(repo, path, delete=True)

dvc/repo/metrics/show.py

@@ -0,0 +1,167 @@
+from __future__ import unicode_literals
+
+import os
+import csv
+import json
+from jsonpath_rw import parse
+
+import dvc.logger as logger
+from dvc.exceptions import OutputNotFoundError, BadMetricError, NoMetricsError
+from dvc.utils.compat import builtin_str, open
+
+
+def _read_metric_json(fd, json_path):
+    parser = parse(json_path)
+    return [x.value for x in parser.find(json.load(fd))]
+
+
+def _get_values(row):
+    if isinstance(row, dict):
+        return list(row.values())
+    else:
+        return row
+
+
+def _do_read_metric_xsv(reader, row, col):
+    if col is not None and row is not None:
+        return [reader[row][col]]
+    elif col is not None:
+        return [r[col] for r in reader]
+    elif row is not None:
+        return _get_values(reader[row])
+    return [_get_values(r) for r in reader]
+
+
+def _read_metric_hxsv(fd, hxsv_path, delimiter):
+    indices = hxsv_path.split(",")
+    row = indices[0]
+    row = int(row) if row else None
+    col = indices[1] if len(indices) > 1 and indices[1] else None
+    reader = list(csv.DictReader(fd, delimiter=builtin_str(delimiter)))
+    return _do_read_metric_xsv(reader, row, col)
+
+
+def _read_metric_xsv(fd, xsv_path, delimiter):
+    indices = xsv_path.split(",")
+    row = indices[0]
+    row = int(row) if row else None
+    col = int(indices[1]) if len(indices) > 1 and indices[1] else None
+    reader = list(csv.reader(fd, delimiter=builtin_str(delimiter)))
+    return _do_read_metric_xsv(reader, row, col)
+
+
+def _read_typed_metric(typ, xpath, fd):
+    if typ == "json":
+        ret = _read_metric_json(fd, xpath)
+    elif typ == "csv":
+        ret = _read_metric_xsv(fd, xpath, ",")
+    elif typ == "tsv":
+        ret = _read_metric_xsv(fd, xpath, "\t")
+    elif typ == "hcsv":
+        ret = _read_metric_hxsv(fd, xpath, ",")
+    elif typ == "htsv":
+        ret = _read_metric_hxsv(fd, xpath, "\t")
+    else:
+        ret = fd.read().strip()
+    return ret
+
+
+def _read_metric(path, typ=None, xpath=None, branch=None):
+    ret = None
+    if not os.path.exists(path):
+        return ret
+
+    typ = typ.lower().strip() if typ else typ
+    xpath = xpath.strip() if xpath else xpath
+    try:
+        with open(path, "r") as fd:
+            if not xpath:
+                ret = fd.read().strip()
+            else:
+                ret = _read_typed_metric(typ, xpath, fd)
+    # Json path library has to be replaced or wrapped in
+    # order to fix this too broad except clause.
+    except Exception:
+        logger.warning(
+            "unable to read metric in '{}' in branch '{}'".format(
+                path, branch
+            ),
+            parse_exception=True,
+        )
+
+    return ret
+
+
+def _collect_metrics(self, path, recursive, typ, xpath, branch):
+    outs = [out for stage in self.stages() for out in stage.outs]
+
+    if path:
+        try:
+            outs = self.find_outs_by_path(path, outs=outs, recursive=recursive)
+        except OutputNotFoundError:
+            logger.debug(
+                "stage file not for found for '{}' in branch '{}'".format(
+                    path, branch
+                )
+            )
+            return []
+
+    res = []
+    for o in outs:
+        if not o.metric:
+            continue
+
+        if not typ and isinstance(o.metric, dict):
+            t = o.metric.get(o.PARAM_METRIC_TYPE, typ)
+            x = o.metric.get(o.PARAM_METRIC_XPATH, xpath)
+        else:
+            t = typ
+            x = xpath
+
+        res.append((o, t, x))
+
+    return res
+
+
+def _read_metrics(self, metrics, branch):
+    res = {}
+    for out, typ, xpath in metrics:
+        assert out.scheme == "local"
+        if out.use_cache:
+            path = self.cache.local.get(out.checksum)
+        else:
+            path = out.path
+
+        metric = _read_metric(path, typ=typ, xpath=xpath, branch=branch)
+        if not metric:
+            continue
+
+        res[out.rel_path] = metric
+
+    return res
+
+
+def show(
+    self,
+    path=None,
+    typ=None,
+    xpath=None,
+    all_branches=False,
+    all_tags=False,
+    recursive=False,
+):
+    res = {}
+    for branch in self.scm.brancher(
+        all_branches=all_branches, all_tags=all_tags
+    ):
+        entries = _collect_metrics(self, path, recursive, typ, xpath, branch)
+        metrics = _read_metrics(self, entries, branch)
+        if metrics:
+            res[branch] = metrics
+
+    if not res:
+        if path:
+            raise BadMetricError(path)
+        raise NoMetricsError()
+
+    return res

dvc/repo/move.py

@@ -0,0 +1,68 @@
+import os
+
+from dvc.exceptions import MoveNotDataSourceError
+from dvc.scm import scm_context
+
+
+def _expand_target_path(from_path, to_path):
+    if os.path.isdir(to_path) and not os.path.isdir(from_path):
+        return os.path.join(to_path, os.path.basename(from_path))
+    return to_path
+
+
+@scm_context
+def move(self, from_path, to_path):
+    """
+    Renames an output file and modifies the stage associated
+    to reflect the change on the pipeline.
+
+    If the output has the same name as its stage, it would
+    also rename the corresponding stage file.
+
+    E.g.
+          Having: (hello, hello.dvc)
+
+          $ dvc move hello greetings
+
+          Result: (greeting, greeting.dvc)
+
+    It only works with outputs generated by `add` or `import`,
+    also known as data sources.
+    """
+    import dvc.output as Output
+    from dvc.stage import Stage
+
+    from_out = Output.loads_from(Stage(self), [from_path])[0]
+
+    to_path = _expand_target_path(from_path, to_path)
+
+    outs = self.find_outs_by_path(from_out.path)
+    assert len(outs) == 1
+    out = outs[0]
+    stage = out.stage
+
+    if not stage.is_data_source:
+        raise MoveNotDataSourceError(stage.relpath)
+
+    stage_name = os.path.splitext(os.path.basename(stage.path))[0]
+    from_name = os.path.basename(from_out.path)
+    if stage_name == from_name:
+        os.unlink(stage.path)
+
+        stage.path = os.path.join(
+            os.path.dirname(to_path),
+            os.path.basename(to_path) + Stage.STAGE_FILE_SUFFIX,
+        )
+
+        stage.wdir = os.path.abspath(
+            os.path.join(os.curdir, os.path.dirname(to_path))
+        )
+
+    to_out = Output.loads_from(
+        stage, [os.path.basename(to_path)], out.cache, out.metric
+    )[0]
+
+    with self.state:
+        out.move(to_out)
+
+    stage.dump()

dvc/repo/pull.py

@@ -0,0 +1,29 @@
+from __future__ import unicode_literals
+
+
+def pull(
+    self,
+    target=None,
+    jobs=1,
+    remote=None,
+    all_branches=False,
+    show_checksums=False,
+    with_deps=False,
+    all_tags=False,
+    force=False,
+    recursive=False,
+):
+    processed_files_count = self.fetch(
+        target,
+        jobs,
+        remote=remote,
+        all_branches=all_branches,
+        all_tags=all_tags,
+        show_checksums=show_checksums,
+        with_deps=with_deps,
+        recursive=recursive,
+    )
+    self.checkout(
+        target=target, with_deps=with_deps, force=force, recursive=recursive
+    )
+    return processed_files_count

dvc/repo/push.py

@@ -0,0 +1,28 @@
+from __future__ import unicode_literals
+
+
+def push(
+    self,
+    target=None,
+    jobs=1,
+    remote=None,
+    all_branches=False,
+    show_checksums=False,
+    with_deps=False,
+    all_tags=False,
+    recursive=False,
+):
+    with self.state:
+        used = self.used_cache(
+            target,
+            all_branches=all_branches,
+            all_tags=all_tags,
+            with_deps=with_deps,
+            force=True,
+            remote=remote,
+            jobs=jobs,
+            recursive=recursive,
+        )["local"]
+        return self.cloud.push(
+            used, jobs, remote=remote, show_checksums=show_checksums
+        )