dvc: bump to 0.24.2

stage: don't exclude md5s for deps/outs

.appveyor.yml

@@ -54,7 +54,7 @@ install:
   - cinst gcloudsdk
   - refreshenv
   - cinst gsutil
-  - cinst openssl.light
+  - cinst openssl.light --version 1.1.1
   - python -m pip install -U pip setuptools wheel
   - pip install -r requirements.txt
   - python setup.py install

.codeclimate.yml

@@ -1,14 +1,5 @@
 languages:
     Python: true
 
-checks:
-    argument-count:
-        enabled: false
-    method-complexity:
-        enabled: false
-    method-lines:
-        config:
-            threshold: 50
-
 exclude_paths:
     - tests/

.codecov.yml

@@ -1 +1 @@
-comment: false
+

.dvc/.gitignore

@@ -1,8 +1,8 @@
-link.state
-link.state.lock
-lock
-config.local
-updater
-cache
-state
-state.lock
+/state
+/lock
+/config.local
+/updater
+/updater.lock
+/state-journal
+/state-wal
+/cache

.gitignore

@@ -3,6 +3,7 @@ neatlynx/__pycache__
 .idea
 cache
 *.pyc
+.env/
 
 cache
 .dvc.conf.lock

.pylintrc

@@ -0,0 +1,568 @@
+[MASTER]
+
+# A comma-separated list of package or module names from where C extensions may
+# be loaded. Extensions are loading into the active Python interpreter and may
+# run arbitrary code.
+extension-pkg-whitelist=
+
+# Add files or directories to the blacklist. They should be base names, not
+# paths.
+ignore=CVS
+
+# Add files or directories matching the regex patterns to the blacklist. The
+# regex matches against base names, not paths.
+ignore-patterns=
+
+# Python code to execute, usually for sys.path manipulation such as
+# pygtk.require().
+#init-hook=
+
+# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
+# number of processors available to use.
+jobs=1
+
+# Control the amount of potential inferred values when inferring a single
+# object. This can help the performance when dealing with large functions or
+# complex, nested conditions.
+limit-inference-results=100
+
+# List of plugins (as comma separated values of python modules names) to load,
+# usually to register additional checkers.
+load-plugins=
+
+# Pickle collected data for later comparisons.
+persistent=yes
+
+# Specify a configuration file.
+#rcfile=
+
+# When enabled, pylint would attempt to guess common misconfiguration and emit
+# user-friendly hints instead of false-positive error messages.
+suggestion-mode=yes
+
+# Allow loading of arbitrary C extensions. Extensions are imported into the
+# active Python interpreter and may run arbitrary code.
+unsafe-load-any-extension=no
+
+
+[MESSAGES CONTROL]
+
+# Only show warnings with the listed confidence levels. Leave empty to show
+# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED.
+confidence=
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifiers separated by comma (,) or put this
+# option multiple times (only on the command line, not in the configuration
+# file where it should appear only once). You can also use "--disable=all" to
+# disable everything first and then reenable specific checks. For example, if
+# you want to run only the similarities checker, you can use "--disable=all
+# --enable=similarities". If you want to run only the classes checker, but have
+# no Warning level messages displayed, use "--disable=all --enable=classes
+# --disable=W".
+disable=print-statement,
+        parameter-unpacking,
+        unpacking-in-except,
+        old-raise-syntax,
+        backtick,
+        long-suffix,
+        old-ne-operator,
+        old-octal-literal,
+        import-star-module-level,
+        non-ascii-bytes-literal,
+        raw-checker-failed,
+        bad-inline-option,
+        locally-disabled,
+        file-ignored,
+        suppressed-message,
+        useless-suppression,
+        deprecated-pragma,
+        use-symbolic-message-instead,
+        apply-builtin,
+        basestring-builtin,
+        buffer-builtin,
+        cmp-builtin,
+        coerce-builtin,
+        execfile-builtin,
+        file-builtin,
+        long-builtin,
+        raw_input-builtin,
+        reduce-builtin,
+        standarderror-builtin,
+        unicode-builtin,
+        xrange-builtin,
+        coerce-method,
+        delslice-method,
+        getslice-method,
+        setslice-method,
+        no-absolute-import,
+        old-division,
+        dict-iter-method,
+        dict-view-method,
+        next-method-called,
+        metaclass-assignment,
+        indexing-exception,
+        raising-string,
+        reload-builtin,
+        oct-method,
+        hex-method,
+        nonzero-method,
+        cmp-method,
+        input-builtin,
+        round-builtin,
+        intern-builtin,
+        unichr-builtin,
+        map-builtin-not-iterating,
+        zip-builtin-not-iterating,
+        range-builtin-not-iterating,
+        filter-builtin-not-iterating,
+        using-cmp-argument,
+        eq-without-hash,
+        div-method,
+        idiv-method,
+        rdiv-method,
+        exception-message-attribute,
+        invalid-str-codec,
+        sys-max-int,
+        bad-python3-import,
+        deprecated-string-function,
+        deprecated-str-translate-call,
+        deprecated-itertools-function,
+        deprecated-types-field,
+        next-method-defined,
+        dict-items-not-iterating,
+        dict-keys-not-iterating,
+        dict-values-not-iterating,
+        deprecated-operator-function,
+        deprecated-urllib-function,
+        xreadlines-attribute,
+        deprecated-sys-function,
+        exception-escape,
+        comprehension-escape,
+        useless-object-inheritance
+
+# Enable the message, report, category or checker with the given id(s). You can
+# either give multiple identifier separated by comma (,) or put this option
+# multiple time (only on the command line, not in the configuration file where
+# it should appear only once). See also the "--disable" option for examples.
+enable=c-extension-no-member
+
+
+[REPORTS]
+
+# Python expression which should return a note less than 10 (10 is the highest
+# note). You have access to the variables errors warning, statement which
+# respectively contain the number of errors / warnings messages and the total
+# number of statements analyzed. This is used by the global evaluation report
+# (RP0004).
+evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
+
+# Template used to display messages. This is a python new-style format string
+# used to format the message information. See doc for all details.
+#msg-template=
+
+# Set the output format. Available formats are text, parseable, colorized, json
+# and msvs (visual studio). You can also give a reporter class, e.g.
+# mypackage.mymodule.MyReporterClass.
+output-format=text
+
+# Tells whether to display a full report or only the messages.
+reports=no
+
+# Activate the evaluation score.
+score=yes
+
+
+[REFACTORING]
+
+# Maximum number of nested blocks for function / method body
+max-nested-blocks=5
+
+# Complete name of functions that never returns. When checking for
+# inconsistent-return-statements if a never returning function is called then
+# it will be considered as an explicit return statement and no message will be
+# printed.
+never-returning-functions=sys.exit
+
+
+[LOGGING]
+
+# Format style used to check logging format string. `old` means using %
+# formatting, while `new` is for `{}` formatting.
+logging-format-style=old
+
+# Logging modules to check that the string format arguments are in logging
+# function parameter format.
+logging-modules=logging
+
+
+[SPELLING]
+
+# Limits count of emitted suggestions for spelling mistakes.
+max-spelling-suggestions=4
+
+# Spelling dictionary name. Available dictionaries: none. To make it working
+# install python-enchant package..
+spelling-dict=
+
+# List of comma separated words that should not be checked.
+spelling-ignore-words=
+
+# A path to a file that contains private dictionary; one word per line.
+spelling-private-dict-file=
+
+# Tells whether to store unknown words to indicated private dictionary in
+# --spelling-private-dict-file option instead of raising a message.
+spelling-store-unknown-words=no
+
+
+[TYPECHECK]
+
+# List of decorators that produce context managers, such as
+# contextlib.contextmanager. Add to this list to register other decorators that
+# produce valid context managers.
+contextmanager-decorators=contextlib.contextmanager
+
+# List of members which are set dynamically and missed by pylint inference
+# system, and so shouldn't trigger E1101 when accessed. Python regular
+# expressions are accepted.
+generated-members=
+
+# Tells whether missing members accessed in mixin class should be ignored. A
+# mixin class is detected if its name ends with "mixin" (case insensitive).
+ignore-mixin-members=yes
+
+# Tells whether to warn about missing members when the owner of the attribute
+# is inferred to be None.
+ignore-none=yes
+
+# This flag controls whether pylint should warn about no-member and similar
+# checks whenever an opaque object is returned when inferring. The inference
+# can return multiple potential results while evaluating a Python object, but
+# some branches might not be evaluated, which results in partial inference. In
+# that case, it might be useful to still emit no-member and other checks for
+# the rest of the inferred objects.
+ignore-on-opaque-inference=yes
+
+# List of class names for which member attributes should not be checked (useful
+# for classes with dynamically set attributes). This supports the use of
+# qualified names.
+ignored-classes=optparse.Values,thread._local,_thread._local
+
+# List of module names for which member attributes should not be checked
+# (useful for modules/projects where namespaces are manipulated during runtime
+# and thus existing member attributes cannot be deduced by static analysis. It
+# supports qualified module names, as well as Unix pattern matching.
+ignored-modules=
+
+# Show a hint with possible names when a member name was not found. The aspect
+# of finding the hint is based on edit distance.
+missing-member-hint=yes
+
+# The minimum edit distance a name should have in order to be considered a
+# similar match for a missing member name.
+missing-member-hint-distance=1
+
+# The total number of similar names that should be taken in consideration when
+# showing a hint for a missing member.
+missing-member-max-choices=1
+
+
+[VARIABLES]
+
+# List of additional names supposed to be defined in builtins. Remember that
+# you should avoid defining new builtins when possible.
+additional-builtins=
+
+# Tells whether unused global variables should be treated as a violation.
+allow-global-unused-variables=yes
+
+# List of strings which can identify a callback function by name. A callback
+# name must start or end with one of those strings.
+callbacks=cb_,
+          _cb
+
+# A regular expression matching the name of dummy variables (i.e. expected to
+# not be used).
+dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_
+
+# Argument names that match this expression will be ignored. Default to name
+# with leading underscore.
+ignored-argument-names=_.*|^ignored_|^unused_
+
+# Tells whether we should check for unused import in __init__ files.
+init-import=no
+
+# List of qualified module names which can have objects that can redefine
+# builtins.
+redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io
+
+
+[FORMAT]
+
+# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
+expected-line-ending-format=
+
+# Regexp for a line that is allowed to be longer than the limit.
+ignore-long-lines=^\s*(# )?<?https?://\S+>?$
+
+# Number of spaces of indent required inside a hanging or continued line.
+indent-after-paren=4
+
+# String used as indentation unit. This is usually "    " (4 spaces) or "\t" (1
+# tab).
+indent-string='    '
+
+# Maximum number of characters on a single line.
+max-line-length=100
+
+# Maximum number of lines in a module.
+max-module-lines=1000
+
+# List of optional constructs for which whitespace checking is disabled. `dict-
+# separator` is used to allow tabulation in dicts, etc.: {1  : 1,\n222: 2}.
+# `trailing-comma` allows a space between comma and closing bracket: (a, ).
+# `empty-line` allows space-only lines.
+no-space-check=trailing-comma,
+               dict-separator
+
+# Allow the body of a class to be on the same line as the declaration if body
+# contains single statement.
+single-line-class-stmt=no
+
+# Allow the body of an if to be on the same line as the test if there is no
+# else.
+single-line-if-stmt=no
+
+
+[BASIC]
+
+# Naming style matching correct argument names.
+argument-naming-style=snake_case
+
+# Regular expression matching correct argument names. Overrides argument-
+# naming-style.
+#argument-rgx=
+
+# Naming style matching correct attribute names.
+attr-naming-style=snake_case
+
+# Regular expression matching correct attribute names. Overrides attr-naming-
+# style.
+#attr-rgx=
+
+# Bad variable names which should always be refused, separated by a comma.
+bad-names=foo,
+          bar,
+          baz,
+          toto,
+          tutu,
+          tata
+
+# Naming style matching correct class attribute names.
+class-attribute-naming-style=any
+
+# Regular expression matching correct class attribute names. Overrides class-
+# attribute-naming-style.
+#class-attribute-rgx=
+
+# Naming style matching correct class names.
+class-naming-style=PascalCase
+
+# Regular expression matching correct class names. Overrides class-naming-
+# style.
+#class-rgx=
+
+# Naming style matching correct constant names.
+const-naming-style=UPPER_CASE
+
+# Regular expression matching correct constant names. Overrides const-naming-
+# style.
+#const-rgx=
+
+# Minimum line length for functions/classes that require docstrings, shorter
+# ones are exempt.
+docstring-min-length=-1
+
+# Naming style matching correct function names.
+function-naming-style=snake_case
+
+# Regular expression matching correct function names. Overrides function-
+# naming-style.
+#function-rgx=
+
+# Good variable names which should always be accepted, separated by a comma.
+good-names=i,
+           j,
+           k,
+           ex,
+           Run,
+           _,
+           s3,
+           gs,
+           x, y,
+           x0, y0,
+           x1, y1,
+           dx, dy
+
+# Include a hint for the correct naming format with invalid-name.
+include-naming-hint=no
+
+# Naming style matching correct inline iteration names.
+inlinevar-naming-style=any
+
+# Regular expression matching correct inline iteration names. Overrides
+# inlinevar-naming-style.
+#inlinevar-rgx=
+
+# Naming style matching correct method names.
+method-naming-style=snake_case
+
+# Regular expression matching correct method names. Overrides method-naming-
+# style.
+#method-rgx=
+
+# Naming style matching correct module names.
+module-naming-style=snake_case
+
+# Regular expression matching correct module names. Overrides module-naming-
+# style.
+#module-rgx=
+
+# Colon-delimited sets of names that determine each other's naming style when
+# the name regexes allow several styles.
+name-group=
+
+# Regular expression which should only match function or class names that do
+# not require a docstring.
+no-docstring-rgx=^_
+
+# List of decorators that produce properties, such as abc.abstractproperty. Add
+# to this list to register other decorators that produce valid properties.
+# These decorators are taken in consideration only for invalid-name.
+property-classes=abc.abstractproperty
+
+# Naming style matching correct variable names.
+variable-naming-style=snake_case
+
+# Regular expression matching correct variable names. Overrides variable-
+# naming-style.
+#variable-rgx=
+
+
+[SIMILARITIES]
+
+# Ignore comments when computing similarities.
+ignore-comments=yes
+
+# Ignore docstrings when computing similarities.
+ignore-docstrings=yes
+
+# Ignore imports when computing similarities.
+ignore-imports=no
+
+# Minimum lines number of a similarity.
+min-similarity-lines=4
+
+
+[MISCELLANEOUS]
+
+# List of note tags to take in consideration, separated by a comma.
+notes=FIXME,
+      XXX,
+      TODO
+
+
+[DESIGN]
+
+# Maximum number of arguments for function / method.
+max-args=5
+
+# Maximum number of attributes for a class (see R0902).
+max-attributes=7
+
+# Maximum number of boolean expressions in an if statement.
+max-bool-expr=5
+
+# Maximum number of branch for function / method body.
+max-branches=12
+
+# Maximum number of locals for function / method body.
+max-locals=15
+
+# Maximum number of parents for a class (see R0901).
+max-parents=7
+
+# Maximum number of public methods for a class (see R0904).
+max-public-methods=20
+
+# Maximum number of return / yield for function / method body.
+max-returns=6
+
+# Maximum number of statements in function / method body.
+max-statements=50
+
+# Minimum number of public methods for a class (see R0903).
+min-public-methods=2
+
+
+[CLASSES]
+
+# List of method names used to declare (i.e. assign) instance attributes.
+defining-attr-methods=__init__,
+                      __new__,
+                      setUp
+
+# List of member names, which should be excluded from the protected access
+# warning.
+exclude-protected=_asdict,
+                  _fields,
+                  _replace,
+                  _source,
+                  _make
+
+# List of valid names for the first argument in a class method.
+valid-classmethod-first-arg=cls
+
+# List of valid names for the first argument in a metaclass class method.
+valid-metaclass-classmethod-first-arg=cls
+
+
+[IMPORTS]
+
+# Allow wildcard imports from modules that define __all__.
+allow-wildcard-with-all=no
+
+# Analyse import fallback blocks. This can be used to support both Python 2 and
+# 3 compatible code, which means that the block might have code that exists
+# only in one or another interpreter, leading to false positives when analysed.
+analyse-fallback-blocks=no
+
+# Deprecated modules which should not be used, separated by a comma.
+deprecated-modules=optparse,tkinter.tix
+
+# Create a graph of external dependencies in the given file (report RP0402 must
+# not be disabled).
+ext-import-graph=
+
+# Create a graph of every (i.e. internal and external) dependencies in the
+# given file (report RP0402 must not be disabled).
+import-graph=
+
+# Create a graph of internal dependencies in the given file (report RP0402 must
+# not be disabled).
+int-import-graph=
+
+# Force import order to recognize a module as part of the standard
+# compatibility libraries.
+known-standard-library=
+
+# Force import order to recognize a module as part of a third party library.
+known-third-party=enchant
+
+
+[EXCEPTIONS]
+
+# Exceptions that will emit a warning when being caught. Defaults to
+# "Exception".
+overgeneral-exceptions=Exception

.travis.yml

@@ -16,18 +16,24 @@ python:
 # version, that causes fpm to crash.
 matrix:
   include:
+  # 3.7 needs Xenial image because of https://github.com/travis-ci/travis-ci/issues/9069
+  - python: "3.7"
+    dist: xenial
+    sudo: true
   - if: type = cron OR tag is present
     os: osx
-    osx_image: xcode9.3
+    osx_image: xcode7.3
     language: ruby
     rvm: 2.4.0
     env:
     - PYTHON_VER=2.7.10
     - PYENV_ROOT="$HOME/.pyenv"
     - PATH="$PYENV_ROOT/bin:$PATH"
+    - CPPFLAGS="-I/usr/local/opt/openssl/include $CPPFLAGS"
+    - LDFLAGS="-L/usr/local/opt/openssl/lib $LDFLAGS"
   - if: type = cron OR tag is present
     os: osx
-    osx_image: xcode9.3
+    osx_image: xcode7.3
     language: ruby
     rvm: 2.4.0
     env:
@@ -36,23 +42,20 @@ matrix:
     - PATH="$PYENV_ROOT/bin:$PATH"
   - if: type = cron OR tag is present
     os: osx
-    osx_image: xcode9.3
+    osx_image: xcode7.3
     language: ruby
     rvm: 2.4.0
     env:
     - PYTHON_VER=3.5.0
     - PYENV_ROOT="$HOME/.pyenv"
     - PATH="$PYENV_ROOT/bin:$PATH"
-  # 3.7 needs Xenial image because of https://github.com/travis-ci/travis-ci/issues/9069
-  - python: "3.7"
-    dist: xenial
-    sudo: true
 before_install:
   - bash ./scripts/ci/before_install.sh
   - if  [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
       eval "$(pyenv init -)";
       PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing --keep --verbose "$PYTHON_VER";
       pyenv shell "$PYTHON_VER";
+      brew install openssl;
       brew cask install google-cloud-sdk;
     fi
   - which python
@@ -78,8 +81,11 @@ install:
   - bash ./scripts/ci/install.sh
   - pip install flake8
 before_script:
+  # stop the build if there are any readme formatting errors
+  - python setup.py checkdocs
   # stop the build if there are any flake8 errors
   - flake8 dvc
+  - flake8 tests
 script:
   - bash ./scripts/ci/script.sh
 deploy:

README.rst

@@ -2,12 +2,15 @@
   :target: https://dvc.org
   :alt: DVC logo
 
-`Website <https://dvc.org>`_ • `Docs <https://dvc.org/doc>`_ • `Discuss <https://discuss.dvc.org/>`_ •
-`Blog <https://blog.dataversioncontrol.com/>`_ • `Twitter <https://twitter.com/iterativeai>`_ •
-`Tutorial <https://dvc.org/doc/tutorial>`_ • `Mailing List <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_
+`Website <https://dvc.org>`_
+• `Docs <https://dvc.org/doc>`_
+• `Twitter <https://twitter.com/iterativeai>`_
+• `Chat (Community & Support) <https://dvc.org/chat>`_
+• `Tutorial <https://dvc.org/doc/tutorial>`_
+• `Mailing List <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_
 
-.. image:: https://travis-ci.org/iterative/dvc.svg?branch=master
-  :target: https://travis-ci.org/iterative/dvc
+.. image:: https://travis-ci.com/iterative/dvc.svg?branch=master
+  :target: https://travis-ci.com/iterative/dvc
   :alt: Travis
 
 .. image:: https://ci.appveyor.com/api/projects/status/github/iterative/dvc?branch=master&svg=true
@@ -24,10 +27,12 @@
 
 |
 
-**Data Science Version Control** or **DVC** is an **open-source** tool for data science projects.
-With a simple and flexible Git-like architecture and interface it helps data scientists:
+**Data Science Version Control** or **DVC** is an **open-source** tool for data science and
+machine learning projects. With a simple and flexible Git-like architecture and interface it
+helps data scientists:
 
-#. manage code together with data files and models;
+#. manage **machine learning models** - versioning, including data sets and transformations (scripts) that were
+   used to generate models;
 
 #. make projects **reproducible**;
 
@@ -42,6 +47,17 @@ data file suffixes and prefixes.
 .. contents:: **Contents**
   :backlinks: none
 
+How DVC works
+=============
+
+DVC is compatible with Git for storing code and the dependency graph (DAG), but not data files cache.
+To store and share data files cache DVC supports remotes - any cloud (S3, Azure, Google Cloud, etc) or any on-premise
+network storage (via SSH, for example).
+
+.. image:: https://dvc.org/static/img/flow.gif
+   :target: https://dvc.org/static/img/flow.gif
+   :alt: how_dvc_works
+
 Quick start
 ===========
 
@@ -64,7 +80,7 @@ Please read `Get Started <https://dvc.org/doc/get-started>`_ for the full versio
 |                                   | | ``$ git push``                                                  |
 +-----------------------------------+-------------------------------------------------------------------+
 | Share data and ML models          | | ``$ dvc remote add myremote s3://mybucket/image_cnn``           |
-|                                   | | ``$ dvc core.remote myremote``                                  |
+|                                   | | ``$ dvc config core.remote myremote``                           |
 |                                   | | ``$ dvc push``                                                  |
 +-----------------------------------+-------------------------------------------------------------------+
 
@@ -73,28 +89,64 @@ Installation
 
 There are three options to install DVC: ``pip``, Homebrew, or an OS-specific package:
 
+pip (PyPI)
+----------
+
+Stable
+^^^^^^
 .. code-block:: bash
 
    pip install dvc
 
+Development
+^^^^^^^^^^^
+.. code-block:: bash
+
+   pip install git+git://github.com/iterative/dvc
+
+Homebrew
+--------
+
+.. code-block:: bash
+
+   brew install iterative/homebrew-dvc/dvc
+
+or:
+
+.. code-block:: bash
+
+   brew cask install iterative/homebrew-dvc/dvc
+
 Package
 -------
 
 Self-contained packages for Windows, Linux, Mac are available. The latest version of the packages can be found at
 GitHub `releases page <https://github.com/iterative/dvc/releases>`_.
 
-Homebrew
---------
+Ubuntu / Debian (apt)
+^^^^^^^^^^^^^^^^^^^^^
+.. code-block:: bash
+
+   sudo wget https://dvc.org/deb/dvc.list -O /etc/apt/sources.list.d/dvc.list
+   sudo apt-get update
+   sudo apt-get install dvc
+
+Fedora / CentOS (rpm)
+^^^^^^^^^^^^^^^^^^^^^
+.. code-block:: bash
+
+   sudo wget https://dvc.org/rpm/dvc.repo -O /etc/yum.repos.d/dvc.repo
+   sudo yum update
+   sudo yum install dvc
+
+Arch linux (AUR)
+^^^^^^^^^^^^^^^^
+*Unofficial package*, any inquiries regarding the AUR package,
+`refer to the maintainer <https://github.com/mroutis/pkgbuilds>`_.
 
 .. code-block:: bash
 
-   $ brew install iterative/homebrew-dvc/dvc
-
-or:
-
-.. code-block:: bash
-
-   $ brew cask install iterative/homebrew-dvc/dvc
+   yay -S dvc
 
 Related technologies
 ====================
@@ -112,17 +164,7 @@ Related technologies
 #. `Workflow Management Systems <https://en.wikipedia.org/wiki/Workflow_management_system>`_. DVC is a workflow
    management system designed specifically to manage machine learning experiments. DVC is built on top of Git.
 
-How DVC works
-=============
-
-DVC is compatible with Git for storing code and the dependency graph (DAG), but not data files cache.
-To store and share data files cache DVC supports remotes - any cloud (S3, Azure, Google Cloud, etc) or any on-premise
-network storage (via SSH, for example).
-
-.. image:: https://dvc.org/static/img/flow.png
-   :target: https://dvc.org/static/img/flow.png
-   :alt: how_dvc_works
-
+#. `DAGsHub <https://dagshub.com/>`_ Is a Github equivalent for DVC - pushing your Git+DVC based repo to DAGsHub will give you a high level dashboard of your project, including DVC pipeline and metrics visualizations, as well as links to DVC managed files if they are in cloud storage.
 
 Contributing
 ============
@@ -139,15 +181,15 @@ details.
 
 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/2
   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/2
-  :alt: 2   
+  :alt: 2
 
 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/3
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/3   
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/3
   :alt: 3
 
 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/4
   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/4
-  :alt: 4   
+  :alt: 4
 
 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/5
   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/5
@@ -155,14 +197,14 @@ details.
 
 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/6
   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/6
-  :alt: 6   
+  :alt: 6
 
 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/7
   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/7
   :alt: 7
 
 Mailing List
-=========
+============
 
 Want to stay up to date? Want to help improve DVC by participating in our ocassional polls? Subscribe to our `mailing list <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_. No spam, really low traffic.
 

dvc/__init__.py

@@ -7,39 +7,53 @@ import os
 import warnings
 
 
-VERSION_BASE = '0.18.1'
+VERSION_BASE = '0.24.2'
 __version__ = VERSION_BASE
 
 PACKAGEPATH = os.path.abspath(os.path.dirname(__file__))
 HOMEPATH = os.path.dirname(PACKAGEPATH)
 VERSIONPATH = os.path.join(PACKAGEPATH, 'version.py')
 
+
+def _update_version_file():
+    """Dynamically update version file."""
+    from git import Repo
+    from git.exc import InvalidGitRepositoryError
+
+    try:
+        repo = Repo(HOMEPATH)
+    except InvalidGitRepositoryError:
+        return __version__
+
+    sha = repo.head.object.hexsha
+    short_sha = repo.git.rev_parse(sha, short=6)
+    dirty = '.mod' if repo.is_dirty() else ''
+    ver = '{}+{}{}'.format(__version__, short_sha, dirty)
+
+    # Write a helper file, that will be installed with the package
+    # and will provide a true version of the installed dvc
+    with open(VERSIONPATH, 'w+') as fobj:
+        fobj.write('# AUTOGENERATED by dvc/__init__.py\n')
+        fobj.write('version = "{}"\n'.format(ver))
+
+    return ver
+
+
+def _remove_version_file():
+    """Remove version.py so that it doesn't get into the release."""
+    if os.path.exists(VERSIONPATH):
+        os.unlink(VERSIONPATH)
+
+
 if os.path.exists(os.path.join(HOMEPATH, 'setup.py')):
     # dvc is run directly from source without installation or
     # __version__ is called from setup.py
     if os.getenv('APPVEYOR_REPO_TAG', '').lower() != 'true' \
        and os.getenv('TRAVIS_TAG', '') == '':
-        # Dynamically update version
-        try:
-            import git
-            repo = git.Repo(HOMEPATH)
-            sha = repo.head.object.hexsha
-            short_sha = repo.git.rev_parse(sha, short=6)
-            dirty = '.mod' if repo.is_dirty() else ''
-            __version__ = '{}+{}{}'.format(__version__, short_sha, dirty)
-
-            # Write a helper file, that will be installed with the package
-            # and will provide a true version of the installed dvc
-            with open(VERSIONPATH, 'w+') as fd:
-                fd.write('# AUTOGENERATED by dvc/__init__.py\n')
-                fd.write('version = "{}"\n'.format(__version__))
-        except Exception:
-            pass
-    else:
-        # Remove version.py so that it doesn't get into the release
-        if os.path.exists(VERSIONPATH):
-            os.unlink(VERSIONPATH)
-else:
+        __version__ = _update_version_file()
+    else:  # pragma: no cover
+        _remove_version_file()
+else:  # pragma: no cover
     # dvc was installed with pip or something. Hopefully we have our
     # auto-generated version.py to help us provide a true version
     try:

dvc/__main__.py

@@ -1,3 +1,5 @@
+"""Main entry point for dvc command line tool."""
+
 import sys
 
 from dvc.main import main

dvc/analytics.py

@@ -0,0 +1,257 @@
+"""Collect and send usage analytics"""
+
+import os
+import json
+import errno
+
+import dvc.logger as logger
+from dvc import VERSION
+
+
+class Analytics(object):
+    """Class for collecting and sending usage analytics.
+
+    Args:
+        info (dict): optional existing analytics report.
+    """
+    URL = 'https://analytics.dvc.org'
+    TIMEOUT_POST = 5
+
+    USER_ID_FILE = 'user_id'
+
+    PARAM_DVC_VERSION = 'dvc_version'
+    PARAM_USER_ID = 'user_id'
+    PARAM_SYSTEM_INFO = 'system_info'
+
+    PARAM_OS = 'os'
+
+    PARAM_WINDOWS_VERSION_MAJOR = 'windows_version_major'
+    PARAM_WINDOWS_VERSION_MINOR = 'windows_version_minor'
+    PARAM_WINDOWS_VERSION_BUILD = 'windows_version_build'
+    PARAM_WINDOWS_VERSION_SERVICE_PACK = 'windows_version_service_pack'
+
+    PARAM_MAC_VERSION = 'mac_version'
+
+    PARAM_LINUX_DISTRO = 'linux_distro'
+    PARAM_LINUX_DISTRO_VERSION = 'linux_distro_version'
+    PARAM_LINUX_DISTRO_LIKE = 'linux_distro_like'
+
+    PARAM_SCM_CLASS = 'scm_class'
+    PARAM_IS_BINARY = 'is_binary'
+    PARAM_CMD_CLASS = 'cmd_class'
+    PARAM_CMD_RETURN_CODE = 'cmd_return_code'
+
+    def __init__(self, info=None):
+        from dvc.config import Config
+        from dvc.lock import Lock
+
+        if info is None:
+            info = {}
+
+        self.info = info
+
+        cdir = Config.get_global_config_dir()
+        try:
+            os.makedirs(cdir)
+        except OSError as exc:
+            if exc.errno != errno.EEXIST:
+                raise
+
+        self.user_id_file = os.path.join(cdir, self.USER_ID_FILE)
+        self.user_id_file_lock = Lock(cdir, self.USER_ID_FILE + '.lock')
+
+    @staticmethod
+    def load(path):
+        """Loads analytics report from json file specified by path.
+
+        Args:
+            path (str): path to json file with analytics report.
+        """
+        with open(path, 'r') as fobj:
+            analytics = Analytics(info=json.load(fobj))
+        os.unlink(path)
+        return analytics
+
+    def _write_user_id(self):
+        import uuid
+
+        with open(self.user_id_file, 'w+') as fobj:
+            user_id = str(uuid.uuid4())
+            info = {self.PARAM_USER_ID: user_id}
+            json.dump(info, fobj)
+            return user_id
+
+    def _read_user_id(self):
+        if not os.path.exists(self.user_id_file):
+            return None
+
+        with open(self.user_id_file, 'r') as fobj:
+            try:
+                info = json.load(fobj)
+                return info[self.PARAM_USER_ID]
+            except json.JSONDecodeError as exc:
+                logger.debug("Failed to load user_id: {}".format(exc))
+                return None
+
+    def _get_user_id(self):
+        from dvc.lock import LockError
+
+        try:
+            with self.user_id_file_lock:
+                user_id = self._read_user_id()
+                if user_id is None:
+                    user_id = self._write_user_id()
+                return user_id
+        except LockError:
+            msg = "Failed to acquire '{}'"
+            logger.debug(msg.format(self.user_id_file_lock.lock_file))
+
+    def _collect_windows(self):
+        import sys
+        version = sys.getwindowsversion()  # pylint: disable=no-member
+        info = {}
+        info[self.PARAM_OS] = 'windows'
+        info[self.PARAM_WINDOWS_VERSION_MAJOR] = version.major
+        info[self.PARAM_WINDOWS_VERSION_MINOR] = version.minor
+        info[self.PARAM_WINDOWS_VERSION_BUILD] = version.build
+        info[self.PARAM_WINDOWS_VERSION_SERVICE_PACK] = version.service_pack
+        return info
+
+    def _collect_darwin(self):
+        import platform
+        info = {}
+        info[self.PARAM_OS] = 'mac'
+        info[self.PARAM_MAC_VERSION] = platform.mac_ver()[0]
+        return info
+
+    def _collect_linux(self):
+        import distro
+        info = {}
+        info[self.PARAM_OS] = 'linux'
+        info[self.PARAM_LINUX_DISTRO] = distro.id()
+        info[self.PARAM_LINUX_DISTRO_VERSION] = distro.version()
+        info[self.PARAM_LINUX_DISTRO_LIKE] = distro.like()
+        return info
+
+    def _collect_system_info(self):
+        import platform
+        system = platform.system()
+
+        if system == 'Windows':
+            return self._collect_windows()
+
+        if system == 'Darwin':
+            return self._collect_darwin()
+
+        if system == 'Linux':
+            return self._collect_linux()
+
+        raise NotImplementedError
+
+    def collect(self):
+        """Collect analytics report."""
+        from dvc.scm import SCM
+        from dvc.utils import is_binary
+        from dvc.project import Project
+        from dvc.exceptions import NotDvcProjectError
+
+        self.info[self.PARAM_DVC_VERSION] = VERSION
+        self.info[self.PARAM_IS_BINARY] = is_binary()
+        self.info[self.PARAM_USER_ID] = self._get_user_id()
+
+        self.info[self.PARAM_SYSTEM_INFO] = self._collect_system_info()
+
+        try:
+            scm = SCM(root_dir=Project.find_root())
+            self.info[self.PARAM_SCM_CLASS] = type(scm).__name__
+        except NotDvcProjectError:
+            pass
+
+    def collect_cmd(self, args, ret):
+        """Collect analytics info from a CLI command."""
+        from dvc.command.daemon import CmdDaemonAnalytics
+
+        assert isinstance(ret, int) or ret is None
+
+        if ret is not None:
+            self.info[self.PARAM_CMD_RETURN_CODE] = ret
+
+        if args is not None and hasattr(args, 'func'):
+            assert args.func != CmdDaemonAnalytics
+            self.info[self.PARAM_CMD_CLASS] = args.func.__name__
+
+    def dump(self):
+        """Save analytics report to a temporary file.
+
+        Returns:
+            str: path to the temporary file that contains the analytics report.
+        """
+        import tempfile
+
+        with tempfile.NamedTemporaryFile(delete=False, mode='w') as fobj:
+            json.dump(self.info, fobj)
+            return fobj.name
+
+    @staticmethod
+    def _is_enabled(cmd=None):
+        from dvc.config import Config
+        from dvc.project import Project
+        from dvc.exceptions import NotDvcProjectError
+        from dvc.command.daemon import CmdDaemonBase
+
+        if os.getenv('DVC_TEST'):
+            return False
+
+        if isinstance(cmd, CmdDaemonBase):
+            return False
+
+        if cmd is None or not hasattr(cmd, 'config'):
+            try:
+                dvc_dir = Project.find_dvc_dir()
+                config = Config(dvc_dir)
+                assert config is not None
+            except NotDvcProjectError:
+                config = Config(validate=False)
+                assert config is not None
+        else:
+            config = cmd.config
+            assert config is not None
+
+        core = config.config.get(Config.SECTION_CORE, {})
+        enabled = core.get(Config.SECTION_CORE_ANALYTICS, True)
+        logger.debug("Analytics is {}abled."
+                     .format('en' if enabled else 'dis'))
+        return enabled
+
+    @staticmethod
+    def send_cmd(cmd, args, ret):
+        """Collect and send analytics for CLI command.
+
+        Args:
+            args (list): parsed args for the CLI command.
+            ret (int): return value of the CLI command.
+        """
+        from dvc.daemon import daemon
+
+        if not Analytics._is_enabled(cmd):
+            return
+
+        analytics = Analytics()
+        analytics.collect_cmd(args, ret)
+        daemon(['analytics', analytics.dump()])
+
+    def send(self):
+        """Collect and send analytics."""
+        import requests
+
+        if not self._is_enabled():
+            return
+
+        self.collect()
+
+        logger.debug("Sending analytics: {}".format(self.info))
+
+        try:
+            requests.post(self.URL, json=self.info, timeout=self.TIMEOUT_POST)
+        except requests.exceptions.RequestException as exc:
+            logger.debug("Failed to send analytics: {}".format(str(exc)))

dvc/cache.py

@@ -1,9 +1,17 @@
+"""Manages cache of a dvc project."""
+
 import os
 
 from dvc.config import Config
 
 
 class Cache(object):
+    """Class that manages cache locations of a dvc project.
+
+    Args:
+        project (dvc.project.Project): project instance that this cache
+            belongs to.
+    """
     CACHE_DIR = 'cache'
 
     def __init__(self, project):
@@ -11,30 +19,63 @@ class Cache(object):
 
         self.project = project
 
-        config = project.config._config[Config.SECTION_CACHE]
+        config = project.config.config[Config.SECTION_CACHE]
+        local = config.get(Config.SECTION_CACHE_LOCAL)
 
-        local = config.get(Config.SECTION_CACHE_LOCAL, None)
         if local:
             name = Config.SECTION_REMOTE_FMT.format(local)
-            sect = project.config._config[name]
+            sect = project.config.config[name]
         else:
-            sect = {}
-            cache_dir = config.get(Config.SECTION_CACHE_DIR, self.CACHE_DIR)
-            if not os.path.isabs(cache_dir):
-                cache_dir = os.path.join(project.dvc_dir, cache_dir)
-                cache_dir = os.path.abspath(os.path.realpath(cache_dir))
-            sect[Config.SECTION_REMOTE_URL] = cache_dir
-            t = config.get(Config.SECTION_CACHE_TYPE, None)
-            if t:
-                sect[Config.SECTION_CACHE_TYPE] = t
+            default_cache_dir = os.path.join(project.dvc_dir, self.CACHE_DIR)
+            cache_dir = config.get(Config.SECTION_CACHE_DIR, default_cache_dir)
+            cache_type = config.get(Config.SECTION_CACHE_TYPE)
+            protected = config.get(Config.SECTION_CACHE_PROTECTED)
 
-        self.local = Remote(project, sect)
+            sect = {
+                Config.PRIVATE_CWD: config.get(Config.PRIVATE_CWD,
+                                               project.dvc_dir),
+                Config.SECTION_REMOTE_URL: cache_dir,
+                Config.SECTION_CACHE_TYPE: cache_type,
+                Config.SECTION_CACHE_PROTECTED: protected
+            }
 
-        self.s3 = self._get_remote(config, Config.SECTION_CACHE_S3)
-        self.gs = self._get_remote(config, Config.SECTION_CACHE_GS)
-        self.ssh = self._get_remote(config, Config.SECTION_CACHE_SSH)
-        self.hdfs = self._get_remote(config, Config.SECTION_CACHE_HDFS)
-        self.azure = self._get_remote(config, Config.SECTION_CACHE_AZURE)
+        self._local = Remote(project, sect)
+
+        self._s3 = self._get_remote(config, Config.SECTION_CACHE_S3)
+        self._gs = self._get_remote(config, Config.SECTION_CACHE_GS)
+        self._ssh = self._get_remote(config, Config.SECTION_CACHE_SSH)
+        self._hdfs = self._get_remote(config, Config.SECTION_CACHE_HDFS)
+        self._azure = self._get_remote(config, Config.SECTION_CACHE_AZURE)
+
+    @property
+    def local(self):
+        """Remote instance for local cache."""
+        return self._local
+
+    @property
+    def s3(self):
+        """Remote instance for AWS S3 cache."""
+        return self._s3
+
+    @property
+    def gs(self):
+        """Remote instance for Google Cloud Storage cache."""
+        return self._gs
+
+    @property
+    def ssh(self):
+        """Remote instance for SSH cache."""
+        return self._ssh
+
+    @property
+    def hdfs(self):
+        """Remote instance for HDFS cache."""
+        return self._hdfs
+
+    @property
+    def azure(self):
+        """Remote instance for azure cache."""
+        return self._azure
 
     def _get_remote(self, config, name):
         from dvc.remote import Remote
@@ -44,5 +85,5 @@ class Cache(object):
             return None
 
         name = Config.SECTION_REMOTE_FMT.format(remote)
-        sect = self.project.config._config[name]
+        sect = self.project.config.config[name]
         return Remote(self.project, sect)

dvc/cli.py

@@ -1,791 +1,141 @@
+"""Command line interface for dvc."""
+
 from __future__ import print_function
 
-import os
 import sys
 import argparse
-from multiprocessing import cpu_count
 
-from dvc.command.init import CmdInit
-from dvc.command.destroy import CmdDestroy
-from dvc.command.remove import CmdRemove
-from dvc.command.move import CmdMove
-from dvc.command.run import CmdRun
-from dvc.command.repro import CmdRepro
-from dvc.command.data_sync import CmdDataPush, CmdDataPull, CmdDataFetch
-from dvc.command.status import CmdDataStatus
-from dvc.command.gc import CmdGC
-from dvc.command.add import CmdAdd
-from dvc.command.imp import CmdImport
-from dvc.command.config import CmdConfig
-from dvc.command.checkout import CmdCheckout
-from dvc.command.remote import CmdRemoteAdd, CmdRemoteRemove
-from dvc.command.remote import CmdRemoteModify, CmdRemoteList
-from dvc.command.metrics import CmdMetricsShow, CmdMetricsAdd
-from dvc.command.metrics import CmdMetricsRemove, CmdMetricsModify
-from dvc.command.install import CmdInstall
-from dvc.command.root import CmdRoot
-from dvc.command.lock import CmdLock, CmdUnlock
-from dvc.command.pipeline import CmdPipelineShow
-from dvc.logger import Logger
+import dvc.logger as logger
+from dvc.command.base import fix_subparsers
+import dvc.command.init as init
+import dvc.command.destroy as destroy
+import dvc.command.remove as remove
+import dvc.command.move as move
+import dvc.command.unprotect as unprotect
+import dvc.command.run as run
+import dvc.command.repro as repro
+import dvc.command.data_sync as data_sync
+import dvc.command.gc as gc
+import dvc.command.add as add
+import dvc.command.imp as imp
+import dvc.command.config as config
+import dvc.command.checkout as checkout
+import dvc.command.remote as remote
+import dvc.command.cache as cache
+import dvc.command.metrics as metrics
+import dvc.command.install as install
+import dvc.command.root as root
+import dvc.command.lock as lock
+import dvc.command.pipeline as pipeline
+import dvc.command.daemon as daemon
+from dvc.exceptions import DvcParserError
 from dvc import VERSION
 
 
-def _fix_subparsers(subparsers):
-    # NOTE: Workaround for bug in Python 3
-    # More info at:
-    #  https://bugs.python.org/issue16308
-    #  https://github.com/iterative/dvc/issues/769
-    if sys.version_info[0] == 3:  # pragma: no cover
-        subparsers.required = True
-        subparsers.dest = 'cmd'
+COMMANDS = [
+    init,
+    destroy,
+    add,
+    remove,
+    move,
+    unprotect,
+    run,
+    repro,
+    data_sync,
+    gc,
+    add,
+    imp,
+    config,
+    checkout,
+    remote,
+    cache,
+    metrics,
+    install,
+    root,
+    lock,
+    pipeline,
+    daemon,
+]
 
 
 class DvcParser(argparse.ArgumentParser):
+    """Custom parser class for dvc CLI."""
     def error(self, message):
-        sys.stderr.write('{}{}\n'.format(Logger.error_prefix(), message))
+        """Custom error method.
+        Args:
+            message (str): error message.
+
+        Raises:
+            dvc.exceptions.DvcParser: dvc parser exception.
+        """
+        logger.error(message)
         self.print_help()
-        sys.exit(2)
+        raise DvcParserError()
 
 
-class VersionAction(argparse.Action):
+class VersionAction(argparse.Action):  # pragma: no cover
+    # pylint: disable=too-few-public-methods
+    """Shows dvc version and exits."""
     def __call__(self, parser, namespace, values, option_string=None):
         print(VERSION)
         sys.exit(0)
 
 
 def parse_args(argv=None):
+    """Parses CLI arguments.
+
+    Args:
+        argv: optional list of arguments to parse. sys.argv is used by default.
+
+    Raises:
+        dvc.exceptions.DvcParserError: raised for argument parsing errors.
+    """
     # Common args
     parent_parser = argparse.ArgumentParser(add_help=False)
-    parent_parser.add_argument(
-                        '-q',
-                        '--quiet',
-                        action='store_true',
-                        default=False,
-                        help='Be quiet.')
-    parent_parser.add_argument(
-                        '-v',
-                        '--verbose',
-                        action='store_true',
-                        default=False,
-                        help='Be verbose.')
+
+    log_level_group = parent_parser.add_mutually_exclusive_group()
+    log_level_group.add_argument(
+        '-q',
+        '--quiet',
+        action='store_true',
+        default=False,
+        help='Be quiet.')
+    log_level_group.add_argument(
+        '-v',
+        '--verbose',
+        action='store_true',
+        default=False,
+        help='Be verbose.')
 
     # Main parser
     desc = 'Data Version Control'
-    parser = DvcParser(prog='dvc',
-                       description=desc,
-                       parents=[parent_parser],
-                       formatter_class=argparse.RawTextHelpFormatter)
+    parser = DvcParser(
+        prog='dvc',
+        description=desc,
+        parents=[parent_parser],
+        formatter_class=argparse.RawTextHelpFormatter)
 
     # NOTE: On some python versions action='version' prints to stderr
     # instead of stdout https://bugs.python.org/issue18920
-    parser.add_argument('-V',
-                        '--version',
-                        action=VersionAction,
-                        nargs=0,
-                        help='Show program\'s version.')
+    parser.add_argument(
+        '-V',
+        '--version',
+        action=VersionAction,
+        nargs=0,
+        help="Show program's version.")
 
     # Sub commands
     subparsers = parser.add_subparsers(
-                        title='Available Commands',
-                        metavar='COMMAND',
-                        dest='cmd',
-                        help='Use dvc COMMAND --help for command-specific '
-                             'help.')
+        title='Available Commands',
+        metavar='COMMAND',
+        dest='cmd',
+        help='Use dvc COMMAND --help for command-specific help.')
 
-    _fix_subparsers(subparsers)
+    fix_subparsers(subparsers)
 
-    # Init
-    INIT_HELP = 'Initialize dvc over a directory ' \
-                '(should already be a git dir).'
-    init_parser = subparsers.add_parser(
-                        'init',
-                        parents=[parent_parser],
-                        description=INIT_HELP,
-                        help=INIT_HELP)
-    init_parser.add_argument(
-                        '--no-scm',
-                        action='store_true',
-                        default=False,
-                        help="Initiate dvc in directory that is "
-                             "not tracked by any scm tool(e.g. git).")
-    init_parser.add_argument(
-                        '-f',
-                        '--force',
-                        action='store_true',
-                        default=False,
-                        help="Overwrite '.dvc' if it exists. Will remove "
-                             "all local cache.")
-    init_parser.set_defaults(func=CmdInit)
-
-    # Destroy
-    DESTROY_HELP = "Destroy dvc. Will remove all project's information, " \
-                   "data files and cache."
-    destroy_parser = subparsers.add_parser(
-                        'destroy',
-                        parents=[parent_parser],
-                        description=DESTROY_HELP,
-                        help=DESTROY_HELP)
-    destroy_parser.add_argument(
-                        '-f',
-                        '--force',
-                        action='store_true',
-                        default=False,
-                        help='Force destruction.')
-    destroy_parser.set_defaults(func=CmdDestroy)
-
-    # Add
-    ADD_HELP = 'Add files/directories to dvc.'
-    add_parser = subparsers.add_parser(
-                        'add',
-                        parents=[parent_parser],
-                        description=ADD_HELP,
-                        help=ADD_HELP)
-    add_parser.add_argument(
-                        '-R',
-                        '--recursive',
-                        action='store_true',
-                        default=False,
-                        help='Recursively add each file under the directory.')
-    add_parser.add_argument(
-                        'targets',
-                        nargs='+',
-                        help='Input files/directories.')
-    add_parser.set_defaults(func=CmdAdd)
-
-    # Import
-    IMPORT_HELP = 'Import files from URL.'
-    import_parser = subparsers.add_parser(
-                        'import',
-                        parents=[parent_parser],
-                        description=IMPORT_HELP,
-                        help=IMPORT_HELP)
-    import_parser.add_argument(
-                        'url',
-                        help='URL.')
-    import_parser.add_argument(
-                        'out',
-                        help='Output.')
-    import_parser.set_defaults(func=CmdImport)
-
-    # Checkout
-    CHECKOUT_HELP = 'Checkout data files from cache.'
-    checkout_parser = subparsers.add_parser(
-                        'checkout',
-                        parents=[parent_parser],
-                        description=CHECKOUT_HELP,
-                        help=CHECKOUT_HELP)
-    checkout_parser.add_argument(
-                        'targets',
-                        nargs='*',
-                        help='DVC files.')
-    checkout_parser.set_defaults(func=CmdCheckout)
-
-    # Run
-    RUN_HELP = 'Generate a stage file from a given ' \
-               'command and execute the command.'
-    run_parser = subparsers.add_parser(
-                        'run',
-                        parents=[parent_parser],
-                        description=RUN_HELP,
-                        help=RUN_HELP)
-    run_parser.add_argument(
-                        '-d',
-                        '--deps',
-                        action='append',
-                        default=[],
-                        help='Declare dependencies for reproducible cmd.')
-    run_parser.add_argument(
-                        '-o',
-                        '--outs',
-                        action='append',
-                        default=[],
-                        help='Declare output data file or data directory.')
-    run_parser.add_argument(
-                        '-O',
-                        '--outs-no-cache',
-                        action='append',
-                        default=[],
-                        help='Declare output regular file or '
-                             'directory (sync to Git, not DVC cache).')
-    run_parser.add_argument(
-                        '-M',
-                        '--metrics-no-cache',
-                        action='append',
-                        default=[],
-                        help='Declare output metric file or '
-                             'directory (not cached by DVC).')
-    run_parser.add_argument(
-                        '-f',
-                        '--file',
-                        help='Specify name of the state file.')
-    run_parser.add_argument(
-                        '-c',
-                        '--cwd',
-                        default=os.path.curdir,
-                        help='Directory to run your command and place state '
-                             'file in.')
-    run_parser.add_argument(
-                        '--no-exec',
-                        action='store_true',
-                        default=False,
-                        help="Only create stage file without actually "
-                             "running it.")
-    run_parser.add_argument(
-                        'command',
-                        nargs=argparse.REMAINDER,
-                        help='Command or command file to execute.')
-    run_parser.set_defaults(func=CmdRun)
-
-    # Parent parser used in pull/push/status
-    parent_cache_parser = argparse.ArgumentParser(
-                        add_help=False,
-                        parents=[parent_parser])
-    parent_cache_parser.add_argument(
-                        '-j',
-                        '--jobs',
-                        type=int,
-                        default=8*cpu_count(),
-                        help='Number of jobs to run simultaneously.')
-    parent_cache_parser.add_argument(
-                        '--show-checksums',
-                        action='store_true',
-                        default=False,
-                        help='Show checksums instead of file names.')
-    parent_cache_parser.add_argument(
-                        'targets',
-                        nargs='*',
-                        default=None,
-                        help='DVC files.')
-
-    # Pull
-    PULL_HELP = 'Pull data files from the cloud.'
-    pull_parser = subparsers.add_parser(
-                        'pull',
-                        parents=[parent_cache_parser],
-                        description=PULL_HELP,
-                        help=PULL_HELP)
-    pull_parser.add_argument(
-                        '-r',
-                        '--remote',
-                        help='Remote repository to pull from.')
-    pull_parser.add_argument(
-                        '-a',
-                        '--all-branches',
-                        action='store_true',
-                        default=False,
-                        help='Fetch cache for all branches.')
-    pull_parser.set_defaults(func=CmdDataPull)
-
-    # Push
-    PUSH_HELP = 'Push data files to the cloud.'
-    push_parser = subparsers.add_parser(
-                        'push',
-                        parents=[parent_cache_parser],
-                        description=PUSH_HELP,
-                        help=PUSH_HELP)
-    push_parser.add_argument(
-                        '-r',
-                        '--remote',
-                        help='Remote repository to push to.')
-    push_parser.add_argument(
-                        '-a',
-                        '--all-branches',
-                        action='store_true',
-                        default=False,
-                        help='Push cache for all branches.')
-    push_parser.set_defaults(func=CmdDataPush)
-
-    # Fetch
-    FETCH_HELP = 'Fetch data files from the cloud.'
-    fetch_parser = subparsers.add_parser(
-                        'fetch',
-                        parents=[parent_cache_parser],
-                        description=FETCH_HELP,
-                        help=FETCH_HELP)
-    fetch_parser.add_argument(
-                        '-r',
-                        '--remote',
-                        help='Remote repository to fetch from.')
-    fetch_parser.add_argument(
-                        '-a',
-                        '--all-branches',
-                        action='store_true',
-                        default=False,
-                        help='Fetch cache for all branches.')
-    fetch_parser.set_defaults(func=CmdDataFetch)
-
-    # Status
-    STATUS_HELP = 'Show the project status.'
-    status_parser = subparsers.add_parser(
-                        'status',
-                        parents=[parent_cache_parser],
-                        description=STATUS_HELP,
-                        help=STATUS_HELP)
-    status_parser.add_argument(
-                        '-c',
-                        '--cloud',
-                        action='store_true',
-                        default=False,
-                        help='Show status of a local cache compared to a '
-                             'remote repository.')
-    status_parser.add_argument(
-                        '-r',
-                        '--remote',
-                        help='Remote repository to compare local cache to.')
-    status_parser.set_defaults(func=CmdDataStatus)
-
-    # Repro
-    REPRO_HELP = 'Reproduce DVC file. Default file name - \'Dvcfile\'.'
-    repro_parser = subparsers.add_parser(
-                        'repro',
-                        parents=[parent_parser],
-                        description=REPRO_HELP,
-                        help=REPRO_HELP)
-    repro_parser.add_argument(
-                        'targets',
-                        nargs='*',
-                        help='DVC file to reproduce.')
-    repro_parser.add_argument(
-                        '-f',
-                        '--force',
-                        action='store_true',
-                        default=False,
-                        help='Reproduce even if dependencies were not '
-                             'changed.')
-    repro_parser.add_argument(
-                        '-s',
-                        '--single-item',
-                        action='store_true',
-                        default=False,
-                        help='Reproduce only single data item without '
-                             'recursive dependencies check.')
-    repro_parser.add_argument(
-                        '-c',
-                        '--cwd',
-                        default=os.path.curdir,
-                        help='Directory to reproduce from.')
-    repro_parser.add_argument(
-                        '-m',
-                        '--metrics',
-                        action='store_true',
-                        default=False,
-                        help='Show metrics after reproduction.')
-    repro_parser.add_argument(
-                        '--dry',
-                        action='store_true',
-                        default=False,
-                        help='Only print the commands that would be executed '
-                             'without actually executing.')
-    repro_parser.add_argument(
-                        '-i',
-                        '--interactive',
-                        action='store_true',
-                        default=False,
-                        help='Ask for confirmation before reproducing each '
-                             'stage.')
-    repro_parser.set_defaults(func=CmdRepro)
-
-    # Remove
-    REMOVE_HELP = 'Remove outputs of DVC file.'
-    remove_parser = subparsers.add_parser(
-                        'remove',
-                        parents=[parent_parser],
-                        description=REMOVE_HELP,
-                        help=REMOVE_HELP)
-    remove_parser_group = remove_parser.add_mutually_exclusive_group()
-    remove_parser_group.add_argument(
-                        '-o',
-                        '--outs',
-                        action='store_true',
-                        default=True,
-                        help='Only remove DVC file outputs.')
-    remove_parser_group.add_argument(
-                        '-p',
-                        '--purge',
-                        action='store_true',
-                        default=False,
-                        help='Remove DVC file and all its outputs.')
-    remove_parser.add_argument(
-                        'targets',
-                        nargs='+',
-                        help='DVC files.')
-    remove_parser.set_defaults(func=CmdRemove)
-
-    # Move
-    MOVE_HELP = 'Move output of DVC file.'
-    move_parser = subparsers.add_parser(
-                        'move',
-                        parents=[parent_parser],
-                        description=MOVE_HELP,
-                        help=MOVE_HELP)
-    move_parser.add_argument(
-                        'src',
-                        help='Source.')
-    move_parser.add_argument(
-                        'dst',
-                        help='Destination.')
-    move_parser.set_defaults(func=CmdMove)
-
-    # Garbage collector
-    GC_HELP = 'Collect garbage.'
-    gc_parser = subparsers.add_parser(
-                        'gc',
-                        parents=[parent_parser],
-                        description=GC_HELP,
-                        help=GC_HELP)
-    gc_parser.add_argument(
-                        '-a',
-                        '--all-branches',
-                        action='store_true',
-                        default=False,
-                        help='Collect garbage for all branches.')
-    gc_parser.add_argument(
-                        '-c',
-                        '--cloud',
-                        action='store_true',
-                        default=False,
-                        help='Collect garbage in remote repository.')
-    gc_parser.add_argument(
-                        '-r',
-                        '--remote',
-                        help='Remote repository to collect garbage in.')
-    gc_parser.set_defaults(func=CmdGC)
-
-    # Config
-    CONFIG_HELP = 'Get or set config options.'
-    config_parser = subparsers.add_parser(
-                        'config',
-                        parents=[parent_parser],
-                        description=CONFIG_HELP,
-                        help=CONFIG_HELP)
-    config_parser.add_argument(
-                        '-u',
-                        '--unset',
-                        default=False,
-                        action='store_true',
-                        help='Unset option.')
-    config_parser.add_argument(
-                        'name',
-                        help='Option name.')
-    config_parser.add_argument(
-                        'value',
-                        nargs='?',
-                        default=None,
-                        help='Option value.')
-    config_parser.add_argument(
-                        '--local',
-                        action='store_true',
-                        default=False,
-                        help='Use local config.')
-    config_parser.set_defaults(func=CmdConfig)
-
-    # Remote
-    REMOTE_HELP = 'Manage set of tracked repositories.'
-    remote_parser = subparsers.add_parser(
-                        'remote',
-                        parents=[parent_parser],
-                        description=REMOTE_HELP,
-                        help=REMOTE_HELP)
-
-    remote_subparsers = remote_parser.add_subparsers(
-                        dest='cmd',
-                        help='Use dvc remote CMD --help for '
-                             'command-specific help.')
-
-    _fix_subparsers(remote_subparsers)
-
-    REMOTE_ADD_HELP = 'Add remote.'
-    remote_add_parser = remote_subparsers.add_parser(
-                        'add',
-                        parents=[parent_parser],
-                        description=REMOTE_ADD_HELP,
-                        help=REMOTE_ADD_HELP)
-    remote_add_parser.add_argument(
-                        'name',
-                        help='Name.')
-    remote_add_parser.add_argument(
-                        'url',
-                        help='URL.')
-    remote_add_parser.add_argument(
-                        '--local',
-                        action='store_true',
-                        default=False,
-                        help='Use local config.')
-    remote_add_parser.add_argument(
-                        '-d',
-                        '--default',
-                        action='store_true',
-                        default=False,
-                        help='Set as default remote.')
-    remote_add_parser.set_defaults(func=CmdRemoteAdd)
-
-    REMOTE_REMOVE_HELP = 'Remove remote.'
-    remote_remove_parser = remote_subparsers.add_parser(
-                        'remove',
-                        parents=[parent_parser],
-                        description=REMOTE_REMOVE_HELP,
-                        help=REMOTE_REMOVE_HELP)
-    remote_remove_parser.add_argument(
-                        'name',
-                        help='Name')
-    remote_remove_parser.add_argument(
-                        '--local',
-                        action='store_true',
-                        default=False,
-                        help='Use local config.')
-    remote_remove_parser.set_defaults(func=CmdRemoteRemove)
-
-    REMOTE_MODIFY_HELP = 'Modify remote.'
-    remote_modify_parser = remote_subparsers.add_parser(
-                        'modify',
-                        parents=[parent_parser],
-                        description=REMOTE_MODIFY_HELP,
-                        help=REMOTE_MODIFY_HELP)
-    remote_modify_parser.add_argument(
-                        'name',
-                        help='Name.')
-    remote_modify_parser.add_argument(
-                        'option',
-                        help='Option.')
-    remote_modify_parser.add_argument(
-                        'value',
-                        nargs='?',
-                        help='Value.')
-    remote_modify_parser.add_argument(
-                        '-u',
-                        '--unset',
-                        default=False,
-                        action='store_true',
-                        help='Unset option.')
-    remote_modify_parser.add_argument(
-                        '--local',
-                        action='store_true',
-                        default=False,
-                        help='Use local config.')
-    remote_modify_parser.set_defaults(func=CmdRemoteModify)
-
-    REMOTE_LIST_HELP = 'List remotes.'
-    remote_list_parser = remote_subparsers.add_parser(
-                        'list',
-                        parents=[parent_parser],
-                        description=REMOTE_LIST_HELP,
-                        help=REMOTE_LIST_HELP)
-    remote_list_parser.add_argument(
-                        '--local',
-                        action='store_true',
-                        default=False,
-                        help='Use local config.')
-    remote_list_parser.set_defaults(func=CmdRemoteList)
-
-    # Metrics
-    METRICS_HELP = 'Get metrics from all branches.'
-    metrics_parser = subparsers.add_parser(
-                        'metrics',
-                        parents=[parent_parser],
-                        description=METRICS_HELP,
-                        help=METRICS_HELP)
-
-    metrics_subparsers = metrics_parser.add_subparsers(
-                        dest='cmd',
-                        help='Use dvc metrics CMD --help for '
-                             'command-specific help.')
-
-    _fix_subparsers(metrics_subparsers)
-
-    METRICS_SHOW_HELP = 'Show metrics.'
-    metrics_show_parser = metrics_subparsers.add_parser(
-                        'show',
-                        parents=[parent_parser],
-                        description=METRICS_SHOW_HELP,
-                        help=METRICS_SHOW_HELP)
-    metrics_show_parser.add_argument(
-                        'path',
-                        nargs='?',
-                        help='Path to metrics file.')
-    metrics_show_parser.add_argument(
-                        '-t',
-                        '--type',
-                        help='Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV).')
-    metrics_show_parser.add_argument(
-                        '-x',
-                        '--xpath',
-                        help='JSON/TSV/HTSV/CSV/HCSV path.')
-    metrics_show_group = metrics_show_parser.add_mutually_exclusive_group()
-    metrics_show_group.add_argument(
-                        '--json-path',
-                        help='JSON path.')
-    metrics_show_group.add_argument(
-                        '--tsv-path',
-                        help='TSV path \'row,column\'(e.g. \'1,2\').')
-    metrics_show_group.add_argument(
-                        '--htsv-path',
-                        help='Headed TSV path \'row,column\'(e.g. \'Name,3\'.')
-    metrics_show_group.add_argument(
-                        '--csv-path',
-                        help='CSV path \'row,column\'(e.g. \'1,2\').')
-    metrics_show_group.add_argument(
-                        '--hcsv-path',
-                        help='Headed CSV path \'row,column\'(e.g. \'Name,3\'.')
-    metrics_show_parser.add_argument(
-                        '-a',
-                        '--all-branches',
-                        action='store_true',
-                        default=False,
-                        help='Show metrics for all branches.')
-    metrics_show_parser.set_defaults(func=CmdMetricsShow)
-
-    METRICS_ADD_HELP = 'Add metrics.'
-    metrics_add_parser = metrics_subparsers.add_parser(
-                        'add',
-                        parents=[parent_parser],
-                        description=METRICS_ADD_HELP,
-                        help=METRICS_ADD_HELP)
-    metrics_add_parser.add_argument(
-                        '-t',
-                        '--type',
-                        help='Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV).')
-    metrics_add_parser.add_argument(
-                        '-x',
-                        '--xpath',
-                        help='JSON/TSV/HTSV/CSV/HCSV path.')
-    metrics_add_parser.add_argument(
-                        'path',
-                        help='Path to metrics file.')
-    metrics_add_parser.set_defaults(func=CmdMetricsAdd)
-
-    METRICS_MODIFY_HELP = 'Modify metrics.'
-    metrics_modify_parser = metrics_subparsers.add_parser(
-                        'modify',
-                        parents=[parent_parser],
-                        description=METRICS_MODIFY_HELP,
-                        help=METRICS_MODIFY_HELP)
-    metrics_modify_parser.add_argument(
-                        '-t',
-                        '--type',
-                        help='Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV).')
-    metrics_modify_parser.add_argument(
-                        '-x',
-                        '--xpath',
-                        help='JSON/TSV/HTSV/CSV/HCSV path.')
-    metrics_modify_parser.add_argument(
-                        'path',
-                        help='Metrics file.')
-    metrics_modify_parser.set_defaults(func=CmdMetricsModify)
-
-    METRICS_REMOVE_HELP = 'Remove metrics.'
-    metrics_remove_parser = metrics_subparsers.add_parser(
-                        'remove',
-                        parents=[parent_parser],
-                        description=METRICS_REMOVE_HELP,
-                        help=METRICS_REMOVE_HELP)
-    metrics_remove_parser.add_argument(
-                        'path',
-                        help='Path to metrics file.')
-    metrics_remove_parser.set_defaults(func=CmdMetricsRemove)
-
-    # Install
-    INSTALL_HELP = 'Install dvc hooks into the repository.'
-    install_parser = subparsers.add_parser(
-                        'install',
-                        parents=[parent_parser],
-                        description=INSTALL_HELP,
-                        help=INSTALL_HELP)
-    install_parser.set_defaults(func=CmdInstall)
-
-    # Root
-    ROOT_HELP = 'Relative path to project\'s directory.'
-    root_parser = subparsers.add_parser(
-                        'root',
-                        parents=[parent_parser],
-                        description=ROOT_HELP,
-                        help=ROOT_HELP)
-    root_parser.set_defaults(func=CmdRoot)
-
-    # Lock
-    LOCK_HELP = 'Lock DVC file.'
-    lock_parser = subparsers.add_parser(
-                        'lock',
-                        parents=[parent_parser],
-                        description=LOCK_HELP,
-                        help=LOCK_HELP)
-    lock_parser.add_argument(
-                        'targets',
-                        nargs='+',
-                        help='DVC files.')
-    lock_parser.set_defaults(func=CmdLock)
-
-    # Unlock
-    UNLOCK_HELP = 'Unlock DVC file.'
-    unlock_parser = subparsers.add_parser(
-                        'unlock',
-                        parents=[parent_parser],
-                        description=UNLOCK_HELP,
-                        help=UNLOCK_HELP)
-    unlock_parser.add_argument(
-                        'targets',
-                        nargs='+',
-                        help='DVC files.')
-    unlock_parser.set_defaults(func=CmdUnlock)
-
-    # Pipeline
-    PIPELINE_HELP = 'Manage pipeline.'
-    pipeline_parser = subparsers.add_parser(
-                        'pipeline',
-                        parents=[parent_parser],
-                        description=PIPELINE_HELP,
-                        help=PIPELINE_HELP)
-
-    pipeline_subparsers = pipeline_parser.add_subparsers(
-                        dest='cmd',
-                        help='Use dvc pipeline CMD --help '
-                             'for command-specific help.')
-
-    _fix_subparsers(pipeline_subparsers)
-
-    PIPELINE_SHOW_HELP = 'Show pipeline.'
-    pipeline_show_parser = pipeline_subparsers.add_parser(
-                        'show',
-                        parents=[parent_parser],
-                        description=PIPELINE_SHOW_HELP,
-                        help=PIPELINE_SHOW_HELP)
-    pipeline_show_group = pipeline_show_parser.add_mutually_exclusive_group()
-    pipeline_show_group.add_argument(
-                        '-c',
-                        '--commands',
-                        action='store_true',
-                        default=False,
-                        help='Print commands instead of paths to DVC files.')
-    pipeline_show_group.add_argument(
-                        '-o',
-                        '--outs',
-                        action='store_true',
-                        default=False,
-                        help='Print output files instead of '
-                             'paths to DVC files.')
-    pipeline_show_parser.add_argument(
-                        '--ascii',
-                        action='store_true',
-                        default=False,
-                        help='Output DAG as ASCII.')
-    pipeline_show_parser.add_argument(
-                        'targets',
-                        nargs='*',
-                        help="DVC files. 'Dvcfile' by default.")
-    pipeline_show_parser.set_defaults(func=CmdPipelineShow)
+    for cmd in COMMANDS:
+        cmd.add_parser(subparsers, parent_parser)
 
     args = parser.parse_args(argv)
 
-    if (issubclass(args.func, CmdRepro)
-        or issubclass(args.func, CmdPipelineShow)) \
-       and hasattr(args, 'targets') \
-       and len(args.targets) == 0:
-        if hasattr(args, 'cwd'):
-            cwd = args.cwd
-        else:
-            cwd = os.curdir
-        path = os.path.join(cwd, 'Dvcfile')
-        if not os.path.exists(path):
-            parser.error("default target '{}' does not exist.".format(path))
-        args.targets = ['Dvcfile']
-
     return args

dvc/command/add.py

@@ -1,6 +1,6 @@
+import dvc.logger as logger
 from dvc.exceptions import DvcException
 from dvc.command.base import CmdBase
-from dvc.logger import Logger
 
 
 class CmdAdd(CmdBase):
@@ -8,7 +8,27 @@ class CmdAdd(CmdBase):
         for target in self.args.targets:
             try:
                 self.project.add(target, recursive=self.args.recursive)
-            except DvcException as ex:
-                Logger.error('Failed to add \'{}\''.format(target), ex)
+            except DvcException:
+                logger.error('failed to add file')
                 return 1
         return 0
+
+
+def add_parser(subparsers, parent_parser):
+    ADD_HELP = 'Add files/directories to dvc.'
+    add_parser = subparsers.add_parser(
+        'add',
+        parents=[parent_parser],
+        description=ADD_HELP,
+        help=ADD_HELP)
+    add_parser.add_argument(
+        '-R',
+        '--recursive',
+        action='store_true',
+        default=False,
+        help='Recursively add each file under the directory.')
+    add_parser.add_argument(
+        'targets',
+        nargs='+',
+        help='Input files/directories.')
+    add_parser.set_defaults(func=CmdAdd)

dvc/command/base.py

@@ -1,45 +1,55 @@
-import os
+import sys
 
-from dvc.logger import Logger
-from dvc.exceptions import DvcException
-from dvc.lock import LockError
+import dvc.logger as logger
+
+
+def fix_subparsers(subparsers):
+    """Workaround for bug in Python 3. See more info at:
+        https://bugs.python.org/issue16308
+        https://github.com/iterative/dvc/issues/769
+
+        Args:
+            subparsers: subparsers to fix.
+    """
+    if sys.version_info[0] == 3:  # pragma: no cover
+        subparsers.required = True
+        subparsers.dest = 'cmd'
 
 
 class CmdBase(object):
     def __init__(self, args):
         from dvc.project import Project
 
-        self.project = Project(self._find_root())
+        self.project = Project()
+        self.config = self.project.config
         self.args = args
-        self._set_loglevel(args)
+        self.set_loglevel(args)
+
+    @property
+    def default_targets(self):
+        """Default targets for `dvc repro` and `dvc pipeline`."""
+        from dvc.stage import Stage
+        msg = "assuming default target '{}'.".format(Stage.STAGE_FILE)
+        logger.warning(msg)
+        return [Stage.STAGE_FILE]
 
     @staticmethod
-    def _set_loglevel(args):
-        if args.quiet and not args.verbose:
-            Logger.be_quiet()
-        elif not args.quiet and args.verbose:
-            Logger.be_verbose()
+    def set_loglevel(args):
+        """Sets log level from CLI arguments."""
+        if args.quiet:
+            logger.be_quiet()
 
-    def _find_root(self):
-        from dvc.project import Project
-
-        root = os.getcwd()
-        while True:
-            dvc_dir = os.path.join(root, Project.DVC_DIR)
-            if os.path.isdir(dvc_dir):
-                return root
-            if os.path.ismount(root):
-                break
-            root = os.path.dirname(root)
-        msg = "Not a dvc repository (checked up to mount point {})"
-        raise DvcException(msg.format(root))
+        elif args.verbose:
+            logger.be_verbose()
 
     def run_cmd(self):
+        from dvc.lock import LockError
+
         try:
             with self.project.lock:
                 return self.run()
-        except LockError as ex:
-            Logger.error('Failed to lock before running a command', ex)
+        except LockError:
+            logger.error('failed to lock before running a command')
             return 1
 
     # Abstract methods that have to be implemented by any inheritance class

dvc/command/cache.py

@@ -0,0 +1,55 @@
+import argparse
+
+from dvc.command.base import fix_subparsers
+from dvc.command.remote import CmdRemoteAdd
+from dvc.command.config import CmdConfig
+
+
+class CmdCacheDir(CmdConfig):
+    def run(self):
+        self.args.name = 'cache.dir'
+        self.args.value = CmdRemoteAdd.resolve_path(self.args.value,
+                                                    self.configobj.filename)
+
+        return super(CmdCacheDir, self).run()
+
+
+def add_parser(subparsers, parent_parser):
+    from dvc.command.config import parent_config_parser
+
+    CACHE_HELP = 'Manage cache settings.'
+    cache_parser = subparsers.add_parser(
+        'cache',
+        parents=[parent_parser],
+        description=CACHE_HELP,
+        help=CACHE_HELP)
+
+    cache_subparsers = cache_parser.add_subparsers(
+        dest='cmd',
+        help='Use dvc cache CMD --help for command-specific help.')
+
+    fix_subparsers(cache_subparsers)
+
+    parent_cache_config_parser = argparse.ArgumentParser(
+        add_help=False,
+        parents=[parent_config_parser])
+    CACHE_DIR_HELP = 'Configure cache directory location.'
+    cache_dir_parser = cache_subparsers.add_parser(
+        'dir',
+        parents=[parent_cache_config_parser],
+        description=CACHE_DIR_HELP,
+        help=CACHE_DIR_HELP)
+    cache_dir_parser.add_argument(
+        '-u',
+        '--unset',
+        default=False,
+        action='store_true',
+        help='Unset option.')
+    cache_dir_parser.add_argument(
+        'value',
+        nargs='?',
+        default=None,
+        help="Path to cache directory. Relative paths are resolved relative "
+             "to the current directory and saved to config relative to the "
+             "config file location.")
+    cache_dir_parser.set_defaults(func=CmdCacheDir)

dvc/command/checkout.py

@@ -4,8 +4,36 @@ from dvc.command.base import CmdBase
 class CmdCheckout(CmdBase):
     def run(self):
         if not self.args.targets:
-            self.project.checkout()
+            self.project.checkout(force=self.args.force)
         else:
             for target in self.args.targets:
-                self.project.checkout(target=target)
+                self.project.checkout(target=target,
+                                      with_deps=self.args.with_deps,
+                                      force=self.args.force)
         return 0
+
+
+def add_parser(subparsers, parent_parser):
+    CHECKOUT_HELP = 'Checkout data files from cache.'
+    checkout_parser = subparsers.add_parser(
+        'checkout',
+        parents=[parent_parser],
+        description=CHECKOUT_HELP,
+        help=CHECKOUT_HELP)
+    checkout_parser.add_argument(
+        '-d',
+        '--with-deps',
+        action='store_true',
+        default=False,
+        help='Checkout all dependencies of the specified target.')
+    checkout_parser.add_argument(
+        '-f',
+        '--force',
+        action='store_true',
+        default=False,
+        help='Do not prompt when removing working directory files.')
+    checkout_parser.add_argument(
+        'targets',
+        nargs='*',
+        help='DVC files.')
+    checkout_parser.set_defaults(func=CmdCheckout)

dvc/command/config.py

@@ -1,102 +1,121 @@
 import os
-import configobj
+import argparse
 
+import dvc.logger as logger
 from dvc.command.base import CmdBase
-from dvc.logger import Logger
 from dvc.config import Config
+from dvc.exceptions import DvcException
 
 
 class CmdConfig(CmdBase):
     def __init__(self, args):
-        from dvc.project import Project
+        from dvc.project import Project, NotDvcProjectError
 
         self.args = args
-        root_dir = self._find_root()
-        if args.local:
-            config = Config.CONFIG_LOCAL
+
+        try:
+            dvc_dir = os.path.join(Project.find_root(), Project.DVC_DIR)
+            saved_exc = None
+        except NotDvcProjectError as exc:
+            dvc_dir = None
+            saved_exc = exc
+
+        self.config = Config(dvc_dir, validate=False)
+        if self.args.system:
+            self.configobj = self.config._system_config
+        elif self.args.glob:
+            self.configobj = self.config._global_config
+        elif self.args.local:
+            if dvc_dir is None:
+                raise saved_exc
+            self.configobj = self.config._local_config
         else:
-            config = Config.CONFIG
-        self.config_file = os.path.join(root_dir, Project.DVC_DIR, config)
-        # Using configobj because it doesn't
-        # drop comments like configparser does.
-        self.configobj = configobj.ConfigObj(self.config_file)
+            if dvc_dir is None:
+                raise saved_exc
+            self.configobj = self.config._project_config
 
     def run_cmd(self):
         return self.run()
 
-    def _get_key(self, d, name, add=False):
-        for k in d.keys():
-            if k.lower() == name.lower():
-                return k
+    def _unset(self, section, opt=None, configobj=None):
+        if configobj is None:
+            configobj = self.configobj
 
-        if add:
-            d[name] = {}
-            return name
-
-        return None
-
-    def save(self):
         try:
-            self.configobj.write()
-        except Exception as exc:
-            msg = "Failed to write config '{}'".format(self.configobj.filename)
-            Logger.error(msg, exc)
+            self.config.unset(configobj, section, opt)
+            self.config.save(configobj)
+        except DvcException:
+            logger.error("failed to unset '{}'".format(self.args.name))
             return 1
         return 0
 
-    def unset(self, section, opt=None):
-        if section not in self.configobj.keys():
-            Logger.error("Section '{}' doesn't exist".format(section))
+    def _show(self, section, opt):
+        try:
+            self.config.show(self.configobj, section, opt)
+        except DvcException:
+            logger.error("failed to show '{}'".format(self.args.name))
             return 1
-
-        if opt in self.configobj[section].keys():
-            del self.configobj[section][opt]
-
-        if len(self.configobj[section]) == 0 or opt is None:
-            del self.configobj[section]
-
-        return self.save()
-
-    def show(self):
-        Logger.info(self.configobj[self.section][self.opt])
         return 0
 
-    def set(self, section, opt, value):
-        if section not in self.configobj.keys():
-            self.configobj[section] = {}
-
-        self.configobj[section][opt] = value
-
-        return self.save()
-
-    def check_opt(self):
-        _section, _opt = self.args.name.strip().split('.', 1)
-        add = (self.args.value is not None and self.args.unset is False)
-
-        section = self._get_key(self.configobj, _section, add)
-
-        if not section:
-            Logger.error('Invalid option name {}'.format(_section))
+    def _set(self, section, opt, value):
+        try:
+            self.config.set(self.configobj, section, opt, value)
+            self.config.save(self.configobj)
+        except DvcException:
+            logger.error("failed to set '{}.{}' to '{}'"
+                         .format(section, opt, value))
             return 1
-
-        opt = self._get_key(self.configobj[section], _opt, add)
-        if not opt:
-            Logger.error('Invalid option value: {}'.format(_opt))
-            return 1
-
-        self.section = section
-        self.opt = opt
-
         return 0
 
     def run(self):
-        if self.check_opt() != 0:
-            return 1
+        section, opt = self.args.name.lower().strip().split('.', 1)
 
         if self.args.unset:
-            return self.unset(self.section, self.opt)
+            return self._unset(section, opt)
+        elif self.args.value is None:
+            return self._show(section, opt)
+        else:
+            return self._set(section, opt, self.args.value)
 
-        if self.args.value is None:
-            return self.show()
 
-        return self.set(self.section, self.opt, self.args.value)
+parent_config_parser = argparse.ArgumentParser(add_help=False)
+parent_config_parser.add_argument(
+   '--global',
+   dest='glob',
+   action='store_true',
+   default=False,
+   help='Use global config.')
+parent_config_parser.add_argument(
+   '--system',
+   action='store_true',
+   default=False,
+   help='Use system config.')
+parent_config_parser.add_argument(
+   '--local',
+   action='store_true',
+   default=False,
+   help='Use local config.')
+
+
+def add_parser(subparsers, parent_parser):
+    CONFIG_HELP = 'Get or set config options.'
+    config_parser = subparsers.add_parser(
+        'config',
+        parents=[parent_config_parser, parent_parser],
+        description=CONFIG_HELP,
+        help=CONFIG_HELP)
+    config_parser.add_argument(
+        '-u',
+        '--unset',
+        default=False,
+        action='store_true',
+        help='Unset option.')
+    config_parser.add_argument(
+        'name',
+        help='Option name.')
+    config_parser.add_argument(
+        'value',
+        nargs='?',
+        default=None,
+        help='Option value.')
+    config_parser.set_defaults(func=CmdConfig)

dvc/command/daemon.py

@@ -0,0 +1,69 @@
+from dvc.command.base import CmdBase, fix_subparsers
+
+
+class CmdDaemonBase(CmdBase):
+    def __init__(self, args):
+        self.args = args
+        self.config = None
+        self.set_loglevel(args)
+
+    def run_cmd(self):
+        return self.run()
+
+
+class CmdDaemonUpdater(CmdDaemonBase):
+    def run(self):
+        import os
+        from dvc.project import Project
+        from dvc.updater import Updater
+
+        root_dir = Project.find_root()
+        dvc_dir = os.path.join(root_dir, Project.DVC_DIR)
+        updater = Updater(dvc_dir)
+        updater.fetch(detach=False)
+
+        return 0
+
+
+class CmdDaemonAnalytics(CmdDaemonBase):
+    def run(self):
+        from dvc.analytics import Analytics
+
+        analytics = Analytics.load(self.args.target)
+        analytics.send()
+
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    DAEMON_HELP = 'Service daemon.'
+    daemon_parser = subparsers.add_parser(
+        'daemon',
+        parents=[parent_parser],
+        description=DAEMON_HELP,
+        help=DAEMON_HELP)
+
+    daemon_subparsers = daemon_parser.add_subparsers(
+        dest='cmd',
+        help='Use dvc daemon CMD --help for command-specific help.')
+
+    fix_subparsers(daemon_subparsers)
+
+    DAEMON_UPDATER_HELP = 'Fetch latest available version.'
+    daemon_updater_parser = daemon_subparsers.add_parser(
+        'updater',
+        parents=[parent_parser],
+        description=DAEMON_UPDATER_HELP,
+        help=DAEMON_UPDATER_HELP)
+    daemon_updater_parser.set_defaults(func=CmdDaemonUpdater)
+
+    DAEMON_ANALYTICS_HELP = 'Send dvc usage analytics.'
+    daemon_analytics_parser = daemon_subparsers.add_parser(
+        'analytics',
+        parents=[parent_parser],
+        description=DAEMON_ANALYTICS_HELP,
+        help=DAEMON_ANALYTICS_HELP)
+    daemon_analytics_parser.add_argument(
+        'target',
+        help="Analytics file.")
+    daemon_analytics_parser.set_defaults(func=CmdDaemonAnalytics)

dvc/command/data_sync.py

@@ -1,3 +1,6 @@
+import argparse
+
+import dvc.logger as logger
 from dvc.command.base import CmdBase
 
 
@@ -23,10 +26,13 @@ class CmdDataPull(CmdDataBase):
                               jobs=self.args.jobs,
                               remote=self.args.remote,
                               show_checksums=self.args.show_checksums,
-                              all_branches=self.args.all_branches)
-        except Exception as exc:
-            self.project.logger.error('Failed to pull data from the cloud',
-                                      exc)
+                              all_branches=self.args.all_branches,
+                              all_tags=self.args.all_tags,
+                              with_deps=self.args.with_deps,
+                              force=self.args.force,
+                              recursive=self.args.recursive)
+        except Exception:
+            logger.error('failed to pull data from the cloud')
             return 1
         return 0
 
@@ -38,9 +44,12 @@ class CmdDataPush(CmdDataBase):
                               jobs=self.args.jobs,
                               remote=self.args.remote,
                               show_checksums=self.args.show_checksums,
-                              all_branches=self.args.all_branches)
-        except Exception as exc:
-            self.project.logger.error('Failed to push data to the cloud', exc)
+                              all_branches=self.args.all_branches,
+                              all_tags=self.args.all_tags,
+                              with_deps=self.args.with_deps,
+                              recursive=self.args.recursive)
+        except Exception:
+            logger.error('failed to push data to the cloud')
             return 1
         return 0
 
@@ -52,9 +61,193 @@ class CmdDataFetch(CmdDataBase):
                                jobs=self.args.jobs,
                                remote=self.args.remote,
                                show_checksums=self.args.show_checksums,
-                               all_branches=self.args.all_branches)
-        except Exception as exc:
-            self.project.logger.error('Failed to fetch data from the cloud',
-                                      exc)
+                               all_branches=self.args.all_branches,
+                               all_tags=self.args.all_tags,
+                               with_deps=self.args.with_deps,
+                               recursive=self.args.recursive)
+        except Exception:
+            logger.error('failed to fetch data from the cloud')
             return 1
         return 0
+
+
+def add_parser(subparsers, parent_parser):
+    from dvc.command.status import CmdDataStatus
+
+    # Parent parser used in pull/push/status
+    parent_cache_parser = argparse.ArgumentParser(
+        add_help=False,
+        parents=[parent_parser])
+    parent_cache_parser.add_argument(
+        '-j',
+        '--jobs',
+        type=int,
+        default=None,
+        help='Number of jobs to run simultaneously.')
+    parent_cache_parser.add_argument(
+        '--show-checksums',
+        action='store_true',
+        default=False,
+        help='Show checksums instead of file names.')
+    parent_cache_parser.add_argument(
+        'targets',
+        nargs='*',
+        default=None,
+        help='DVC files.')
+
+    # Pull
+    PULL_HELP = 'Pull data files from the cloud.'
+    pull_parser = subparsers.add_parser(
+        'pull',
+        parents=[parent_cache_parser],
+        description=PULL_HELP,
+        help=PULL_HELP)
+    pull_parser.add_argument(
+        '-r',
+        '--remote',
+        help='Remote repository to pull from.')
+    pull_parser.add_argument(
+        '-a',
+        '--all-branches',
+        action='store_true',
+        default=False,
+        help='Fetch cache for all branches.')
+    pull_parser.add_argument(
+        '-T',
+        '--all-tags',
+        action='store_true',
+        default=False,
+        help='Fetch cache for all tags.')
+    pull_parser.add_argument(
+        '-d',
+        '--with-deps',
+        action='store_true',
+        default=False,
+        help='Fetch cache for all dependencies of the specified target.')
+    pull_parser.add_argument(
+        '-f',
+        '--force',
+        action='store_true',
+        default=False,
+        help='Do not prompt when removing working directory files.')
+    pull_parser.add_argument(
+        '-R',
+        '--recursive',
+        action='store_true',
+        default=False,
+        help='Pull cache for subdirectories of the specified directory.')
+    pull_parser.set_defaults(func=CmdDataPull)
+
+    # Push
+    PUSH_HELP = 'Push data files to the cloud.'
+    push_parser = subparsers.add_parser(
+        'push',
+        parents=[parent_cache_parser],
+        description=PUSH_HELP,
+        help=PUSH_HELP)
+    push_parser.add_argument(
+        '-r',
+        '--remote',
+        help='Remote repository to push to.')
+    push_parser.add_argument(
+        '-a',
+        '--all-branches',
+        action='store_true',
+        default=False,
+        help='Push cache for all branches.')
+    push_parser.add_argument(
+        '-T',
+        '--all-tags',
+        action='store_true',
+        default=False,
+        help='Push cache for all tags.')
+    push_parser.add_argument(
+        '-d',
+        '--with-deps',
+        action='store_true',
+        default=False,
+        help='Push cache for all dependencies of the specified target.')
+    push_parser.add_argument(
+        '-R',
+        '--recursive',
+        action='store_true',
+        default=False,
+        help='Push cache from subdirectories of specified directory.')
+    push_parser.set_defaults(func=CmdDataPush)
+
+    # Fetch
+    FETCH_HELP = 'Fetch data files from the cloud.'
+    fetch_parser = subparsers.add_parser(
+        'fetch',
+        parents=[parent_cache_parser],
+        description=FETCH_HELP,
+        help=FETCH_HELP)
+    fetch_parser.add_argument(
+        '-r',
+        '--remote',
+        help='Remote repository to fetch from.')
+    fetch_parser.add_argument(
+        '-a',
+        '--all-branches',
+        action='store_true',
+        default=False,
+        help='Fetch cache for all branches.')
+    fetch_parser.add_argument(
+        '-T',
+        '--all-tags',
+        action='store_true',
+        default=False,
+        help='Fetch cache for all tags.')
+    fetch_parser.add_argument(
+        '-d',
+        '--with-deps',
+        action='store_true',
+        default=False,
+        help='Fetch cache for all dependencies of the '
+        'specified target.')
+    fetch_parser.add_argument(
+        '-R',
+        '--recursive',
+        action='store_true',
+        default=False,
+        help='Fetch cache for subdirectories of specified directory.')
+    fetch_parser.set_defaults(func=CmdDataFetch)
+
+    # Status
+    STATUS_HELP = 'Show the project status.'
+    status_parser = subparsers.add_parser(
+        'status',
+        parents=[parent_cache_parser],
+        description=STATUS_HELP,
+        help=STATUS_HELP)
+    status_parser.add_argument(
+        '-c',
+        '--cloud',
+        action='store_true',
+        default=False,
+        help='Show status of a local cache compared to a remote repository.')
+    status_parser.add_argument(
+        '-r',
+        '--remote',
+        help='Remote repository to compare local cache to.')
+    status_parser.add_argument(
+        '-a',
+        '--all-branches',
+        action='store_true',
+        default=False,
+        help='Show status of a local cache compared to a remote repository '
+             'for all branches.')
+    status_parser.add_argument(
+        '-T',
+        '--all-tags',
+        action='store_true',
+        default=False,
+        help='Show status of a local cache compared to a remote repository '
+             'for all tags.')
+    status_parser.add_argument(
+        '-d',
+        '--with-deps',
+        action='store_true',
+        default=False,
+        help='Show status for all dependencies of the specified target.')
+    status_parser.set_defaults(func=CmdDataStatus)

dvc/command/destroy.py

@@ -1,22 +1,44 @@
+import dvc.prompt as prompt
+import dvc.logger as logger
 from dvc.command.base import CmdBase
-from dvc.prompt import prompt
 from dvc.exceptions import DvcException
 
 
 class CmdDestroy(CmdBase):
     def run_cmd(self):
         try:
-            msg = u'This will destroy all information about your pipelines, ' \
-                  u'all data files, as well as cache in .dvc/cache.\n' \
-                  u'Are you sure you want to continue?'
+            statement = (
+                'This will destroy all information about your pipelines,'
+                ' all data files, as well as cache in .dvc/cache.'
+                '\n'
+                'Are you sure you want to continue?'
+            )
 
-            if not self.args.force and not prompt(msg, False):
-                msg = u'Cannot destroy without a confirmation from the ' \
-                      u'user. Use \'-f\' to force.'
-                raise DvcException(msg)
+            if not self.args.force and not prompt.confirm(statement):
+                raise DvcException(
+                    "cannot destroy without a confirmation from the user."
+                    " Use '-f' to force."
+                )
 
             self.project.destroy()
-        except Exception as exc:
-            self.project.logger.error('Failed to destroy DVC', exc)
+        except Exception:
+            logger.error('failed to destroy DVC')
             return 1
         return 0
+
+
+def add_parser(subparsers, parent_parser):
+    DESTROY_HELP = "Destroy dvc. Will remove all project's information, " \
+                   "data files and cache."
+    destroy_parser = subparsers.add_parser(
+        'destroy',
+        parents=[parent_parser],
+        description=DESTROY_HELP,
+        help=DESTROY_HELP)
+    destroy_parser.add_argument(
+        '-f',
+        '--force',
+        action='store_true',
+        default=False,
+        help='Force destruction.')
+    destroy_parser.set_defaults(func=CmdDestroy)

dvc/command/gc.py

@@ -1,9 +1,92 @@
+import os
+import dvc.prompt as prompt
+import dvc.logger as logger
+
 from dvc.command.base import CmdBase
 
 
 class CmdGC(CmdBase):
     def run(self):
+        msg = 'this will remove all cache except the cache that is used in '
+        if not self.args.all_branches and not self.args.all_tags:
+            msg += 'the current git branch'
+        elif self.args.all_branches and not self.args.all_tags:
+            msg += 'all git branches'
+        elif not self.args.all_branches and self.args.all_tags:
+            msg += 'all git tags'
+        else:
+            msg += 'all git branches and all git tags'
+
+        if self.args.projects is not None and len(self.args.projects) > 0:
+            msg += ' of the current and the following projects:'
+
+            for project_path in self.args.projects:
+                msg += '\n  - %s' % os.path.abspath(project_path)
+        else:
+            msg += ' of the current project.'
+
+        logger.warning(msg)
+
+        msg = 'Are you sure you want to proceed?'
+        if not self.args.force and not prompt.confirm(msg):
+            return 1
+
         self.project.gc(all_branches=self.args.all_branches,
+                        all_tags=self.args.all_tags,
                         cloud=self.args.cloud,
-                        remote=self.args.remote)
+                        remote=self.args.remote,
+                        force=self.args.force,
+                        jobs=self.args.jobs,
+                        projects=self.args.projects)
         return 0
+
+
+def add_parser(subparsers, parent_parser):
+    GC_HELP = 'Collect garbage.'
+    gc_parser = subparsers.add_parser(
+        'gc',
+        parents=[parent_parser],
+        description=GC_HELP,
+        help=GC_HELP)
+    gc_parser.add_argument(
+        '-a',
+        '--all-branches',
+        action='store_true',
+        default=False,
+        help='Collect garbage for all branches.')
+    gc_parser.add_argument(
+        '-T',
+        '--all-tags',
+        action='store_true',
+        default=False,
+        help='Collect garbage for all tags.')
+    gc_parser.add_argument(
+        '-c',
+        '--cloud',
+        action='store_true',
+        default=False,
+        help='Collect garbage in remote repository.')
+    gc_parser.add_argument(
+        '-r',
+        '--remote',
+        help='Remote repository to collect garbage in.')
+    gc_parser.add_argument(
+        '-f',
+        '--force',
+        action='store_true',
+        default=False,
+        help='Force garbage collection.')
+    gc_parser.add_argument(
+        '-j',
+        '--jobs',
+        type=int,
+        default=None,
+        help='Number of jobs to run simultaneously.')
+    gc_parser.add_argument(
+        '-p',
+        '--projects',
+        type=str,
+        nargs='*',
+        default=None,
+        help='Collect garbage for all given projects.')
+    gc_parser.set_defaults(func=CmdGC)

dvc/command/imp.py

@@ -1,13 +1,50 @@
+import os
+
+import dvc.logger as logger
+from dvc.utils.compat import urlparse
 from dvc.exceptions import DvcException
 from dvc.command.base import CmdBase
-from dvc.logger import Logger
 
 
 class CmdImport(CmdBase):
     def run(self):
         try:
-            self.project.imp(self.args.url, self.args.out)
-        except DvcException as ex:
-            Logger.error('Failed to import {}'.format(self.args.url), ex)
+            default_out = os.path.basename(
+                urlparse(self.args.url).path
+            )
+
+            out = self.args.out or default_out
+
+            self.project.imp(self.args.url, out)
+        except DvcException:
+            logger.error('failed to import {}. You could also try downloading '
+                         'it manually and adding it with `dvc add` command.'
+                         .format(self.args.url))
             return 1
         return 0
+
+
+def add_parser(subparsers, parent_parser):
+    IMPORT_HELP = 'Import files from URL.'
+    import_parser = subparsers.add_parser(
+        'import',
+        parents=[parent_parser],
+        description=IMPORT_HELP,
+        help=IMPORT_HELP)
+    import_parser.add_argument(
+        'url',
+        help="URL. Supported urls: "
+        "'/path/to/file', "
+        "'C:\\\\path\\to\\file', "
+        "'https://example.com/path/to/file', "
+        "'s3://bucket/path/to/file', "
+        "'gs://bucket/path/to/file', "
+        "'hdfs://example.com/path/to/file', "
+        "'ssh://example.com:/path/to/file', "
+        "'remote://myremote/path/to/file'(see "
+        "`dvc remote` commands). ")
+    import_parser.add_argument(
+        'out',
+        nargs='?',
+        help='Output.')
+    import_parser.set_defaults(func=CmdImport)

dvc/command/init.py

@@ -1,4 +1,4 @@
-from dvc.logger import Logger
+import dvc.logger as logger
 
 
 class CmdInit(object):
@@ -9,8 +9,35 @@ class CmdInit(object):
         from dvc.project import Project, InitError
 
         try:
-            Project.init('.', no_scm=self.args.no_scm, force=self.args.force)
-        except InitError as e:
-            Logger.error('Failed to initiate dvc', e)
+            self.project = Project.init('.',
+                                        no_scm=self.args.no_scm,
+                                        force=self.args.force)
+            self.config = self.project.config
+        except InitError:
+            logger.error('failed to initiate dvc')
             return 1
         return 0
+
+
+def add_parser(subparsers, parent_parser):
+    """Setup parser for `dvc init`."""
+    INIT_HELP = 'Initialize dvc over a directory ' \
+                '(should already be a git dir).'
+    init_parser = subparsers.add_parser(
+        'init',
+        parents=[parent_parser],
+        description=INIT_HELP,
+        help=INIT_HELP)
+    init_parser.add_argument(
+        '--no-scm',
+        action='store_true',
+        default=False,
+        help="Initiate dvc in directory that is "
+        "not tracked by any scm tool (e.g. git).")
+    init_parser.add_argument(
+        '-f',
+        '--force',
+        action='store_true',
+        default=False,
+        help="Overwrite '.dvc' if it exists. Will remove all local cache.")
+    init_parser.set_defaults(func=CmdInit)

dvc/command/install.py

@@ -1,4 +1,4 @@
-from dvc.logger import Logger
+import dvc.logger as logger
 from dvc.command.base import CmdBase
 
 
@@ -6,7 +6,17 @@ class CmdInstall(CmdBase):
     def run_cmd(self):
         try:
             self.project.install()
-        except Exception as e:
-            Logger.error('Failed to install dvc hooks', e)
+        except Exception:
+            logger.error('failed to install dvc hooks')
             return 1
         return 0
+
+
+def add_parser(subparsers, parent_parser):
+    INSTALL_HELP = 'Install dvc hooks into the repository.'
+    install_parser = subparsers.add_parser(
+        'install',
+        parents=[parent_parser],
+        description=INSTALL_HELP,
+        help=INSTALL_HELP)
+    install_parser.set_defaults(func=CmdInstall)

dvc/command/lock.py

@@ -1,3 +1,4 @@
+import dvc.logger as logger
 from dvc.exceptions import DvcException
 from dvc.command.base import CmdBase
 
@@ -7,10 +8,12 @@ class CmdLockBase(CmdBase):
         for target in self.args.targets:
             try:
                 self.project.lock_stage(target, unlock=unlock)
-            except DvcException as ex:
-                msg = 'Failed to {}lock \'{}\''
-                msg.format('un' if unlock else '', target)
-                self.project.logger.error(msg, ex)
+            except DvcException:
+                logger.error(
+                    "failed to {}lock '{}'"
+                    .format('un' if unlock else '', target)
+                )
+
                 return 1
         return 0
 
@@ -23,3 +26,29 @@ class CmdLock(CmdLockBase):
 class CmdUnlock(CmdLockBase):
     def run(self):
         return super(CmdUnlock, self).run(True)
+
+
+def add_parser(subparsers, parent_parser):
+    LOCK_HELP = 'Lock DVC file.'
+    lock_parser = subparsers.add_parser(
+        'lock',
+        parents=[parent_parser],
+        description=LOCK_HELP,
+        help=LOCK_HELP)
+    lock_parser.add_argument(
+        'targets',
+        nargs='+',
+        help='DVC files.')
+    lock_parser.set_defaults(func=CmdLock)
+
+    UNLOCK_HELP = 'Unlock DVC file.'
+    unlock_parser = subparsers.add_parser(
+        'unlock',
+        parents=[parent_parser],
+        description=UNLOCK_HELP,
+        help=UNLOCK_HELP)
+    unlock_parser.add_argument(
+        'targets',
+        nargs='+',
+        help='DVC files.')
+    unlock_parser.set_defaults(func=CmdUnlock)

dvc/command/metrics.py

@@ -1,5 +1,6 @@
-from dvc.command.base import CmdBase
+import dvc.logger as logger
 from dvc.exceptions import DvcException
+from dvc.command.base import CmdBase, fix_subparsers
 
 
 class CmdMetricsShow(CmdBase):
@@ -28,9 +29,10 @@ class CmdMetricsShow(CmdBase):
             self.project.metrics_show(self.args.path,
                                       typ=typ,
                                       xpath=xpath,
-                                      all_branches=self.args.all_branches)
-        except DvcException as exc:
-            self.project.logger.error('Failed to show metrics', exc)
+                                      all_branches=self.args.all_branches,
+                                      all_tags=self.args.all_tags)
+        except DvcException:
+            logger.error('failed to show metrics')
             return 1
 
         return 0
@@ -42,8 +44,8 @@ class CmdMetricsModify(CmdBase):
             self.project.metrics_modify(self.args.path,
                                         typ=self.args.type,
                                         xpath=self.args.xpath)
-        except DvcException as exc:
-            self.project.logger.error('Failed to modify metrics', exc)
+        except DvcException:
+            logger.error('failed to modify metrics')
             return 1
 
         return 0
@@ -55,9 +57,9 @@ class CmdMetricsAdd(CmdBase):
             self.project.metrics_add(self.args.path,
                                      self.args.type,
                                      self.args.xpath)
-        except DvcException as exc:
-            msg = 'Failed to add metric file \'{}\''.format(self.args.path)
-            self.project.logger.error(msg, exc)
+        except DvcException:
+            msg = "failed to add metric file '{}'".format(self.args.path)
+            logger.error(msg)
             return 1
 
         return 0
@@ -67,9 +69,121 @@ class CmdMetricsRemove(CmdBase):
     def run(self):
         try:
             self.project.metrics_remove(self.args.path)
-        except DvcException as exc:
-            msg = 'Failed to remove metric file \'{}\''.format(self.args.path)
-            self.project.logger.error(msg, exc)
+        except DvcException:
+            msg = "failed to remove metric file '{}'".format(self.args.path)
+            logger.error(msg)
             return 1
 
         return 0
+
+
+def add_parser(subparsers, parent_parser):
+    METRICS_HELP = 'Get metrics from all branches.'
+    metrics_parser = subparsers.add_parser(
+        'metrics',
+        parents=[parent_parser],
+        description=METRICS_HELP,
+        help=METRICS_HELP)
+
+    metrics_subparsers = metrics_parser.add_subparsers(
+        dest='cmd',
+        help="Use dvc metrics CMD --help for command-specific help.")
+
+    fix_subparsers(metrics_subparsers)
+
+    METRICS_SHOW_HELP = 'Show metrics.'
+    metrics_show_parser = metrics_subparsers.add_parser(
+        'show',
+        parents=[parent_parser],
+        description=METRICS_SHOW_HELP,
+        help=METRICS_SHOW_HELP)
+    metrics_show_parser.add_argument(
+        'path',
+        nargs='?',
+        help='Path to metrics file.')
+    metrics_show_parser.add_argument(
+        '-t',
+        '--type',
+        help='Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV).')
+    metrics_show_parser.add_argument(
+        '-x',
+        '--xpath',
+        help='JSON/TSV/HTSV/CSV/HCSV path.')
+    metrics_show_group = metrics_show_parser.add_mutually_exclusive_group()
+    metrics_show_group.add_argument(
+        '--json-path',
+        help='JSON path.')
+    metrics_show_group.add_argument(
+        '--tsv-path',
+        help="TSV path 'row,column' (e.g. '1,2').")
+    metrics_show_group.add_argument(
+        '--htsv-path',
+        help="Headed TSV path 'row,column (e.g. 'Name,3').")
+    metrics_show_group.add_argument(
+        '--csv-path',
+        help="CSV path 'row,column' (e.g. '1,2').")
+    metrics_show_group.add_argument(
+        '--hcsv-path',
+        help="Headed CSV path 'row,column' (e.g. 'Name,3').")
+    metrics_show_parser.add_argument(
+        '-a',
+        '--all-branches',
+        action='store_true',
+        default=False,
+        help='Show metrics for all branches.')
+    metrics_show_parser.add_argument(
+        '-T',
+        '--all-tags',
+        action='store_true',
+        default=False,
+        help='Show metrics for all tags.')
+    metrics_show_parser.set_defaults(func=CmdMetricsShow)
+
+    METRICS_ADD_HELP = 'Add metrics.'
+    metrics_add_parser = metrics_subparsers.add_parser(
+        'add',
+        parents=[parent_parser],
+        description=METRICS_ADD_HELP,
+        help=METRICS_ADD_HELP)
+    metrics_add_parser.add_argument(
+        '-t',
+        '--type',
+        help='Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV).')
+    metrics_add_parser.add_argument(
+        '-x',
+        '--xpath',
+        help='JSON/TSV/HTSV/CSV/HCSV path.')
+    metrics_add_parser.add_argument(
+        'path',
+        help='Path to metrics file.')
+    metrics_add_parser.set_defaults(func=CmdMetricsAdd)
+
+    METRICS_MODIFY_HELP = 'Modify metrics.'
+    metrics_modify_parser = metrics_subparsers.add_parser(
+        'modify',
+        parents=[parent_parser],
+        description=METRICS_MODIFY_HELP,
+        help=METRICS_MODIFY_HELP)
+    metrics_modify_parser.add_argument(
+        '-t',
+        '--type',
+        help='Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV).')
+    metrics_modify_parser.add_argument(
+        '-x',
+        '--xpath',
+        help='JSON/TSV/HTSV/CSV/HCSV path.')
+    metrics_modify_parser.add_argument(
+        'path',
+        help='Metrics file.')
+    metrics_modify_parser.set_defaults(func=CmdMetricsModify)
+
+    METRICS_REMOVE_HELP = 'Remove metrics.'
+    metrics_remove_parser = metrics_subparsers.add_parser(
+        'remove',
+        parents=[parent_parser],
+        description=METRICS_REMOVE_HELP,
+        help=METRICS_REMOVE_HELP)
+    metrics_remove_parser.add_argument(
+        'path',
+        help='Path to metrics file.')
+    metrics_remove_parser.set_defaults(func=CmdMetricsRemove)

dvc/command/move.py

@@ -1,3 +1,4 @@
+import dvc.logger as logger
 from dvc.exceptions import DvcException
 from dvc.command.base import CmdBase
 
@@ -6,9 +7,21 @@ class CmdMove(CmdBase):
     def run(self):
         try:
             self.project.move(self.args.src, self.args.dst)
-        except DvcException as ex:
-            msg = 'Failed to move \'{}\' -> \'{}\''.format(self.args.src,
-                                                           self.args.dst)
-            self.project.logger.error(msg, ex)
+        except DvcException:
+            msg = "failed to move '{}' -> '{}'".format(self.args.src,
+                                                       self.args.dst)
+            logger.error(msg)
             return 1
         return 0
+
+
+def add_parser(subparsers, parent_parser):
+    MOVE_HELP = 'Move output of DVC file.'
+    move_parser = subparsers.add_parser(
+        'move',
+        parents=[parent_parser],
+        description=MOVE_HELP,
+        help=MOVE_HELP)
+    move_parser.add_argument('src', help='Source.')
+    move_parser.add_argument('dst', help='Destination.')
+    move_parser.set_defaults(func=CmdMove)

dvc/command/pipeline.py

@@ -1,8 +1,8 @@
 import os
 
+import dvc.logger as logger
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase
-from dvc.dagascii import Dagascii
+from dvc.command.base import CmdBase, fix_subparsers
 
 
 class CmdPipelineShow(CmdBase):
@@ -17,14 +17,14 @@ class CmdPipelineShow(CmdBase):
 
         for n in networkx.dfs_postorder_nodes(G, node):
             if commands:
-                self.project.logger.info(stages[n].cmd)
+                logger.info(stages[n].cmd)
             elif outs:
                 for out in stages[n].outs:
-                    self.project.logger.info(out.rel_path)
+                    logger.info(str(out))
             else:
-                self.project.logger.info(n)
+                logger.info(n)
 
-    def _show_ascii(self, target, commands, outs):
+    def __build_graph(self, target, commands, outs):
         import networkx
         from dvc.stage import Stage
 
@@ -33,14 +33,24 @@ class CmdPipelineShow(CmdBase):
 
         pipelines = list(filter(lambda g: node in g.nodes(),
                                 self.project.pipelines()))
-        if len(pipelines) == 0:
-            return
 
         assert len(pipelines) == 1
         G = pipelines[0]
         stages = networkx.get_node_attributes(G, 'stage')
 
-        nodes = set()
+        nodes = []
+        for n in G.nodes():
+            stage = stages[n]
+            if commands:
+                if stage.cmd is None:
+                    continue
+                nodes.append(stage.cmd)
+            elif outs:
+                for out in stage.outs:
+                    nodes.append(str(out))
+            else:
+                nodes.append(stage.relpath)
+
         edges = []
         for e in G.edges():
             from_stage = stages[e[0]]
@@ -48,37 +58,131 @@ class CmdPipelineShow(CmdBase):
             if commands:
                 if to_stage.cmd is None:
                     continue
-                nodes.add(from_stage.cmd)
-                nodes.add(to_stage.cmd)
                 edges.append((from_stage.cmd, to_stage.cmd))
             elif outs:
                 for from_out in from_stage.outs:
                     for to_out in to_stage.outs:
-                        nodes.add(from_out.rel_path)
-                        nodes.add(to_out.rel_path)
-                        edges.append((from_out.rel_path,
-                                      to_out.rel_path))
+                        edges.append((str(from_out),
+                                      str(to_out)))
             else:
-                nodes.add(from_stage.relpath)
-                nodes.add(to_stage.relpath)
                 edges.append((from_stage.relpath, to_stage.relpath))
 
-        d = Dagascii(nodes, edges)
-        d.draw()
+        return nodes, edges
+
+    def _show_ascii(self, target, commands, outs):
+        from dvc.dagascii import draw
+
+        nodes, edges = self.__build_graph(target, commands, outs)
+
+        if not nodes:
+            return
+
+        draw(nodes, edges)
+
+    def __write_dot(self, target, commands, outs, filename):
+        import networkx
+        from networkx.drawing.nx_pydot import write_dot
+
+        _, edges = self.__build_graph(target, commands, outs)
+        edges = [edge[::-1] for edge in edges]
+
+        simple_g = networkx.DiGraph()
+        simple_g.add_edges_from(edges)
+        write_dot(simple_g, filename)
 
     def run(self, unlock=False):
+        if not self.args.targets:
+            self.args.targets = self.default_targets
+
         for target in self.args.targets:
             try:
                 if self.args.ascii:
                     self._show_ascii(target,
                                      self.args.commands,
                                      self.args.outs)
+                elif self.args.dot:
+                    self.__write_dot(target,
+                                     self.args.commands,
+                                     self.args.outs,
+                                     self.args.dot)
                 else:
                     self._show(target,
                                self.args.commands,
                                self.args.outs)
-            except DvcException as ex:
-                msg = 'Failed to show pipeline for \'{}\''.format(target)
-                self.project.logger.error(msg, ex)
+            except DvcException:
+                msg = "failed to show pipeline for '{}'".format(target)
+                logger.error(msg)
                 return 1
         return 0
+
+
+class CmdPipelineList(CmdBase):
+    def run(self):
+        import networkx
+
+        pipelines = self.project.pipelines()
+        for p in pipelines:
+            stages = networkx.get_node_attributes(p, 'stage')
+            for stage in stages:
+                logger.info(stage)
+            if len(stages) != 0:
+                logger.info('=' * 80)
+        logger.info('{} pipeline(s) total'.format(len(pipelines)))
+
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    PIPELINE_HELP = 'Manage pipeline.'
+    pipeline_parser = subparsers.add_parser(
+        'pipeline',
+        parents=[parent_parser],
+        description=PIPELINE_HELP,
+        help=PIPELINE_HELP)
+
+    pipeline_subparsers = pipeline_parser.add_subparsers(
+        dest='cmd',
+        help="Use dvc pipeline CMD --help for command-specific help.")
+
+    fix_subparsers(pipeline_subparsers)
+
+    PIPELINE_SHOW_HELP = 'Show pipeline.'
+    pipeline_show_parser = pipeline_subparsers.add_parser(
+        'show',
+        parents=[parent_parser],
+        description=PIPELINE_SHOW_HELP,
+        help=PIPELINE_SHOW_HELP)
+    pipeline_show_group = pipeline_show_parser.add_mutually_exclusive_group()
+    pipeline_show_group.add_argument(
+        '-c',
+        '--commands',
+        action='store_true',
+        default=False,
+        help='Print commands instead of paths to DVC files.')
+    pipeline_show_group.add_argument(
+        '-o',
+        '--outs',
+        action='store_true',
+        default=False,
+        help='Print output files instead of paths to DVC files.')
+    pipeline_show_parser.add_argument(
+        '--ascii',
+        action='store_true',
+        default=False,
+        help='Output DAG as ASCII.')
+    pipeline_show_parser.add_argument(
+        '--dot',
+        help='Write DAG in .dot format.')
+    pipeline_show_parser.add_argument(
+        'targets',
+        nargs='*',
+        help="DVC files. 'Dvcfile' by default.")
+    pipeline_show_parser.set_defaults(func=CmdPipelineShow)
+
+    PIPELINE_LIST_HELP = 'List pipelines.'
+    pipeline_list_parser = pipeline_subparsers.add_parser(
+        'list',
+        parents=[parent_parser],
+        description=PIPELINE_LIST_HELP,
+        help=PIPELINE_LIST_HELP)
+    pipeline_list_parser.set_defaults(func=CmdPipelineList)

dvc/command/remote.py

@@ -1,31 +1,84 @@
+import os
 import re
 
+import dvc.logger as logger
+from dvc.command.base import fix_subparsers
 from dvc.config import Config
+from dvc.remote import _get, RemoteLOCAL
 from dvc.command.config import CmdConfig
-from dvc.logger import Logger
 
 
 class CmdRemoteAdd(CmdConfig):
+    @staticmethod
+    def resolve_path(path, config_file):
+        """Resolve path relative to config file location.
+
+        Args:
+            path: Path to be resolved.
+            config_file: Path to config file, which `path` is specified
+                relative to.
+
+        Returns:
+            Path relative to the `config_file` location. If `path` is an
+            absolute path then it will be returned without change.
+
+        """
+        if os.path.isabs(path):
+            return path
+        return os.path.relpath(path, os.path.dirname(config_file))
+
     def run(self):
+        remote = _get({Config.SECTION_REMOTE_URL: self.args.url})
+        if remote == RemoteLOCAL:
+            self.args.url = self.resolve_path(self.args.url,
+                                              self.configobj.filename)
+
         section = Config.SECTION_REMOTE_FMT.format(self.args.name)
-        ret = self.set(section, Config.SECTION_REMOTE_URL, self.args.url)
+        ret = self._set(section, Config.SECTION_REMOTE_URL, self.args.url)
         if ret != 0:
             return ret
 
         if self.args.default:
-            msg = 'Setting \'{}\' as a default remote.'.format(self.args.name)
-            Logger.info(msg)
-            ret = self.set(Config.SECTION_CORE,
-                           Config.SECTION_CORE_REMOTE,
-                           self.args.name)
+            msg = "Setting '{}' as a default remote.".format(self.args.name)
+            logger.info(msg)
+            ret = self._set(Config.SECTION_CORE,
+                            Config.SECTION_CORE_REMOTE,
+                            self.args.name)
 
         return ret
 
 
 class CmdRemoteRemove(CmdConfig):
+    def _remove_default(self, config):
+        core = config.get(Config.SECTION_CORE, None)
+        if core is None:
+            return 0
+
+        default = core.get(Config.SECTION_CORE_REMOTE, None)
+        if default is None:
+            return 0
+
+        if default == self.args.name:
+            return self._unset(Config.SECTION_CORE,
+                               opt=Config.SECTION_CORE_REMOTE,
+                               configobj=config)
+
     def run(self):
         section = Config.SECTION_REMOTE_FMT.format(self.args.name)
-        return self.unset(section, Config.SECTION_REMOTE_URL)
+        ret = self._unset(section)
+        if ret != 0:
+            return ret
+
+        for configobj in [self.config._local_config,
+                          self.config._project_config,
+                          self.config._global_config,
+                          self.config._system_config]:
+            self._remove_default(configobj)
+            self.config.save(configobj)
+            if configobj == self.configobj:
+                break
+
+        return 0
 
 
 class CmdRemoteModify(CmdConfig):
@@ -35,6 +88,13 @@ class CmdRemoteModify(CmdConfig):
         return super(CmdRemoteModify, self).run()
 
 
+class CmdRemoteDefault(CmdConfig):
+    def run(self):
+        self.args.value = self.args.name
+        self.args.name = 'core.remote'
+        return super(CmdRemoteDefault, self).run()
+
+
 class CmdRemoteList(CmdConfig):
     def run(self):
         for section in self.configobj.keys():
@@ -43,5 +103,105 @@ class CmdRemoteList(CmdConfig):
                 name = r.group('name')
                 url = self.configobj[section].get(Config.SECTION_REMOTE_URL,
                                                   '')
-                Logger.info('{}\t{}'.format(name, url))
+                logger.info('{}\t{}'.format(name, url))
         return 0
+
+
+def add_parser(subparsers, parent_parser):
+    from dvc.command.config import parent_config_parser
+
+    REMOTE_HELP = 'Manage set of tracked repositories.'
+    remote_parser = subparsers.add_parser(
+        'remote',
+        parents=[parent_parser],
+        description=REMOTE_HELP,
+        help=REMOTE_HELP)
+
+    remote_subparsers = remote_parser.add_subparsers(
+        dest='cmd',
+        help='Use dvc remote CMD --help for '
+        'command-specific help.')
+
+    fix_subparsers(remote_subparsers)
+
+    REMOTE_ADD_HELP = 'Add remote.'
+    remote_add_parser = remote_subparsers.add_parser(
+        'add',
+        parents=[parent_config_parser, parent_parser],
+        description=REMOTE_ADD_HELP,
+        help=REMOTE_ADD_HELP)
+    remote_add_parser.add_argument(
+        'name',
+        help='Name.')
+    remote_add_parser.add_argument(
+        'url',
+        help='URL. See full list of supported urls at '
+        'https://dvc.org/doc/commands-reference/remote')
+    remote_add_parser.add_argument(
+        '-d',
+        '--default',
+        action='store_true',
+        default=False,
+        help='Set as default remote.')
+    remote_add_parser.set_defaults(func=CmdRemoteAdd)
+
+    REMOTE_DEFAULT_HELP = 'Set/unset default remote.'
+    remote_default_parser = remote_subparsers.add_parser(
+        'default',
+        parents=[parent_config_parser, parent_parser],
+        description=REMOTE_DEFAULT_HELP,
+        help=REMOTE_DEFAULT_HELP)
+    remote_default_parser.add_argument(
+        'name',
+        nargs='?',
+        help='Name of the remote.')
+    remote_default_parser.add_argument(
+        '-u',
+        '--unset',
+        action='store_true',
+        default=False,
+        help='Unset default remote.')
+    remote_default_parser.set_defaults(func=CmdRemoteDefault)
+
+    REMOTE_REMOVE_HELP = 'Remove remote.'
+    remote_remove_parser = remote_subparsers.add_parser(
+        'remove',
+        parents=[parent_config_parser, parent_parser],
+        description=REMOTE_REMOVE_HELP,
+        help=REMOTE_REMOVE_HELP)
+    remote_remove_parser.add_argument(
+        'name',
+        help='Name')
+    remote_remove_parser.set_defaults(func=CmdRemoteRemove)
+
+    REMOTE_MODIFY_HELP = 'Modify remote.'
+    remote_modify_parser = remote_subparsers.add_parser(
+        'modify',
+        parents=[parent_config_parser, parent_parser],
+        description=REMOTE_MODIFY_HELP,
+        help=REMOTE_MODIFY_HELP)
+    remote_modify_parser.add_argument(
+        'name',
+        help='Name.')
+    remote_modify_parser.add_argument(
+        'option',
+        help='Option.')
+    remote_modify_parser.add_argument(
+        'value',
+        nargs='?',
+        help='Value.')
+    remote_modify_parser.add_argument(
+        '-u',
+        '--unset',
+        default=False,
+        action='store_true',
+        help='Unset option.')
+    remote_modify_parser.set_defaults(func=CmdRemoteModify)
+
+    REMOTE_LIST_HELP = 'List remotes.'
+    remote_list_parser = remote_subparsers.add_parser(
+        'list',
+        parents=[parent_config_parser, parent_parser],
+        description=REMOTE_LIST_HELP,
+        help=REMOTE_LIST_HELP)
+    remote_list_parser.set_defaults(func=CmdRemoteList)

dvc/command/remove.py

@@ -1,15 +1,69 @@
+import dvc.prompt as prompt
+import dvc.logger as logger
 from dvc.exceptions import DvcException
 from dvc.command.base import CmdBase
-from dvc.logger import Logger
 
 
 class CmdRemove(CmdBase):
+    def _is_outs_only(self, target):
+        if not self.args.purge:
+            return True
+
+        if self.args.force:
+            return False
+
+        msg = (
+            u'Are you sure you want to remove {} with its outputs?'
+            .format(target)
+        )
+
+        if prompt.confirm(msg):
+            return False
+
+        raise DvcException(
+            u'Cannot purge without a confirmation from the user.'
+            u" Use '-f' to force."
+        )
+
     def run(self):
-        outs_only = not self.args.purge
         for target in self.args.targets:
             try:
+                outs_only = self._is_outs_only(target)
                 self.project.remove(target, outs_only=outs_only)
-            except DvcException as ex:
-                Logger.error('Failed to remove {}'.format(target), ex)
+            except DvcException:
+                logger.error('failed to remove {}'.format(target))
                 return 1
         return 0
+
+
+def add_parser(subparsers, parent_parser):
+    REMOVE_HELP = 'Remove outputs of DVC file.'
+    remove_parser = subparsers.add_parser(
+        'remove',
+        parents=[parent_parser],
+        description=REMOVE_HELP,
+        help=REMOVE_HELP)
+    remove_parser_group = remove_parser.add_mutually_exclusive_group()
+    remove_parser_group.add_argument(
+        '-o',
+        '--outs',
+        action='store_true',
+        default=True,
+        help='Only remove DVC file outputs.(default)')
+    remove_parser_group.add_argument(
+        '-p',
+        '--purge',
+        action='store_true',
+        default=False,
+        help='Remove DVC file and all its outputs.')
+    remove_parser.add_argument(
+        '-f',
+        '--force',
+        action='store_true',
+        default=False,
+        help='Force purge.')
+    remove_parser.add_argument(
+        'targets',
+        nargs='+',
+        help='DVC files.')
+    remove_parser.set_defaults(func=CmdRemove)

dvc/command/repro.py

@@ -1,6 +1,8 @@
 import os
 
+import dvc.logger as logger
 from dvc.command.base import CmdBase
+from dvc.command.status import CmdDataStatus
 from dvc.exceptions import DvcException
 
 
@@ -11,21 +13,103 @@ class CmdRepro(CmdBase):
         if self.args.cwd:
             os.chdir(self.args.cwd)
 
+        # Dirty hack so the for loop below can at least enter once
+        if self.args.all_pipelines:
+            self.args.targets = [None]
+        elif not self.args.targets:
+            self.args.targets = self.default_targets
+
         ret = 0
         for target in self.args.targets:
             try:
-                self.project.reproduce(target,
-                                       recursive=recursive,
-                                       force=self.args.force,
-                                       dry=self.args.dry,
-                                       interactive=self.args.interactive)
+                stages = self.project.reproduce(
+                               target,
+                               recursive=recursive,
+                               force=self.args.force,
+                               dry=self.args.dry,
+                               interactive=self.args.interactive,
+                               pipeline=self.args.pipeline,
+                               all_pipelines=self.args.all_pipelines,
+                               ignore_build_cache=self.args.ignore_build_cache)
+
+                if len(stages) == 0:
+                    logger.info(CmdDataStatus.UP_TO_DATE_MSG)
+
                 if self.args.metrics:
                     self.project.metrics_show()
-            except DvcException as ex:
-                msg = 'Failed to reproduce \'{}\''.format(target)
-                self.project.logger.error(msg, ex)
+            except DvcException:
+                logger.error()
                 ret = 1
                 break
 
         os.chdir(saved_dir)
         return ret
+
+
+def add_parser(subparsers, parent_parser):
+    REPRO_HELP = "Reproduce DVC file. Default file name - 'Dvcfile'."
+    repro_parser = subparsers.add_parser(
+        'repro',
+        parents=[parent_parser],
+        description=REPRO_HELP,
+        help=REPRO_HELP)
+    repro_parser.add_argument(
+        'targets',
+        nargs='*',
+        help='DVC file to reproduce.')
+    repro_parser.add_argument(
+        '-f',
+        '--force',
+        action='store_true',
+        default=False,
+        help='Reproduce even if dependencies were not changed.')
+    repro_parser.add_argument(
+        '-s',
+        '--single-item',
+        action='store_true',
+        default=False,
+        help='Reproduce only single data item without recursive dependencies '
+             'check.')
+    repro_parser.add_argument(
+        '-c',
+        '--cwd',
+        default=os.path.curdir,
+        help='Directory within your project to reroduce from.')
+    repro_parser.add_argument(
+        '-m',
+        '--metrics',
+        action='store_true',
+        default=False,
+        help='Show metrics after reproduction.')
+    repro_parser.add_argument(
+        '--dry',
+        action='store_true',
+        default=False,
+        help='Only print the commands that would be executed without '
+             'actually executing.')
+    repro_parser.add_argument(
+        '-i',
+        '--interactive',
+        action='store_true',
+        default=False,
+        help='Ask for confirmation before reproducing each stage.')
+    repro_parser.add_argument(
+        '-p',
+        '--pipeline',
+        action='store_true',
+        default=False,
+        help='Reproduce the whole pipeline that the specified stage file '
+             'belongs to.')
+    repro_parser.add_argument(
+        '-P',
+        '--all-pipelines',
+        action='store_true',
+        default=False,
+        help='Reproduce all pipelines in the project.')
+    repro_parser.add_argument(
+        '--ignore-build-cache',
+        action='store_true',
+        default=False,
+        help="Reproduce all descendants of a changed stage even if their "
+             "direct dependencies didn't change.")
+    repro_parser.set_defaults(func=CmdRepro)

dvc/command/root.py

@@ -1,9 +1,23 @@
 import os
 
+import dvc.logger as logger
 from dvc.command.base import CmdBase
 
 
 class CmdRoot(CmdBase):
+    def run_cmd(self):
+        return self.run()
+
     def run(self):
-        self.project.logger.info(os.path.relpath(self.project.root_dir))
+        logger.info(os.path.relpath(self.project.root_dir))
         return 0
+
+
+def add_parser(subparsers, parent_parser):
+    ROOT_HELP = "Relative path to project's directory."
+    root_parser = subparsers.add_parser(
+        'root',
+        parents=[parent_parser],
+        description=ROOT_HELP,
+        help=ROOT_HELP)
+    root_parser.set_defaults(func=CmdRoot)

dvc/command/run.py

@@ -1,20 +1,143 @@
+import os
+import argparse
+
+import dvc.logger as logger
 from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
 
 
 class CmdRun(CmdBase):
     def run(self):
+        overwrite = (self.args.yes or self.args.overwrite_dvcfile)
+
+        if not any([self.args.deps,
+                    self.args.outs,
+                    self.args.outs_no_cache,
+                    self.args.command]):  # pragma: no cover
+            logger.error("too few arguments. Specify at least one: '-d', "
+                         "'-o', '-O', 'command'.")
+            return 1
+
         try:
-            self.project.run(cmd=' '.join(self.args.command),
+            self.project.run(cmd=self._parsed_cmd(),
                              outs=self.args.outs,
                              outs_no_cache=self.args.outs_no_cache,
                              metrics_no_cache=self.args.metrics_no_cache,
                              deps=self.args.deps,
                              fname=self.args.file,
                              cwd=self.args.cwd,
-                             no_exec=self.args.no_exec)
-        except DvcException as ex:
-            self.project.logger.error('Failed to run command', ex)
+                             no_exec=self.args.no_exec,
+                             overwrite=overwrite,
+                             ignore_build_cache=self.args.ignore_build_cache,
+                             remove_outs=self.args.remove_outs)
+        except DvcException:
+            logger.error('failed to run command')
             return 1
 
         return 0
+
+    def _parsed_cmd(self):
+        """
+        We need to take into account two cases:
+
+        - ['python code.py foo bar']: Used mainly with dvc as a library
+        - ['echo', 'foo bar']: List of arguments received from the CLI
+
+        The second case would need quoting, as it was passed through:
+                dvc run echo "foo bar"
+        """
+        if len(self.args.command) < 2:
+            return ' '.join(self.args.command)
+
+        return ' '.join(self._quote_argument(arg) for arg in self.args.command)
+
+    def _quote_argument(self, argument):
+        if ' ' not in argument or '"' in argument:
+            return argument
+
+        return '"{}"'.format(argument)
+
+
+def add_parser(subparsers, parent_parser):
+    RUN_HELP = 'Generate a stage file from a given ' \
+               'command and execute the command.'
+    run_parser = subparsers.add_parser(
+        'run',
+        parents=[parent_parser],
+        description=RUN_HELP,
+        help=RUN_HELP)
+    run_parser.add_argument(
+        '-d',
+        '--deps',
+        action='append',
+        default=[],
+        help='Declare dependencies for reproducible cmd.')
+    run_parser.add_argument(
+        '-o',
+        '--outs',
+        action='append',
+        default=[],
+        help='Declare output data file or data directory.')
+    run_parser.add_argument(
+        '-O',
+        '--outs-no-cache',
+        action='append',
+        default=[],
+        help='Declare output regular file or directory '
+             '(sync to Git, not DVC cache).')
+    run_parser.add_argument(
+        '-M',
+        '--metrics-no-cache',
+        action='append',
+        default=[],
+        help='Declare output metric file or '
+             'directory (not cached by DVC).')
+    run_parser.add_argument(
+        '-f',
+        '--file',
+        help="Specify name of the stage file. It should be "
+             "either 'Dvcfile' or have a '.dvc' suffix (e.g. "
+             "'prepare.dvc', 'clean.dvc', etc) in order for "
+             "dvc to be able to find it later. By default "
+             "the first output basename + .dvc is used as "
+             "a stage filename.")
+    run_parser.add_argument(
+        '-c',
+        '--cwd',
+        default=os.path.curdir,
+        help="Directory within your project to run your command and place "
+             "stage file in.")
+    run_parser.add_argument(
+        '--no-exec',
+        action='store_true',
+        default=False,
+        help="Only create stage file without actually running it.")
+    run_parser.add_argument(
+        '-y',
+        '--yes',
+        action='store_true',
+        default=False,
+        help="(OBSOLETED, use --overwrite-dvcfile instead) Automatic 'yes' "
+             "answer to all prompts. E.g. when '.dvc' file exists and dvc "
+             "asks if you want to overwrite it.")
+    run_parser.add_argument(
+        '--overwrite-dvcfile',
+        action='store_true',
+        default=False,
+        help="Overwrite existing dvc file without asking for confirmation.")
+    run_parser.add_argument(
+        '--ignore-build-cache',
+        action='store_true',
+        default=False,
+        help="Run this stage even if it has been already ran with the same "
+             "command/dependencies/outputs/etc before.")
+    run_parser.add_argument(
+        '--remove-outs',
+        action='store_true',
+        default=False,
+        help="Remove outputs before running the command.")
+    run_parser.add_argument(
+        'command',
+        nargs=argparse.REMAINDER,
+        help='Command to execute.')
+    run_parser.set_defaults(func=CmdRun)

dvc/command/status.py

@@ -1,9 +1,11 @@
+import dvc.logger as logger
 from dvc.command.data_sync import CmdDataBase
 
 
 class CmdDataStatus(CmdDataBase):
     STATUS_LEN = 10
     STATUS_INDENT = '\t'
+    UP_TO_DATE_MSG = "Pipeline is up to date. Nothing to reproduce."
 
     def _normalize(self, s):
         s += ':'
@@ -15,11 +17,11 @@ class CmdDataStatus(CmdDataBase):
 
         for key, value in status.items():
             if isinstance(value, dict):
-                self.project.logger.info('{}{}'.format(ind, key))
+                logger.info('{}{}'.format(ind, key))
                 self._show(value, indent+1)
             else:
                 msg = '{}{}{}'.format(ind, self._normalize(value), key)
-                self.project.logger.info(msg)
+                logger.info(msg)
 
     def do_run(self, target=None):
         indent = 1 if self.args.cloud else 0
@@ -28,13 +30,16 @@ class CmdDataStatus(CmdDataBase):
                                      jobs=self.args.jobs,
                                      cloud=self.args.cloud,
                                      show_checksums=self.args.show_checksums,
-                                     remote=self.args.remote)
+                                     remote=self.args.remote,
+                                     all_branches=self.args.all_branches,
+                                     all_tags=self.args.all_tags,
+                                     with_deps=self.args.with_deps)
             if st:
                 self._show(st, indent)
             else:
-                self.project.logger.info("Nothing to reproduce. "
-                                         "Pipeline is up to date.")
-        except Exception as exc:
-            self.project.logger.error('Failed to obtain data status', exc)
+                logger.info(self.UP_TO_DATE_MSG)
+
+        except Exception:
+            logger.error('failed to obtain data status')
             return 1
         return 0

dvc/command/unprotect.py

@@ -0,0 +1,29 @@
+import dvc.logger as logger
+from dvc.exceptions import DvcException
+from dvc.command.base import CmdBase
+
+
+class CmdUnprotect(CmdBase):
+    def run(self):
+        for target in self.args.targets:
+            try:
+                self.project.unprotect(target)
+            except DvcException:
+                msg = "failed to unprotect '{}'".format(target)
+                logger.error(msg)
+                return 1
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    UNPROTECT_HELP = 'Unprotect data file/directory.'
+    unprotect_parser = subparsers.add_parser(
+        'unprotect',
+        parents=[parent_parser],
+        description=UNPROTECT_HELP,
+        help=UNPROTECT_HELP)
+    unprotect_parser.add_argument(
+        'targets',
+        nargs='+',
+        help='Data files/directory.')
+    unprotect_parser.set_defaults(func=CmdUnprotect)

dvc/config.py

@@ -2,49 +2,127 @@
 DVC config objects.
 """
 import os
+import errno
 import configobj
 from schema import Schema, Optional, And, Use, Regex
 
+import dvc.logger as logger
 from dvc.exceptions import DvcException
 
 
 class ConfigError(DvcException):
-    """ DVC config exception """
-    def __init__(self, ex=None):
-        super(ConfigError, self).__init__('Config file error', ex)
+    """DVC config exception.
 
-
-def supported_url(url):
-    from dvc.remote import supported_url as supported
-    return supported(url)
+    Args:
+        msg (str): error message.
+        ex (Exception): optional exception that has caused this error.
+    """
+    def __init__(self, msg, ex=None):
+        super(ConfigError, self).__init__('config file error: {}'.format(msg),
+                                          ex)
 
 
 def supported_cache_type(types):
+    """Checks if link type config option has a valid value.
+
+    Args:
+        types (list/string): type(s) of links that dvc should try out.
+    """
     if isinstance(types, str):
-        types = [t.strip() for t in types.split(',')]
-    for t in types:
-        if t not in ['reflink', 'hardlink', 'symlink', 'copy']:
+        types = [typ.strip() for typ in types.split(',')]
+    for typ in types:
+        if typ not in ['reflink', 'hardlink', 'symlink', 'copy']:
             return False
     return True
 
 
 def supported_loglevel(level):
+    """Checks if log level config option has a valid value.
+
+    Args:
+        level (str): log level name.
+    """
     return level in ['info', 'debug', 'warning', 'error']
 
 
 def supported_cloud(cloud):
+    """Checks if obsoleted cloud option has a valid value.
+
+    Args:
+        cloud (str): cloud type name.
+    """
     return cloud in ['aws', 'gcp', 'local', '']
 
 
 def is_bool(val):
+    """Checks that value is a boolean.
+
+    Args:
+        val (str): string value verify.
+
+    Returns:
+        bool: True if value stands for boolean, False otherwise.
+    """
     return val.lower() in ['true', 'false']
 
 
 def to_bool(val):
+    """Converts value to boolean.
+
+    Args:
+        val (str): string to convert to boolean.
+
+    Returns:
+        bool: True if value.lower() == 'true', False otherwise.
+    """
     return val.lower() == 'true'
 
 
-class Config(object):
+def is_whole(val):
+    """Checks that value is a whole integer.
+
+    Args:
+        val (str): number string to verify.
+
+    Returns:
+        bool: True if val is a whole number, False otherwise.
+    """
+    return int(val) >= 0
+
+
+def is_percent(val):
+    """Checks that value is a percent.
+
+    Args:
+        val (str): number string to verify.
+
+    Returns:
+        bool: True if 0<=value<=100, False otherwise.
+    """
+    return int(val) >= 0 and int(val) <= 100
+
+
+class Config(object):  # pylint: disable=too-many-instance-attributes
+    """Class that manages configuration files for a dvc project.
+
+    Args:
+        dvc_dir (str): optional path to `.dvc` directory, that is used to
+            access project-specific configs like .dvc/config and
+            .dvc/config.local.
+        validate (bool): optional flag to tell dvc if it should validate the
+            config or just load it as is. 'True' by default.
+
+
+    Raises:
+        ConfigError: thrown when config has an invalid format.
+    """
+    APPNAME = 'dvc'
+    APPAUTHOR = 'iterative'
+
+    # NOTE: used internally in RemoteLOCAL to know config
+    # location, that url should resolved relative to.
+    PRIVATE_CWD = '_cwd'
+
     CONFIG = 'config'
     CONFIG_LOCAL = 'config.local'
 
@@ -54,11 +132,14 @@ class Config(object):
     SECTION_CORE_REMOTE = 'remote'
     SECTION_CORE_INTERACTIVE_SCHEMA = And(str, is_bool, Use(to_bool))
     SECTION_CORE_INTERACTIVE = 'interactive'
+    SECTION_CORE_ANALYTICS = 'analytics'
+    SECTION_CORE_ANALYTICS_SCHEMA = And(str, is_bool, Use(to_bool))
 
     SECTION_CACHE = 'cache'
     SECTION_CACHE_DIR = 'dir'
     SECTION_CACHE_TYPE = 'type'
     SECTION_CACHE_TYPE_SCHEMA = supported_cache_type
+    SECTION_CACHE_PROTECTED = 'protected'
     SECTION_CACHE_LOCAL = 'local'
     SECTION_CACHE_S3 = 's3'
     SECTION_CACHE_GS = 'gs'
@@ -73,9 +154,12 @@ class Config(object):
         Optional(SECTION_CACHE_SSH): str,
         Optional(SECTION_CACHE_AZURE): str,
 
-        # backward compatibility
-        Optional(SECTION_CACHE_DIR, default='cache'): str,
+        Optional(SECTION_CACHE_DIR): str,
         Optional(SECTION_CACHE_TYPE, default=None): SECTION_CACHE_TYPE_SCHEMA,
+        Optional(SECTION_CACHE_PROTECTED,
+                 default=False): And(str, is_bool, Use(to_bool)),
+
+        Optional(PRIVATE_CWD): str,
     }
 
     # backward compatibility
@@ -90,6 +174,8 @@ class Config(object):
         Optional(SECTION_CORE_REMOTE, default=''): And(str, Use(str.lower)),
         Optional(SECTION_CORE_INTERACTIVE,
                  default=False): SECTION_CORE_INTERACTIVE_SCHEMA,
+        Optional(SECTION_CORE_ANALYTICS,
+                 default=True): SECTION_CORE_ANALYTICS_SCHEMA,
 
         # backward compatibility
         Optional(SECTION_CORE_CLOUD, default=''): SECTION_CORE_CLOUD_SCHEMA,
@@ -103,12 +189,15 @@ class Config(object):
     SECTION_AWS_ENDPOINT_URL = 'endpointurl'
     SECTION_AWS_REGION = 'region'
     SECTION_AWS_PROFILE = 'profile'
+    SECTION_AWS_USE_SSL = 'use_ssl'
     SECTION_AWS_SCHEMA = {
         SECTION_AWS_STORAGEPATH: str,
         Optional(SECTION_AWS_REGION): str,
-        Optional(SECTION_AWS_PROFILE, default='default'): str,
-        Optional(SECTION_AWS_CREDENTIALPATH, default=''): str,
-        Optional(SECTION_AWS_ENDPOINT_URL, default=None): str,
+        Optional(SECTION_AWS_PROFILE): str,
+        Optional(SECTION_AWS_CREDENTIALPATH): str,
+        Optional(SECTION_AWS_ENDPOINT_URL): str,
+        Optional(SECTION_AWS_USE_SSL,
+                 default=True): And(str, is_bool, Use(to_bool)),
     }
 
     # backward compatibility
@@ -127,25 +216,53 @@ class Config(object):
         SECTION_LOCAL_STORAGEPATH: str,
     }
 
+    SECTION_AZURE_CONNECTION_STRING = 'connection_string'
+
     SECTION_REMOTE_REGEX = r'^\s*remote\s*"(?P<name>.*)"\s*$'
     SECTION_REMOTE_FMT = 'remote "{}"'
     SECTION_REMOTE_URL = 'url'
     SECTION_REMOTE_USER = 'user'
+    SECTION_REMOTE_PORT = 'port'
+    SECTION_REMOTE_KEY_FILE = 'keyfile'
+    SECTION_REMOTE_TIMEOUT = 'timeout'
+    SECTION_REMOTE_PASSWORD = 'password'
+    SECTION_REMOTE_ASK_PASSWORD = 'ask_password'
     SECTION_REMOTE_SCHEMA = {
-        SECTION_REMOTE_URL: And(supported_url, error="Unsupported URL"),
+        SECTION_REMOTE_URL: str,
         Optional(SECTION_AWS_REGION): str,
-        Optional(SECTION_AWS_PROFILE, default='default'): str,
-        Optional(SECTION_AWS_CREDENTIALPATH, default=''): str,
-        Optional(SECTION_AWS_ENDPOINT_URL, default=None): str,
+        Optional(SECTION_AWS_PROFILE): str,
+        Optional(SECTION_AWS_CREDENTIALPATH): str,
+        Optional(SECTION_AWS_ENDPOINT_URL): str,
+        Optional(SECTION_AWS_USE_SSL,
+                 default=True): And(str, is_bool, Use(to_bool)),
         Optional(SECTION_GCP_PROJECTNAME): str,
         Optional(SECTION_CACHE_TYPE): SECTION_CACHE_TYPE_SCHEMA,
+        Optional(SECTION_CACHE_PROTECTED,
+                 default=False): And(str, is_bool, Use(to_bool)),
         Optional(SECTION_REMOTE_USER): str,
+        Optional(SECTION_REMOTE_PORT): Use(int),
+        Optional(SECTION_REMOTE_KEY_FILE): str,
+        Optional(SECTION_REMOTE_TIMEOUT): Use(int),
+        Optional(SECTION_REMOTE_PASSWORD): str,
+        Optional(SECTION_REMOTE_ASK_PASSWORD): And(str, is_bool, Use(to_bool)),
+        Optional(SECTION_AZURE_CONNECTION_STRING): str,
+
+        Optional(PRIVATE_CWD): str,
+    }
+
+    SECTION_STATE = 'state'
+    SECTION_STATE_ROW_LIMIT = 'row_limit'
+    SECTION_STATE_ROW_CLEANUP_QUOTA = 'row_cleanup_quota'
+    SECTION_STATE_SCHEMA = {
+        Optional(SECTION_STATE_ROW_LIMIT): And(Use(int), is_whole),
+        Optional(SECTION_STATE_ROW_CLEANUP_QUOTA): And(Use(int), is_percent),
     }
 
     SCHEMA = {
         Optional(SECTION_CORE, default={}): SECTION_CORE_SCHEMA,
         Optional(Regex(SECTION_REMOTE_REGEX)): SECTION_REMOTE_SCHEMA,
         Optional(SECTION_CACHE, default={}): SECTION_CACHE_SCHEMA,
+        Optional(SECTION_STATE, default={}): SECTION_STATE_SCHEMA,
 
         # backward compatibility
         Optional(SECTION_AWS, default={}): SECTION_AWS_SCHEMA,
@@ -153,30 +270,261 @@ class Config(object):
         Optional(SECTION_LOCAL, default={}): SECTION_LOCAL_SCHEMA,
     }
 
-    def __init__(self, dvc_dir):
-        self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
-        self.config_file = os.path.join(dvc_dir, self.CONFIG)
-        self.config_local_file = os.path.join(dvc_dir, self.CONFIG_LOCAL)
+    def __init__(self, dvc_dir=None, validate=True):
+        self.system_config_file = os.path.join(self.get_system_config_dir(),
+                                               self.CONFIG)
+        self.global_config_file = os.path.join(self.get_global_config_dir(),
+                                               self.CONFIG)
 
+        if dvc_dir is not None:
+            self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
+            self.config_file = os.path.join(dvc_dir, self.CONFIG)
+            self.config_local_file = os.path.join(dvc_dir, self.CONFIG_LOCAL)
+        else:
+            self.dvc_dir = None
+            self.config_file = None
+            self.config_local_file = None
+
+        self._system_config = None
+        self._global_config = None
+        self._project_config = None
+        self._local_config = None
+
+        self.config = None
+
+        self.load(validate=validate)
+
+    @staticmethod
+    def get_global_config_dir():
+        """Returns global config location. E.g. ~/.config/dvc/config.
+
+        Returns:
+            str: path to the global config directory.
+        """
+        from appdirs import user_config_dir
+        return user_config_dir(appname=Config.APPNAME,
+                               appauthor=Config.APPAUTHOR)
+
+    @staticmethod
+    def get_system_config_dir():
+        """Returns system config location. E.g. /etc/dvc.conf.
+
+        Returns:
+            str: path to the system config directory.
+        """
+        from appdirs import site_config_dir
+        return site_config_dir(appname=Config.APPNAME,
+                               appauthor=Config.APPAUTHOR)
+
+    @staticmethod
+    def init(dvc_dir):
+        """Initializes dvc config.
+
+        Args:
+            dvc_dir (str): path to .dvc directory.
+
+        Returns:
+            dvc.config.Config: config object.
+        """
+        config_file = os.path.join(dvc_dir, Config.CONFIG)
+        open(config_file, 'w+').close()
+        return Config(dvc_dir)
+
+    def _load(self):
+        self._system_config = configobj.ConfigObj(self.system_config_file)
+        self._global_config = configobj.ConfigObj(self.global_config_file)
+
+        if self.config_file is not None:
+            self._project_config = configobj.ConfigObj(self.config_file)
+        else:
+            self._project_config = configobj.ConfigObj()
+
+        if self.config_local_file is not None:
+            self._local_config = configobj.ConfigObj(self.config_local_file)
+        else:
+            self._local_config = configobj.ConfigObj()
+
+        self.config = None
+
+    def _load_config(self, path):
+        config = configobj.ConfigObj(path)
+        config = self._lower(config)
+        self._resolve_paths(config, path)
+        return config
+
+    @staticmethod
+    def _resolve_path(path, config_file):
+        assert os.path.isabs(config_file)
+        config_dir = os.path.dirname(config_file)
+        return os.path.abspath(os.path.join(config_dir, path))
+
+    def _resolve_cache_path(self, config, fname):
+        cache = config.get(self.SECTION_CACHE)
+        if cache is None:
+            return
+
+        cache_dir = cache.get(self.SECTION_CACHE_DIR)
+        if cache_dir is None:
+            return
+
+        cache[self.PRIVATE_CWD] = os.path.dirname(fname)
+
+    def _resolve_paths(self, config, fname):
+        self._resolve_cache_path(config, fname)
+        for section in config.values():
+            if self.SECTION_REMOTE_URL not in section.keys():
+                continue
+
+            section[self.PRIVATE_CWD] = os.path.dirname(fname)
+
+    def load(self, validate=True):
+        """Loads config from all the config files.
+
+        Args:
+            validate (bool): optional flag to tell dvc if it should validate
+                the config or just load it as is. 'True' by default.
+
+
+        Raises:
+            dvc.config.ConfigError: thrown if config has invalid format.
+        """
+        self._load()
         try:
-            self._config = configobj.ConfigObj(self.config_file)
-            local = configobj.ConfigObj(self.config_local_file)
+            self.config = self._load_config(self.system_config_file)
+            user = self._load_config(self.global_config_file)
+            config = self._load_config(self.config_file)
+            local = self._load_config(self.config_local_file)
 
             # NOTE: schema doesn't support ConfigObj.Section validation, so we
             # need to convert our config to dict before passing it to
-            self._config = self._lower(self._config)
-            local = self._lower(local)
-            self._config.update(local)
+            for conf in [user, config, local]:
+                self.config = self._merge(self.config, conf)
 
-            self._config = Schema(self.SCHEMA).validate(self._config)
+            if validate:
+                self.config = Schema(self.SCHEMA).validate(self.config)
 
             # NOTE: now converting back to ConfigObj
-            self._config = configobj.ConfigObj(self._config,
-                                               write_empty_values=True)
-            self._config.filename = self.config_file
+            self.config = configobj.ConfigObj(self.config,
+                                              write_empty_values=True)
+            self.config.filename = self.config_file
+            self._resolve_paths(self.config, self.config_file)
         except Exception as ex:
             raise ConfigError(ex)
 
+    @staticmethod
+    def _get_key(conf, name, add=False):
+        for k in conf.keys():
+            if k.lower() == name.lower():
+                return k
+
+        if add:
+            conf[name] = {}
+            return name
+
+        return None
+
+    def save(self, config=None):
+        """Saves config to config files.
+
+        Args:
+            config (configobj.ConfigObj): optional config object to save.
+
+        Raises:
+            dvc.config.ConfigError: thrown if failed to write config file.
+        """
+        if config is not None:
+            clist = [config]
+        else:
+            clist = [self._system_config,
+                     self._global_config,
+                     self._project_config,
+                     self._local_config]
+
+        for conf in clist:
+            if conf.filename is None:
+                continue
+
+            try:
+                logger.debug("Writing '{}'.".format(conf.filename))
+                dname = os.path.dirname(os.path.abspath(conf.filename))
+                try:
+                    os.makedirs(dname)
+                except OSError as exc:
+                    if exc.errno != errno.EEXIST:
+                        raise
+                conf.write()
+            except Exception as exc:
+                msg = "failed to write config '{}'".format(conf.filename)
+                raise ConfigError(msg, exc)
+
+    @staticmethod
+    def unset(config, section, opt=None):
+        """Unsets specified option and/or section in the config.
+
+        Args:
+            config (configobj.ConfigObj): config to work on.
+            section (str): section name.
+            opt (str): optional option name.
+        """
+        if section not in config.keys():
+            raise ConfigError("section '{}' doesn't exist".format(section))
+
+        if opt is None:
+            del config[section]
+            return
+
+        if opt not in config[section].keys():
+            raise ConfigError("option '{}.{}' doesn't exist".format(section,
+                                                                    opt))
+        del config[section][opt]
+
+        if not config[section]:
+            del config[section]
+
+    @staticmethod
+    def set(config, section, opt, value):
+        """Sets specified option in the config.
+
+        Args:
+            config (configobj.ConfigObj): config to work on.
+            section (str): section name.
+            opt (str): option name.
+            value: value to set option to.
+        """
+        if section not in config.keys():
+            config[section] = {}
+
+        config[section][opt] = value
+
+    @staticmethod
+    def show(config, section, opt):
+        """Prints option value from the config.
+
+        Args:
+            config (configobj.ConfigObj): config to work on.
+            section (str): section name.
+            opt (str): option name.
+        """
+        if section not in config.keys():
+            raise ConfigError("section '{}' doesn't exist".format(section))
+
+        if opt not in config[section].keys():
+            raise ConfigError("option '{}.{}' doesn't exist".format(section,
+                                                                    opt))
+
+        logger.info(config[section][opt])
+
+    @staticmethod
+    def _merge(first, second):
+        res = {}
+        sections = list(first.keys()) + list(second.keys())
+        for section in sections:
+            first_copy = first.get(section, {}).copy()
+            second_copy = second.get(section, {}).copy()
+            first_copy.update(second_copy)
+            res[section] = first_copy
+        return res
+
     @staticmethod
     def _lower(config):
         new_config = {}
@@ -186,9 +534,3 @@ class Config(object):
                 new_s[key.lower()] = value
             new_config[s_key.lower()] = new_s
         return new_config
-
-    @staticmethod
-    def init(dvc_dir):
-        config_file = os.path.join(dvc_dir, Config.CONFIG)
-        open(config_file, 'w+').close()
-        return Config(dvc_dir)

dvc/daemon.py

@@ -0,0 +1,83 @@
+"""Launch `dvc daemon` command in a separate detached process."""
+
+import os
+import sys
+from subprocess import Popen
+
+import dvc.logger as logger
+from dvc.utils import is_binary, fix_env
+
+
+CREATE_NEW_PROCESS_GROUP = 0x00000200
+DETACHED_PROCESS = 0x00000008
+
+
+def _spawn_windows(cmd):
+    from subprocess import STARTUPINFO, STARTF_USESHOWWINDOW
+
+    creationflags = CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS
+
+    startupinfo = STARTUPINFO()
+    startupinfo.dwFlags |= STARTF_USESHOWWINDOW
+
+    Popen(cmd,
+          env=fix_env(),
+          close_fds=True,
+          shell=False,
+          creationflags=creationflags,
+          startupinfo=startupinfo).communicate()
+
+
+def _spawn_posix(cmd):
+    # NOTE: using os._exit instead of sys.exit, because dvc built
+    # with PyInstaller has trouble with SystemExit exeption and throws
+    # errors such as "[26338] Failed to execute script __main__"
+    try:
+        pid = os.fork()
+        if pid > 0:
+            return
+    except OSError:
+        logger.error("failed at first fork")
+        os._exit(1)  # pylint: disable=protected-access
+
+    os.setsid()
+    os.umask(0)
+
+    try:
+        pid = os.fork()
+        if pid > 0:
+            os._exit(0)  # pylint: disable=protected-access
+    except OSError:
+        logger.error("failed at second fork")
+        os._exit(1)  # pylint: disable=protected-access
+
+    sys.stdin.close()
+    sys.stdout.close()
+    sys.stderr.close()
+
+    Popen(cmd, env=fix_env(), close_fds=True, shell=False).communicate()
+
+    os._exit(0)  # pylint: disable=protected-access
+
+
+def daemon(args):
+    """Launch a `dvc daemon` command in a detached process.
+
+    Args:
+        args (list): list of arguments to append to `dvc daemon` command.
+    """
+    cmd = [sys.executable]
+    if not is_binary():
+        cmd += ['-m', 'dvc']
+    cmd += ['daemon', '-q'] + args
+
+    logger.debug("Trying to spawn '{}'".format(cmd))
+
+    if os.name == 'nt':
+        _spawn_windows(cmd)
+    elif os.name == 'posix':
+        _spawn_posix(cmd)
+    else:
+        raise NotImplementedError
+
+    logger.debug("Spawned '{}'".format(cmd))

dvc/dagascii.py

@@ -1,121 +1,361 @@
+"""Draws DAG in ASCII."""
+import sys
 import math
 
-from asciicanvas.asciicanvas import AsciiCanvas
-from asciicanvas.style import Style
-
 from grandalf.graphs import Vertex, Edge, Graph
 from grandalf.layouts import SugiyamaLayout
 from grandalf.routing import route_with_lines, EdgeViewer
 
 
-class Dagascii(object):
-    def __init__(self, vertexes, edges):
-        #
-        # Just a reminder about naming conventions:
-        # +------------X
-        # |
-        # |
-        # |
-        # |
-        # Y
-        #
+class VertexViewer(object):
+    """Class to define vertex box boundaries that will be accounted for during
+    graph building by grandalf.
 
-        V = {v: Vertex(" {} ".format(v)) for v in vertexes}
-        # NOTE: reverting edges to correctly orientate the graph
-        E = [Edge(V[e], V[s]) for s, e in edges]
-        V = V.values()
-        g = Graph(V, E)
+    Args:
+        name (str): name of the vertex.
+    """
+    HEIGHT = 3  # top and bottom box edges + text
 
-        class VertexViewer(object):
-            h = 3  # top and bottom box edges + text
+    def __init__(self, name):
+        # pylint: disable=invalid-name
+        self._h = self.HEIGHT  # top and bottom box edges + text
+        self._w = len(name) + 2  # right and left bottom edges + text
 
-            def __init__(self, name):
-                self.w = len(name) + 2  # right and left bottom edges + text
+    @property
+    def h(self):  # pylint: disable=invalid-name
+        """Height of the box."""
+        return self._h
 
-        for v in V:
-            v.view = VertexViewer(v.data)
+    @property
+    def w(self):  # pylint: disable=invalid-name
+        """Width of the box."""
+        return self._w
 
-        # NOTE: determine min box length to create the best layout
-        minw = min([v.view.w for v in V])
 
-        for e in E:
-            e.view = EdgeViewer()
+class AsciiCanvas(object):
+    """Class for drawing in ASCII.
 
-        sug = SugiyamaLayout(g.C[0])
-        gr = g.C[0]
-        r = list(filter(lambda x: len(x.e_in()) == 0, gr.sV))
+    Args:
+        cols (int): number of columns in the canvas. Should be > 1.
+        lines (int): number of lines in the canvas. Should be > 1.
+    """
+    TIMEOUT = 10
 
-        sug.init_all(roots=r, optimize=True)
+    def __init__(self, cols, lines):
+        assert cols > 1
+        assert lines > 1
 
-        sug.yspace = VertexViewer.h
-        sug.xspace = minw
-        sug.route_edge = route_with_lines
+        self.cols = cols
+        self.lines = lines
 
-        sug.draw()
-
-        self.sug = sug
+        self.canvas = [[' '] * cols for l in range(lines)]
 
     def draw(self):
-        # NOTE: coordinates might me negative, so we need to shift
-        # everything to the positive plane before we actually draw it.
-        Xs = []
-        Ys = []
+        """Draws ASCII canvas on the screen."""
+        if sys.stdout.isatty():  # pragma: no cover
+            from asciimatics.screen import Screen
+            Screen.wrapper(self._do_draw)
+        else:
+            for line in self.canvas:
+                print(''.join(line))
 
-        for v in self.sug.g.sV:
-            # NOTE: moving boxes w/2 to the left
-            Xs.append(v.view.xy[0] - v.view.w/2.0)
-            Xs.append(v.view.xy[0] + v.view.w/2.0)
-            Ys.append(v.view.xy[1])
-            Ys.append(v.view.xy[1] + v.view.h)
+    def _do_draw(self, screen):  # pragma: no cover
+        # pylint: disable=too-many-locals
+        # pylint: disable=too-many-branches, too-many-statements
+        from dvc.system import System
+        from asciimatics.event import KeyboardEvent
 
-        for e in self.sug.g.sE:
-            for xy in e.view._pts:
-                Xs.append(xy[0])
-                Ys.append(xy[1])
+        offset_x = 0
+        offset_y = 0
+        smaxrow, smaxcol = screen.dimensions
+        assert smaxrow > 1
+        assert smaxcol > 1
+        smaxrow -= 1
+        smaxcol -= 1
 
-        minx = min(Xs)
-        miny = min(Ys)
-        maxx = max(Xs)
-        maxy = max(Ys)
+        if self.lines + 1 > smaxrow:
+            max_y = self.lines + 1 - smaxrow
+        else:
+            max_y = 0
 
-        canvas_cols = int(math.ceil(math.ceil(maxx) - math.floor(minx))) + 1
-        canvas_lines = int(round(maxy - miny))
+        if self.cols + 1 > smaxcol:
+            max_x = self.cols + 1 - smaxcol
+        else:
+            max_x = 0
 
-        canvas = AsciiCanvas(canvas_cols, canvas_lines)
+        while True:
+            for y in range(smaxrow + 1):
+                y_index = offset_y + y
+                line = []
+                for x in range(smaxcol + 1):
+                    x_index = offset_x + x
+                    if len(self.canvas) > y_index \
+                       and len(self.canvas[y_index]) > x_index:
+                        line.append(self.canvas[y_index][x_index])
+                    else:
+                        line.append(' ')
+                assert len(line) == (smaxcol + 1)
+                screen.print_at(''.join(line), 0, y)
 
-        # NOTE: first draw edges so that node boxes could overwrite them
-        for e in self.sug.g.sE:
-            assert len(e.view._pts) > 1
-            for index in range(1, len(e.view._pts)):
-                start = e.view._pts[index - 1]
-                end = e.view._pts[index]
+            screen.refresh()
 
-                start_x = int(round(start[0] - minx))
-                start_y = int(round(start[1] - miny))
-                end_x = int(round(end[0] - minx))
-                end_y = int(round(end[1] - miny))
+            # NOTE: get_event() doesn't block by itself,
+            # so we have to do the blocking ourselves.
+            #
+            # NOTE: using this workaround while waiting for PR [1]
+            # to get merged and released. After that need to adjust
+            # asciimatics version requirements.
+            #
+            # [1] https://github.com/peterbrittain/asciimatics/pull/188
+            System.wait_for_input(self.TIMEOUT)
 
-                assert start_x >= 0
-                assert start_y >= 0
-                assert end_x >= 0
-                assert end_y >= 0
+            event = screen.get_event()
+            if not isinstance(event, KeyboardEvent):
+                continue
 
-                canvas.add_line(start_x, start_y,
-                                end_x, end_y,
-                                style=Style('*'))
+            k = event.key_code
+            if k == screen.KEY_DOWN or k == ord('s'):
+                offset_y += 1
+            elif k == screen.KEY_PAGE_DOWN or k == ord('S'):
+                offset_y += smaxrow
+            elif k == screen.KEY_UP or k == ord('w'):
+                offset_y -= 1
+            elif k == screen.KEY_PAGE_UP or k == ord('W'):
+                offset_y -= smaxrow
+            elif k == screen.KEY_RIGHT or k == ord('d'):
+                offset_x += 1
+            elif k == ord('D'):
+                offset_x += smaxcol
+            elif k == screen.KEY_LEFT or k == ord('a'):
+                offset_x -= 1
+            elif k == ord('A'):
+                offset_x -= smaxcol
+            elif k == ord('q') or k == ord('Q'):
+                break
 
-        for v in self.sug.g.sV:
-            # NOTE: moving boxes w/2 to the left
-            x = v.view.xy[0] - v.view.w/2.0
-            y = v.view.xy[1]
+            if offset_y > max_y:
+                offset_y = max_y
+            elif offset_y < 0:
+                offset_y = 0
 
-            canvas.add_nine_patch_rect(int(round(x - minx)),
-                                       int(round(y - miny)),
-                                       v.view.w,
-                                       v.view.h)
+            if offset_x > max_x:
+                offset_x = max_x
+            elif offset_x < 0:
+                offset_x = 0
 
-            canvas.add_text(int(round(x - minx)) + 1,
-                            int(round(y - miny)) + 1,
-                            v.data)
+    def point(self, x, y, char):
+        """Create a point on ASCII canvas.
 
-        canvas.print_out()
+        Args:
+            x (int): x coordinate. Should be >= 0 and < number of columns in
+                the canvas.
+            y (int): y coordinate. Should be >= 0 an < number of lines in the
+                canvas.
+            char (str): character to place in the specified point on the
+                canvas.
+        """
+        assert len(char) == 1
+        assert x >= 0
+        assert x < self.cols
+        assert y >= 0
+        assert y < self.lines
+
+        self.canvas[y][x] = char
+
+    def line(self, x0, y0, x1, y1, char):
+        """Create a line on ASCII canvas.
+
+        Args:
+            x0 (int): x coordinate where the line should start.
+            y0 (int): y coordinate where the line should start.
+            x1 (int): x coordinate where the line should end.
+            y1 (int): y coordinate where the line should end.
+            char (str): character to draw the line with.
+        """
+        # pylint: disable=too-many-arguments, too-many-branches
+        if x0 > x1:
+            x1, x0 = x0, x1
+            y1, y0 = y0, y1
+
+        dx = x1 - x0
+        dy = y1 - y0
+
+        if dx == 0 and dy == 0:
+            self.point(x0, y0, char)
+        elif abs(dx) >= abs(dy):
+            for x in range(x0, x1 + 1):
+                if dx == 0:
+                    y = y0
+                else:
+                    y = y0 + int(round((x - x0) * dy / float((dx))))
+                self.point(x, y, char)
+        elif y0 < y1:
+            for y in range(y0, y1 + 1):
+                if dy == 0:
+                    x = x0
+                else:
+                    x = x0 + int(round((y - y0) * dx / float((dy))))
+                self.point(x, y, char)
+        else:
+            for y in range(y1, y0 + 1):
+                if dy == 0:
+                    x = x0
+                else:
+                    x = x1 + int(round((y - y1) * dx / float((dy))))
+                self.point(x, y, char)
+
+    def text(self, x, y, text):
+        """Print a text on ASCII canvas.
+
+        Args:
+            x (int): x coordinate where the text should start.
+            y (int): y coordinate where the text should start.
+            text (str): string that should be printed.
+        """
+        for i, char in enumerate(text):
+            self.point(x + i, y, char)
+
+    def box(self, x0, y0, width, height):
+        """Create a box on ASCII canvas.
+
+        Args:
+            x0 (int): x coordinate of the box corner.
+            y0 (int): y coordinate of the box corner.
+            width (int): box width.
+            height (int): box height.
+        """
+        assert width > 1
+        assert height > 1
+
+        width -= 1
+        height -= 1
+
+        for x in range(x0, x0 + width):
+            self.point(x, y0, '-')
+            self.point(x, y0 + height, '-')
+
+        for y in range(y0, y0 + height):
+            self.point(x0, y, '|')
+            self.point(x0 + width, y, '|')
+
+        self.point(x0, y0, '+')
+        self.point(x0 + width, y0, '+')
+        self.point(x0, y0 + height, '+')
+        self.point(x0 + width, y0 + height, '+')
+
+
+def _build_sugiyama_layout(vertexes, edges):
+    #
+    # Just a reminder about naming conventions:
+    # +------------X
+    # |
+    # |
+    # |
+    # |
+    # Y
+    #
+
+    vertexes = {v: Vertex(" {} ".format(v)) for v in vertexes}
+    # NOTE: reverting edges to correctly orientate the graph
+    edges = [Edge(vertexes[e], vertexes[s]) for s, e in edges]
+    vertexes = vertexes.values()
+    graph = Graph(vertexes, edges)
+
+    for vertex in vertexes:
+        vertex.view = VertexViewer(vertex.data)
+
+    # NOTE: determine min box length to create the best layout
+    minw = min([v.view.w for v in vertexes])
+
+    for edge in edges:
+        edge.view = EdgeViewer()
+
+    sug = SugiyamaLayout(graph.C[0])
+    graph = graph.C[0]
+    roots = list(filter(lambda x: len(x.e_in()) == 0, graph.sV))
+
+    sug.init_all(roots=roots, optimize=True)
+
+    sug.yspace = VertexViewer.HEIGHT
+    sug.xspace = minw
+    sug.route_edge = route_with_lines
+
+    sug.draw()
+
+    return sug
+
+
+def draw(vertexes, edges):
+    """Build a DAG and draw it in ASCII.
+
+    Args:
+        vertexes (list): list of graph vertexes.
+        edges (list): list of graph edges.
+    """
+    # pylint: disable=too-many-locals
+    # NOTE: coordinates might me negative, so we need to shift
+    # everything to the positive plane before we actually draw it.
+    Xs = []  # pylint: disable=invalid-name
+    Ys = []  # pylint: disable=invalid-name
+
+    sug = _build_sugiyama_layout(vertexes, edges)
+
+    for vertex in sug.g.sV:
+        # NOTE: moving boxes w/2 to the left
+        Xs.append(vertex.view.xy[0] - vertex.view.w/2.0)
+        Xs.append(vertex.view.xy[0] + vertex.view.w/2.0)
+        Ys.append(vertex.view.xy[1])
+        Ys.append(vertex.view.xy[1] + vertex.view.h)
+
+    for edge in sug.g.sE:
+        for x, y in edge.view._pts:  # pylint: disable=protected-access
+            Xs.append(x)
+            Ys.append(y)
+
+    minx = min(Xs)
+    miny = min(Ys)
+    maxx = max(Xs)
+    maxy = max(Ys)
+
+    canvas_cols = int(math.ceil(math.ceil(maxx) - math.floor(minx))) + 1
+    canvas_lines = int(round(maxy - miny))
+
+    canvas = AsciiCanvas(canvas_cols, canvas_lines)
+
+    # NOTE: first draw edges so that node boxes could overwrite them
+    for edge in sug.g.sE:
+        # pylint: disable=protected-access
+        assert len(edge.view._pts) > 1
+        for index in range(1, len(edge.view._pts)):
+            start = edge.view._pts[index - 1]
+            end = edge.view._pts[index]
+
+            start_x = int(round(start[0] - minx))
+            start_y = int(round(start[1] - miny))
+            end_x = int(round(end[0] - minx))
+            end_y = int(round(end[1] - miny))
+
+            assert start_x >= 0
+            assert start_y >= 0
+            assert end_x >= 0
+            assert end_y >= 0
+
+            canvas.line(start_x, start_y,
+                        end_x, end_y,
+                        '*')
+
+    for vertex in sug.g.sV:
+        # NOTE: moving boxes w/2 to the left
+        x = vertex.view.xy[0] - vertex.view.w/2.0
+        y = vertex.view.xy[1]
+
+        canvas.box(int(round(x - minx)),
+                   int(round(y - miny)),
+                   vertex.view.w,
+                   vertex.view.h)
+
+        canvas.text(int(round(x - minx)) + 1,
+                    int(round(y - miny)) + 1,
+                    vertex.data)
+
+    canvas.draw()

dvc/data_cloud.py

@@ -1,15 +1,29 @@
-from dvc.logger import Logger
+"""Manages dvc remotes that user can use with push/pull/status comamnds."""
+
+import dvc.logger as logger
 from dvc.config import Config, ConfigError
 
+from dvc.remote import Remote
 from dvc.remote.s3 import RemoteS3
 from dvc.remote.gs import RemoteGS
 from dvc.remote.azure import RemoteAzure
 from dvc.remote.ssh import RemoteSSH
 from dvc.remote.hdfs import RemoteHDFS
 from dvc.remote.local import RemoteLOCAL
+from dvc.remote.http import RemoteHTTP
 
 
 class DataCloud(object):
+    """Class that manages dvc remotes.
+
+    Args:
+        project (dvc.project.Project): project instance that belongs to the
+            project that we are working on.
+        config (dict): config of the project.
+
+    Raises:
+        config.ConfigError: thrown when config has invalid format.
+    """
     CLOUD_MAP = {
         'aws': RemoteS3,
         'gcp': RemoteGS,
@@ -17,6 +31,8 @@ class DataCloud(object):
         'ssh': RemoteSSH,
         'hdfs': RemoteHDFS,
         'local': RemoteLOCAL,
+        'http': RemoteHTTP,
+        'https': RemoteHTTP,
     }
 
     def __init__(self, project, config=None):
@@ -24,61 +40,56 @@ class DataCloud(object):
         self._config = config
         self._core = self._config[Config.SECTION_CORE]
 
+    @property
+    def _cloud(self):
         remote = self._core.get(Config.SECTION_CORE_REMOTE, '')
-        if remote == '':
-            if self._core.get(Config.SECTION_CORE_CLOUD, None):
-                # backward compatibility
-                msg = 'Using obsoleted config format. Consider updating.'
-                Logger.warn(msg)
-                self._cloud = self.__init__compat()
-            else:
-                self._cloud = None
-            return
+        if remote != '':
+            return self._init_remote(remote)
 
-        self._cloud = self._init_remote(remote)
+        if self._core.get(Config.SECTION_CORE_CLOUD, None):
+            # backward compatibility
+            msg = 'using obsoleted config format. Consider updating.'
+            logger.warning(msg)
+            return self._init_compat()
 
-    @staticmethod
-    def supported(config):
-        for cloud in DataCloud.CLOUD_MAP.values():
-            if cloud.supported(config):
-                return cloud
         return None
 
     def _init_remote(self, remote):
         section = Config.SECTION_REMOTE_FMT.format(remote)
         cloud_config = self._config.get(section, None)
         if not cloud_config:
-            msg = "Can't find remote section '{}' in config"
+            msg = "can't find remote section '{}' in config"
             raise ConfigError(msg.format(section))
 
-        cloud_type = self.supported(cloud_config)
-        if not cloud_type:
-            raise ConfigError("Unsupported cloud '{}'".format(cloud_config))
+        return Remote(self.project, cloud_config)
 
-        return self._init_cloud(cloud_config, cloud_type)
-
-    def __init__compat(self):
+    def _init_compat(self):
         name = self._core.get(Config.SECTION_CORE_CLOUD, '').strip().lower()
         if name == '':
             self._cloud = None
-            return
+            return None
 
         cloud_type = self.CLOUD_MAP.get(name, None)
         if not cloud_type:
-            msg = "Wrong cloud type {} specified"
-            raise ConfigError(msg.format(name))
+            msg = "wrong cloud type '{}' specified".format(name)
+            raise ConfigError(msg)
 
         cloud_config = self._config.get(name, None)
         if not cloud_config:
-            msg = "Can't find cloud section '{}' in config"
-            raise ConfigError(msg.format(name))
+            msg = "can't find cloud section '{}' in config".format(name)
+            raise ConfigError(msg)
+
+        # NOTE: check if the class itself has everything needed for operation.
+        # E.g. all the imported packages.
+        if not cloud_type.supported(cloud_type.compat_config(cloud_config)):
+            raise ConfigError("unsupported cloud '{}'".format(name))
 
         return self._init_cloud(cloud_config, cloud_type)
 
     def _init_cloud(self, cloud_config, cloud_type):
         global_storage_path = self._core.get(Config.SECTION_CORE_STORAGEPATH)
         if global_storage_path:
-            Logger.warn('Using obsoleted config format. Consider updating.')
+            logger.warning('using obsoleted config format. Consider updating.')
 
         cloud = cloud_type(self.project, cloud_config)
         return cloud
@@ -90,16 +101,24 @@ class DataCloud(object):
         if self._cloud:
             return self._cloud
 
-        msg = "No remote repository specified. Setup default " \
-              "repository with\n"\
-              "    dvc config core.remote <name>\n" \
-              "or use:\n" \
-              "    dvc {} -r <name>".format(cmd)
-        raise ConfigError(msg)
+        raise ConfigError(
+            'No remote repository specified. Setup default repository with\n'
+            '    dvc config core.remote <name>\n'
+            'or use:\n'
+            '    dvc {} -r <name>\n'
+            .format(cmd)
+        )
 
-    def push(self, targets, jobs=1, remote=None, show_checksums=False):
-        """
-        Push data items in a cloud-agnostic way.
+    def push(self, targets, jobs=None, remote=None, show_checksums=False):
+        """Push data items in a cloud-agnostic way.
+
+        Args:
+            targets (list): list of targets to push to the cloud.
+            jobs (int): number of jobs that can be running simultaneously.
+            remote (dvc.remote.base.RemoteBase): optional remote to push to.
+                By default remote from core.remote config option is used.
+            show_checksums (bool): show checksums instead of file names in
+                information messages.
         """
         return self.project.cache.local.push(targets,
                                              jobs=jobs,
@@ -107,9 +126,16 @@ class DataCloud(object):
                                                                     'push'),
                                              show_checksums=show_checksums)
 
-    def pull(self, targets, jobs=1, remote=None, show_checksums=False):
-        """
-        Pull data items in a cloud-agnostic way.
+    def pull(self, targets, jobs=None, remote=None, show_checksums=False):
+        """Pull data items in a cloud-agnostic way.
+
+        Args:
+            targets (list): list of targets to pull from the cloud.
+            jobs (int): number of jobs that can be running simultaneously.
+            remote (dvc.remote.base.RemoteBase): optional remote to pull from.
+                By default remote from core.remote config option is used.
+            show_checksums (bool): show checksums instead of file names in
+                information messages.
         """
         return self.project.cache.local.pull(targets,
                                              jobs=jobs,
@@ -117,9 +143,17 @@ class DataCloud(object):
                                                                     'pull'),
                                              show_checksums=show_checksums)
 
-    def status(self, targets, jobs=1, remote=None, show_checksums=False):
-        """
-        Check status of data items in a cloud-agnostic way.
+    def status(self, targets, jobs=None, remote=None, show_checksums=False):
+        """Check status of data items in a cloud-agnostic way.
+
+        Args:
+            targets (list): list of targets to check status for.
+            jobs (int): number of jobs that can be running simultaneously.
+            remote (dvc.remote.base.RemoteBase): optional remote to compare
+                targets to. By default remote from core.remote config option
+                is used.
+            show_checksums (bool): show checksums instead of file names in
+                information messages.
         """
         cloud = self._get_cloud(remote, 'status')
         return self.project.cache.local.status(targets,

dvc/dependency/__init__.py

@@ -1,63 +1,66 @@
 import schema
 
-try:
-    from urlparse import urlparse
-except ImportError:
-    from urllib.parse import urlparse
-
-from dvc.exceptions import DvcException
 from dvc.config import Config
+from dvc.utils.compat import urlparse
 
-from dvc.dependency.base import DependencyBase
+import dvc.output as output
+from dvc.output.base import OutputBase
 from dvc.dependency.s3 import DependencyS3
 from dvc.dependency.gs import DependencyGS
 from dvc.dependency.local import DependencyLOCAL
 from dvc.dependency.hdfs import DependencyHDFS
 from dvc.dependency.ssh import DependencySSH
+from dvc.dependency.http import DependencyHTTP
 
 from dvc.remote import Remote
-from dvc.remote.local import RemoteLOCAL
-from dvc.remote.s3 import RemoteS3
-from dvc.remote.hdfs import RemoteHDFS
 
-DEPS = [DependencyHDFS,
-        DependencyS3,
-        DependencyGS,
-        DependencySSH,
-        DependencyLOCAL]
+DEPS = [
+    DependencyGS,
+    DependencyHDFS,
+    DependencyHTTP,
+    DependencyS3,
+    DependencySSH,
+    # NOTE: DependencyLOCAL is the default choice
+]
 
-DEP_MAP = {'': DependencyLOCAL,
-           'ssh': DependencySSH,
-           's3': DependencyS3,
-           'gs': DependencyGS,
-           'hdfs': DependencyHDFS}
-
-SCHEMA = {
-    DependencyBase.PARAM_PATH: str,
-    schema.Optional(RemoteLOCAL.PARAM_MD5): schema.Or(str, None),
-    schema.Optional(RemoteS3.PARAM_ETAG): schema.Or(str, None),
-    schema.Optional(RemoteHDFS.PARAM_CHECKSUM): schema.Or(str, None),
+DEP_MAP = {
+    'local': DependencyLOCAL,
+    'ssh': DependencySSH,
+    's3': DependencyS3,
+    'gs': DependencyGS,
+    'hdfs': DependencyHDFS,
+    'http': DependencyHTTP,
+    'https': DependencyHTTP,
 }
 
 
+# NOTE: schema for dependencies is basically the same as for outputs, but
+# without output-specific entries like 'cache' (whether or not output is
+# cached, see -o and -O flags for `dvc run`) and 'metric' (whether or not
+# output is a metric file and how to parse it, see `-M` flag for `dvc run`).
+SCHEMA = output.SCHEMA.copy()
+del SCHEMA[schema.Optional(OutputBase.PARAM_CACHE)]
+del SCHEMA[schema.Optional(OutputBase.PARAM_METRIC)]
+
+
 def _get(stage, p, info):
     parsed = urlparse(p)
     if parsed.scheme == 'remote':
         name = Config.SECTION_REMOTE_FMT.format(parsed.netloc)
-        sect = stage.project.config._config[name]
+        sect = stage.project.config.config[name]
         remote = Remote(stage.project, sect)
         return DEP_MAP[remote.scheme](stage, p, info, remote=remote)
 
     for d in DEPS:
         if d.supported(p):
             return d(stage, p, info)
-    raise DvcException('Dependency \'{}\' is not supported'.format(p))
+    return DependencyLOCAL(stage, p, info)
 
 
 def loadd_from(stage, d_list):
     ret = []
     for d in d_list:
-        p = d.pop(DependencyBase.PARAM_PATH)
+        p = d.pop(OutputBase.PARAM_PATH)
         ret.append(_get(stage, p, d))
     return ret
 

dvc/dependency/base.py

@@ -1,72 +1,20 @@
-import re
-
 from dvc.exceptions import DvcException
 
 
-class DependencyError(DvcException):
-    def __init__(self, path, msg):
-        msg = 'Dependency \'{}\' error: {}'
-        super(DependencyError, self).__init__(msg.format(path, msg))
-
-
-class DependencyDoesNotExistError(DependencyError):
+class DependencyDoesNotExistError(DvcException):
     def __init__(self, path):
-        msg = 'does not exist'
-        super(DependencyDoesNotExistError, self).__init__(path, msg)
+        msg = "dependency '{}' does not exist".format(path)
+        super(DependencyDoesNotExistError, self).__init__(msg)
 
 
-class DependencyIsNotFileOrDirError(DependencyError):
+class DependencyIsNotFileOrDirError(DvcException):
     def __init__(self, path):
-        msg = 'not a file or directory'
-        super(DependencyIsNotFileOrDirError, self).__init__(path, msg)
+        msg = "dependency '{}' is not a file or directory".format(path)
+        super(DependencyIsNotFileOrDirError, self).__init__(msg)
 
 
 class DependencyBase(object):
-    REGEX = None
+    IS_DEPENDENCY = True
 
-    PARAM_PATH = 'path'
-
-    def __init__(self, stage, path):
-        self.stage = stage
-        self.project = stage.project
-        self.path = path
-
-    @classmethod
-    def match(cls, url):
-        return re.match(cls.REGEX, url)
-
-    def group(self, name):
-        match = self.match(self.path)
-        if not match:
-            return None
-        return match.group(name)
-
-    @classmethod
-    def supported(cls, url):
-        return cls.match(url) is not None
-
-    @property
-    def sep(self):
-        return '/'
-
-    @property
-    def exists(self):
-        return self.remote.exists([self.path_info])
-
-    def changed(self):
-        raise NotImplementedError
-
-    def status(self):
-        if self.changed():
-            # FIXME better msgs
-            return {self.rel_path: 'changed'}
-        return {}
-
-    def save(self):
-        raise NotImplementedError
-
-    def dumpd(self):
-        return {self.PARAM_PATH: self.path}
-
-    def download(self, to_info):
-        self.remote.download([self.path_info], [to_info])
+    DoesNotExistError = DependencyDoesNotExistError
+    IsNotFileOrDirError = DependencyIsNotFileOrDirError

dvc/dependency/gs.py

@@ -1,25 +1,6 @@
-import posixpath
-
-try:
-    from urlparse import urlparse
-except ImportError:
-    from urllib.parse import urlparse
-
-from dvc.dependency.s3 import DependencyS3
-from dvc.remote.gs import RemoteGS
+from dvc.output.gs import OutputGS
+from dvc.dependency.base import DependencyBase
 
 
-class DependencyGS(DependencyS3):
-    REGEX = RemoteGS.REGEX
-
-    def __init__(self, stage, path, info=None, remote=None):
-        super(DependencyGS, self).__init__(stage, path)
-        self.info = info
-        self.remote = remote if remote else RemoteGS(stage.project, {})
-        bucket = remote.bucket if remote else urlparse(path).netloc
-        key = urlparse(path).path.lstrip('/')
-        if remote:
-            key = posixpath.join(remote.prefix, key)
-        self.path_info = {'scheme': 'gs',
-                          'bucket': bucket,
-                          'key': key}
+class DependencyGS(DependencyBase, OutputGS):
+    pass

dvc/dependency/hdfs.py

@@ -1,35 +1,6 @@
-import posixpath
-
-try:
-    from urlparse import urlparse
-except ImportError:
-    from urllib.parse import urlparse
-
+from dvc.output.hdfs import OutputHDFS
 from dvc.dependency.base import DependencyBase
-from dvc.remote.hdfs import RemoteHDFS
 
 
-class DependencyHDFS(DependencyBase):
-    REGEX = RemoteHDFS.REGEX
-
-    def __init__(self, stage, path, info=None, remote=None):
-        super(DependencyHDFS, self).__init__(stage, path)
-        self.info = info
-        self.remote = remote if remote else RemoteHDFS(stage.project, {})
-        if remote:
-            path = posixpath.join(remote.url, urlparse(path).path.lstrip('/'))
-        user = remote.user if remote else self.group('user')
-        self.path_info = {'scheme': 'hdfs',
-                          'user': user,
-                          'url': path}
-
-    def changed(self):
-        return self.info != self.remote.save_info(self.path_info)
-
-    def save(self):
-        self.info = self.remote.save_info(self.path_info)
-
-    def dumpd(self):
-        ret = self.info
-        ret[self.PARAM_PATH] = self.path
-        return ret
+class DependencyHDFS(DependencyBase, OutputHDFS):
+    pass

dvc/dependency/http.py

@@ -0,0 +1,21 @@
+from dvc.utils.compat import urlparse, urljoin
+from dvc.output.base import OutputBase
+from dvc.remote.http import RemoteHTTP
+from dvc.dependency.base import DependencyBase
+
+
+class DependencyHTTP(DependencyBase, OutputBase):
+    REMOTE = RemoteHTTP
+
+    def __init__(self, stage, path, info=None, remote=None):
+        super(DependencyHTTP, self).__init__(stage,
+                                             path,
+                                             info=info,
+                                             remote=remote)
+        if path.startswith('remote'):
+            path = urljoin(self.remote.cache_dir, urlparse(path).path)
+
+        self.path_info = {
+            'scheme': urlparse(path).scheme,
+            'path': path,
+        }

dvc/dependency/local.py

@@ -1,79 +1,6 @@
-import os
-
-try:
-    from urlparse import urlparse
-except ImportError:
-    from urllib.parse import urlparse
-
 from dvc.dependency.base import DependencyBase
-from dvc.dependency.base import DependencyDoesNotExistError
-from dvc.dependency.base import DependencyIsNotFileOrDirError
-from dvc.logger import Logger
-from dvc.remote.local import RemoteLOCAL
+from dvc.output.local import OutputLOCAL
 
 
-class DependencyLOCAL(DependencyBase):
-    REGEX = r'^(?P<path>(/+|.:\\+)?[^:]*)$'
-
-    DoesNotExistError = DependencyDoesNotExistError
-    IsNotFileOrDirError = DependencyIsNotFileOrDirError
-
-    def __init__(self, stage, path, info=None, remote=None):
-        self.stage = stage
-        self.project = stage.project
-        self.info = info
-        if remote is not None:
-            self.remote = remote
-        else:
-            self.remote = RemoteLOCAL(stage.project, {})
-
-        if remote:
-            path = os.path.join(remote.prefix, urlparse(path).path.lstrip('/'))
-
-        if not os.path.isabs(path):
-            path = self.remote.ospath(path)
-            path = os.path.join(stage.cwd, path)
-        self.path = os.path.abspath(os.path.normpath(path))
-
-        self.path_info = {'scheme': 'local',
-                          'path': self.path}
-
-    @property
-    def sep(self):
-        return os.sep
-
-    @property
-    def rel_path(self):
-        return os.path.relpath(self.path)
-
-    def changed(self):
-        if not self.exists:
-            return True
-
-        info = self.remote.save_info(self.path_info)
-
-        return self.info != info
-
-    def save(self):
-        if not self.exists:
-            raise self.DoesNotExistError(self.rel_path)
-
-        if not os.path.isfile(self.path) and not os.path.isdir(self.path):
-            raise self.IsNotFileOrDirError(self.rel_path)
-
-        if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \
-           (os.path.isdir(self.path) and len(os.listdir(self.path)) == 0):
-            Logger.warn("File/directory '{}' is empty.".format(self.rel_path))
-
-        self.info = self.remote.save_info(self.path_info)
-
-    def dumpd(self):
-        if self.path.startswith(self.stage.project.root_dir):
-            path = self.remote.unixpath(os.path.relpath(self.path,
-                                                        self.stage.cwd))
-        else:
-            path = self.path
-
-        info = self.info.copy()
-        info[self.PARAM_PATH] = path
-        return info
+class DependencyLOCAL(DependencyBase, OutputLOCAL):
+    pass

dvc/dependency/s3.py

@@ -1,37 +1,6 @@
-import posixpath
-
-try:
-    from urlparse import urlparse
-except ImportError:
-    from urllib.parse import urlparse
-
+from dvc.output.s3 import OutputS3
 from dvc.dependency.base import DependencyBase
-from dvc.remote.s3 import RemoteS3
 
 
-class DependencyS3(DependencyBase):
-    REGEX = RemoteS3.REGEX
-
-    def __init__(self, stage, path, info=None, remote=None):
-        super(DependencyS3, self).__init__(stage, path)
-        self.info = info
-        self.remote = remote if remote else RemoteS3(stage.project, {})
-
-        bucket = remote.bucket if remote else urlparse(path).netloc
-        key = urlparse(path).path.lstrip('/')
-        if remote:
-            key = posixpath.join(remote.prefix, key)
-        self.path_info = {'scheme': 's3',
-                          'bucket': bucket,
-                          'key': key}
-
-    def changed(self):
-        return self.info != self.remote.save_info(self.path_info)
-
-    def save(self):
-        self.info = self.remote.save_info(self.path_info)
-
-    def dumpd(self):
-        ret = self.info
-        ret[self.PARAM_PATH] = self.path
-        return ret
+class DependencyS3(DependencyBase, OutputS3):
+    pass

dvc/dependency/ssh.py

@@ -1,42 +1,6 @@
-import posixpath
-
-try:
-    from urlparse import urlparse
-except ImportError:
-    from urllib.parse import urlparse
-
+from dvc.output.ssh import OutputSSH
 from dvc.dependency.base import DependencyBase
-from dvc.remote.ssh import RemoteSSH
 
 
-class DependencySSH(DependencyBase):
-    REGEX = RemoteSSH.REGEX
-
-    def __init__(self, stage, path, info=None, remote=None):
-        super(DependencySSH, self).__init__(stage, path)
-        self.info = info
-        self.remote = remote if remote else RemoteSSH(stage.project, {})
-
-        host = remote.host if remote else self.group('host')
-        user = remote.user if remote else self.group('user')
-        if remote:
-            path = posixpath.join(remote.prefix,
-                                  urlparse(path).path.lstrip('/'))
-        else:
-            path = self.match(self.path).group('path')
-
-        self.path_info = {'scheme': 'ssh',
-                          'host': host,
-                          'user': user,
-                          'path': path}
-
-    def changed(self):
-        return self.info != self.remote.save_info(self.path_info)
-
-    def save(self):
-        self.info = self.remote.save_info(self.path_info)
-
-    def dumpd(self):
-        ret = self.info
-        ret[self.PARAM_PATH] = self.path
-        return ret
+class DependencySSH(DependencyBase, OutputSSH):
+    pass

dvc/exceptions.py

@@ -1,31 +1,133 @@
+"""Exceptions raised by the dvc."""
+
 import traceback
 
 
 class DvcException(Exception):
+    """Base class for all dvc exceptions.
+
+    Args:
+        msg (str): message for this exception.
+        cause (Exception): optional cause exception.
+    """
     def __init__(self, msg, cause=None):
+        # NOTE: unlike python 3, python 2 doesn't have built-in support
+        # for chained exceptions, so we are using our own implementation.
         self.cause = cause
-        self.case_tb = None
+        self.cause_tb = None
         if cause:
             try:
                 self.cause_tb = traceback.format_exc()
-            except Exception:
+            except AttributeError:  # pragma: no cover
                 pass
         super(DvcException, self).__init__(msg)
 
 
-class UnsupportedRemoteError(DvcException):
-    def __init__(self, config):
-        msg = "Remote '{}' is not supported.".format(config)
-        super(UnsupportedRemoteError, self).__init__(msg)
-
-
 class OutputDuplicationError(DvcException):
+    """Thrown if a file/directory is specified as an output in more than one
+    stage.
+
+    Args:
+        output (str): path to the file/directory.
+        stages (list): list of paths to stages.
+    """
     def __init__(self, output, stages):
         assert isinstance(output, str)
         assert isinstance(stages, list)
         assert all(isinstance(stage, str) for stage in stages)
-        msg = "File '{}' is specified as an output in more than one stage:{}"
-        s = ""
-        for stage in stages:
-            s += "\n    " + stage
-        super(OutputDuplicationError, self).__init__(msg.format(output, s))
+        msg = ("file/directory '{}' is specified as an output in more than one"
+               "stage: {}").format(output, "\n    ".join(stages))
+        super(OutputDuplicationError, self).__init__(msg)
+
+
+class WorkingDirectoryAsOutputError(DvcException):
+    """Thrown if directory that stage is going to be saved in is specified as
+    an output of anothe stage.
+
+    Args:
+        cwd (str): path to the directory.
+        fname (str): path to the stage file that has cwd specified as an
+            output.
+    """
+    def __init__(self, cwd, fname):
+        assert isinstance(cwd, str)
+        assert isinstance(fname, str)
+        msg = (
+            "current working directory '{cwd}' is specified as an output in"
+            " '{fname}'. Use another CWD to prevent any data removal."
+            .format(cwd=cwd, fname=fname)
+        )
+        super(WorkingDirectoryAsOutputError, self).__init__(msg)
+
+
+class CircularDependencyError(DvcException):
+    """Thrown if a file/directory specified both as an output and as a
+    dependency.
+
+    Args:
+        dependency (str): path to the dependency.
+    """
+    def __init__(self, dependency):
+        assert isinstance(dependency, str)
+        msg = ("file/directory '{}' is specified as an output and as a "
+               "dependency.")
+        super(CircularDependencyError, self).__init__(msg.format(dependency))
+
+
+class ArgumentDuplicationError(DvcException):
+    """Thrown if a file/directory is specified as a dependency/output more
+    than once.
+
+    Args:
+        path (str): path to the file/directory.
+    """
+    def __init__(self, path):
+        assert isinstance(path, str)
+        msg = "file '{}' is specified more than once."
+        super(ArgumentDuplicationError, self).__init__(msg.format(path))
+
+
+class MoveNotDataSourceError(DvcException):
+    """Thrown if attempted to move a file/directory that is not an output
+    in a data source stage.
+
+    Args:
+        path (str): path to the file/directory.
+    """
+    def __init__(self, path):
+        msg = "move is not permitted for stages that are not data sources. " \
+              "You need to either move '{path}' to a new location and edit " \
+              "it by hand, or remove '{path}' and create a new one at the " \
+              "desired location."
+        super(MoveNotDataSourceError, self).__init__(msg.format(path=path))
+
+
+class NotDvcProjectError(DvcException):
+    """Thrown if a directory is not a dvc project.
+
+    Args:
+        root (str): path to the directory.
+    """
+    def __init__(self, root):
+        msg = "not a dvc repository (checked up to mount point '{}')"
+        super(NotDvcProjectError, self).__init__(msg.format(root))
+
+
+class DvcParserError(DvcException):
+    """Base class for CLI parser errors."""
+    def __init__(self):
+        super(DvcParserError, self).__init__("parser error")
+
+
+class CyclicGraphError(DvcException):
+    def __init__(self, stages):
+        assert isinstance(stages, list)
+        stages = '\n'.join('\t- {}'.format(stage) for stage in stages)
+        msg = (
+            "you've introduced a cycle in your pipeline that involves"
+            ' the following stages:'
+            '\n'
+            '{stages}'
+            .format(stages=stages)
+        )
+        super(CyclicGraphError, self).__init__(msg)

dvc/istextfile.py

@@ -1,19 +1,23 @@
+"""Use heuristics to guess if it is a text file or a binary file."""
+
 # Based on https://eli.thegreenplace.net/2011/10/19/
 # perls-guess-if-file-is-text-or-binary-implemented-in-python
 
 import sys
 PY3 = sys.version_info[0] == 3
 
+
 # A function that takes an integer in the 8-bit range and returns
 # a single-character byte object in py3 / a single-character string
 # in py2.
 #
-int2byte = (lambda x: bytes((x,))) if PY3 else chr
+def _int2byte(i):
+    if PY3:
+        return bytes((i,))
+    return chr(i)
 
 
-_text_characters = (
-        b''.join(int2byte(i) for i in range(32, 127)) +
-        b'\n\r\t\f\b')
+TEXT_CHARS = (b''.join(_int2byte(i) for i in range(32, 127)) + b'\n\r\t\f\b')
 
 
 def istextfile(fname, blocksize=512):
@@ -22,17 +26,18 @@ def istextfile(fname, blocksize=512):
         If more than 30% of the chars in the block are non-text, or there
         are NUL ('\x00') bytes in the block, assume this is a binary file.
     """
-    with open(fname, 'rb') as fd:
-        block = fd.read(blocksize)
+    with open(fname, 'rb') as fobj:
+        block = fobj.read(blocksize)
+
+    if not block:
+        # An empty file is considered a valid text file
+        return True
 
     if b'\x00' in block:
         # Files with null bytes are binary
         return False
-    elif not block:
-        # An empty file is considered a valid text file
-        return True
 
     # Use translate's 'deletechars' argument to efficiently remove all
-    # occurrences of _text_characters from the block
-    nontext = block.translate(None, _text_characters)
+    # occurrences of TEXT_CHARS from the block
+    nontext = block.translate(None, TEXT_CHARS)
     return float(len(nontext)) / len(block) <= 0.30

dvc/lock.py

@@ -1,3 +1,5 @@
+"""Manages dvc lock file."""
+
 import os
 import time
 import zc.lockfile
@@ -6,10 +8,17 @@ from dvc.exceptions import DvcException
 
 
 class LockError(DvcException):
-    pass
+    """Thrown when unable to acquire the lock for dvc project."""
 
 
 class Lock(object):
+    """Class for dvc project lock.
+
+    Args:
+        dvc_dir (str): path to the directory that the lock should be created
+            in.
+        name (str): name of the lock file.
+    """
     LOCK_FILE = 'lock'
     TIMEOUT = 5
 
@@ -21,10 +30,11 @@ class Lock(object):
         try:
             self._lock = zc.lockfile.LockFile(self.lock_file)
         except zc.lockfile.LockError:
-            raise LockError('Cannot perform the cmd since DVC is busy and '
+            raise LockError('cannot perform the cmd since DVC is busy and '
                             'locked. Please retry the cmd later.')
 
     def lock(self):
+        """Acquire lock for dvc project."""
         try:
             self._do_lock()
             return
@@ -34,11 +44,12 @@ class Lock(object):
         self._do_lock()
 
     def unlock(self):
+        """Release lock for dvc project."""
         self._lock.close()
         self._lock = None
 
     def __enter__(self):
         self.lock()
 
-    def __exit__(self, type, value, tb):
+    def __exit__(self, typ, value, tbck):
         self.unlock()

dvc/logger.py

@@ -1,120 +1,296 @@
+"""Manages logger for dvc project."""
+
+import re
 import sys
 import logging
-import colorama
 import traceback
 
+from contextlib import contextmanager
 
-colorama.init()
+import colorama
 
 
-class Logger(object):
-    FMT = '%(message)s'
-    DEFAULT_LEVEL = logging.INFO
+def info(message):
+    """Prints an info message."""
+    logger.info(message)
 
-    LEVEL_MAP = {
-        'debug': logging.DEBUG,
+
+def debug(message):
+    """Prints a debug message."""
+    prefix = colorize('Debug', color='blue')
+
+    out = '{prefix}: {message}'.format(prefix=prefix, message=message)
+
+    logger.debug(out)
+
+
+def warning(message):
+    """Prints a warning message."""
+    prefix = colorize('Warning', color='yellow')
+
+    out = '{prefix}: {message}'.format(prefix=prefix, message=message)
+
+    logger.warning(out)
+
+
+def error(message=None):
+    """Prints an error message."""
+    prefix = colorize('Error', color='red')
+
+    exception, stack_trace = _parse_exc()
+
+    out = (
+        '{prefix}: {description}'
+        '\n'
+        '{stack_trace}'
+        '\n'
+        '{footer}'.format(
+            prefix=prefix,
+            description=_description(message, exception),
+            stack_trace=stack_trace,
+            footer=_footer(),
+        )
+    )
+
+    logger.error(out)
+
+
+def box(message, border_color=None):
+    """Prints a message in a box.
+
+    Args:
+        message (str): message to print.
+        border_color (str): name of a color to outline the box with.
+    """
+    lines = message.split('\n')
+    max_width = max(_visual_width(line) for line in lines)
+
+    padding_horizontal = 5
+    padding_vertical = 1
+
+    box_size_horizontal = (max_width + (padding_horizontal * 2))
+
+    chars = {
+        'corner':     '+',
+        'horizontal': '-',
+        'vertical':   '|',
+        'empty':      ' ',
+    }
+
+    margin = "{corner}{line}{corner}\n".format(
+        corner=chars['corner'],
+        line=chars['horizontal'] * box_size_horizontal,
+    )
+
+    padding_lines = [
+        "{border}{space}{border}\n".format(
+            border=colorize(chars['vertical'], color=border_color),
+            space=chars['empty'] * box_size_horizontal,
+        ) * padding_vertical
+    ]
+
+    content_lines = [
+        "{border}{space}{content}{space}{border}\n".format(
+            border=colorize(chars['vertical'], color=border_color),
+            space=chars['empty'] * padding_horizontal,
+            content=_visual_center(line, max_width),
+        ) for line in lines
+    ]
+
+    box_str = "{margin}{padding}{content}{padding}{margin}".format(
+        margin=colorize(margin, color=border_color),
+        padding=''.join(padding_lines),
+        content=''.join(content_lines),
+    )
+
+    logger.info(box_str)
+
+
+def level():
+    """Returns current log level."""
+    return logger.getEffectiveLevel()
+
+
+def set_level(level_name):
+    """Sets log level.
+
+    Args:
+        level_name (str): log level name. E.g. info, debug, warning, error,
+            critical.
+    """
+    if not level_name:
+        return
+
+    levels = {
         'info': logging.INFO,
-        'warn': logging.WARNING,
-        'error': logging.ERROR
+        'debug': logging.DEBUG,
+        'warning': logging.WARNING,
+        'error': logging.ERROR,
+        'critical': logging.CRITICAL,
     }
 
-    COLOR_MAP = {
-        'debug': colorama.Fore.BLUE,
-        'warn': colorama.Fore.YELLOW,
-        'error': colorama.Fore.RED
+    logger.setLevel(levels.get(level_name))
+
+
+def be_quiet():
+    """Disables all messages except critical ones."""
+    logger.setLevel(logging.CRITICAL)
+
+
+def be_verbose():
+    """Enables all messages."""
+    logger.setLevel(logging.DEBUG)
+
+
+@contextmanager
+def verbose():
+    """Enables verbose mode for the context."""
+    previous_level = level()
+    be_verbose()
+    yield
+    logger.setLevel(previous_level)
+
+
+@contextmanager
+def quiet():
+    """Enables quiet mode for the context."""
+    previous_level = level()
+    be_quiet()
+    yield
+    logger.setLevel(previous_level)
+
+
+def is_quiet():
+    """Returns whether or not all messages except critical ones are
+    disabled.
+    """
+    return level() == logging.CRITICAL
+
+
+def is_verbose():
+    """Returns whether or not all messages are enabled."""
+    return level() == logging.DEBUG
+
+
+def colorize(message, color=None):
+    """Returns a message in a specified color."""
+    if not color:
+        return message
+
+    colors = {
+        'green': colorama.Fore.GREEN,
+        'yellow': colorama.Fore.YELLOW,
+        'blue': colorama.Fore.BLUE,
+        'red': colorama.Fore.RED,
     }
 
-    def __init__(self, loglevel=None):
-        if loglevel:
-            Logger.set_level(loglevel)
+    return u'{color}{message}{nc}'.format(
+        color=colors.get(color, ''),
+        message=message,
+        nc=colorama.Fore.RESET,
+    )
 
-    @staticmethod
-    def init():
-        sh = logging.StreamHandler(sys.stdout)
-        sh.setFormatter(logging.Formatter(Logger.FMT))
-        sh.setLevel(logging.DEBUG)
 
-        Logger.logger().addHandler(sh)
-        Logger.set_level()
+def _init_colorama():
+    colorama.init()
 
-    @staticmethod
-    def logger():
-        return logging.getLogger('dvc')
 
-    @staticmethod
-    def set_level(level=None):
-        if not level:
-            lvl = Logger.DEFAULT_LEVEL
-        else:
-            lvl = Logger.LEVEL_MAP.get(level.lower(), Logger.DEFAULT_LEVEL)
-        Logger.logger().setLevel(lvl)
+def set_default_level():
+    """Sets default log level."""
+    logger.setLevel(logging.INFO)
 
-    @staticmethod
-    def be_quiet():
-        Logger.logger().setLevel(logging.CRITICAL)
 
-    @staticmethod
-    def be_verbose():
-        Logger.logger().setLevel(logging.DEBUG)
+def _add_handlers():
+    formatter = '%(message)s'
 
-    @staticmethod
-    def colorize(msg, typ):
-        header = ''
-        footer = ''
+    class _LogLevelFilter(logging.Filter):
+        # pylint: disable=too-few-public-methods
+        def filter(self, record):
+            return record.levelno <= logging.WARNING
 
-        if sys.stdout.isatty():  # pragma: no cover
-            header = Logger.COLOR_MAP.get(typ.lower(), '')
-            footer = colorama.Style.RESET_ALL
+    sh_out = logging.StreamHandler(sys.stdout)
+    sh_out.setFormatter(logging.Formatter(formatter))
+    sh_out.setLevel(logging.DEBUG)
+    sh_out.addFilter(_LogLevelFilter())
 
-        return u'{}{}{}'.format(header, msg, footer)
+    sh_err = logging.StreamHandler(sys.stderr)
+    sh_err.setFormatter(logging.Formatter(formatter))
+    sh_err.setLevel(logging.ERROR)
 
-    @staticmethod
-    def parse_exc(exc, tb=None):
-        str_tb = tb if tb else None
-        str_exc = ': {}'.format(str(exc)) if exc else ""
+    logger.addHandler(sh_out)
+    logger.addHandler(sh_err)
 
-        if exc and hasattr(exc, 'cause') and exc.cause:
-            cause_tb = exc.cause_tb if hasattr(exc, 'cause_tb') else None
-            cause_str_exc, cause_str_tb = Logger.parse_exc(exc.cause, cause_tb)
 
-            str_tb = cause_str_tb
-            str_exc = '{}{}'.format(str_exc, cause_str_exc)
+def _walk_exc(exc):
+    exc_list = [str(exc)]
+    tb_list = [traceback.format_exc()]
 
-        return (str_exc, str_tb)
+    # NOTE: parsing chained exceptions. See dvc/exceptions.py for more info.
+    while hasattr(exc, 'cause') and exc.cause is not None:
+        exc_list.append(str(exc.cause))
+        if hasattr(exc, 'cause_tb') and exc.cause_tb is not None:
+            tb_list.insert(0, str(exc.cause_tb))
+        exc = exc.cause
 
-    @staticmethod
-    def _prefix(msg, typ):
-        return Logger.colorize('{}: '.format(msg), typ)
+    return (exc_list, tb_list)
 
-    @staticmethod
-    def error_prefix():
-        return Logger._prefix('Error', 'error')
 
-    @staticmethod
-    def warning_prefix():
-        return Logger._prefix('Warning', 'warn')
+def _parse_exc():
+    exc = sys.exc_info()[1]
+    if not exc:
+        return (None, '')
 
-    @staticmethod
-    def debug_prefix():
-        return Logger._prefix('Debug', 'debug')
+    exc_list, tb_list = _walk_exc(exc)
 
-    @staticmethod
-    def error(msg, exc=None):
-        prefix = Logger.error_prefix()
-        str_exc, str_tb = Logger.parse_exc(exc)
-        if Logger.logger().getEffectiveLevel() == logging.DEBUG and exc:
-            str_tb = str_tb if str_tb else traceback.format_exc()
-            Logger.logger().error(prefix + str_tb)
-        return Logger.logger().error(prefix + msg + str_exc)
+    exception = ': '.join(exc_list)
 
-    @staticmethod
-    def warn(msg):
-        return Logger.logger().warn(Logger.warning_prefix() + msg)
+    if is_verbose():
+        stack_trace = '{line}\n{stack_trace}{line}\n'.format(
+            line=colorize('-' * 60, color='red'),
+            stack_trace='\n'.join(tb_list),
+        )
+    else:
+        stack_trace = ''
 
-    @staticmethod
-    def debug(msg):
-        return Logger.logger().debug(Logger.debug_prefix() + msg)
+    return (exception, stack_trace)
 
-    @staticmethod
-    def info(msg):
-        return Logger.logger().info(msg)
+
+def _description(message, exception):
+    if exception and message:
+        description = '{message} - {exception}'
+    elif exception:
+        description = '{exception}'
+    elif message:
+        description = '{message}'
+
+    return description.format(message=message, exception=exception)
+
+
+def _footer():
+    return '{phrase} Hit us up at {url}, we are always happy to help!'.format(
+        phrase=colorize('Having any troubles?', 'yellow'),
+        url=colorize('https://dvc.org/support', 'blue'),
+    )
+
+
+def _visual_width(line):
+    """Get the the number of columns required to display a string"""
+
+    return len(re.sub(colorama.ansitowin32.AnsiToWin32.ANSI_CSI_RE, '', line))
+
+
+def _visual_center(line, width):
+    """Center align string according to it's visual width"""
+
+    spaces = max(width - _visual_width(line), 0)
+    left_padding = int(spaces / 2)
+    right_padding = (spaces - left_padding)
+
+    return (left_padding * ' ') + line + (right_padding * ' ')
+
+
+logger = logging.getLogger('dvc')  # pylint: disable=invalid-name
+
+set_default_level()
+_add_handlers()
+_init_colorama()

dvc/main.py

@@ -1,27 +1,45 @@
-from dvc.logger import Logger
+"""Main entry point for dvc CLI."""
+
+import dvc.logger as logger
 from dvc.cli import parse_args
 from dvc.command.base import CmdBase
+from dvc.analytics import Analytics
+from dvc.exceptions import NotDvcProjectError, DvcParserError
 
 
 def main(argv=None):
-    Logger.init()
+    """Run dvc CLI command.
 
-    args = parse_args(argv)
-
-    # Init loglevel early in case we'll run
-    # into errors before setting it properly
-    CmdBase._set_loglevel(args)
+    Args:
+        argv: optional list of arguments to parse. sys.argv is used by default.
 
+    Returns:
+        int: command's return code.
+    """
+    args = None
+    cmd = None
     try:
+        args = parse_args(argv)
+
+        # Init loglevel early in case we'll run
+        # into errors before setting it properly
+        CmdBase.set_loglevel(args)
+
         cmd = args.func(args)
-    except Exception as ex:
-        Logger.error('Initialization error', ex)
-        return 255
 
-    try:
         ret = cmd.run_cmd()
-    except Exception as ex:
-        Logger.error('Unexpected error', ex)
-        return 254
+    except KeyboardInterrupt:
+        logger.error("interrupted by the user")
+        ret = 252
+    except NotDvcProjectError:
+        logger.error()
+        ret = 253
+    except DvcParserError:
+        ret = 254
+    except Exception:  # pylint: disable=broad-except
+        logger.error('unexpected error')
+        ret = 255
+
+    Analytics().send_cmd(cmd, args, ret)
 
     return ret

dvc/output/__init__.py

@@ -1,10 +1,9 @@
 import schema
 
-from dvc.exceptions import DvcException
 from dvc.config import Config
+from dvc.utils.compat import urlparse
 
-from dvc.dependency import SCHEMA, urlparse
-from dvc.dependency.base import DependencyBase
+from dvc.output.base import OutputBase
 from dvc.output.s3 import OutputS3
 from dvc.output.gs import OutputGS
 from dvc.output.local import OutputLOCAL
@@ -12,25 +11,49 @@ from dvc.output.hdfs import OutputHDFS
 from dvc.output.ssh import OutputSSH
 
 from dvc.remote import Remote
+from dvc.remote.s3 import RemoteS3
+from dvc.remote.hdfs import RemoteHDFS
+from dvc.remote.local import RemoteLOCAL
 
-
-OUTS = [OutputHDFS, OutputS3, OutputGS, OutputSSH, OutputLOCAL]
+OUTS = [
+    OutputHDFS,
+    OutputS3,
+    OutputGS,
+    OutputSSH,
+    # NOTE: OutputLOCAL is the default choice
+]
 
 OUTS_MAP = {'hdfs': OutputHDFS,
             's3': OutputS3,
             'gs': OutputGS,
             'ssh': OutputSSH,
-            '': OutputLOCAL}
+            'local': OutputLOCAL}
 
-SCHEMA[schema.Optional(OutputLOCAL.PARAM_CACHE)] = bool
-SCHEMA[schema.Optional(OutputLOCAL.PARAM_METRIC)] = OutputLOCAL.METRIC_SCHEMA
+SCHEMA = {
+        OutputBase.PARAM_PATH: str,
+
+        # NOTE: currently there are only 3 possible checksum names:
+        #
+        #    1) md5 (LOCAL, SSH, GS);
+        #    2) etag (S3);
+        #    3) checksum (HDFS);
+        #
+        # so when a few types of outputs share the same name, we only need
+        # specify it once.
+        schema.Optional(RemoteLOCAL.PARAM_CHECKSUM): schema.Or(str, None),
+        schema.Optional(RemoteS3.PARAM_CHECKSUM): schema.Or(str, None),
+        schema.Optional(RemoteHDFS.PARAM_CHECKSUM): schema.Or(str, None),
+
+        schema.Optional(OutputBase.PARAM_CACHE): bool,
+        schema.Optional(OutputBase.PARAM_METRIC): OutputBase.METRIC_SCHEMA,
+}
 
 
 def _get(stage, p, info, cache, metric):
     parsed = urlparse(p)
     if parsed.scheme == 'remote':
         name = Config.SECTION_REMOTE_FMT.format(parsed.netloc)
-        sect = stage.project.config._config[name]
+        sect = stage.project.config.config[name]
         remote = Remote(stage.project, sect)
         return OUTS_MAP[remote.scheme](stage,
                                        p,
@@ -42,15 +65,15 @@ def _get(stage, p, info, cache, metric):
     for o in OUTS:
         if o.supported(p):
             return o(stage, p, info, cache=cache, remote=None, metric=metric)
-    raise DvcException('Output \'{}\' is not supported'.format(p))
+    return OutputLOCAL(stage, p, info, cache=cache, remote=None, metric=metric)
 
 
 def loadd_from(stage, d_list):
     ret = []
     for d in d_list:
-        p = d.pop(DependencyBase.PARAM_PATH)
-        cache = d.pop(OutputLOCAL.PARAM_CACHE, True)
-        metric = d.pop(OutputLOCAL.PARAM_METRIC, False)
+        p = d.pop(OutputBase.PARAM_PATH)
+        cache = d.pop(OutputBase.PARAM_CACHE, True)
+        metric = d.pop(OutputBase.PARAM_METRIC, False)
         ret.append(_get(stage, p, info=d, cache=cache, metric=metric))
     return ret
 

dvc/output/base.py

@@ -1,24 +1,184 @@
+import re
+from schema import Or, Optional
+
 from dvc.exceptions import DvcException
 
 
-class OutputError(DvcException):
-    def __init__(self, path, msg):
-        msg = 'Output \'{}\' error: {}'.format(path, msg)
-        super(OutputError, self).__init__(msg)
-
-
-class OutputDoesNotExistError(OutputError):
+class OutputDoesNotExistError(DvcException):
     def __init__(self, path):
-        super(OutputDoesNotExistError, self).__init__(path, 'does not exist')
+        msg = "output '{}' does not exist".format(path)
+        super(OutputDoesNotExistError, self).__init__(msg)
 
 
-class OutputIsNotFileOrDirError(OutputError):
+class OutputIsNotFileOrDirError(DvcException):
     def __init__(self, path):
-        msg = 'not a file or directory'
-        super(OutputIsNotFileOrDirError, self).__init__(path, msg)
+        msg = "output '{}' is not a file or directory".format(path)
+        super(OutputIsNotFileOrDirError, self).__init__(msg)
 
 
-class OutputAlreadyTrackedError(OutputError):
+class OutputAlreadyTrackedError(DvcException):
     def __init__(self, path):
-        msg = 'already tracked by scm(e.g. git)'
-        super(OutputAlreadyTrackedError, self).__init__(path, msg)
+        msg = "output '{}' is already tracked by scm (e.g. git)".format(path)
+        super(OutputAlreadyTrackedError, self).__init__(msg)
+
+
+class OutputBase(object):
+    IS_DEPENDENCY = False
+
+    REMOTE = None
+
+    PARAM_PATH = 'path'
+    PARAM_CACHE = 'cache'
+    PARAM_METRIC = 'metric'
+    PARAM_METRIC_TYPE = 'type'
+    PARAM_METRIC_XPATH = 'xpath'
+
+    METRIC_SCHEMA = Or(None, bool,
+                       {Optional(PARAM_METRIC_TYPE): Or(str, None),
+                        Optional(PARAM_METRIC_XPATH): Or(str, None)})
+
+    DoesNotExistError = OutputDoesNotExistError
+    IsNotFileOrDirError = OutputIsNotFileOrDirError
+
+    def __init__(self,
+                 stage,
+                 path,
+                 info=None,
+                 remote=None,
+                 cache=True,
+                 metric=False):
+        self.stage = stage
+        self.project = stage.project
+        self.url = path
+        self.info = info
+        self.remote = remote or self.REMOTE(self.project, {})
+        self.use_cache = False if self.IS_DEPENDENCY else cache
+        self.metric = False if self.IS_DEPENDENCY else metric
+
+        if self.use_cache and getattr(self.project.cache,
+                                      self.REMOTE.scheme) is None:
+            raise DvcException("no cache location setup for '{}' outputs."
+                               .format(self.REMOTE.scheme))
+
+    def __repr__(self):
+        return "{class_name}: '{url}'".format(
+            class_name=type(self).__name__,
+            url=(self.url or 'No url')
+        )
+
+    def __str__(self):
+        return self.url
+
+    @classmethod
+    def match(cls, url):
+        return re.match(cls.REMOTE.REGEX, url)
+
+    def group(self, name):
+        match = self.match(self.url)
+        if not match:
+            return None
+        return match.group(name)
+
+    @classmethod
+    def supported(cls, url):
+        return cls.match(url) is not None
+
+    @property
+    def scheme(self):
+        return self.REMOTE.scheme
+
+    @property
+    def path(self):
+        return self.path_info['path']
+
+    @property
+    def sep(self):
+        return '/'
+
+    @property
+    def checksum(self):
+        return self.info.get(self.remote.PARAM_CHECKSUM)
+
+    @property
+    def exists(self):
+        return self.remote.exists(self.path_info)
+
+    def changed(self):
+        if not self.exists:
+            return True
+
+        if not self.use_cache:
+            return self.info != self.remote.save_info(self.path_info)
+
+        return getattr(self.project.cache, self.scheme).changed(self.path_info,
+                                                                self.info)
+
+    def changed_cache(self):
+        if not self.use_cache or not self.checksum:
+            return True
+
+        cache = self.project.cache.__getattribute__(self.scheme)
+
+        return cache.changed_cache(self.checksum)
+
+    def status(self):
+        if self.changed():
+            # FIXME better msgs
+            return {str(self): 'changed'}
+        return {}
+
+    def save(self):
+        if not self.use_cache:
+            self.info = self.remote.save_info(self.path_info)
+        else:
+            self.info = getattr(self.project.cache,
+                                self.scheme).save(self.path_info)
+
+    def dumpd(self):
+        ret = self.info.copy()
+        ret[self.PARAM_PATH] = self.url
+
+        if self.IS_DEPENDENCY:
+            return ret
+
+        ret[self.PARAM_CACHE] = self.use_cache
+
+        if isinstance(self.metric, dict):
+            if self.PARAM_METRIC_XPATH in self.metric \
+               and not self.metric[self.PARAM_METRIC_XPATH]:
+                del self.metric[self.PARAM_METRIC_XPATH]
+
+        ret[self.PARAM_METRIC] = self.metric
+
+        return ret
+
+    def download(self, to_info):
+        self.remote.download([self.path_info], [to_info])
+
+    def checkout(self, force=False):
+        if not self.use_cache:
+            return
+
+        getattr(self.project.cache, self.scheme).checkout(self.path_info,
+                                                          self.info,
+                                                          force=force)
+
+    def remove(self, ignore_remove=False):
+        self.remote.remove(self.path_info)
+        if self.scheme != 'local':
+            return
+
+        if ignore_remove and self.use_cache and self.is_local:
+            self.project.scm.ignore_remove(self.path)
+
+    def move(self, out):
+        if self.scheme == 'local' and self.use_cache and self.is_local:
+            self.project.scm.ignore_remove(self.path)
+
+        self.remote.move(self.path_info, out.path_info)
+        self.url = out.url
+        self.path_info = out.path_info
+        self.save()
+
+        if self.scheme == 'local' and self.use_cache and self.is_local:
+            self.project.scm.ignore(self.path)

dvc/output/gs.py

@@ -1,53 +1,6 @@
-from dvc.dependency.gs import DependencyGS
-from dvc.exceptions import DvcException
+from dvc.remote.gs import RemoteGS
+from dvc.output.s3 import OutputS3
 
 
-class OutputGS(DependencyGS):
-    PARAM_CACHE = 'cache'
-    PARAM_METRIC = 'metric'
-
-    def __init__(self,
-                 stage,
-                 path,
-                 info=None,
-                 remote=None,
-                 cache=True,
-                 metric=False):
-        super(OutputGS, self).__init__(stage, path, info, remote=remote)
-        self.use_cache = cache
-        self.metric = metric
-        if cache and self.project.cache.gs is None:
-            raise DvcException("No cache location setup for \'gs\' outputs.")
-
-    def dumpd(self):
-        ret = super(OutputGS, self).dumpd()
-        ret[self.PARAM_CACHE] = self.use_cache
-        ret[self.PARAM_METRIC] = self.metric
-        return ret
-
-    def changed(self):
-        if super(OutputGS, self).changed():
-            return True
-
-        if self.use_cache \
-           and self.project.cache.gs.changed(self.path_info, self.info):
-            return True
-
-        return False
-
-    def checkout(self):
-        if not self.use_cache:
-            return
-
-        self.project.cache.gs.checkout(self.path_info, self.info)
-
-    def save(self):
-        super(OutputGS, self).save()
-
-        if not self.use_cache:
-            return
-
-        self.info = self.project.cache.gs.save(self.path_info)
-
-    def remove(self, ignore_remove=False):
-        self.remote.remove(self.path_info)
+class OutputGS(OutputS3):
+    REMOTE = RemoteGS

dvc/output/hdfs.py

@@ -1,10 +1,12 @@
-from dvc.dependency.hdfs import DependencyHDFS
-from dvc.exceptions import DvcException
+import posixpath
+
+from dvc.utils.compat import urlparse
+from dvc.output.base import OutputBase
+from dvc.remote.hdfs import RemoteHDFS
 
 
-class OutputHDFS(DependencyHDFS):
-    PARAM_CACHE = 'cache'
-    PARAM_METRIC = 'metric'
+class OutputHDFS(OutputBase):
+    REMOTE = RemoteHDFS
 
     def __init__(self,
                  stage,
@@ -13,41 +15,15 @@ class OutputHDFS(DependencyHDFS):
                  remote=None,
                  cache=True,
                  metric=False):
-        super(OutputHDFS, self).__init__(stage, path, info, remote=remote)
-        self.use_cache = cache
-        self.metric = metric
-        if cache and self.project.cache.hdfs is None:
-            raise DvcException("No cache location setup for \'hdfs\' outputs.")
-
-    def dumpd(self):
-        ret = super(OutputHDFS, self).dumpd()
-        ret[self.PARAM_CACHE] = self.use_cache
-        ret[self.PARAM_METRIC] = self.metric
-        return ret
-
-    def changed(self):
-        if super(OutputHDFS, self).changed():
-            return True
-
-        if self.use_cache \
-           and self.project.cache.hdfs.changed(self.path_info, self.info):
-            return True
-
-        return False
-
-    def checkout(self):
-        if not self.use_cache:
-            return
-
-        self.project.cache.hdfs.checkout(self.path_info, self.info)
-
-    def save(self):
-        super(OutputHDFS, self).save()
-
-        if not self.use_cache:
-            return
-
-        self.info = self.project.cache.hdfs.save(self.path_info)
-
-    def remove(self, ignore_remove=False):
-        self.remote.remove(self.path_info)
+        super(OutputHDFS, self).__init__(stage,
+                                         path,
+                                         info=info,
+                                         remote=remote,
+                                         cache=cache,
+                                         metric=metric)
+        if remote:
+            path = posixpath.join(remote.url, urlparse(path).path.lstrip('/'))
+        user = remote.user if remote else self.group('user')
+        self.path_info = {'scheme': 'hdfs',
+                          'user': user,
+                          'path': path}

dvc/output/local.py

@@ -1,25 +1,15 @@
 import os
-from schema import Optional, Or
 
-from dvc.output.base import OutputDoesNotExistError, OutputIsNotFileOrDirError
-from dvc.output.base import OutputAlreadyTrackedError
-from dvc.dependency.local import DependencyLOCAL
-from dvc.exceptions import DvcException
+import dvc.logger as logger
+from dvc.utils.compat import urlparse
 from dvc.istextfile import istextfile
+from dvc.exceptions import DvcException
+from dvc.remote.local import RemoteLOCAL
+from dvc.output.base import OutputBase, OutputAlreadyTrackedError
 
 
-class OutputLOCAL(DependencyLOCAL):
-    PARAM_CACHE = 'cache'
-    PARAM_METRIC = 'metric'
-    PARAM_METRIC_TYPE = 'type'
-    PARAM_METRIC_XPATH = 'xpath'
-
-    METRIC_SCHEMA = Or(None, bool,
-                       {Optional(PARAM_METRIC_TYPE): Or(str, None),
-                        Optional(PARAM_METRIC_XPATH): Or(str, None)})
-
-    DoesNotExistError = OutputDoesNotExistError
-    IsNotFileOrDirError = OutputIsNotFileOrDirError
+class OutputLOCAL(OutputBase):
+    REMOTE = RemoteLOCAL
 
     def __init__(self,
                  stage,
@@ -28,82 +18,98 @@ class OutputLOCAL(DependencyLOCAL):
                  remote=None,
                  cache=True,
                  metric=False):
-        super(OutputLOCAL, self).__init__(stage, path, info, remote=remote)
-        self.use_cache = cache
-        self.metric = metric
+        super(OutputLOCAL, self).__init__(stage,
+                                          path,
+                                          info,
+                                          remote=remote,
+                                          cache=cache,
+                                          metric=metric)
+        if remote:
+            p = os.path.join(remote.prefix,
+                             urlparse(self.url).path.lstrip('/'))
+        else:
+            p = path
 
-    @property
-    def md5(self):
-        return self.info.get(self.project.cache.local.PARAM_MD5, None)
+        if not os.path.isabs(p):
+            p = self.remote.to_ospath(p)
+            p = os.path.join(stage.cwd, p)
+        p = os.path.abspath(os.path.normpath(p))
 
-    @property
-    def cache(self):
-        return self.project.cache.local.get(self.md5)
+        self.path_info = {'scheme': 'local',
+                          'path': p}
+
+    def __str__(self):
+        return self.rel_path
 
     @property
     def is_local(self):
-        return self.path.startswith(self.project.root_dir)
+        return (urlparse(self.url).scheme != 'remote'
+                and not os.path.isabs(self.url))
+
+    @property
+    def sep(self):
+        return os.sep
+
+    @property
+    def rel_path(self):
+        return os.path.relpath(self.path)
+
+    @property
+    def cache(self):
+        return self.project.cache.local.get(self.checksum)
 
     def dumpd(self):
         ret = super(OutputLOCAL, self).dumpd()
-        ret[self.PARAM_CACHE] = self.use_cache
+        if self.is_local:
+            path = self.remote.unixpath(os.path.relpath(self.path,
+                                                        self.stage.cwd))
+        else:
+            path = self.url
 
-        if isinstance(self.metric, dict):
-            if self.PARAM_METRIC_XPATH in self.metric and \
-               not self.metric[self.PARAM_METRIC_XPATH]:
-                del self.metric[self.PARAM_METRIC_XPATH]
-
-        if self.metric:
-            ret[self.PARAM_METRIC] = self.metric
+        ret[self.PARAM_PATH] = path
 
         return ret
 
-    def changed(self):
-        if not self.use_cache:
-            return super(OutputLOCAL, self).changed()
-
-        return self.project.cache.local.changed(self.path_info, self.info)
-
-    def checkout(self):
-        if not self.use_cache:
-            return
-
-        self.project.cache.local.checkout(self.path_info, self.info)
-
     def _verify_metric(self):
         if not self.metric:
             return
 
+        if not os.path.exists(self.path):
+            return
+
         if os.path.isdir(self.path):
-            msg = 'Directory \'{}\' cannot be used as metrics.'
+            msg = "directory '{}' cannot be used as metrics."
             raise DvcException(msg.format(self.rel_path))
 
         if not istextfile(self.path):
-            msg = 'Binary file \'{}\' cannot be used as metrics.'
+            msg = "binary file '{}' cannot be used as metrics."
             raise DvcException(msg.format(self.rel_path))
 
     def save(self):
-        if not self.use_cache:
-            super(OutputLOCAL, self).save()
-            self._verify_metric()
-            msg = 'Output \'{}\' doesn\'t use cache. Skipping saving.'
-            self.project.logger.debug(msg.format(self.rel_path))
-            return
-
         if not os.path.exists(self.path):
             raise self.DoesNotExistError(self.rel_path)
 
-        if not os.path.isfile(self.path) and not os.path.isdir(self.path):
+        if (not os.path.isfile(self.path) and not os.path.isdir(self.path)):
             raise self.IsNotFileOrDirError(self.rel_path)
 
         if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \
            (os.path.isdir(self.path) and len(os.listdir(self.path)) == 0):
-            msg = "File/directory '{}' is empty.".format(self.rel_path)
-            self.project.logger.warn(msg)
+            msg = "file/directory '{}' is empty.".format(self.rel_path)
+            logger.warning(msg)
+
+        if not self.use_cache:
+            self.info = self.remote.save_info(self.path_info)
+            self._verify_metric()
+            if not self.IS_DEPENDENCY:
+                msg = "Output '{}' doesn't use cache. Skipping saving."
+                logger.info(msg.format(self.rel_path))
+            return
+
+        assert not self.IS_DEPENDENCY
 
         if not self.changed():
-            msg = 'Output \'{}\' didn\'t change. Skipping saving.'
-            self.project.logger.debug(msg.format(self.rel_path))
+            msg = "Output '{}' didn't change. Skipping saving."
+            logger.info(msg.format(self.rel_path))
             return
 
         if self.is_local:
@@ -114,20 +120,3 @@ class OutputLOCAL(DependencyLOCAL):
                 self.project.scm.ignore(self.path)
 
         self.info = self.project.cache.local.save(self.path_info)
-
-    def remove(self, ignore_remove=False):
-        self.remote.remove(self.path_info)
-        if ignore_remove and self.use_cache and self.is_local:
-            self.project.scm.ignore_remove(self.path)
-
-    def move(self, out):
-        if self.use_cache and self.is_local:
-            self.project.scm.ignore_remove(self.path)
-
-        self.remote.move(self.path_info, out.path_info)
-        self.path = out.path
-        self.path_info = out.path_info
-        self.save()
-
-        if self.use_cache and self.is_local:
-            self.project.scm.ignore(self.path)

dvc/output/s3.py

@@ -1,10 +1,12 @@
-from dvc.dependency.s3 import DependencyS3
-from dvc.exceptions import DvcException
+import posixpath
+
+from dvc.remote.s3 import RemoteS3
+from dvc.utils.compat import urlparse
+from dvc.output.base import OutputBase
 
 
-class OutputS3(DependencyS3):
-    PARAM_CACHE = 'cache'
-    PARAM_METRIC = 'metric'
+class OutputS3(OutputBase):
+    REMOTE = RemoteS3
 
     def __init__(self,
                  stage,
@@ -13,41 +15,16 @@ class OutputS3(DependencyS3):
                  remote=None,
                  cache=True,
                  metric=False):
-        super(OutputS3, self).__init__(stage, path, info, remote=remote)
-        self.use_cache = cache
-        self.metric = metric
-        if cache and self.project.cache.s3 is None:
-            raise DvcException("No cache location setup for \'s3\' outputs.")
-
-    def dumpd(self):
-        ret = super(OutputS3, self).dumpd()
-        ret[self.PARAM_CACHE] = self.use_cache
-        ret[self.PARAM_METRIC] = self.metric
-        return ret
-
-    def changed(self):
-        if super(OutputS3, self).changed():
-            return True
-
-        if self.use_cache \
-           and self.project.cache.s3.changed(self.path_info, self.info):
-            return True
-
-        return False
-
-    def checkout(self):
-        if not self.use_cache:
-            return
-
-        self.project.cache.s3.checkout(self.path_info, self.info)
-
-    def save(self):
-        super(OutputS3, self).save()
-
-        if not self.use_cache:
-            return
-
-        self.info = self.project.cache.s3.save(self.path_info)
-
-    def remove(self, ignore_remove=False):
-        self.remote.remove(self.path_info)
+        super(OutputS3, self).__init__(stage,
+                                       path,
+                                       info=info,
+                                       remote=remote,
+                                       cache=cache,
+                                       metric=metric)
+        bucket = remote.bucket if remote else urlparse(path).netloc
+        path = urlparse(path).path.lstrip('/')
+        if remote:
+            path = posixpath.join(remote.prefix, path)
+        self.path_info = {'scheme': self.scheme,
+                          'bucket': bucket,
+                          'path': path}

dvc/output/ssh.py

@@ -1,10 +1,13 @@
-from dvc.dependency.ssh import DependencySSH
-from dvc.exceptions import DvcException
+import getpass
+import posixpath
+
+from dvc.utils.compat import urlparse
+from dvc.output.base import OutputBase
+from dvc.remote.ssh import RemoteSSH
 
 
-class OutputSSH(DependencySSH):
-    PARAM_CACHE = 'cache'
-    PARAM_METRIC = 'metric'
+class OutputSSH(OutputBase):
+    REMOTE = RemoteSSH
 
     def __init__(self,
                  stage,
@@ -13,41 +16,27 @@ class OutputSSH(DependencySSH):
                  remote=None,
                  cache=True,
                  metric=False):
-        super(OutputSSH, self).__init__(stage, path, info, remote=remote)
-        self.use_cache = cache
-        self.metric = metric
-        if cache and self.project.cache.ssh is None:
-            raise DvcException("No cache location setup for \'ssh\' outputs.")
+        super(OutputSSH, self).__init__(stage,
+                                        path,
+                                        info=info,
+                                        remote=remote,
+                                        cache=cache,
+                                        metric=metric)
+        parsed = urlparse(path)
+        host = remote.host if remote else parsed.hostname
+        port = remote.port if remote else (parsed.port
+                                           or RemoteSSH.DEFAULT_PORT)
+        user = remote.user if remote else (parsed.username
+                                           or getpass.getuser())
 
-    def dumpd(self):
-        ret = super(OutputSSH, self).dumpd()
-        ret[self.PARAM_CACHE] = self.use_cache
-        ret[self.PARAM_METRIC] = self.metric
-        return ret
+        if remote:
+            path = posixpath.join(remote.prefix,
+                                  urlparse(path).path.lstrip('/'))
+        else:
+            path = parsed.path
 
-    def changed(self):
-        if super(OutputSSH, self).changed():
-            return True
-
-        if self.use_cache \
-           and self.project.cache.ssh.changed(self.path_info, self.info):
-            return True
-
-        return False
-
-    def checkout(self):
-        if not self.use_cache:
-            return
-
-        self.project.cache.ssh.checkout(self.path_info, self.info)
-
-    def save(self):
-        super(OutputSSH, self).save()
-
-        if not self.use_cache:
-            return
-
-        self.info = self.project.cache.ssh.save(self.path_info)
-
-    def remove(self, ignore_remove=False):
-        self.remote.remove(self.path_info)
+        self.path_info = {'scheme': 'ssh',
+                          'host': host,
+                          'port': port,
+                          'user': user,
+                          'path': path}

dvc/progress.py

@@ -1,6 +1,9 @@
+"""Manages progress bars for dvc project."""
+
 from __future__ import print_function
 import sys
 import threading
+import dvc.logger as logger
 
 
 class Progress(object):
@@ -11,42 +14,58 @@ class Progress(object):
         self._n_total = 0
         self._n_finished = 0
         self._lock = threading.Lock()
+        self._line = None
 
     def set_n_total(self, total):
+        """Sets total number of targets."""
         self._n_total = total
         self._n_finished = 0
 
+    @property
+    def is_finished(self):
+        """Returns if all targets have finished."""
+        return self._n_total == self._n_finished
+
     def _clearln(self):
-        print('\r\x1b[K', end='')
+        self._print('\r\x1b[K', end='')
 
     def _writeln(self, line):
         self._clearln()
-        print(line, end='')
+        self._print(line, end='')
         sys.stdout.flush()
 
-    def update_target(self, name, current, total):
+    def refresh(self, line=None):
+        """Refreshes progress bar."""
         # Just go away if it is locked. Will update next time
         if not self._lock.acquire(False):
             return
 
-        bar = self._bar(name, current, total)
+        if line is None:
+            line = self._line
 
-        if sys.stdout.isatty():
-            self._writeln(bar)
+        if sys.stdout.isatty() and line is not None:
+            self._writeln(line)
+            self._line = line
 
         self._lock.release()
 
+    def update_target(self, name, current, total):
+        """Updates progress bar for a specified target."""
+        self.refresh(self._bar(name, current, total))
+
     def finish_target(self, name):
+        """Finishes progress bar for a specified target."""
         # We have to write a msg about finished target
         with self._lock:
-            bar = self._bar(name, 100, 100)
+            pbar = self._bar(name, 100, 100)
 
             if sys.stdout.isatty():
                 self._clearln()
 
-            print(bar)
+            self._print(pbar)
 
             self._n_finished += 1
+            self._line = None
 
     def _bar(self, target_name, current, total):
         """
@@ -56,23 +75,40 @@ class Progress(object):
         bar_len = 30
 
         if total is None:
-            progress = 0
+            state = 0
             percent = "?% "
         else:
             total = int(total)
-            progress = int((100 * current)/total) if current < total else 100
-            percent = str(progress) + "% "
+            state = int((100 * current)/total) if current < total else 100
+            percent = str(state) + "% "
 
-        if self._n_total:
+        if self._n_total > 1:
             num = "({}/{}): ".format(self._n_finished + 1, self._n_total)
         else:
             num = ""
 
-        n_sh = int((progress * bar_len)/100)
+        n_sh = int((state * bar_len)/100)
         n_sp = bar_len - n_sh
-        bar = "[" + '#'*n_sh + ' '*n_sp + "] "
+        pbar = "[" + '#'*n_sh + ' '*n_sp + "] "
 
-        return num + bar + percent + target_name
+        return num + pbar + percent + target_name
+
+    @staticmethod
+    def _print(*args, **kwargs):
+        if logger.is_quiet():
+            return
+
+        print(*args, **kwargs)
+
+    def __enter__(self):
+        self._lock.acquire(True)
+        if self._line is not None:
+            self._clearln()
+
+    def __exit__(self, typ, value, tbck):
+        if self._line is not None:
+            self.refresh()
+        self._lock.release()
 
 
-progress = Progress()
+progress = Progress()  # pylint: disable=invalid-name

dvc/prompt.py

@@ -1,18 +1,54 @@
+"""Manages user prompts."""
+
 import sys
+from getpass import getpass
 
 try:
     # NOTE: in Python3 raw_input() was renamed to input()
-    input = raw_input
+    input = raw_input  # pylint: disable=redefined-builtin, invalid-name
 except NameError:
     pass
 
 
-def prompt(msg, default=False):  # pragma: no cover
+def _ask(prompt, limited_to=None):
     if not sys.stdout.isatty():
-        return default
+        return None
 
-    answer = input(msg + u' (y/n)\n').lower()
-    while answer not in ['yes', 'no', 'y', 'n']:
-        answer = input('Enter \'yes\' or \'no\'.\n').lower()
+    while True:
+        answer = input(prompt + ' ').lower()
 
-    return answer[0] == "y"
+        if not limited_to:
+            return answer
+
+        if answer in limited_to:
+            return answer
+
+        print("Your response must be one of: {options}. Please try again."
+              .format(options=limited_to))
+
+
+def confirm(statement):
+    """Ask the user for confirmation about the specified statement.
+
+    Args:
+        statement (str): statement to ask the user confirmation about.
+
+    Returns:
+        bool: whether or not specified statement was confirmed.
+    """
+    prompt = '{statement} [y/n]'.format(statement=statement)
+    answer = _ask(prompt, limited_to=['yes', 'no', 'y', 'n'])
+    return answer and answer.startswith('y')
+
+
+def password(statement):
+    """Ask the user for a password.
+
+    Args:
+        statement (str): string to prompt the user with.
+
+    Returns:
+        str: password entered by the user.
+    """
+    prompt = '{statement}: '.format(statement=statement)
+    return getpass(prompt)

dvc/remote/__init__.py

@@ -1,27 +1,29 @@
-from dvc.remote.local import RemoteLOCAL
-from dvc.remote.s3 import RemoteS3
+from dvc.remote.azure import RemoteAzure
 from dvc.remote.gs import RemoteGS
 from dvc.remote.hdfs import RemoteHDFS
+from dvc.remote.local import RemoteLOCAL
+from dvc.remote.s3 import RemoteS3
 from dvc.remote.ssh import RemoteSSH
-from dvc.remote.azure import RemoteAzure
-
-from dvc.config import Config
-from dvc.exceptions import UnsupportedRemoteError
+from dvc.remote.http import RemoteHTTP
 
 
-REMOTES = [RemoteHDFS, RemoteSSH, RemoteS3, RemoteGS, RemoteAzure, RemoteLOCAL]
+REMOTES = [
+    RemoteAzure,
+    RemoteGS,
+    RemoteHDFS,
+    RemoteHTTP,
+    RemoteS3,
+    RemoteSSH,
+    # NOTE: RemoteLOCAL is the default
+]
 
 
-def supported_url(url):
-    config = {Config.SECTION_REMOTE_URL: url}
-    for r in REMOTES:
-        if r.supported(config):
-            return True
-    return False
+def _get(config):
+    for remote in REMOTES:
+        if remote.supported(config):
+            return remote
+    return RemoteLOCAL
 
 
 def Remote(project, config):
-    for r in REMOTES:
-        if r.supported(config):
-            return r(project, config)
-    raise UnsupportedRemoteError(str(config))
+    return _get(config)(project, config)

dvc/remote/azure.py

@@ -7,7 +7,8 @@ try:
 except ImportError:
     BlockBlobService = None
 
-from dvc.logger import Logger
+import dvc.logger as logger
+from dvc.utils.compat import urlparse
 from dvc.progress import progress
 from dvc.config import Config
 from dvc.remote.base import RemoteBase
@@ -23,133 +24,81 @@ class Callback(object):
 
 class RemoteAzure(RemoteBase):
     scheme = 'azure'
-    REGEX = (r'^azure://'
+    REGEX = (r'azure://((?P<path>[^=;]*)?|('
+             # backward compatibility
              r'(ContainerName=(?P<container_name>[^;]+);?)?'
-             r'(?P<connection_string>.+)?$')
+             r'(?P<connection_string>.+)?)?)$')
     REQUIRES = {'azure-storage-blob': BlockBlobService}
-    PARAM_ETAG = 'etag'
+    PARAM_CHECKSUM = 'etag'
     COPY_POLL_SECONDS = 5
 
     def __init__(self, project, config):
         super(RemoteAzure, self).__init__(project, config)
-        self.project = project
 
-        self.url = config.get(Config.SECTION_REMOTE_URL)
-        match = re.match(self.REGEX, self.url)
+        self.url = config.get(Config.SECTION_REMOTE_URL, 'azure://')
+        match = re.match(self.REGEX, self.url)  # backward compatibility
 
+        path = match.group('path')
         self.bucket = (
-            match.group('container_name')
+            urlparse(self.url if path else '').netloc
+            or match.group('container_name')  # backward compatibility
             or os.getenv('AZURE_STORAGE_CONTAINER_NAME'))
 
+        self.prefix = urlparse(self.url).path.lstrip('/') if path else ''
+
         self.connection_string = (
-            match.group('connection_string')
+            config.get(Config.SECTION_AZURE_CONNECTION_STRING)
+            or match.group('connection_string')  # backward compatibility
             or os.getenv('AZURE_STORAGE_CONNECTION_STRING'))
 
         if not self.bucket:
-            raise ValueError('Azure Storage container name missing')
+            raise ValueError('azure storage container name missing')
 
         if not self.connection_string:
-            raise ValueError('Azure Storage connection string missing')
+            raise ValueError('azure storage connection string missing')
 
         self.__blob_service = None
 
+        self.path_info = {'scheme': self.scheme, 'bucket': self.bucket}
+
     @property
     def blob_service(self):
         if self.__blob_service is None:
+            logger.debug('URL {}'.format(self.url))
+            logger.debug('Connection string {}'.format(self.connection_string))
             self.__blob_service = BlockBlobService(
                 connection_string=self.connection_string)
+            logger.debug('Container name {}'.format(self.bucket))
             self.__blob_service.create_container(self.bucket)
         return self.__blob_service
 
-# FIXME: temporarily disabled because of the lack of test for external azure
-# dependencies/outputs/cache.
-#
-#    def _get_etag(self, bucket, key):
-#        try:
-#            blob = self.blob_service.get_blob_properties(bucket, key)
-#            return blob.properties.etag
-#        except Exception:
-#            return None
-#
-#    def save_info(self, path_info):
-#        if path_info['scheme'] != self.scheme:
-#            raise NotImplementedError
-#
-#        return {self.PARAM_ETAG: self._get_etag(
-#            path_info['bucket'], path_info['key'])}
-#
-#    def save(self, path_info):
-#        if path_info['scheme'] != self.scheme:
-#            raise NotImplementedError
-#
-#        etag = self._get_etag(path_info['bucket'], path_info['key'])
-#        dest_key = '{}/{}'.format(etag[0:2], etag[2:])
-#
-#        self._copy_blob(to_bucket=self.bucket,
-#                        to_key=dest_key,
-#                        from_bucket=path_info['bucket'],
-#                        from_key=path_info['key'])
-#
-#        return {self.PARAM_ETAG: etag}
-#
-#    def changed(self, path_info, checksum_info):
-#        return checksum_info != self.save_info(path_info)
-#
-#    def _copy_blob(self, to_bucket, to_key, from_bucket, from_key):
-#        source = self.blob_service.make_blob_url(from_bucket, from_key)
-#
-#        copy = self.blob_service.copy_blob(to_bucket, to_key, source)
-#
-#        if self.COPY_POLL_SECONDS <= 0:
-#            return
-#
-#        while copy.status != 'success':
-#            time.sleep(self.COPY_POLL_SECONDS)
-#            copy = self.blob_service.get_blob_properties(
-#                to_bucket, to_key).properties.copy
-#
-#    def checkout(self, path_info, checksum_info):
-#        if path_info['scheme'] != self.scheme:
-#            raise NotImplementedError
-#
-#        etag = checksum_info.get(self.PARAM_ETAG, None)
-#        if not etag:
-#            return
-#
-#        self._copy_blob(to_bucket=path_info['bucket'],
-#                        to_key=path_info['key'],
-#                        from_bucket=self.bucket,
-#                        from_key='{}/{}'.format(etag[0:2], etag[2:]))
-#
-#    def remove(self, path_info):
-#        if path_info['scheme'] != self.scheme:
-#            raise NotImplementedError
-#
-#        Logger.debug('Removing azure://{}/{}'.format(path_info['bucket'],
-#                                                     path_info['key']))
-#
-#        self.blob_service.delete_blob(path_info['bucket'], path_info['key'])
+    def remove(self, path_info):
+        if path_info['scheme'] != self.scheme:
+            raise NotImplementedError
 
-    def md5s_to_path_infos(self, md5s):
-        return [{
-            'scheme': self.scheme,
-            'bucket': self.bucket,
-            'key': '{}/{}'.format(md5[0:2], md5[2:])
-        } for md5 in md5s]
+        logger.debug('Removing azure://{}/{}'.format(path_info['bucket'],
+                                                     path_info['path']))
 
-    def exists(self, path_infos):
-        keys = {blob.name
-                for blob in self.blob_service.list_blobs(self.bucket)}
+        self.blob_service.delete_blob(path_info['bucket'], path_info['path'])
 
-        ret = []
-        for path_info in path_infos:
-            if path_info['scheme'] != self.scheme:
-                raise NotImplementedError
+    def _list_paths(self, bucket, prefix):
+        blob_service = self.blob_service
+        next_marker = None
+        while True:
+            blobs = blob_service.list_blobs(bucket,
+                                            prefix=prefix,
+                                            marker=next_marker)
 
-            key = path_info['key']
-            ret.append(key in keys)
+            for blob in blobs:
+                yield blob.name
 
-        return ret
+            if not blobs.next_marker:
+                break
+
+            next_marker = blobs.next_marker
+
+    def list_cache_paths(self):
+        return self._list_paths(self.bucket, self.prefix)
 
     def upload(self, from_infos, to_infos, names=None):
         names = self._verify_path_args(to_infos, from_infos, names)
@@ -162,10 +111,10 @@ class RemoteAzure(RemoteBase):
                 raise NotImplementedError
 
             bucket = to_info['bucket']
-            key = to_info['key']
+            path = to_info['path']
 
-            Logger.debug("Uploading '{}' to '{}/{}'".format(
-                from_info['path'], bucket, key))
+            logger.debug("Uploading '{}' to '{}/{}'".format(
+                from_info['path'], bucket, path))
 
             if not name:
                 name = os.path.basename(from_info['path'])
@@ -174,10 +123,10 @@ class RemoteAzure(RemoteBase):
 
             try:
                 self.blob_service.create_blob_from_path(
-                    bucket, key, from_info['path'], progress_callback=cb)
-            except Exception as ex:
-                Logger.error("Failed to upload '{}'".format(from_info['path']),
-                             ex)
+                    bucket, path, from_info['path'], progress_callback=cb)
+            except Exception:
+                msg = "failed to upload '{}'".format(from_info['path'])
+                logger.warning(msg)
             else:
                 progress.finish_target(name)
 
@@ -196,10 +145,10 @@ class RemoteAzure(RemoteBase):
                 raise NotImplementedError
 
             bucket = from_info['bucket']
-            key = from_info['key']
+            path = from_info['path']
 
-            Logger.debug("Downloading '{}/{}' to '{}'".format(
-                bucket, key, to_info['path']))
+            logger.debug("Downloading '{}/{}' to '{}'".format(
+                bucket, path, to_info['path']))
 
             tmp_file = self.tmp_file(to_info['path'])
             if not name:
@@ -211,34 +160,12 @@ class RemoteAzure(RemoteBase):
 
             try:
                 self.blob_service.get_blob_to_path(
-                    bucket, key, tmp_file, progress_callback=cb)
-            except Exception as exc:
-                Logger.error("Failed to download '{}/{}'".format(
-                    bucket, key), exc)
+                    bucket, path, tmp_file, progress_callback=cb)
+            except Exception:
+                msg = "failed to download '{}/{}'".format(bucket, path)
+                logger.warning(msg)
             else:
                 os.rename(tmp_file, to_info['path'])
 
                 if not no_progress_bar:
                     progress.finish_target(name)
-
-# FIXME: temporarily disabled because of the lack of test for external azure
-# dependencies/outputs/cache.
-#
-#    def gc(self, cinfos):
-#        used = [info[self.PARAM_ETAG] for info in cinfos['azure']]
-#        used += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
-#
-#        all_blobs = self.blob_service.list_blobs(self.bucket)
-#
-#        removed = False
-#        for blob in all_blobs:
-#            etag = blob.properties.etag
-#            if etag in used:
-#                continue
-#            path_info = {'scheme': self.scheme,
-#                         'key': blob.name,
-#                         'bucket': self.bucket}
-#            self.remove(path_info)
-#            removed = True
-#
-#        return removed

dvc/remote/base.py

@@ -1,9 +1,12 @@
 import os
 import re
 import errno
+import posixpath
+from multiprocessing import cpu_count
 
+import dvc.prompt as prompt
+import dvc.logger as logger
 from dvc.config import Config
-from dvc.logger import Logger
 from dvc.exceptions import DvcException
 
 
@@ -27,40 +30,63 @@ class DataCloudError(DvcException):
         super(DataCloudError, self).__init__('Data sync error: {}'.format(msg))
 
 
+class RemoteBaseCmdError(DvcException):
+    def __init__(self, cmd, ret, err):
+        m = "SSH command '{}' finished with non-zero return code '{}': {}"
+        super(RemoteBaseCmdError, self).__init__(m.format(cmd, ret, err))
+
+
+class RemoteMissingDepsError(DvcException):
+    pass
+
+
 class RemoteBase(object):
+    scheme = None
     REGEX = None
     REQUIRES = {}
+    JOBS = 4 * cpu_count()
 
     def __init__(self, project, config):
-        pass
+        self.project = project
+        deps_ok = all(self.REQUIRES.values())
+        if not deps_ok:
+            missing = [k for k, v in self.REQUIRES.items() if v is None]
+            url = config.get(Config.SECTION_REMOTE_URL,
+                             '{}://'.format(self.scheme))
+            msg = (
+                "URL '{}' is supported but requires these missing "
+                "dependencies: {}. If you have installed dvc using pip, "
+                "choose one of these options to proceed: \n"
+                "\n"
+                "    1) Install specific missing dependencies:\n"
+                "        pip install {}\n"
+                "    2) Install dvc package that includes those missing "
+                "dependencies: \n"
+                "        pip install dvc[{}]\n"
+                "    3) Install dvc package with all possible "
+                "dependencies included: \n"
+                "        pip install dvc[all]\n"
+                "\n"
+                "If you have installed dvc from a binary package and you "
+                "are still seeing this message, please report it to us "
+                "using https://github.com/iterative/dvc/issues. Thank you!"
+                .format(url, missing, " ".join(missing), self.scheme)
+            )
+            raise RemoteMissingDepsError(msg)
+
+    def __repr__(self):
+        return "{class_name}: '{url}'".format(
+            class_name=type(self).__name__,
+            url=(self.url or 'No url')
+        )
+
+    def compat_config(config):
+        return config.copy()
 
     @classmethod
     def supported(cls, config):
         url = config[Config.SECTION_REMOTE_URL]
-        url_ok = cls.match(url)
-        deps_ok = all(cls.REQUIRES.values())
-        if url_ok and not deps_ok:
-            missing = [k for k, v in cls.REQUIRES.items() if v is None]
-            msg = "URL \'{}\' is supported but requires these missing " \
-                  "dependencies: {}. If you have installed dvc using pip, " \
-                  "choose one of these options to proceed: \n" \
-                  "\n" \
-                  "    1) Install specific missing dependencies:\n" \
-                  "        pip install {}\n" \
-                  "    2) Install dvc package that includes those missing " \
-                  "dependencies: \n" \
-                  "        pip install dvc[{}]\n" \
-                  "    3) Install dvc package with all possible " \
-                  "dependencies included: \n" \
-                  "        pip install dvc[all]\n" \
-                  "\n" \
-                  "If you have installed dvc from a binary package and you " \
-                  "are still seeing this message, please report it to us " \
-                  "using https://github.com/iterative/dvc/issues. Thank you!"
-            msg = msg.format(url, missing, " ".join(missing), cls.scheme)
-            Logger.warn(msg)
-
-        return url_ok and deps_ok
+        return cls.match(url) is not None
 
     @classmethod
     def match(cls, url):
@@ -75,25 +101,62 @@ class RemoteBase(object):
     @staticmethod
     def tmp_file(fname):
         """ Temporary name for a partial download """
-        # FIXME probably better use uuid()
-        return fname + '.part'
+        import uuid
+        return fname + '.' + str(uuid.uuid4())
 
     def save_info(self, path_info):
         raise NotImplementedError
 
     def changed(self, path_info, checksum_info):
-        raise NotImplementedError
+        """Checks if data has changed.
+
+        A file is considered changed if:
+            - It doesn't exist on the working directory (was unlinked)
+            - Checksum is not computed (saving a new file)
+            - The checkusm stored in the State is different from the given one
+            - There's no file in the cache
+
+        Args:
+            path_info: dict with path information.
+            checksum: expected checksum for this data.
+
+        Returns:
+            bool: True if data has changed, False otherwise.
+        """
+
+        logger.debug("checking if '{}'('{}') has changed."
+                     .format(path_info, checksum_info))
+
+        if not self.exists(path_info):
+            logger.debug("'{}' doesn't exist.".format(path_info))
+            return True
+
+        checksum = checksum_info.get(self.PARAM_CHECKSUM)
+        if checksum is None:
+            logger.debug("checksum for '{}' is missing.".format(path_info))
+            return True
+
+        if self.changed_cache(checksum):
+            logger.debug("cache for '{}'('{}') has changed."
+                         .format(path_info, checksum))
+            return True
+
+        actual = self.save_info(path_info)[self.PARAM_CHECKSUM]
+        if checksum != actual:
+            logger.debug("checksum '{}'(actual '{}') for '{}' has changed."
+                         .format(checksum, actual, path_info))
+            return True
+
+        logger.debug("'{}' hasn't changed.".format(path_info))
+        return False
 
     def save(self, path_info):
         raise NotImplementedError
 
-    def checkout(self, path_info, checksum_info):
-        raise NotImplementedError
-
     def download(self, from_infos, to_infos, no_progress_bar=False, name=None):
         raise NotImplementedError
 
-    def upload(self, from_infos, to_infos, path_info, name=None):
+    def upload(self, from_infos, to_infos, path_info, names=None):
         raise NotImplementedError
 
     def remove(self, path_info):
@@ -102,6 +165,9 @@ class RemoteBase(object):
     def move(self, path_info):
         raise NotImplementedError
 
+    def copy(self, from_info, to_info):
+        raise NotImplementedError
+
     def _makedirs(self, fname):
         dname = os.path.dirname(fname)
 
@@ -111,9 +177,6 @@ class RemoteBase(object):
             if e.errno != errno.EEXIST:
                 raise
 
-    def md5s_to_path_infos(self, md5s):
-        raise NotImplementedError
-
     def exists(self, path_infos):
         raise NotImplementedError
 
@@ -130,3 +193,147 @@ class RemoteBase(object):
             assert len(names) == len(to_infos)
 
         return names
+
+    @property
+    def ospath(self):
+        return posixpath
+
+    def checksum_to_path(self, checksum):
+        return self.ospath.join(self.prefix, checksum[0:2], checksum[2:])
+
+    def path_to_checksum(self, path):
+        relpath = self.ospath.relpath(path, self.prefix)
+        return self.ospath.dirname(relpath) + self.ospath.basename(relpath)
+
+    def checksum_to_path_info(self, checksum):
+        path_info = self.path_info.copy()
+        path_info['path'] = self.checksum_to_path(checksum)
+        return path_info
+
+    def md5s_to_path_infos(self, md5s):
+        return [
+            self.checksum_to_path_info(md5)
+            for md5 in md5s
+        ]
+
+    def list_cache_paths(self):
+        raise NotImplementedError
+
+    def all(self):
+        # NOTE: The list might be way too big(e.g. 100M entries, md5 for each
+        # is 32 bytes, so ~3200Mb list) and we don't really need all of it at
+        # the same time, so it makes sense to use a generator to gradually
+        # iterate over it, without keeping all of it in memory.
+        return (
+            self.path_to_checksum(path)
+            for path in self.list_cache_paths()
+        )
+
+    def gc(self, cinfos):
+        from dvc.remote.local import RemoteLOCAL
+
+        used = [info[RemoteLOCAL.PARAM_CHECKSUM] for info in cinfos['local']]
+
+        if self.scheme != '':
+            used += [info[self.PARAM_CHECKSUM] for info in cinfos[self.scheme]]
+
+        removed = False
+        for checksum in self.all():
+            if checksum in used:
+                continue
+            path_info = self.checksum_to_path_info(checksum)
+            self.remove(path_info)
+            removed = True
+        return removed
+
+    def changed_cache(self, checksum):
+        cache = self.checksum_to_path_info(checksum)
+        expected = {self.PARAM_CHECKSUM: checksum}
+        actual = self.save_info(cache)
+
+        logger.debug("Cache '{}' actual '{}'.".format(str(expected),
+                                                      str(actual)))
+
+        if expected != actual:
+            if self.exists(cache):
+                msg = 'corrupted cache file {}'
+                logger.warning(msg.format(str(cache)))
+                self.remove(cache)
+            return True
+
+        return False
+
+    def cache_exists(self, checksums):
+        # NOTE: The reason for such an odd logic is that most of the remotes
+        # take much shorter time to just retrieve everything they have under
+        # a certain prefix(e.g. s3, gs, ssh, hdfs). Other remotes that can
+        # check if particular file exists much quicker, use their own
+        # implementation of cache_exists(see http, local).
+        #
+        # Result of all() might be way too big, so we should walk through
+        # it in one pass.
+        return list(filter(lambda checksum: checksum in checksums, self.all()))
+
+    def already_cached(self, path_info):
+        current = self.save_info(path_info)[self.PARAM_CHECKSUM]
+
+        if not current:
+            return False
+
+        return not self.changed_cache(current)
+
+    def safe_remove(self, path_info, force=False):
+        if not self.exists(path_info):
+            return
+
+        if not force and not self.already_cached(path_info):
+            msg = (
+                "file '{}' is going to be removed."
+                ' Are you sure you want to proceed?'
+                .format(str(path_info))
+            )
+
+            if not prompt.confirm(msg):
+                raise DvcException(
+                    "unable to remove '{}' without a confirmation"
+                    " from the user. Use '-f' to force."
+                    .format(str(path_info))
+                )
+
+        self.remove(path_info)
+
+    def do_checkout(self, path_info, checksum, force=False):
+        if self.exists(path_info):
+            msg = "data '{}' exists. Removing before checkout."
+            logger.warning(msg.format(str(path_info)))
+            self.safe_remove(path_info, force=force)
+
+        from_info = self.checksum_to_path_info(checksum)
+        self.copy(from_info, path_info)
+
+    def checkout(self, path_info, checksum_info, force=False):
+        scheme = path_info['scheme']
+        if scheme not in ['', 'local'] and scheme != self.scheme:
+            raise NotImplementedError
+
+        checksum = checksum_info.get(self.PARAM_CHECKSUM)
+        if not checksum:
+            msg = "No checksum info for '{}'."
+            logger.info(msg.format(str(path_info)))
+            return
+
+        if not self.changed(path_info, checksum_info):
+            msg = "Data '{}' didn't change."
+            logger.info(msg.format(str(path_info)))
+            return
+
+        if self.changed_cache(checksum):
+            msg = "Cache '{}' not found. File '{}' won't be created."
+            logger.warning(msg.format(checksum, str(path_info)))
+            self.safe_remove(path_info, force=force)
+            return
+
+        msg = "Checking out '{}' with cache '{}'."
+        logger.info(msg.format(str(path_info), checksum))
+
+        self.do_checkout(path_info, checksum, force=force)

dvc/remote/gs.py

@@ -1,19 +1,13 @@
 import os
-import posixpath
 
 try:
     from google.cloud import storage
 except ImportError:
     storage = None
 
-try:
-    from urlparse import urlparse
-except ImportError:
-    from urllib.parse import urlparse
-
-from dvc.logger import Logger
+import dvc.logger as logger
+from dvc.utils.compat import urlparse
 from dvc.remote.base import RemoteBase
-from dvc.remote.local import RemoteLOCAL
 from dvc.config import Config
 from dvc.progress import progress
 from dvc.exceptions import DvcException
@@ -23,162 +17,103 @@ class RemoteGS(RemoteBase):
     scheme = 'gs'
     REGEX = r'^gs://(?P<path>.*)$'
     REQUIRES = {'google.cloud.storage': storage}
-    PARAM_ETAG = 'etag'
+    PARAM_CHECKSUM = 'md5'
 
     def __init__(self, project, config):
-        self.project = project
+        super(RemoteGS, self).__init__(project, config)
         storagepath = 'gs://'
         storagepath += config.get(Config.SECTION_AWS_STORAGEPATH, '/')
         storagepath.lstrip('/')
         self.url = config.get(Config.SECTION_REMOTE_URL, storagepath)
         self.projectname = config.get(Config.SECTION_GCP_PROJECTNAME, None)
 
-    @property
-    def bucket(self):
-        return urlparse(self.url).netloc
+        parsed = urlparse(self.url)
+        self.bucket = parsed.netloc
+        self.prefix = parsed.path.lstrip('/')
 
-    @property
-    def prefix(self):
-        return urlparse(self.url).path.lstrip('/')
+        self.path_info = {'scheme': 'gs', 'bucket': self.bucket}
+
+    @staticmethod
+    def compat_config(config):
+        ret = config.copy()
+        url = 'gs://' + ret.pop(Config.SECTION_AWS_STORAGEPATH, '').lstrip('/')
+        ret[Config.SECTION_REMOTE_URL] = url
+        return ret
 
     @property
     def gs(self):
         return storage.Client()
 
-    def get_etag(self, bucket, key):
-        blob = self.gs.bucket(bucket).get_blob(key)
+    def get_md5(self, bucket, path):
+        import base64
+        import codecs
+
+        blob = self.gs.bucket(bucket).get_blob(path)
         if not blob:
             return None
 
-        return blob.etag
+        b64_md5 = blob.md5_hash
+        md5 = base64.b64decode(b64_md5)
+        return codecs.getencoder('hex')(md5)[0].decode('utf-8')
 
     def save_info(self, path_info):
         if path_info['scheme'] != 'gs':
             raise NotImplementedError
 
-        return {self.PARAM_ETAG: self.get_etag(path_info['bucket'],
-                                               path_info['key'])}
+        return {self.PARAM_CHECKSUM: self.get_md5(path_info['bucket'],
+                                                  path_info['path'])}
 
-    @staticmethod
-    def to_string(path_info):
-        return "{}://{}/{}".format(path_info['scheme'],
-                                   path_info['bucket'],
-                                   path_info['key'])
-
-    def changed_cache(self, etag):
-        key = posixpath.join(self.prefix, etag[0:2], etag[2:])
-        cache = {'scheme': 'gs', 'bucket': self.bucket, 'key': key}
-
-        if {self.PARAM_ETAG: etag} != self.save_info(cache):
-            if self.exists([cache])[0]:
-                msg = 'Corrupted cache file {}'
-                Logger.warn(msg.format(self.to_string(cache)))
-                self.remove(cache)
-            return True
-        return False
-
-    def changed(self, path_info, checksum_info):
-        if not self.exists([path_info])[0]:
-            return True
-
-        etag = checksum_info.get(self.PARAM_ETAG, None)
-        if etag is None:
-            return True
-
-        if self.changed_cache(etag):
-            return True
-
-        return checksum_info != self.save_info(path_info)
-
-    def _copy(self, from_info, to_info, gs=None):
+    def copy(self, from_info, to_info, gs=None):
         gs = gs if gs else self.gs
 
-        blob = gs.bucket(from_info['bucket']).get_blob(from_info['key'])
+        blob = gs.bucket(from_info['bucket']).get_blob(from_info['path'])
         if not blob:
-            msg = '{} doesn\'t exist in the cloud'
-            raise DvcException(msg.format(from_info['key']))
+            msg = "'{}' doesn't exist in the cloud".format(from_info['path'])
+            raise DvcException(msg)
 
         bucket = self.gs.bucket(to_info['bucket'])
         bucket.copy_blob(blob,
                          self.gs.bucket(to_info['bucket']),
-                         new_name=to_info['key'])
+                         new_name=to_info['path'])
 
     def save(self, path_info):
         if path_info['scheme'] != 'gs':
             raise NotImplementedError
 
-        etag = self.get_etag(path_info['bucket'], path_info['key'])
-        key = posixpath.join(self.prefix, etag[0:2], etag[2:])
-        to_info = {'scheme': 'gs', 'bucket': self.bucket, 'key': key}
+        md5 = self.get_md5(path_info['bucket'], path_info['path'])
+        path = self.checksum_to_path(md5)
+        to_info = {'scheme': 'gs', 'bucket': self.bucket, 'path': path}
 
-        self._copy(path_info, to_info)
+        self.copy(path_info, to_info)
 
-        return {self.PARAM_ETAG: etag}
-
-    def checkout(self, path_info, checksum_info):
-        if path_info['scheme'] != 'gs':
-            raise NotImplementedError
-
-        etag = checksum_info.get(self.PARAM_ETAG, None)
-        if not etag:
-            return
-
-        if not self.changed(path_info, checksum_info):
-            msg = "Data '{}' didn't change."
-            Logger.info(msg.format(self.to_string(path_info)))
-            return
-
-        if self.changed_cache(etag):
-            msg = "Cache '{}' not found. File '{}' won't be created."
-            Logger.warn(msg.format(etag, self.to_string(path_info)))
-            return
-
-        if self.exists([path_info])[0]:
-            msg = "Data '{}' exists. Removing before checkout."
-            Logger.warn(msg.format(self.to_string(path_info)))
-            self.remove(path_info)
-            return
-
-        msg = "Checking out '{}' with cache '{}'."
-        Logger.info(msg.format(self.to_string(path_info), etag))
-
-        key = posixpath.join(self.prefix, etag[0:2], etag[2:])
-        from_info = {'scheme': 'gs', 'bucket': self.bucket, 'key': key}
-
-        self._copy(from_info, path_info)
+        return {self.PARAM_CHECKSUM: md5}
 
     def remove(self, path_info):
         if path_info['scheme'] != 'gs':
             raise NotImplementedError
 
-        Logger.debug("Removing gs://{}/{}".format(path_info['bucket'],
-                                                  path_info['key']))
+        logger.debug("Removing gs://{}/{}".format(path_info['bucket'],
+                                                  path_info['path']))
 
-        blob = self.gs.bucket(path_info['bucket']).get_blob(path_info['key'])
+        blob = self.gs.bucket(path_info['bucket']).get_blob(path_info['path'])
         if not blob:
             return
 
         blob.delete()
 
-    def md5s_to_path_infos(self, md5s):
-        return [{'scheme': 'gs',
-                 'bucket': self.bucket,
-                 'key': posixpath.join(self.prefix,
-                                       md5[0:2], md5[2:])} for md5 in md5s]
+    def _list_paths(self, bucket, prefix):
+        for blob in self.gs.bucket(bucket).list_blobs(prefix=prefix):
+            yield blob.name
 
-    def exists(self, path_infos):
-        ret = []
-        gs = self.gs
+    def list_cache_paths(self):
+        return self._list_paths(self.bucket, self.prefix)
 
-        bucket = gs.bucket(self.bucket)
-        keys = [blob.name for blob in bucket.list_blobs(prefix=self.prefix)]
+    def exists(self, path_info):
+        assert not isinstance(path_info, list)
+        assert path_info['scheme'] == 'gs'
 
-        for path_info in path_infos:
-            exists = False
-            if path_info['key'] in keys:
-                exists = True
-            ret.append(exists)
-        return ret
+        paths = self._list_paths(path_info['bucket'], path_info['path'])
+        return any(path_info['path'] == path for path in paths)
 
     def upload(self, from_infos, to_infos, names=None):
         names = self._verify_path_args(to_infos, from_infos, names)
@@ -192,9 +127,9 @@ class RemoteGS(RemoteBase):
             if from_info['scheme'] != 'local':
                 raise NotImplementedError
 
-            Logger.debug("Uploading '{}' to '{}/{}'".format(from_info['path'],
+            logger.debug("Uploading '{}' to '{}/{}'".format(from_info['path'],
                                                             to_info['bucket'],
-                                                            to_info['key']))
+                                                            to_info['path']))
 
             if not name:
                 name = os.path.basename(from_info['path'])
@@ -203,13 +138,13 @@ class RemoteGS(RemoteBase):
 
             try:
                 bucket = gs.bucket(to_info['bucket'])
-                blob = bucket.blob(to_info['key'])
+                blob = bucket.blob(to_info['path'])
                 blob.upload_from_filename(from_info['path'])
-            except Exception as exc:
-                msg = "Failed to upload '{}' to '{}/{}'"
-                Logger.error(msg.format(from_info['path'],
+            except Exception:
+                msg = "failed to upload '{}' to '{}/{}'"
+                logger.error(msg.format(from_info['path'],
                                         to_info['bucket'],
-                                        to_info['key']), exc)
+                                        to_info['path']))
                 continue
 
             progress.finish_target(name)
@@ -228,16 +163,16 @@ class RemoteGS(RemoteBase):
                 raise NotImplementedError
 
             if to_info['scheme'] == 'gs':
-                self._copy(from_info, to_info, gs=gs)
+                self.copy(from_info, to_info, gs=gs)
                 continue
 
             if to_info['scheme'] != 'local':
                 raise NotImplementedError
 
             msg = "Downloading '{}/{}' to '{}'".format(from_info['bucket'],
-                                                       from_info['key'],
+                                                       from_info['path'],
                                                        to_info['path'])
-            Logger.debug(msg)
+            logger.debug(msg)
 
             tmp_file = self.tmp_file(to_info['path'])
             if not name:
@@ -245,49 +180,23 @@ class RemoteGS(RemoteBase):
 
             if not no_progress_bar:
                 # percent_cb is not available for download_to_filename, so
-                # lets at least update progress at keypoints(start, finish)
+                # lets at least update progress at pathpoints(start, finish)
                 progress.update_target(name, 0, None)
 
             self._makedirs(to_info['path'])
 
             try:
                 bucket = gs.bucket(from_info['bucket'])
-                blob = bucket.get_blob(from_info['key'])
+                blob = bucket.get_blob(from_info['path'])
                 blob.download_to_filename(tmp_file)
-            except Exception as exc:
-                msg = "Failed to download '{}/{}' to '{}'"
-                Logger.error(msg.format(from_info['bucket'],
-                                        from_info['key'],
-                                        to_info['path']), exc)
+            except Exception:
+                msg = "failed to download '{}/{}' to '{}'"
+                logger.error(msg.format(from_info['bucket'],
+                                        from_info['path'],
+                                        to_info['path']))
                 continue
 
             os.rename(tmp_file, to_info['path'])
 
             if not no_progress_bar:
                 progress.finish_target(name)
-
-    def _path_to_etag(self, path):
-        relpath = posixpath.relpath(path, self.prefix)
-        return posixpath.dirname(relpath) + posixpath.basename(relpath)
-
-    def _all_etags(self):
-        blobs = self.gs.bucket(self.bucket).list_blobs(prefix=self.prefix)
-        blobs = list(blobs)
-        return [self._path_to_etag(blob.name) for blob in blobs]
-
-    def gc(self, cinfos):
-        used = [info[self.PARAM_ETAG] for info in cinfos['gs']]
-        used += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
-
-        removed = False
-        for etag in self._all_etags():
-            if etag in used:
-                continue
-            path_info = {'scheme': 'gs',
-                         'bucket': self.bucket,
-                         'key': posixpath.join(self.prefix,
-                                               etag[0:2], etag[2:])}
-            self.remove(path_info)
-            removed = True
-
-        return removed

dvc/remote/hdfs.py

@@ -4,42 +4,53 @@ import getpass
 import posixpath
 from subprocess import Popen, PIPE
 
+import dvc.logger as logger
 from dvc.config import Config
-from dvc.remote.base import RemoteBase
-from dvc.remote.local import RemoteLOCAL
-from dvc.exceptions import DvcException
-from dvc.logger import Logger
+from dvc.remote.base import RemoteBase, RemoteBaseCmdError
 from dvc.utils import fix_env
 
 
+class RemoteHDFSCmdError(RemoteBaseCmdError):
+    pass
+
+
 class RemoteHDFS(RemoteBase):
     scheme = 'hdfs'
     REGEX = r'^hdfs://((?P<user>.*)@)?.*$'
     PARAM_CHECKSUM = 'checksum'
 
     def __init__(self, project, config):
-        self.project = project
+        super(RemoteHDFS, self).__init__(project, config)
         self.url = config.get(Config.SECTION_REMOTE_URL, '/')
+        self.prefix = self.url
         self.user = self.group('user')
         if not self.user:
             self.user = config.get(Config.SECTION_REMOTE_USER,
                                    getpass.getuser())
 
+        self.path_info = {'scheme': 'hdfs', 'user': self.user}
+
     def hadoop_fs(self, cmd, user=None):
         cmd = 'hadoop fs -' + cmd
         if user:
             cmd = 'HADOOP_USER_NAME={} '.format(user) + cmd
+
+        # NOTE: close_fds doesn't work with redirected stdin/stdout/stderr.
+        # See https://github.com/iterative/dvc/issues/1197.
+        close_fds = (os.name != 'nt')
+
+        executable = os.getenv('SHELL') if os.name != 'nt' else None
         p = Popen(cmd,
                   shell=True,
-                  close_fds=True,
-                  executable=os.getenv('SHELL'),
+                  close_fds=close_fds,
+                  executable=executable,
                   env=fix_env(os.environ),
                   stdin=PIPE,
                   stdout=PIPE,
                   stderr=PIPE)
         out, err = p.communicate()
         if p.returncode != 0:
-            raise DvcException('HDFS command failed: {}: {}'.format(cmd, err))
+            raise RemoteHDFSCmdError(cmd, p.returncode, err)
         return out.decode('utf-8')
 
     @staticmethod
@@ -50,146 +61,62 @@ class RemoteHDFS(RemoteBase):
 
     def checksum(self, path_info):
         regex = r'.*\t.*\t(?P<checksum>.*)'
-        stdout = self.hadoop_fs('checksum {}'.format(path_info['url']),
+        stdout = self.hadoop_fs('checksum {}'.format(path_info['path']),
                                 user=path_info['user'])
         return self._group(regex, stdout, 'checksum')
 
-    def cp(self, from_info, to_info):
-        self.hadoop_fs('mkdir -p {}'.format(posixpath.dirname(to_info['url'])),
-                       user=to_info['user'])
-        self.hadoop_fs('cp -f {} {}'.format(from_info['url'], to_info['url']),
+    def copy(self, from_info, to_info):
+        dname = posixpath.dirname(to_info['path'])
+        self.hadoop_fs('mkdir -p {}'.format(dname), user=to_info['user'])
+        self.hadoop_fs('cp -f {} {}'.format(from_info['path'],
+                                            to_info['path']),
                        user=to_info['user'])
 
     def rm(self, path_info):
-        self.hadoop_fs('rm {}'.format(path_info['url']),
+        self.hadoop_fs('rm {}'.format(path_info['path']),
                        user=path_info['user'])
 
     def save_info(self, path_info):
         if path_info['scheme'] != 'hdfs':
             raise NotImplementedError
 
-        assert path_info.get('url')
+        assert path_info.get('path')
 
         return {self.PARAM_CHECKSUM: self.checksum(path_info)}
 
-    @staticmethod
-    def to_string(path_info):
-        return "{}://{}".format(path_info['scheme'],
-                                path_info['url'])
-
-    def changed_cache(self, checksum):
-        cache = {}
-        cache['scheme'] = 'hdfs'
-        cache['user'] = self.user
-        cache['url'] = posixpath.join(self.url, checksum[0:2], checksum[2:])
-
-        if {self.PARAM_CHECKSUM: checksum} != self.save_info(cache):
-            if self.exists([cache])[0]:
-                msg = 'Corrupted cache file {}'
-                Logger.warn(msg.format(self.to_string(cache)))
-                self.remove(cache)
-            return True
-
-        return False
-
-    def changed(self, path_info, checksum_info):
-        if not self.exists([path_info])[0]:
-            return True
-
-        checksum = checksum_info.get(self.PARAM_CHECKSUM, None)
-        if checksum is None:
-            return True
-
-        if self.changed_cache(self, checksum):
-            return True
-
-        return checksum_info != self.save_info(path_info)
-
     def save(self, path_info):
         if path_info['scheme'] != 'hdfs':
             raise NotImplementedError
 
-        assert path_info.get('url')
+        assert path_info.get('path')
 
         checksum = self.checksum(path_info)
         dest = path_info.copy()
-        dest['url'] = posixpath.join(self.url, checksum[0:2], checksum[2:])
+        dest['path'] = self.checksum_to_path(checksum)
 
-        self.cp(path_info, dest)
+        self.copy(path_info, dest)
 
         return {self.PARAM_CHECKSUM: checksum}
 
-    def checkout(self, path_info, checksum_info):
-        if path_info['scheme'] != 'hdfs':
-            raise NotImplementedError
-
-        assert path_info.get('url')
-
-        checksum = checksum_info.get(self.PARAM_CHECKSUM, None)
-        if not checksum:
-            return
-
-        if not self.changed(path_info, checksum_info):
-            msg = "Data '{}' didn't change."
-            Logger.info(msg.format(self.to_string(path_info)))
-            return
-
-        if self.changed_cache(checksum):
-            msg = "Cache '{}' not found. File '{}' won't be created."
-            Logger.warn(msg.format(checksum, self.to_string(path_info)))
-            return
-
-        if self.exists([path_info])[0]:
-            msg = "Data '{}' exists. Removing before checkout."
-            Logger.warn(msg.format(self.to_string(path_info)))
-            self.remove(path_info)
-            return
-
-        msg = "Checking out '{}' with cache '{}'."
-        Logger.info(msg.format(self.to_string(path_info), checksum))
-
-        src = path_info.copy()
-        src['url'] = posixpath.join(self.url, checksum[0:2], checksum[2:])
-
-        self.cp(src, path_info)
-
     def remove(self, path_info):
         if path_info['scheme'] != 'hdfs':
             raise NotImplementedError
 
-        assert path_info.get('url')
+        assert path_info.get('path')
 
-        Logger.debug('Removing {}'.format(path_info['url']))
+        logger.debug('Removing {}'.format(path_info['path']))
 
         self.rm(path_info)
 
-    def md5s_to_path_infos(self, md5s):
-        return [{'scheme': 'hdfs',
-                 'user': self.user,
-                 'url': posixpath.join(self.url,
-                                       md5[0:2], md5[2:])} for md5 in md5s]
+    def exists(self, path_info):
+        assert not isinstance(path_info, list)
+        assert path_info['scheme'] == 'hdfs'
 
-    def exists(self, path_infos):
         try:
-            stdout = self.hadoop_fs('ls -R {}'.format(self.url))
-        except DvcException:
-            return len(path_infos) * [False]
-
-        lines = stdout.split('\n')
-        lurl = []
-        for line in lines:
-            if not line.startswith('-'):
-                continue
-            lurl.append(line.split()[-1])
-
-        ret = []
-        for path_info in path_infos:
-            exists = False
-            if path_info['url'] in lurl:
-                exists = True
-            ret.append(exists)
-
-        return ret
+            self.hadoop_fs('test -e {}'.format(path_info['path']))
+            return True
+        except RemoteHDFSCmdError:
+            return False
 
     def upload(self, from_infos, to_infos, names=None):
         names = self._verify_path_args(to_infos, from_infos, names)
@@ -201,11 +128,11 @@ class RemoteHDFS(RemoteBase):
             if from_info['scheme'] != 'local':
                 raise NotImplementedError
 
-            cmd = 'mkdir -p {}'.format(posixpath.dirname(to_info['url']))
+            cmd = 'mkdir -p {}'.format(posixpath.dirname(to_info['path']))
             self.hadoop_fs(cmd, user=to_info['user'])
 
             cmd = 'copyFromLocal {} {}'.format(from_info['path'],
-                                               to_info['url'])
+                                               to_info['path'])
             self.hadoop_fs(cmd, user=to_info['user'])
 
     def download(self,
@@ -220,7 +147,7 @@ class RemoteHDFS(RemoteBase):
                 raise NotImplementedError
 
             if to_info['scheme'] == 'hdfs':
-                self.cp(from_info, to_info)
+                self.copy(from_info, to_info)
                 continue
 
             if to_info['scheme'] != 'local':
@@ -230,36 +157,21 @@ class RemoteHDFS(RemoteBase):
             if not os.path.exists(dname):
                 os.makedirs(dname)
 
-            cmd = 'copyToLocal {} {}'.format(from_info['url'], to_info['path'])
+            cmd = 'copyToLocal {} {}'.format(from_info['path'],
+                                             to_info['path'])
             self.hadoop_fs(cmd, user=from_info['user'])
 
-    def _path_to_checksum(self, path):
-        relpath = posixpath.relpath(path, self.url)
-        return posixpath.dirname(relpath) + posixpath.basename(relpath)
+    def list_cache_paths(self):
+        try:
+            self.hadoop_fs('test -e {}'.format(self.prefix))
+        except RemoteHDFSCmdError:
+            return []
 
-    def _all_checksums(self):
-        stdout = self.hadoop_fs('ls -R {}'.format(self.url))
+        stdout = self.hadoop_fs('ls -R {}'.format(self.prefix))
         lines = stdout.split('\n')
         flist = []
         for line in lines:
             if not line.startswith('-'):
                 continue
             flist.append(line.split()[-1])
-        return [self._path_to_checksum(path) for path in flist]
-
-    def gc(self, cinfos):
-        used = [info[self.PARAM_CHECKSUM] for info in cinfos['hdfs']]
-        used += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
-
-        removed = False
-        for checksum in self._all_checksums():
-            if checksum in used:
-                continue
-            path_info = {'scheme': 'hdfs',
-                         'user': self.user,
-                         'url': posixpath.join(self.url,
-                                               checksum[0:2], checksum[2:])}
-            self.remove(path_info)
-            removed = True
-
-        return removed
+        return flist

dvc/remote/http.py

@@ -0,0 +1,144 @@
+import os
+import threading
+import requests
+
+import dvc.logger as logger
+from dvc.progress import progress
+from dvc.exceptions import DvcException
+from dvc.config import Config
+from dvc.remote.base import RemoteBase
+
+
+class ProgressBarCallback(object):
+    def __init__(self, name, total):
+        self.name = name
+        self.total = total
+        self.current = 0
+        self.lock = threading.Lock()
+
+    def __call__(self, byts):
+        with self.lock:
+            self.current += byts
+            progress.update_target(self.name, self.current, self.total)
+
+
+class RemoteHTTP(RemoteBase):
+    scheme = 'http'
+    REGEX = r'^https?://.*$'
+    REQUEST_TIMEOUT = 10
+    CHUNK_SIZE = 1000000  # Megabyte
+    PARAM_CHECKSUM = 'etag'
+
+    def __init__(self, project, config):
+        super(RemoteHTTP, self).__init__(project, config)
+        self.cache_dir = config.get(Config.SECTION_REMOTE_URL)
+        self.url = self.cache_dir
+        self.path_info = {'scheme': 'http'}
+
+    @property
+    def prefix(self):
+        return self.cache_dir
+
+    def download(self,
+                 from_infos,
+                 to_infos,
+                 no_progress_bar=False,
+                 names=None):
+        names = self._verify_path_args(to_infos, from_infos, names)
+
+        for to_info, from_info, name in zip(to_infos, from_infos, names):
+            if from_info['scheme'] not in ['http', 'https']:
+                raise NotImplementedError
+
+            if to_info['scheme'] != 'local':
+                raise NotImplementedError
+
+            msg = "Downloading '{}' to '{}'".format(from_info['path'],
+                                                    to_info['path'])
+            logger.debug(msg)
+
+            tmp_file = self.tmp_file(to_info['path'])
+            if not name:
+                name = os.path.basename(to_info['path'])
+
+            self._makedirs(to_info['path'])
+
+            total = self._content_length(from_info['path'])
+
+            if no_progress_bar or not total:
+                cb = None
+            else:
+                cb = ProgressBarCallback(name, total)
+
+            try:
+                self._download_to(from_info['path'], tmp_file, callback=cb)
+            except Exception:
+                msg = "failed to download '{}'".format(from_info['path'])
+                logger.error(msg)
+                continue
+
+            os.rename(tmp_file, to_info['path'])
+
+            if not no_progress_bar:
+                progress.finish_target(name)
+
+    def exists(self, path_info):
+        assert not isinstance(path_info, list)
+        assert path_info['scheme'] in ['http', 'https']
+        return bool(self._request('HEAD', path_info.get('path')))
+
+    def cache_exists(self, md5s):
+        assert isinstance(md5s, list)
+
+        def func(md5):
+            return bool(self._request('HEAD', self.checksum_to_path(md5)))
+
+        return list(filter(func, md5s))
+
+    def save_info(self, path_info):
+        if path_info['scheme'] not in ['http', 'https']:
+            raise NotImplementedError
+
+        return {self.PARAM_CHECKSUM: self._etag(path_info['path'])}
+
+    def _content_length(self, url):
+        return self._request('HEAD', url).headers.get('Content-Length')
+
+    def _etag(self, url):
+        etag = self._request('HEAD', url).headers.get('ETag')
+
+        if not etag:
+            raise DvcException("could not find an ETag for '{}'".format(url))
+
+        if etag.startswith('W/'):
+            raise DvcException(
+                "Weak ETags are not supported."
+                " (Etag: '{etag}', URL: '{url}')".format(etag=etag, url=url)
+            )
+
+        return etag
+
+    def _download_to(self, url, file, callback=None):
+        r = self._request('GET', url, stream=True)
+
+        with open(file, 'wb') as fd:
+            bytes_transfered = 0
+
+            for chunk in r.iter_content(chunk_size=self.CHUNK_SIZE):
+                fd.write(chunk)
+
+                if callback:
+                    bytes_transfered += len(chunk)
+                    callback(bytes_transfered)
+
+    def _request(self, method, url, **kwargs):
+        kwargs.setdefault('allow_redirects', True)
+        kwargs.setdefault('timeout', self.REQUEST_TIMEOUT)
+
+        try:
+            return requests.request(method, url, **kwargs)
+        except requests.exceptions.RequestException:
+            raise DvcException('could not perform a {} request'.format(method))
+
+    def gc(self):
+        raise NotImplementedError

dvc/remote/local.py

@@ -1,4 +1,5 @@
 import os
+import stat
 import uuid
 import json
 import ntpath
@@ -6,9 +7,9 @@ import shutil
 import posixpath
 from operator import itemgetter
 
+import dvc.logger as logger
 from dvc.system import System
-from dvc.remote.base import RemoteBase, STATUS_MAP
-from dvc.logger import Logger
+from dvc.remote.base import RemoteBase, STATUS_MAP, STATUS_NEW, STATUS_DELETED
 from dvc.utils import remove, move, copyfile, dict_md5, to_chunks
 from dvc.utils import LARGE_DIR_SIZE
 from dvc.config import Config
@@ -18,9 +19,9 @@ from concurrent.futures import ThreadPoolExecutor
 
 
 class RemoteLOCAL(RemoteBase):
-    scheme = ''
-    REGEX = r'^(?P<path>(/+|.:\\+).*)$'
-    PARAM_MD5 = 'md5'
+    scheme = 'local'
+    REGEX = r'^(?P<path>.*)$'
+    PARAM_CHECKSUM = 'md5'
     PARAM_PATH = 'path'
     PARAM_RELPATH = 'relpath'
     MD5_DIR_SUFFIX = '.dir'
@@ -34,12 +35,16 @@ class RemoteLOCAL(RemoteBase):
     }
 
     def __init__(self, project, config):
-        self.project = project
-        self.state = self.project.state
-        self.link_state = project.link_state
+        super(RemoteLOCAL, self).__init__(project, config)
+        self.state = self.project.state if self.project else None
+        self.protected = config.get(Config.SECTION_CACHE_PROTECTED, False)
         storagepath = config.get(Config.SECTION_AWS_STORAGEPATH, None)
         self.cache_dir = config.get(Config.SECTION_REMOTE_URL, storagepath)
 
+        if self.cache_dir is not None and not os.path.isabs(self.cache_dir):
+            cwd = config[Config.PRIVATE_CWD]
+            self.cache_dir = os.path.abspath(os.path.join(cwd, self.cache_dir))
+
         types = config.get(Config.SECTION_CACHE_TYPE, None)
         if types:
             if isinstance(types, str):
@@ -51,6 +56,15 @@ class RemoteLOCAL(RemoteBase):
         if self.cache_dir is not None and not os.path.exists(self.cache_dir):
             os.mkdir(self.cache_dir)
 
+        self.path_info = {'scheme': 'local'}
+
+    @staticmethod
+    def compat_config(config):
+        ret = config.copy()
+        url = ret.pop(Config.SECTION_AWS_STORAGEPATH, '')
+        ret[Config.SECTION_REMOTE_URL] = url
+        return ret
+
     @property
     def url(self):
         return self.cache_dir
@@ -59,7 +73,7 @@ class RemoteLOCAL(RemoteBase):
     def prefix(self):
         return self.cache_dir
 
-    def all(self):
+    def list_cache_paths(self):
         clist = []
         for entry in os.listdir(self.cache_dir):
             subdir = os.path.join(self.cache_dir, entry)
@@ -67,8 +81,7 @@ class RemoteLOCAL(RemoteBase):
                 continue
 
             for cache in os.listdir(subdir):
-                path = os.path.join(subdir, cache)
-                clist.append(self.path_to_md5(path))
+                clist.append(os.path.join(subdir, cache))
 
         return clist
 
@@ -78,18 +91,40 @@ class RemoteLOCAL(RemoteBase):
 
         return os.path.join(self.cache_dir, md5[0:2], md5[2:])
 
-    def path_to_md5(self, path):
-        relpath = os.path.relpath(path, self.cache_dir)
-        return os.path.dirname(relpath) + os.path.basename(relpath)
-
-    def changed_cache(self, md5):
+    def changed_cache_file(self, md5):
         cache = self.get(md5)
         if self.state.changed(cache, md5=md5):
             if os.path.exists(cache):
-                msg = 'Corrupted cache file {}'
-                Logger.warn(msg.format(os.path.relpath(cache)))
+                msg = 'Corrupted cache file {}.'
+                logger.warning(msg.format(os.path.relpath(cache)))
                 remove(cache)
             return True
+        return False
+
+    def exists(self, path_info):
+        assert not isinstance(path_info, list)
+        assert path_info['scheme'] == 'local'
+        return os.path.exists(path_info['path'])
+
+    def changed_cache(self, md5):
+        cache = self.get(md5)
+        clist = [(cache, md5)]
+
+        while True:
+            if len(clist) == 0:
+                break
+
+            cache, md5 = clist.pop()
+            if self.changed_cache_file(md5):
+                return True
+
+            if not self.is_dir_cache(cache):
+                continue
+
+            for entry in self.load_dir_cache(md5):
+                md5 = entry[self.PARAM_CHECKSUM]
+                cache = self.get(md5)
+                clist.append((cache, md5))
 
         return False
 
@@ -100,25 +135,45 @@ class RemoteLOCAL(RemoteBase):
         if not os.path.exists(dname):
             os.makedirs(dname)
 
+        # NOTE: just create an empty file for an empty cache
+        if os.path.getsize(cache) == 0:
+            open(path, 'w+').close()
+
+            msg = "Created empty file: {} -> {}".format(cache, path)
+            logger.debug(msg)
+            return
+
         i = len(self.cache_types)
         while i > 0:
             try:
                 self.CACHE_TYPE_MAP[self.cache_types[0]](cache, path)
-                msg = "Created '{}': {} -> {}"
-                Logger.info(msg.format(self.cache_types[0],
-                                       os.path.relpath(cache),
-                                       os.path.relpath(path)))
+
+                if self.protected:
+                    os.chmod(path, stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH)
+
+                msg = "Created {}'{}': {} -> {}".format(
+                    'protected ' if self.protected else '',
+                    self.cache_types[0], cache, path)
+
+                logger.debug(msg)
                 return
-            except Exception as exc:
-                msg = 'Cache type \'{}\' is not supported: {}'
-                Logger.debug(msg.format(self.cache_types[0], str(exc)))
+
+            except DvcException as exc:
+                msg = "Cache type '{}' is not supported: {}"
+                logger.debug(msg.format(self.cache_types[0], str(exc)))
                 del self.cache_types[0]
                 i -= 1
 
-        raise DvcException('No possible cache types left to try out.')
+        raise DvcException('no possible cache types left to try out.')
+
+    @property
+    def ospath(self):
+        if os.name == 'nt':
+            return ntpath
+        return posixpath
 
     @classmethod
-    def ospath(cls, path):
+    def to_ospath(cls, path):
         if os.name == 'nt':
             return cls.ntpath(path)
         return cls.unixpath(path)
@@ -140,12 +195,13 @@ class RemoteLOCAL(RemoteBase):
 
         for root, dirs, files in os.walk(dname):
             bar = False
+
             if len(files) > LARGE_DIR_SIZE:
                 msg = "Computing md5 for a large directory {}. " \
                       "This is only done once."
-                Logger.info(msg.format(os.path.relpath(dname)))
+                logger.info(msg.format(os.path.relpath(root)))
                 bar = True
-                title = os.path.relpath(dname)
+                title = os.path.relpath(root)
                 processed = 0
                 total = len(files)
                 progress.update_target(title, 0, total)
@@ -158,16 +214,9 @@ class RemoteLOCAL(RemoteBase):
                     progress.update_target(title, processed, total)
                     processed += 1
 
-                # FIXME: we could've used md5 = state.update(path, dump=False)
-                # here, but it is around twice as slow(on ssd, don't know about
-                # hdd) for a directory with small files. What we could do here
-                # is introduce some kind of a limit for file size, after which
-                # we would actually register it in our state file.
-                md5 = self.state.update(path, dump=False)
+                md5 = self.state.update(path)
                 dir_info.append({self.PARAM_RELPATH: relpath,
-                                 self.PARAM_MD5: md5})
-
-            self.state.dump()
+                                 self.PARAM_CHECKSUM: md5})
 
             if bar:
                 progress.finish_target(title)
@@ -176,7 +225,7 @@ class RemoteLOCAL(RemoteBase):
         dir_info = sorted(dir_info, key=itemgetter(self.PARAM_RELPATH))
 
         md5 = dict_md5(dir_info) + self.MD5_DIR_SUFFIX
-        if self.changed_cache(md5):
+        if self.changed_cache_file(md5):
             self.dump_dir_cache(md5, dir_info)
 
         return (md5, dir_info)
@@ -189,18 +238,18 @@ class RemoteLOCAL(RemoteBase):
         try:
             with open(path, 'r') as fd:
                 d = json.load(fd)
-        except Exception as exc:
-            msg = u'Failed to load dir cache \'{}\''
-            Logger.error(msg.format(os.path.relpath(path)), exc)
+        except Exception:
+            msg = u"Failed to load dir cache '{}'"
+            logger.error(msg.format(os.path.relpath(path)))
             return []
 
         if not isinstance(d, list):
-            msg = u'Dir cache file format error \'{}\': skipping the file'
-            Logger.error(msg.format(os.path.relpath(path)))
+            msg = u"dir cache file format error '{}' [skipping the file]"
+            logger.error(msg.format(os.path.relpath(path)))
             return []
 
         for info in d:
-            info['relpath'] = self.ospath(info['relpath'])
+            info['relpath'] = self.to_ospath(info['relpath'])
 
         return d
 
@@ -225,38 +274,17 @@ class RemoteLOCAL(RemoteBase):
     def is_dir_cache(cls, cache):
         return cache.endswith(cls.MD5_DIR_SUFFIX)
 
-    def checkout(self, path_info, checksum_info):
+    def do_checkout(self, path_info, checksum, force=False):
         path = path_info['path']
-        md5 = checksum_info.get(self.PARAM_MD5, None)
+        md5 = checksum
         cache = self.get(md5)
 
-        if not cache:
-            msg = 'No cache info for \'{}\'. Skipping checkout.'
-            Logger.warn(msg.format(os.path.relpath(path)))
-            return
-
-        if not self.changed(path_info, checksum_info):
-            msg = "Data '{}' didn't change."
-            Logger.info(msg.format(os.path.relpath(path)))
-            return
-
-        if self.changed_cache(md5):
-            msg = u'Cache \'{}\' not found. File \'{}\' won\'t be created.'
-            Logger.warn(msg.format(md5, os.path.relpath(path)))
-            remove(path)
-            return
-
-        if os.path.exists(path):
-            msg = u'Data \'{}\' exists. Removing before checkout.'
-            Logger.warn(msg.format(os.path.relpath(path)))
-            remove(path)
-
-        msg = u'Checking out \'{}\' with cache \'{}\'.'
-        Logger.info(msg.format(os.path.relpath(path), md5))
-
         if not self.is_dir_cache(cache):
+            if os.path.exists(path):
+                self.safe_remove(path_info, force=force)
+
             self.link(cache, path)
-            self.link_state.update(path)
+            self.state.update_link(path)
             return
 
         # Create dir separately so that dir is created
@@ -264,13 +292,65 @@ class RemoteLOCAL(RemoteBase):
         if not os.path.exists(path):
             os.makedirs(path)
 
-        for entry in self.load_dir_cache(md5):
-            md5 = entry[self.PARAM_MD5]
-            c = self.get(md5)
+        dir_info = self.load_dir_cache(md5)
+        dir_relpath = os.path.relpath(path)
+        dir_size = len(dir_info)
+        bar = dir_size > LARGE_DIR_SIZE
+
+        logger.info("Linking directory '{}'.".format(dir_relpath))
+
+        for processed, entry in enumerate(dir_info):
             relpath = entry[self.PARAM_RELPATH]
+            m = entry[self.PARAM_CHECKSUM]
             p = os.path.join(path, relpath)
-            self.link(c, p)
-        self.link_state.update(path)
+            c = self.get(m)
+
+            entry_info = {'scheme': path_info['scheme'], self.PARAM_PATH: p}
+
+            entry_checksum_info = {self.PARAM_CHECKSUM: m}
+
+            if self.changed(entry_info, entry_checksum_info):
+                if os.path.exists(p):
+                    self.safe_remove(entry_info, force=force)
+
+                self.link(c, p)
+
+            if bar:
+                progress.update_target(dir_relpath, processed, dir_size)
+
+        self._discard_working_directory_changes(path, dir_info, force=force)
+
+        self.state.update_link(path)
+
+        if bar:
+            progress.finish_target(dir_relpath)
+
+    def already_cached(self, path_info):
+        assert path_info['scheme'] in ['', 'local']
+
+        current_md5 = self.state.update(path_info['path'])
+
+        if not current_md5:
+            return False
+
+        return not self.changed_cache(current_md5)
+
+    def _discard_working_directory_changes(self, path, dir_info, force=False):
+        working_dir_files = set(
+            os.path.join(root, file)
+            for root, _, files in os.walk(path)
+            for file in files
+        )
+
+        cached_files = set(
+            os.path.join(path, file['relpath'])
+            for file in dir_info
+        )
+
+        delta = working_dir_files - cached_files
+
+        for file in delta:
+            self.safe_remove({'scheme': 'local', 'path': file}, force=force)
 
     def _move(self, inp, outp):
         # moving in two stages to make last the move atomic in
@@ -292,17 +372,22 @@ class RemoteLOCAL(RemoteBase):
             remove(path)
 
         self.link(cache, path)
-        self.link_state.update(path)
+        self.state.update_link(path)
 
-        return {self.PARAM_MD5: md5}
+        return {self.PARAM_CHECKSUM: md5}
 
     def _save_dir(self, path_info):
         path = path_info['path']
         md5, dir_info = self.state.update_info(path)
+        dir_relpath = os.path.relpath(path)
+        dir_size = len(dir_info)
+        bar = dir_size > LARGE_DIR_SIZE
 
-        for entry in dir_info:
+        logger.info("Linking directory '{}'.".format(dir_relpath))
+
+        for processed, entry in enumerate(dir_info):
             relpath = entry[self.PARAM_RELPATH]
-            m = entry[self.PARAM_MD5]
+            m = entry[self.PARAM_CHECKSUM]
             p = os.path.join(path, relpath)
             c = self.get(m)
 
@@ -313,9 +398,15 @@ class RemoteLOCAL(RemoteBase):
 
             self.link(c, p)
 
-        self.link_state.update(path)
+            if bar:
+                progress.update_target(dir_relpath, processed, dir_size)
 
-        return {self.PARAM_MD5: md5}
+        self.state.update_link(path)
+
+        if bar:
+            progress.finish_target(dir_relpath)
+
+        return {self.PARAM_CHECKSUM: md5}
 
     def save(self, path_info):
         if path_info['scheme'] != 'local':
@@ -324,7 +415,7 @@ class RemoteLOCAL(RemoteBase):
         path = path_info['path']
 
         msg = "Saving '{}' to cache '{}'."
-        Logger.info(msg.format(os.path.relpath(path),
+        logger.info(msg.format(os.path.relpath(path),
                                os.path.relpath(self.cache_dir)))
 
         if os.path.isdir(path):
@@ -336,20 +427,7 @@ class RemoteLOCAL(RemoteBase):
         if path_info['scheme'] != 'local':
             raise NotImplementedError
 
-        return {self.PARAM_MD5: self.state.update(path_info['path'])}
-
-    def changed(self, path_info, checksum_info):
-        if not self.exists([path_info])[0]:
-            return True
-
-        md5 = checksum_info.get(self.PARAM_MD5, None)
-        if md5 is None:
-            return True
-
-        if self.changed_cache(md5):
-            return True
-
-        return checksum_info != self.save_info(path_info)
+        return {self.PARAM_CHECKSUM: self.state.update(path_info['path'])}
 
     def remove(self, path_info):
         if path_info['scheme'] != 'local':
@@ -363,16 +441,9 @@ class RemoteLOCAL(RemoteBase):
 
         move(from_info['path'], to_info['path'])
 
-    def md5s_to_path_infos(self, md5s):
-        return [{'scheme': 'local',
-                 'path': os.path.join(self.prefix,
-                                      md5[0:2], md5[2:])} for md5 in md5s]
-
-    def exists(self, path_infos):
-        ret = []
-        for path_info in path_infos:
-            ret.append(os.path.exists(path_info['path']))
-        return ret
+    def cache_exists(self, md5s):
+        assert isinstance(md5s, list)
+        return list(filter(lambda md5: not self.changed_cache_file(md5), md5s))
 
     def upload(self, from_infos, to_infos, names=None):
         names = self._verify_path_args(to_infos, from_infos, names)
@@ -384,7 +455,7 @@ class RemoteLOCAL(RemoteBase):
             if from_info['scheme'] != 'local':
                 raise NotImplementedError
 
-            Logger.debug("Uploading '{}' to '{}'".format(from_info['path'],
+            logger.debug("Uploading '{}' to '{}'".format(from_info['path'],
                                                          to_info['path']))
 
             if not name:
@@ -394,10 +465,11 @@ class RemoteLOCAL(RemoteBase):
 
             try:
                 copyfile(from_info['path'], to_info['path'], name=name)
-            except Exception as exc:
-                msg = "Failed to upload '{}' tp '{}'"
-                Logger.error(msg.format(from_info['path'],
-                                        to_info['path']), exc)
+            except Exception:
+                logger.error(
+                    "failed to upload '{}' to '{}'"
+                    .format(from_info['path'], to_info['path'])
+                )
 
     def download(self,
                  from_infos,
@@ -413,7 +485,7 @@ class RemoteLOCAL(RemoteBase):
             if to_info['scheme'] != 'local':
                 raise NotImplementedError
 
-            Logger.debug("Downloading '{}' to '{}'".format(from_info['path'],
+            logger.debug("Downloading '{}' to '{}'".format(from_info['path'],
                                                            to_info['path']))
 
             if not name:
@@ -426,46 +498,24 @@ class RemoteLOCAL(RemoteBase):
                          tmp_file,
                          no_progress_bar=no_progress_bar,
                          name=name)
-            except Exception as exc:
-                msg = "Failed to download '{}' to '{}'"
-                Logger.error(msg.format(from_info['path'],
-                                        to_info['path']), exc)
+            except Exception:
+                logger.error(
+                    "failed to download '{}' to '{}'"
+                    .format(from_info['path'], to_info['path'])
+                )
+
                 continue
 
             os.rename(tmp_file, to_info['path'])
 
-    def _collect(self, checksum_infos, push=False):
-        missing = []
-        collected = []
-        for info in checksum_infos:
-            md5 = info[self.PARAM_MD5]
-            cache = self.get(md5)
-
-            if not self.is_dir_cache(info[self.PARAM_MD5]):
-                continue
-
-            if not os.path.exists(cache):
-                missing.append(info)
-                continue
-
-            for i in self.load_dir_cache(md5):
-                if info.get('branch'):
-                    i['branch'] = info['branch']
-                i[self.PARAM_PATH] = posixpath.join(info[self.PARAM_PATH],
-                                                    i[self.PARAM_RELPATH])
-                collected.append(i.copy())
-
-        collected.extend(checksum_infos)
-        return collected, missing
-
     def _group(self, checksum_infos, show_checksums=False):
         by_md5 = {}
 
         for info in checksum_infos:
-            md5 = info[self.PARAM_MD5]
+            md5 = info[self.PARAM_CHECKSUM]
 
             if show_checksums:
-                by_md5[md5] = md5
+                by_md5[md5] = {'name': md5}
                 continue
 
             name = info[self.PARAM_PATH]
@@ -474,195 +524,110 @@ class RemoteLOCAL(RemoteBase):
                 name += '({})'.format(branch)
 
             if md5 not in by_md5.keys():
-                by_md5[md5] = ''
+                by_md5[md5] = {'name': name}
             else:
-                by_md5[md5] += ' '
+                by_md5[md5]['name'] += ' ' + name
 
-            by_md5[md5] += name
+        return by_md5
 
-        return list(by_md5.keys()), list(by_md5.values())
-
-    def gc(self, checksum_infos):
-        checksum_infos = self._collect(checksum_infos['local'])[0]
-        used_md5s = [info[self.PARAM_MD5] for info in checksum_infos]
-
-        removed = False
-        for md5 in self.all():
-            if md5 in used_md5s:
-                continue
-            remove(self.get(md5))
-            removed = True
-
-        return removed
-
-    def status(self, checksum_infos, remote, jobs=1, show_checksums=False):
-        Logger.info("Preparing to pull data from {}".format(remote.url))
+    def status(self, checksum_infos, remote, jobs=None, show_checksums=False):
+        logger.info("Preparing to collect status from {}".format(remote.url))
         title = "Collecting information"
 
-        progress.set_n_total(0)
-        progress.update_target(title, 0, 100)
+        ret = {}
 
-        checksum_infos, missing = self._collect(checksum_infos)
-        checksum_infos += missing
+        progress.set_n_total(1)
+        progress.update_target(title, 0, 100)
 
         progress.update_target(title, 10, 100)
 
-        md5s, names = self._group(checksum_infos,
-                                  show_checksums=show_checksums)
-
-        progress.update_target(title, 20, 100)
-
-        path_infos = remote.md5s_to_path_infos(md5s)
+        ret = self._group(checksum_infos, show_checksums=show_checksums)
+        md5s = list(ret.keys())
 
         progress.update_target(title, 30, 100)
 
-        remote_exists = remote.exists(path_infos)
+        remote_exists = list(remote.cache_exists(md5s))
 
         progress.update_target(title, 90, 100)
 
-        local_exists = [not self.changed_cache(md5) for md5 in md5s]
+        local_exists = self.cache_exists(md5s)
 
         progress.finish_target(title)
 
-        return [(name, STATUS_MAP[l, r]) for name, l, r in zip(names,
-                                                               local_exists,
-                                                               remote_exists)]
+        for md5, info in ret.items():
+            info['status'] = STATUS_MAP[(md5 in local_exists,
+                                         md5 in remote_exists)]
 
-    def _do_pull(self,
+        return ret
+
+    def _get_chunks(self, download, remote, status_info, status, jobs):
+        title = "Analysing status."
+
+        progress.set_n_total(1)
+        total = len(status_info)
+        current = 0
+
+        cache = []
+        path_infos = []
+        names = []
+        for md5, info in status_info.items():
+            if info['status'] == status:
+                cache.append(self.checksum_to_path_info(md5))
+                path_infos.append(remote.checksum_to_path_info(md5))
+                names.append(info['name'])
+            current += 1
+            progress.update_target(title, current, total)
+
+        progress.finish_target(title)
+
+        progress.set_n_total(len(names))
+
+        if download:
+            to_infos = cache
+            from_infos = path_infos
+        else:
+            to_infos = path_infos
+            from_infos = cache
+
+        return list(zip(to_chunks(from_infos, jobs),
+                        to_chunks(to_infos, jobs),
+                        to_chunks(names, jobs)))
+
+    def _process(self,
                  checksum_infos,
                  remote,
-                 jobs=1,
-                 show_checksums=False):
-        title = "Collecting information"
+                 jobs=None,
+                 show_checksums=False,
+                 download=False):
+        msg = "Preparing to {} data {} '{}'"
+        logger.info(msg.format('download' if download else 'upload',
+                               'from' if download else 'to',
+                               remote.url))
 
-        progress.set_n_total(0)
-        progress.update_target(title, 0, 100)
+        if download:
+            func = remote.download
+            status = STATUS_DELETED
+        else:
+            func = remote.upload
+            status = STATUS_NEW
 
-        grouped = zip(*self._group(checksum_infos,
-                                   show_checksums=show_checksums))
+        if jobs is None:
+            jobs = remote.JOBS
 
-        progress.update_target(title, 10, 100)
-
-        md5s = []
-        names = []
-        # NOTE: filter files that are not corrupted
-        for md5, name in grouped:
-            if self.changed_cache(md5):
-                md5s.append(md5)
-                names.append(name)
-
-        progress.update_target(title, 30, 100)
-
-        cache = [{'scheme': 'local', 'path': self.get(md5)} for md5 in md5s]
-
-        progress.update_target(title, 50, 100)
-
-        path_infos = remote.md5s_to_path_infos(md5s)
-
-        progress.update_target(title, 60, 100)
-
-        assert len(path_infos) == len(cache) == len(md5s) == len(names)
-
-        chunks = list(zip(to_chunks(path_infos, jobs),
-                          to_chunks(cache, jobs),
-                          to_chunks(names, jobs)))
-
-        progress.finish_target(title)
-
-        progress.set_n_total(len(names))
-
-        if len(chunks) == 0:
-            return
-
-        futures = []
-        with ThreadPoolExecutor(max_workers=len(chunks)) as executor:
-            for from_infos, to_infos, names in chunks:
-                res = executor.submit(remote.download,
-                                      from_infos,
-                                      to_infos,
-                                      names=names)
-                futures.append(res)
-
-        for f in futures:
-            f.result()
-
-    def pull(self, checksum_infos, remote, jobs=1, show_checksums=False):
-        Logger.info("Preparing to pull data from {}".format(remote.url))
-
-        # NOTE: try fetching missing dir info
-        checksum_infos, missing = self._collect(checksum_infos)
-        if len(missing) > 0:
-            self._do_pull(missing,
-                          remote,
-                          jobs,
-                          show_checksums=show_checksums)
-            checksum_infos += self._collect(missing)[0]
-
-        self._do_pull(checksum_infos,
-                      remote,
-                      jobs,
-                      show_checksums=show_checksums)
-
-    def push(self, checksum_infos, remote, jobs=1, show_checksums=False):
-        Logger.info("Preparing to push data to {}".format(remote.url))
-        title = "Collecting information"
-
-        progress.set_n_total(0)
-        progress.update_target(title, 0, 100)
-
-        checksum_infos = self._collect(checksum_infos)[0]
-
-        progress.update_target(title, 10, 100)
-
-        # NOTE: verifying that our cache is not corrupted
-        def func(info):
-            return not self.changed_cache(info[self.PARAM_MD5])
-        checksum_infos = list(filter(func, checksum_infos))
-
-        progress.update_target(title, 20, 100)
-
-        # NOTE: filter files that are already uploaded
-        md5s = [i[self.PARAM_MD5] for i in checksum_infos]
-        exists = remote.exists(remote.md5s_to_path_infos(md5s))
-
-        progress.update_target(title, 30, 100)
-
-        def func(entry):
-            return not entry[0]
-
-        assert len(exists) == len(checksum_infos)
-        infos_exist = list(filter(func, zip(exists, checksum_infos)))
-        checksum_infos = [i for e, i in infos_exist]
-
-        progress.update_target(title, 70, 100)
-
-        md5s, names = self._group(checksum_infos,
+        status_info = self.status(checksum_infos,
+                                  remote,
+                                  jobs=jobs,
                                   show_checksums=show_checksums)
-        cache = [{'scheme': 'local', 'path': self.get(md5)} for md5 in md5s]
 
-        progress.update_target(title, 80, 100)
-
-        path_infos = remote.md5s_to_path_infos(md5s)
-
-        assert len(path_infos) == len(cache) == len(md5s) == len(names)
-
-        progress.update_target(title, 90, 100)
-
-        chunks = list(zip(to_chunks(path_infos, jobs),
-                          to_chunks(cache, jobs),
-                          to_chunks(names, jobs)))
-
-        progress.finish_target(title)
-
-        progress.set_n_total(len(names))
+        chunks = self._get_chunks(download, remote, status_info, status, jobs)
 
         if len(chunks) == 0:
             return
 
         futures = []
-        with ThreadPoolExecutor(max_workers=len(chunks)) as executor:
-            for to_infos, from_infos, names in chunks:
-                res = executor.submit(remote.upload,
+        with ThreadPoolExecutor(max_workers=jobs) as executor:
+            for from_infos, to_infos, names in chunks:
+                res = executor.submit(func,
                                       from_infos,
                                       to_infos,
                                       names=names)
@@ -670,3 +635,25 @@ class RemoteLOCAL(RemoteBase):
 
         for f in futures:
             f.result()
+
+    def push(self,
+             checksum_infos,
+             remote,
+             jobs=None,
+             show_checksums=False):
+        self._process(checksum_infos,
+                      remote,
+                      jobs=jobs,
+                      show_checksums=show_checksums,
+                      download=False)
+
+    def pull(self,
+             checksum_infos,
+             remote,
+             jobs=None,
+             show_checksums=False):
+        self._process(checksum_infos,
+                      remote,
+                      jobs=jobs,
+                      show_checksums=show_checksums,
+                      download=True)

dvc/remote/s3.py

@@ -1,22 +1,16 @@
 import os
 import threading
-import posixpath
 
 try:
     import boto3
 except ImportError:
     boto3 = None
 
-try:
-    from urlparse import urlparse
-except ImportError:
-    from urllib.parse import urlparse
-
-from dvc.logger import Logger
+import dvc.logger as logger
+from dvc.utils.compat import urlparse
 from dvc.progress import progress
 from dvc.config import Config
 from dvc.remote.base import RemoteBase
-from dvc.remote.local import RemoteLOCAL
 from dvc.exceptions import DvcException
 
 
@@ -37,52 +31,63 @@ class RemoteS3(RemoteBase):
     scheme = 's3'
     REGEX = r'^s3://(?P<path>.*)$'
     REQUIRES = {'boto3': boto3}
-    PARAM_ETAG = 'etag'
+    PARAM_CHECKSUM = 'etag'
 
     def __init__(self, project, config):
-        import configobj
+        super(RemoteS3, self).__init__(project, config)
+
+        storagepath = 's3://{}'.format(
+            config.get(Config.SECTION_AWS_STORAGEPATH, '').lstrip('/')
+        )
 
-        self.project = project
-        storagepath = 's3://' + config.get(Config.SECTION_AWS_STORAGEPATH,
-                                           '').lstrip('/')
         self.url = config.get(Config.SECTION_REMOTE_URL, storagepath)
-        self.region = config.get(Config.SECTION_AWS_REGION, None)
-        self.profile = config.get(Config.SECTION_AWS_PROFILE, 'default')
-        self.endpoint_url = config.get(Config.SECTION_AWS_ENDPOINT_URL, None)
 
-        credentialpath = config.get(Config.SECTION_AWS_CREDENTIALPATH, None)
-        if credentialpath:
-            creds_conf = configobj.ConfigObj(credentialpath)
-            creds = creds_conf.get(self.profile, {})
-        else:
-            creds = {}
+        self.region = (
+            os.environ.get('AWS_DEFAULT_REGION') or
+            config.get(Config.SECTION_AWS_REGION)
+        )
 
-        self.region = creds.get('region', self.region)
-        self.aws_access_key_id = creds.get('aws_access_key_id', None)
-        self.aws_secret_access_key = creds.get('aws_secret_access_key', None)
+        self.profile = (
+            os.environ.get('AWS_PROFILE') or
+            config.get(Config.SECTION_AWS_PROFILE)
+        )
 
-    @property
-    def bucket(self):
-        return urlparse(self.url).netloc
+        self.endpoint_url = config.get(Config.SECTION_AWS_ENDPOINT_URL)
 
-    @property
-    def prefix(self):
-        return urlparse(self.url).path.lstrip('/')
+        self.use_ssl = config.get(Config.SECTION_AWS_USE_SSL, True)
+
+        shared_creds = config.get(Config.SECTION_AWS_CREDENTIALPATH)
+        if shared_creds:
+            os.environ.setdefault('AWS_SHARED_CREDENTIALS_FILE', shared_creds)
+
+        parsed = urlparse(self.url)
+        self.bucket = parsed.netloc
+        self.prefix = parsed.path.lstrip('/')
+
+        self.path_info = {'scheme': self.scheme, 'bucket': self.bucket}
+
+    @staticmethod
+    def compat_config(config):
+        ret = config.copy()
+        url = 's3://' + ret.pop(Config.SECTION_AWS_STORAGEPATH, '').lstrip('/')
+        ret[Config.SECTION_REMOTE_URL] = url
+        return ret
 
     @property
     def s3(self):
-        session = boto3.session.Session(profile_name=self.profile)
-        return session.client('s3',
-                              aws_access_key_id=self.aws_access_key_id,
-                              aws_secret_access_key=self.aws_secret_access_key,
-                              region_name=self.region,
-                              endpoint_url=self.endpoint_url)
+        session = boto3.session.Session(profile_name=self.profile,
+                                        region_name=self.region)
 
-    def get_etag(self, bucket, key):
+        return session.client('s3',
+                              endpoint_url=self.endpoint_url,
+                              use_ssl=self.use_ssl)
+
+    def get_etag(self, bucket, path):
         try:
-            obj = self.s3.head_object(Bucket=bucket, Key=key)
+            obj = self.s3.head_object(Bucket=bucket, Key=path)
         except Exception:
-            raise DvcException('s3://{}/{} does not exist'.format(bucket, key))
+            raise DvcException('s3://{}/{} does not exist'.format(bucket,
+                                                                  path))
 
         return obj['ETag'].strip('"')
 
@@ -90,120 +95,55 @@ class RemoteS3(RemoteBase):
         if path_info['scheme'] != 's3':
             raise NotImplementedError
 
-        return {self.PARAM_ETAG: self.get_etag(path_info['bucket'],
-                                               path_info['key'])}
+        return {self.PARAM_CHECKSUM: self.get_etag(path_info['bucket'],
+                                                   path_info['path'])}
 
-    def changed(self, path_info, checksum_info):
-        if not self.exists([path_info])[0]:
-            return True
-
-        etag = checksum_info.get(self.PARAM_ETAG, None)
-        if etag is None:
-            return True
-
-        if self.changed_cache(etag):
-            return True
-
-        return checksum_info != self.save_info(path_info)
-
-    def _copy(self, from_info, to_info, s3=None):
+    def copy(self, from_info, to_info, s3=None):
         s3 = s3 if s3 else self.s3
 
         source = {'Bucket': from_info['bucket'],
-                  'Key': from_info['key']}
-        self.s3.copy(source, to_info['bucket'], to_info['key'])
+                  'Key': from_info['path']}
+        self.s3.copy(source, to_info['bucket'], to_info['path'])
 
     def save(self, path_info):
         if path_info['scheme'] != 's3':
             raise NotImplementedError
 
-        etag = self.get_etag(path_info['bucket'], path_info['key'])
-        key = posixpath.join(self.prefix, etag[0:2], etag[2:])
-        to_info = {'scheme': 's3', 'bucket': self.bucket, 'key': key}
+        etag = self.get_etag(path_info['bucket'], path_info['path'])
+        path = self.checksum_to_path(etag)
+        to_info = {'scheme': 's3', 'bucket': self.bucket, 'path': path}
 
-        self._copy(path_info, to_info)
+        self.copy(path_info, to_info)
 
-        return {self.PARAM_ETAG: etag}
-
-    @staticmethod
-    def to_string(path_info):
-        return "s3://{}/{}".format(path_info['bucket'], path_info['key'])
-
-    def changed_cache(self, etag):
-        key = posixpath.join(self.prefix, etag[0:2], etag[2:])
-        cache = {'scheme': 's3', 'bucket': self.bucket, 'key': key}
-
-        if {self.PARAM_ETAG: etag} != self.save_info(cache):
-            if self.exists([cache])[0]:
-                msg = 'Corrupted cache file {}'
-                Logger.warn(msg.format(self.to_string(cache)))
-                self.remove(cache)
-            return True
-
-        return False
-
-    def checkout(self, path_info, checksum_info):
-        if path_info['scheme'] != 's3':
-            raise NotImplementedError
-
-        etag = checksum_info.get(self.PARAM_ETAG, None)
-        if not etag:
-            return
-
-        if not self.changed(path_info, checksum_info):
-            msg = "Data '{}' didn't change."
-            Logger.info(msg.format(self.to_string(path_info)))
-            return
-
-        if self.changed_cache(etag):
-            msg = "Cache '{}' not found. File '{}' won't be created."
-            Logger.warn(msg.format(etag, self.to_string(path_info)))
-            return
-
-        if self.exists([path_info])[0]:
-            msg = "Data '{}' exists. Removing before checkout."
-            Logger.warn(msg.format(self.to_string(path_info)))
-            self.remove(path_info)
-            return
-
-        msg = "Checking out '{}' with cache '{}'."
-        Logger.info(msg.format(self.to_string(path_info), etag))
-
-        key = posixpath.join(self.prefix, etag[0:2], etag[2:])
-        from_info = {'scheme': 's3', 'bucket': self.bucket, 'key': key}
-
-        self._copy(from_info, path_info)
+        return {self.PARAM_CHECKSUM: etag}
 
     def remove(self, path_info):
         if path_info['scheme'] != 's3':
             raise NotImplementedError
 
-        Logger.debug('Removing s3://{}/{}'.format(path_info['bucket'],
-                                                  path_info['key']))
+        logger.debug('Removing s3://{}/{}'.format(path_info['bucket'],
+                                                  path_info['path']))
 
         self.s3.delete_object(Bucket=path_info['bucket'],
-                              Key=path_info['key'])
+                              Key=path_info['path'])
 
-    def md5s_to_path_infos(self, md5s):
-        return [{'scheme': self.scheme,
-                 'bucket': self.bucket,
-                 'key': posixpath.join(self.prefix,
-                                       md5[0:2], md5[2:])} for md5 in md5s]
-
-    def _all_keys(self):
+    def _list_paths(self, bucket, prefix):
         s3 = self.s3
 
-        keys = []
-        kwargs = {'Bucket': self.bucket,
-                  'Prefix': self.prefix}
+        kwargs = {'Bucket': bucket,
+                  'Prefix': prefix}
         while True:
+            # NOTE: list_objects_v2() is 90% faster than head_object [1]
+            #
+            # [1] https://www.peterbe.com/plog/
+            #     fastest-way-to-find-out-if-a-file-exists-in-s3
             resp = s3.list_objects_v2(**kwargs)
             contents = resp.get('Contents', None)
             if not contents:
                 break
 
             for obj in contents:
-                keys.append(obj['Key'])
+                yield obj['Key']
 
             token = resp.get('NextContinuationToken', None)
             if not token:
@@ -211,24 +151,15 @@ class RemoteS3(RemoteBase):
 
             kwargs['ContinuationToken'] = token
 
-        return keys
+    def list_cache_paths(self):
+        return self._list_paths(self.bucket, self.prefix)
 
-    def exists(self, path_infos):
-        # NOTE: We mostly use exists() method when filtering a bulk of cache
-        # files to decide if we need to download/upload them and in s3
-        # list_objects_v2() is much-much faster than trying to check keys
-        # one-by-one.
-        ret = []
+    def exists(self, path_info):
+        assert not isinstance(path_info, list)
+        assert path_info['scheme'] == 's3'
 
-        keys = self._all_keys()
-
-        for path_info in path_infos:
-            exists = False
-            if path_info['key'] in keys:
-                exists = True
-            ret.append(exists)
-
-        return ret
+        paths = self._list_paths(path_info['bucket'], path_info['path'])
+        return any(path_info['path'] == path for path in paths)
 
     def upload(self, from_infos, to_infos, names=None):
         names = self._verify_path_args(to_infos, from_infos, names)
@@ -242,9 +173,9 @@ class RemoteS3(RemoteBase):
             if from_info['scheme'] != 'local':
                 raise NotImplementedError
 
-            Logger.debug("Uploading '{}' to '{}/{}'".format(from_info['path'],
+            logger.debug("Uploading '{}' to '{}/{}'".format(from_info['path'],
                                                             to_info['bucket'],
-                                                            to_info['key']))
+                                                            to_info['path']))
 
             if not name:
                 name = os.path.basename(from_info['path'])
@@ -255,11 +186,11 @@ class RemoteS3(RemoteBase):
             try:
                 s3.upload_file(from_info['path'],
                                to_info['bucket'],
-                               to_info['key'],
+                               to_info['path'],
                                Callback=cb)
-            except Exception as exc:
-                msg = "Failed to upload '{}'".format(from_info['path'])
-                Logger.error(msg, exc)
+            except Exception:
+                msg = "failed to upload '{}'".format(from_info['path'])
+                logger.error(msg)
                 continue
 
             progress.finish_target(name)
@@ -278,67 +209,43 @@ class RemoteS3(RemoteBase):
                 raise NotImplementedError
 
             if to_info['scheme'] == 's3':
-                self._copy(from_info, to_info, s3=s3)
+                self.copy(from_info, to_info, s3=s3)
                 continue
 
             if to_info['scheme'] != 'local':
                 raise NotImplementedError
 
             msg = "Downloading '{}/{}' to '{}'".format(from_info['bucket'],
-                                                       from_info['key'],
+                                                       from_info['path'],
                                                        to_info['path'])
-            Logger.debug(msg)
+            logger.debug(msg)
 
             tmp_file = self.tmp_file(to_info['path'])
             if not name:
                 name = os.path.basename(to_info['path'])
 
-            if no_progress_bar:
-                cb = None
-            else:
-                total = s3.head_object(Bucket=from_info['bucket'],
-                                       Key=from_info['key'])['ContentLength']
-                cb = Callback(name, total)
-
             self._makedirs(to_info['path'])
 
             try:
+                if no_progress_bar:
+                    cb = None
+                else:
+                    total = s3.head_object(
+                                    Bucket=from_info['bucket'],
+                                    Key=from_info['path'])['ContentLength']
+                    cb = Callback(name, total)
+
                 s3.download_file(from_info['bucket'],
-                                 from_info['key'],
+                                 from_info['path'],
                                  tmp_file,
                                  Callback=cb)
-            except Exception as exc:
-                msg = "Failed to download '{}/{}'".format(from_info['bucket'],
-                                                          from_info['key'])
-                Logger.error(msg, exc)
-                return
+            except Exception:
+                msg = "failed to download '{}/{}'".format(from_info['bucket'],
+                                                          from_info['path'])
+                logger.error(msg)
+                continue
 
             os.rename(tmp_file, to_info['path'])
 
             if not no_progress_bar:
                 progress.finish_target(name)
-
-    def _path_to_etag(self, path):
-        relpath = posixpath.relpath(path, self.prefix)
-        return posixpath.dirname(relpath) + posixpath.basename(relpath)
-
-    def _all(self):
-        keys = self._all_keys()
-        return [self._path_to_etag(key) for key in keys]
-
-    def gc(self, cinfos):
-        used_etags = [info[self.PARAM_ETAG] for info in cinfos['s3']]
-        used_etags += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
-
-        removed = False
-        for etag in self._all():
-            if etag in used_etags:
-                continue
-            path_info = {'scheme': 's3',
-                         'key': posixpath.join(self.prefix,
-                                               etag[0:2], etag[2:]),
-                         'bucket': self.bucket}
-            self.remove(path_info)
-            removed = True
-
-        return removed

dvc/remote/ssh.py

@@ -7,12 +7,14 @@ try:
 except ImportError:
     paramiko = None
 
-from dvc.logger import Logger
-from dvc.progress import progress
-from dvc.remote.base import RemoteBase
-from dvc.remote.local import RemoteLOCAL
+import dvc.prompt as prompt
+import dvc.logger as logger
+
 from dvc.config import Config
+from dvc.progress import progress
+from dvc.utils.compat import urlparse
 from dvc.exceptions import DvcException
+from dvc.remote.base import RemoteBase, RemoteBaseCmdError
 
 
 def sizeof_fmt(num, suffix='B'):
@@ -26,7 +28,7 @@ def sizeof_fmt(num, suffix='B'):
 
 def percent_cb(name, complete, total):
     """ Callback for updating target progress """
-    Logger.debug('{}: {} transferred out of {}'.format(name,
+    logger.debug('{}: {} transferred out of {}'.format(name,
                                                        sizeof_fmt(complete),
                                                        sizeof_fmt(total)))
     progress.update_target(name, complete, total)
@@ -37,73 +39,140 @@ def create_cb(name):
     return (lambda cur, tot: percent_cb(name, cur, tot))
 
 
+class RemoteSSHCmdError(RemoteBaseCmdError):
+    pass
+
+
 class RemoteSSH(RemoteBase):
     scheme = 'ssh'
 
-    # NOTE: temporarily only absolute paths are allowed
-    REGEX = r'^ssh://((?P<user>.*)@)?(?P<host>[^/]*):(?P<path>/.*)$'
+    # NOTE: we support both URL-like (ssh://[user@]host.xz[:port]/path) and
+    # SCP-like (ssh://[user@]host.xz:path) urls.
+    REGEX = r'^ssh://.*$'
 
     REQUIRES = {'paramiko': paramiko}
-    PARAM_MD5 = 'md5'
+
+    JOBS = 4
+
+    PARAM_CHECKSUM = 'md5'
+
+    DEFAULT_PORT = 22
+    TIMEOUT = 1800
 
     def __init__(self, project, config):
-        self.project = project
-        self.url = config.get(Config.SECTION_REMOTE_URL, '/')
-        self.host = self.group('host')
-        self.user = self.group('user')
-        if not self.user:
-            self.user = config.get(Config.SECTION_REMOTE_USER,
-                                   getpass.getuser())
-        self.prefix = self.group('path')
+        super(RemoteSSH, self).__init__(project, config)
+        self.url = config.get(Config.SECTION_REMOTE_URL, 'ssh://')
 
-    def md5s_to_path_infos(self, md5s):
-        return [{'scheme': 'ssh',
-                 'host': self.host,
-                 'user': self.user,
-                 'path': posixpath.join(self.prefix,
-                                        md5[0:2], md5[2:])} for md5 in md5s]
+        parsed = urlparse(self.url)
+        self.host = parsed.hostname
+        self.user = (config.get(Config.SECTION_REMOTE_USER)
+                     or parsed.username
+                     or getpass.getuser())
+        self.prefix = parsed.path
+        self.port = (config.get(Config.SECTION_REMOTE_PORT)
+                     or parsed.port
+                     or self.DEFAULT_PORT)
+        self.keyfile = config.get(Config.SECTION_REMOTE_KEY_FILE, None)
+        self.timeout = config.get(Config.SECTION_REMOTE_TIMEOUT, self.TIMEOUT)
+        self.password = config.get(Config.SECTION_REMOTE_PASSWORD, None)
+        self.ask_password = config.get(Config.SECTION_REMOTE_ASK_PASSWORD,
+                                       False)
 
-    def ssh(self, host=None, user=None):
-        msg = "Establishing ssh connection with '{}' as user '{}'"
-        Logger.debug(msg.format(host, user))
+        self.path_info = {'scheme': 'ssh',
+                          'host': self.host,
+                          'user': self.user,
+                          'port': self.port}
+
+    def ssh(self, host=None, user=None, port=None):
+        msg = "Establishing ssh connection with '{}' " \
+              "through port '{}' as user '{}'"
+        logger.debug(msg.format(host, port, user))
 
         ssh = paramiko.SSHClient()
 
         ssh.load_system_host_keys()
         ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
 
-        ssh.connect(host, username=user)
+        if self.ask_password and self.password is None:
+            msg = ("Enter a private key passphrase or a password for "
+                   "host '{}' port '{}' user '{}'").format(host, port, user)
+            self.password = prompt.password(msg)
+
+        ssh.connect(host,
+                    username=user,
+                    port=port,
+                    key_filename=self.keyfile,
+                    timeout=self.timeout,
+                    password=self.password)
 
         return ssh
 
-    def exists(self, path_infos):
-        ret = []
-        ssh = self.ssh(host=self.host, user=self.user)
-        cmd = 'find {} -type f -follow -print'.format(self.prefix)
-        stdout = self._exec(ssh, cmd)
-        plist = stdout.split()
-        ssh.close()
+    def exists(self, path_info):
+        assert not isinstance(path_info, list)
+        assert path_info['scheme'] == 'ssh'
 
-        for path_info in path_infos:
-            exists = False
-            if path_info['path'] in plist:
+        with self.ssh(path_info['host'],
+                      path_info['user'],
+                      path_info['port']) as ssh:
+            try:
+                self._exec(ssh, 'test -e {}'.format(path_info['path']))
                 exists = True
-            ret.append(exists)
+            except RemoteSSHCmdError:
+                exists = False
 
-        return ret
+        return exists
 
     def _exec(self, ssh, cmd):
         stdin, stdout, stderr = ssh.exec_command(cmd)
-        if stdout.channel.recv_exit_status() != 0:
-            DvcException('SSH command \'{}\' failed: {}'.format(cmd,
-                                                                stderr.read()))
-        return stdout.read().decode('utf-8')
+        channel = stdout.channel
+
+        stdin.close()
+        channel.shutdown_write()
+
+        stdout_chunks = []
+        stderr_chunks = []
+        while (not channel.closed
+               or channel.recv_ready()
+               or channel.recv_stderr_ready()):
+            import select
+            got_chunk = False
+            readq, _, _ = select.select([stdout.channel], [], [], self.timeout)
+            for c in readq:
+                if c.recv_ready():
+                    stdout_chunks.append(stdout.channel.recv(len(c.in_buffer)))
+                    got_chunk = True
+
+                if c.recv_stderr_ready():
+                    stderr_len = len(c.in_stderr_buffer)
+                    s = stderr.channel.recv_stderr(stderr_len)
+                    stderr_chunks.append(s)
+                    got_chunk = True
+
+            if not got_chunk \
+               and stdout.channel.exit_status_ready() \
+               and not stderr.channel.recv_stderr_ready() \
+               and not stdout.channel.recv_ready():
+                stdout.channel.shutdown_read()
+                stdout.channel.close()
+                break
+
+        stdout.close()
+        stderr.close()
+
+        ret = stdout.channel.recv_exit_status()
+        if ret != 0:
+            err = b''.join(stderr_chunks).decode('utf-8')
+            raise RemoteSSHCmdError(cmd, ret, err)
+
+        return b''.join(stdout_chunks).decode('utf-8')
 
     def md5(self, path_info):
         if path_info['scheme'] != 'ssh':
             raise NotImplementedError
 
-        ssh = self.ssh(host=path_info['host'], user=path_info['user'])
+        ssh = self.ssh(host=path_info['host'],
+                       user=path_info['user'],
+                       port=path_info['port'])
 
         # Use different md5 commands depending on os
         stdout = self._exec(ssh, 'uname').strip()
@@ -114,7 +183,7 @@ class RemoteSSH(RemoteBase):
             md5cmd = 'md5sum'
             index = 0
         else:
-            msg = '\'{}\' is not supported as a remote'.format(stdout)
+            msg = "'{}' is not supported as a remote".format(stdout)
             raise DvcException(msg)
 
         stdout = self._exec(ssh, '{} {}'.format(md5cmd, path_info['path']))
@@ -125,7 +194,7 @@ class RemoteSSH(RemoteBase):
 
         return md5
 
-    def cp(self, from_info, to_info, ssh=None):
+    def copy(self, from_info, to_info, ssh=None):
         if from_info['scheme'] != 'ssh' or to_info['scheme'] != 'ssh':
             raise NotImplementedError
 
@@ -133,7 +202,8 @@ class RemoteSSH(RemoteBase):
         assert from_info['user'] == to_info['user']
 
         s = ssh if ssh else self.ssh(host=from_info['host'],
-                                     user=from_info['user'])
+                                     user=from_info['user'],
+                                     port=from_info['port'])
 
         dname = posixpath.dirname(to_info['path'])
         self._exec(s, 'mkdir -p {}'.format(dname))
@@ -146,43 +216,7 @@ class RemoteSSH(RemoteBase):
         if path_info['scheme'] != 'ssh':
             raise NotImplementedError
 
-        return {self.PARAM_MD5: self.md5(path_info)}
-
-    @staticmethod
-    def to_string(path_info):
-        return "{}://{}@{}:{}".format(path_info['scheme'],
-                                      path_info['user'],
-                                      path_info['host'],
-                                      path_info['path'])
-
-    def changed_cache(self, md5):
-        cache = {}
-        cache['scheme'] = 'ssh'
-        cache['host'] = self.host
-        cache['user'] = self.user
-        cache['path'] = posixpath.join(self.prefix, md5[0:2], md5[2:])
-
-        if {self.PARAM_MD5: md5} != self.save_info(cache):
-            if self.exists([cache])[0]:
-                msg = 'Corrupted cache file {}'
-                Logger.warn(msg.format(self.to_string(cache)))
-                self.remove(cache)
-            return True
-
-        return False
-
-    def changed(self, path_info, checksum_info):
-        if not self.exists([path_info])[0]:
-            return True
-
-        md5 = checksum_info.get(self.PARAM_MD5, None)
-        if md5 is None:
-            return True
-
-        if self.changed_code(md5):
-            return True
-
-        return checksum_info != self.save_info(path_info)
+        return {self.PARAM_CHECKSUM: self.md5(path_info)}
 
     def save(self, path_info):
         if path_info['scheme'] != 'ssh':
@@ -190,53 +224,23 @@ class RemoteSSH(RemoteBase):
 
         md5 = self.md5(path_info)
         dest = path_info.copy()
-        dest['path'] = posixpath.join(self.prefix, md5[0:2], md5[2:])
+        dest['path'] = self.checksum_to_path(md5)
 
-        self.cp(path_info, dest)
+        self.copy(path_info, dest)
 
-        return {self.PARAM_MD5: md5}
-
-    def checkout(self, path_info, checksum_info):
-        if path_info['scheme'] != 'ssh':
-            raise NotImplementedError
-
-        md5 = checksum_info.get(self.PARAM_MD5, None)
-        if not md5:
-            return
-
-        if not self.changed(path_info, checksum_info):
-            msg = "Data '{}' didn't change."
-            Logger.info(msg.format(self.to_string(path_info)))
-            return
-
-        if self.changed_cache(md5):
-            msg = "Cache '{}' not found. File '{}' won't be created."
-            Logger.warn(msg.format(md5, self.to_string(path_info)))
-            return
-
-        if self.exists([path_info])[0]:
-            msg = "Data '{}' exists. Removing before checkout."
-            Logger.warn(msg.format(self.to_string(path_info)))
-            self.remove(path_info)
-            return
-
-        msg = "Checking out '{}' with cache '{}'."
-        Logger.info(msg.format(self.to_string(path_info), md5))
-
-        src = path_info.copy()
-        src['path'] = posixpath.join(self.prefix, md5[0:2], md5[2:])
-
-        self.cp(src, path_info)
+        return {self.PARAM_CHECKSUM: md5}
 
     def remove(self, path_info):
         if path_info['scheme'] != 'ssh':
             raise NotImplementedError
 
-        Logger.debug('Removing ssh://{}@{}/{}'.format(path_info['user'],
+        logger.debug('Removing ssh://{}@{}/{}'.format(path_info['user'],
                                                       path_info['host'],
                                                       path_info['path']))
 
-        ssh = self.ssh(host=path_info['host'], user=path_info['user'])
+        ssh = self.ssh(host=path_info['host'],
+                       user=path_info['user'],
+                       port=path_info['port'])
         ssh.open_sftp().remove(path_info['path'])
         ssh.close()
 
@@ -247,7 +251,9 @@ class RemoteSSH(RemoteBase):
                  names=None):
         names = self._verify_path_args(from_infos, to_infos, names)
 
-        ssh = self.ssh(host=from_infos[0]['host'], user=from_infos[0]['user'])
+        ssh = self.ssh(host=from_infos[0]['host'],
+                       user=from_infos[0]['user'],
+                       port=from_infos[0]['port'])
 
         for to_info, from_info, name in zip(to_infos, from_infos, names):
             if from_info['scheme'] != 'ssh':
@@ -255,8 +261,9 @@ class RemoteSSH(RemoteBase):
 
             if to_info['scheme'] == 'ssh':
                 assert from_info['host'] == to_info['host']
+                assert from_info['port'] == to_info['port']
                 assert from_info['user'] == to_info['user']
-                self.cp(from_info, to_info, ssh=ssh)
+                self.copy(from_info, to_info, ssh=ssh)
                 continue
 
             if to_info['scheme'] != 'local':
@@ -265,7 +272,7 @@ class RemoteSSH(RemoteBase):
             msg = "Downloading '{}/{}' to '{}'".format(from_info['host'],
                                                        from_info['path'],
                                                        to_info['path'])
-            Logger.debug(msg)
+            logger.debug(msg)
 
             if not name:
                 name = os.path.basename(to_info['path'])
@@ -276,11 +283,11 @@ class RemoteSSH(RemoteBase):
                 ssh.open_sftp().get(from_info['path'],
                                     tmp_file,
                                     callback=create_cb(name))
-            except Exception as exc:
-                msg = "Failed to download '{}/{}' to '{}'"
-                Logger.error(msg.format(from_info['host'],
+            except Exception:
+                msg = "failed to download '{}/{}' to '{}'"
+                logger.error(msg.format(from_info['host'],
                                         from_info['path'],
-                                        to_info['path']), exc)
+                                        to_info['path']))
                 continue
 
             os.rename(tmp_file, to_info['path'])
@@ -291,7 +298,9 @@ class RemoteSSH(RemoteBase):
     def upload(self, from_infos, to_infos, names=None):
         names = self._verify_path_args(to_infos, from_infos, names)
 
-        ssh = self.ssh(host=to_infos[0]['host'], user=to_infos[0]['user'])
+        ssh = self.ssh(host=to_infos[0]['host'],
+                       user=to_infos[0]['user'],
+                       port=to_infos[0]['port'])
         sftp = ssh.open_sftp()
 
         for from_info, to_info, name in zip(from_infos, to_infos, names):
@@ -301,7 +310,7 @@ class RemoteSSH(RemoteBase):
             if from_info['scheme'] != 'local':
                 raise NotImplementedError
 
-            Logger.debug("Uploading '{}' to '{}/{}'".format(from_info['path'],
+            logger.debug("Uploading '{}' to '{}/{}'".format(from_info['path'],
                                                             to_info['host'],
                                                             to_info['path']))
 
@@ -315,11 +324,11 @@ class RemoteSSH(RemoteBase):
                 sftp.put(from_info['path'],
                          to_info['path'],
                          callback=create_cb(name))
-            except Exception as exc:
-                msg = "Failed to upload '{}' to '{}/{}'"
-                Logger.error(msg.format(from_info['path'],
+            except Exception:
+                msg = "failed to upload '{}' to '{}/{}'"
+                logger.error(msg.format(from_info['path'],
                                         to_info['host'],
-                                        to_info['path'], exc))
+                                        to_info['path']))
                 continue
 
             progress.finish_target(name)
@@ -327,33 +336,12 @@ class RemoteSSH(RemoteBase):
         sftp.close()
         ssh.close()
 
-    def _path_to_md5(self, path):
-        relpath = posixpath.relpath(path, self.prefix)
-        return posixpath.dirname(relpath) + posixpath.basename(relpath)
-
-    def _all_md5s(self):
-        ssh = self.ssh(host=self.host, user=self.user)
+    def list_cache_paths(self):
+        ssh = self.ssh(host=self.host,
+                       user=self.user,
+                       port=self.port)
         cmd = 'find {} -type f -follow -print'.format(self.prefix)
         stdout = self._exec(ssh, cmd)
         flist = stdout.split()
         ssh.close()
-
-        return [self._path_to_md5(path) for path in flist]
-
-    def gc(self, cinfos):
-        used = [info[self.PARAM_MD5] for info in cinfos['ssh']]
-        used += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
-
-        removed = False
-        for md5 in self._all_md5s():
-            if md5 in used:
-                continue
-            path_info = {'scheme': 'ssh',
-                         'user': self.user,
-                         'host': self.host,
-                         'path': posixpath.join(self.prefix,
-                                                md5[0:2], md5[2:])}
-            self.remove(path_info)
-            removed = True
-
-        return removed
+        return flist

dvc/scm.py

@@ -1,205 +0,0 @@
-import os
-
-from dvc.exceptions import DvcException
-from dvc.logger import Logger
-from dvc.utils import fix_env
-
-
-class SCMError(DvcException):
-    pass
-
-
-class FileNotInRepoError(DvcException):
-    pass
-
-
-class Base(object):
-    def __init__(self, root_dir=os.curdir):
-        self.root_dir = os.path.abspath(os.path.realpath(root_dir))
-
-    @staticmethod
-    def is_repo(root_dir):
-        return True
-
-    def ignore(self, path):
-        pass
-
-    def ignore_remove(self, path):
-        pass
-
-    def ignore_file(self):
-        pass
-
-    def ignore_list(self, p_list):
-        return [self.ignore(path) for path in p_list]
-
-    def add(self, paths):
-        pass
-
-    def commit(self, msg):
-        pass
-
-    def checkout(self, branch):
-        pass
-
-    def branch(self, branch):
-        pass
-
-    def brancher(self, branches=None, all_branches=False):
-        if not branches and not all_branches:
-            yield ''
-            return
-
-        saved = self.active_branch()
-        if not branches:
-            branches = self.list_branches() if all_branches else [saved]
-        for branch in branches:
-            self.checkout(branch)
-            yield branch
-        self.checkout(saved)
-
-    def untracked_files(self):
-        pass
-
-    def is_tracked(self, path):
-        pass
-
-    def active_branch(self):
-        pass
-
-    def list_branches(self):
-        pass
-
-    def install(self):
-        pass
-
-
-class Git(Base):
-    GITIGNORE = '.gitignore'
-    GIT_DIR = '.git'
-
-    def __init__(self, root_dir=os.curdir):
-        super(Git, self).__init__(root_dir)
-
-        import git
-        from git.exc import InvalidGitRepositoryError
-        try:
-            self.repo = git.Repo(root_dir)
-        except InvalidGitRepositoryError:
-            raise SCMError('{} is not a git repository'.format(root_dir))
-
-        # NOTE: fixing LD_LIBRARY_PATH for binary built by PyInstaller.
-        # http://pyinstaller.readthedocs.io/en/stable/runtime-information.html
-        env = fix_env(None)
-        lp = env.get('LD_LIBRARY_PATH', None)
-        self.repo.git.update_environment(LD_LIBRARY_PATH=lp)
-
-    @staticmethod
-    def is_repo(root_dir):
-        git_dir = os.path.join(root_dir, Git.GIT_DIR)
-        return os.path.isdir(git_dir)
-
-    def ignore_file(self):
-        return self.GITIGNORE
-
-    def _get_gitignore(self, path):
-        entry = os.path.basename(path)
-        gitignore = os.path.join(os.path.abspath(os.path.dirname(path)),
-                                 self.GITIGNORE)
-        gitignore = os.path.abspath(os.path.realpath(gitignore))
-
-        if not gitignore.startswith(self.root_dir):
-            raise FileNotInRepoError(path)
-
-        return entry, gitignore
-
-    def ignore(self, path):
-        entry, gitignore = self._get_gitignore(path)
-
-        ignore_list = []
-        if os.path.exists(gitignore):
-            ignore_list = open(gitignore, 'r').readlines()
-            filtered = list(filter(lambda x: x.strip() == entry.strip(),
-                                   ignore_list))
-            if len(filtered) != 0:
-                return
-
-        msg = "Adding '{}' to '{}'.".format(os.path.relpath(path),
-                                            os.path.relpath(gitignore))
-        Logger.info(msg)
-
-        content = entry
-        if len(ignore_list) > 0:
-            content = '\n' + content
-
-        open(gitignore, 'a').write(content)
-
-    def ignore_remove(self, path):
-        entry, gitignore = self._get_gitignore(path)
-
-        if not os.path.exists(gitignore):
-            return
-
-        with open(gitignore, 'r') as fd:
-            lines = fd.readlines()
-
-        filtered = list(filter(lambda x: x.strip() != entry.strip(), lines))
-
-        with open(gitignore, 'w') as fd:
-            fd.writelines(filtered)
-
-    def add(self, paths):
-        # NOTE: GitPython is not currently able to handle index version >= 3.
-        # See https://github.com/iterative/dvc/issues/610 for more details.
-        try:
-            self.repo.index.add(paths)
-        except AssertionError as exc:
-            msg = 'Failed to add \'{}\' to git. You can add those files '
-            msg += 'manually using \'git add\'. '
-            msg += 'See \'https://github.com/iterative/dvc/issues/610\' '
-            msg += 'for more details.'
-            Logger.error(msg.format(str(paths)), exc)
-
-    def commit(self, msg):
-        self.repo.index.commit(msg)
-
-    def checkout(self, branch, create_new=False):
-        if create_new:
-            self.repo.git.checkout('HEAD', b=branch)
-        else:
-            self.repo.git.checkout(branch)
-
-    def branch(self, branch):
-        self.repo.git.branch(branch)
-
-    def untracked_files(self):
-        files = self.repo.untracked_files
-        return [os.path.join(self.repo.working_dir, fname) for fname in files]
-
-    def is_tracked(self, path):
-        return len(self.repo.git.ls_files(path)) != 0
-
-    def active_branch(self):
-        return self.repo.active_branch.name
-
-    def list_branches(self):
-        return [h.name for h in self.repo.heads]
-
-    def install(self):
-        hook = os.path.join(self.root_dir,
-                            self.GIT_DIR,
-                            'hooks',
-                            'post-checkout')
-        if os.path.isfile(hook):
-            msg = 'Git hook \'{}\' already exists.'
-            raise SCMError(msg.format(os.path.relpath(hook)))
-        with open(hook, 'w+') as fd:
-            fd.write('#!/bin/sh\nexec dvc checkout\n')
-        os.chmod(hook, 0o777)
-
-
-def SCM(root_dir=os.curdir, no_scm=False):
-    if Git.is_repo(root_dir):
-        return Git(root_dir)
-
-    return Base(root_dir)

dvc/scm/__init__.py

@@ -0,0 +1,22 @@
+"""Manages source control systems(e.g. Git)."""
+
+from dvc.scm.base import Base
+from dvc.scm.git import Git
+
+
+def SCM(root_dir, project=None):  # pylint: disable=invalid-name
+    """Returns SCM instance that corresponds to a project at the specified
+    path.
+
+    Args:
+        root_dir (str): path to a root directory of the project.
+        project (dvc.project.Project): dvc project instance that root_dir
+            belongs to.
+
+    Returns:
+        dvc.scm.base.Base: SCM instance.
+    """
+    if Git.is_repo(root_dir) or Git.is_submodule(root_dir):
+        return Git(root_dir, project=project)
+
+    return Base(root_dir, project=project)

dvc/scm/base.py

@@ -0,0 +1,142 @@
+"""Manages source control systems(e.g. Git) in dvc."""
+
+import os
+
+from dvc.exceptions import DvcException
+
+
+class SCMError(DvcException):
+    """Base class for source control management errors."""
+
+
+class FileNotInRepoError(DvcException):
+    """Thrown when trying to find .gitignore for a file that is not in a scm
+    repository.
+    """
+
+
+class Base(object):
+    """Base class for source control management driver implementations."""
+    def __init__(self, root_dir=os.curdir, project=None):
+        self.project = project
+        self.root_dir = root_dir
+
+    def __repr__(self):
+        return "{class_name}: '{directory}'".format(
+            class_name=type(self).__name__,
+            directory=self.dir
+        )
+
+    @property
+    def dir(self):
+        """Path to a directory with SCM specific information."""
+        return None
+
+    @staticmethod
+    def is_repo(root_dir):  # pylint: disable=unused-argument
+        """Returns whether or not root_dir is a valid SCM repository."""
+        return True
+
+    @staticmethod
+    def is_submodule(root_dir):  # pylint: disable=unused-argument
+        """Returns whether or not root_dir is a valid SCM repository
+        submodule.
+        """
+        return True
+
+    def ignore(self, path):  # pylint: disable=unused-argument
+        """Makes SCM ignore a specified path."""
+
+    def ignore_remove(self, path):  # pylint: disable=unused-argument
+        """Makes SCM stop ignoring a specified path."""
+
+    @property
+    def ignore_file(self):
+        """Filename for a file that contains ignored paths for this SCM."""
+
+    def ignore_list(self, p_list):
+        """Makes SCM ignore all paths specified in a list."""
+        return [self.ignore(path) for path in p_list]
+
+    def add(self, paths):
+        """Makes SCM start tracking every path from a specified list of paths.
+        """
+
+    def commit(self, msg):
+        """Makes SCM create a commit."""
+
+    def checkout(self, branch, create_new=False):
+        """Makes SCM checkout a branch."""
+
+    def branch(self, branch):
+        """Makes SCM create a branch with a specified name."""
+
+    def tag(self, tag):
+        """Makes SCM create a tag with a specified name."""
+
+    def brancher(self,
+                 branches=None,
+                 all_branches=False,
+                 tags=None,
+                 all_tags=False):
+        """Generator that iterates over specified revisions.
+
+        Args:
+            branches (list): a list of branches to iterate over.
+            all_branches (bool): iterate over all available branches.
+            tags (list): a list of tags to iterate over.
+            all_tags (bool): iterate over all available tags.
+
+        Yields:
+            str: the current revision.
+        """
+        if not branches and not all_branches \
+           and not tags and not all_tags:
+            yield ''
+            return
+
+        saved = self.active_branch()
+        revs = []
+
+        if all_branches:
+            branches = self.list_branches()
+
+        if all_tags:
+            tags = self.list_tags()
+
+        if branches is None:
+            revs.extend([saved])
+        else:
+            revs.extend(branches)
+
+        if tags is not None:
+            revs.extend(tags)
+
+        for rev in revs:
+            self.checkout(rev)
+            yield rev
+
+        self.checkout(saved)
+
+    def untracked_files(self):  # pylint: disable=no-self-use
+        """Returns a list of untracked files."""
+        return []
+
+    def is_tracked(self, path):  # pylint: disable=no-self-use, unused-argument
+        """Returns whether or not a specified path is tracked."""
+        return False
+
+    def active_branch(self):  # pylint: disable=no-self-use
+        """Returns current branch in the project."""
+        return ''
+
+    def list_branches(self):  # pylint: disable=no-self-use
+        """Returns a list of available branches in the project."""
+        return []
+
+    def list_tags(self):  # pylint: disable=no-self-use
+        """Returns a list of available tags in the project."""
+        return []
+
+    def install(self):
+        """Adds dvc commands to SCM hooks for the project."""

dvc/scm/git.py

@@ -0,0 +1,170 @@
+"""Manages Git."""
+
+# NOTE: need this one because python 2 would re-import this module
+# (called git.py) instead of importing gitpython's git module.
+from __future__ import absolute_import
+
+import os
+
+import dvc.logger as logger
+from dvc.utils import fix_env
+from dvc.scm.base import Base, SCMError, FileNotInRepoError
+
+
+class Git(Base):
+    """Class for managing Git."""
+    GITIGNORE = '.gitignore'
+    GIT_DIR = '.git'
+
+    def __init__(self, root_dir=os.curdir, project=None):
+        super(Git, self).__init__(root_dir, project=project)
+
+        import git
+        from git.exc import InvalidGitRepositoryError
+        try:
+            self.repo = git.Repo(root_dir)
+        except InvalidGitRepositoryError:
+            msg = '{} is not a git repository'
+            raise SCMError(msg.format(root_dir))
+
+        # NOTE: fixing LD_LIBRARY_PATH for binary built by PyInstaller.
+        # http://pyinstaller.readthedocs.io/en/stable/runtime-information.html
+        env = fix_env(None)
+        libpath = env.get('LD_LIBRARY_PATH', None)
+        self.repo.git.update_environment(LD_LIBRARY_PATH=libpath)
+
+    @staticmethod
+    def is_repo(root_dir):
+        return os.path.isdir(Git._get_git_dir(root_dir))
+
+    @staticmethod
+    def is_submodule(root_dir):
+        return os.path.isfile(Git._get_git_dir(root_dir))
+
+    @staticmethod
+    def _get_git_dir(root_dir):
+        return os.path.join(root_dir, Git.GIT_DIR)
+
+    @property
+    def dir(self):
+        return self.repo.git_dir
+
+    @property
+    def ignore_file(self):
+        return self.GITIGNORE
+
+    def _get_gitignore(self, path):
+        assert os.path.isabs(path)
+        # NOTE: using '/' prefix to make path unambiguous
+        entry = '/' + os.path.basename(path)
+        gitignore = os.path.join(os.path.dirname(path), self.GITIGNORE)
+
+        if not gitignore.startswith(self.root_dir):
+            raise FileNotInRepoError(path)
+
+        return entry, gitignore
+
+    def ignore(self, path):
+        entry, gitignore = self._get_gitignore(path)
+
+        ignore_list = []
+        if os.path.exists(gitignore):
+            ignore_list = open(gitignore, 'r').readlines()
+            filtered = list(filter(lambda x: x.strip() == entry.strip(),
+                                   ignore_list))
+            if filtered:
+                return
+
+        msg = "Adding '{}' to '{}'.".format(os.path.relpath(path),
+                                            os.path.relpath(gitignore))
+        logger.info(msg)
+
+        content = entry
+        if ignore_list:
+            content = '\n' + content
+
+        with open(gitignore, 'a') as fobj:
+            fobj.write(content)
+
+        if self.project is not None:
+            self.project.files_to_git_add.append(os.path.relpath(gitignore))
+
+    def ignore_remove(self, path):
+        entry, gitignore = self._get_gitignore(path)
+
+        if not os.path.exists(gitignore):
+            return
+
+        with open(gitignore, 'r') as fobj:
+            lines = fobj.readlines()
+
+        filtered = list(filter(lambda x: x.strip() != entry.strip(), lines))
+
+        with open(gitignore, 'w') as fobj:
+            fobj.writelines(filtered)
+
+        if self.project is not None:
+            self.project.files_to_git_add.append(os.path.relpath(gitignore))
+
+    def add(self, paths):
+        # NOTE: GitPython is not currently able to handle index version >= 3.
+        # See https://github.com/iterative/dvc/issues/610 for more details.
+        try:
+            self.repo.index.add(paths)
+        except AssertionError:
+            msg = (
+                "failed to add '{}' to git. You can add those files"
+                " manually using 'git add'."
+                " See 'https://github.com/iterative/dvc/issues/610'"
+                " for more details."
+                .format(str(paths))
+            )
+
+            logger.error(msg)
+
+    def commit(self, msg):
+        self.repo.index.commit(msg)
+
+    def checkout(self, branch, create_new=False):
+        if create_new:
+            self.repo.git.checkout('HEAD', b=branch)
+        else:
+            self.repo.git.checkout(branch)
+
+    def branch(self, branch):
+        self.repo.git.branch(branch)
+
+    def tag(self, tag):
+        self.repo.git.tag(tag)
+
+    def untracked_files(self):
+        files = self.repo.untracked_files
+        return [os.path.join(self.repo.working_dir, fname) for fname in files]
+
+    def is_tracked(self, path):
+        return len(self.repo.git.ls_files(path)) != 0
+
+    def active_branch(self):
+        return self.repo.active_branch.name
+
+    def list_branches(self):
+        return [h.name for h in self.repo.heads]
+
+    def list_tags(self):
+        return [t.name for t in self.repo.tags]
+
+    def _install_hook(self, name, cmd):
+        hook = os.path.join(self.root_dir,
+                            self.GIT_DIR,
+                            'hooks',
+                            name)
+        if os.path.isfile(hook):
+            msg = "git hook '{}' already exists."
+            raise SCMError(msg.format(os.path.relpath(hook)))
+        with open(hook, 'w+') as fobj:
+            fobj.write('#!/bin/sh\nexec dvc {}\n'.format(cmd))
+        os.chmod(hook, 0o777)
+
+    def install(self):
+        self._install_hook('post-checkout', 'checkout')
+        self._install_hook('pre-commit', 'status')

dvc/signal_handler.py

@@ -1,17 +0,0 @@
-import signal
-import threading
-
-from dvc.logger import Logger
-
-
-class SignalHandler(object):
-    def __enter__(self):
-        if isinstance(threading.current_thread(), threading._MainThread):
-            self.old_handler = signal.signal(signal.SIGINT, self.handler)
-
-    def handler(self, sig, frame):
-        Logger.debug('Ignoring SIGINT during critical parts of code...')
-
-    def __exit__(self, type, value, traceback):
-        if isinstance(threading.current_thread(), threading._MainThread):
-            signal.signal(signal.SIGINT, self.old_handler)

dvc/stage.py

@@ -1,41 +1,81 @@
 import os
 import yaml
-import itertools
 import posixpath
 import subprocess
 from schema import Schema, SchemaError, Optional, Or, And
 
+import dvc.prompt as prompt
+import dvc.logger as logger
 import dvc.dependency as dependency
 import dvc.output as output
 from dvc.exceptions import DvcException
-from dvc.prompt import prompt
-from dvc.logger import Logger
 from dvc.utils import dict_md5, fix_env
 
 
 class StageCmdFailedError(DvcException):
     def __init__(self, stage):
-        msg = u'Stage \'{}\' cmd {} failed'.format(stage.relpath, stage.cmd)
+        msg = u"stage '{}' cmd {} failed".format(stage.relpath, stage.cmd)
         super(StageCmdFailedError, self).__init__(msg)
 
 
 class StageFileFormatError(DvcException):
-    def __init__(self):
-        super(StageFileFormatError, self).__init__('Stage file format error')
+    def __init__(self, fname, e):
+        msg = "stage file '{}' format error: {}".format(fname, str(e))
+        super(StageFileFormatError, self).__init__(msg)
 
 
 class StageFileDoesNotExistError(DvcException):
     def __init__(self, fname):
         msg = "'{}' does not exist.".format(fname)
+
+        sname = fname + Stage.STAGE_FILE_SUFFIX
+        if Stage.is_stage_file(sname):
+            msg += " Do you mean '{}'?".format(sname)
+
         super(StageFileDoesNotExistError, self).__init__(msg)
 
 
+class StageFileAlreadyExistsError(DvcException):
+    def __init__(self, relpath):
+        msg = "stage '{}' already exists".format(relpath)
+        super(StageFileAlreadyExistsError, self).__init__(msg)
+
+
 class StageFileIsNotDvcFileError(DvcException):
     def __init__(self, fname):
         msg = "'{}' is not a dvc file".format(fname)
+
+        sname = fname + Stage.STAGE_FILE_SUFFIX
+        if Stage.is_stage_file(sname):
+            msg += " Do you mean '{}'?".format(sname)
+
         super(StageFileIsNotDvcFileError, self).__init__(msg)
 
 
+class StageFileBadNameError(DvcException):
+    def __init__(self, msg):
+        super(StageFileBadNameError, self).__init__(msg)
+
+
+class StageBadCwdError(DvcException):
+    def __init__(self, cwd):
+        msg = "stage cwd '{}' is outside of the current dvc project"
+        super(StageBadCwdError, self).__init__(msg.format(cwd))
+
+
+class MissingDep(DvcException):
+    def __init__(self, deps):
+        assert len(deps) > 0
+
+        if len(deps) > 1:
+            dep = 'dependencies'
+        else:
+            dep = 'dependency'
+
+        msg = u'missing {}: {}'.format(dep, ', '.join(map(str, deps)))
+        super(MissingDep, self).__init__(msg)
+
+
 class MissingDataSource(DvcException):
     def __init__(self, missing_files):
         assert len(missing_files) > 0
@@ -84,80 +124,113 @@ class Stage(object):
         self.md5 = md5
         self.locked = locked
 
+    def __repr__(self):
+        return "Stage: '{path}'".format(
+            path=self.relpath if self.path else 'No path'
+        )
+
     @property
     def relpath(self):
         return os.path.relpath(self.path)
 
     @property
     def is_data_source(self):
+        """Whether the stage file was created with `dvc add` or `dvc import`"""
         return self.cmd is None
 
+    @staticmethod
+    def is_valid_filename(path):
+        return (path.endswith(Stage.STAGE_FILE_SUFFIX)
+                or os.path.basename(path) == Stage.STAGE_FILE)
+
     @staticmethod
     def is_stage_file(path):
-        if not os.path.isfile(path):
-            return False
-
-        if not path.endswith(Stage.STAGE_FILE_SUFFIX) \
-           and os.path.basename(path) != Stage.STAGE_FILE:
-            return False
-
-        return True
+        return os.path.isfile(path) and Stage.is_valid_filename(path)
 
     def changed_md5(self):
-        md5 = self.dumpd().get(self.PARAM_MD5, None)
-
-        # backward compatibility
-        if self.md5 is None:
-            return False
-
-        if self.md5 and md5 and self.md5 == md5:
-            return False
-
-        msg = "Dvc file '{}' md5 changed(expected '{}', actual '{}')"
-        self.project.logger.debug(msg.format(self.relpath, self.md5, md5))
-        return True
+        return self.md5 != self._compute_md5()
 
     @property
     def is_callback(self):
+        """
+        A callback stage is always considered as changed,
+        so it runs on every `dvc repro` call.
+        """
         return not self.is_data_source and len(self.deps) == 0
 
     @property
     def is_import(self):
-        return not self.cmd and \
-               len(self.deps) == 1 and \
-               len(self.outs) == 1
+        """Whether the stage file was created with `dvc import`."""
+        return (not self.cmd
+                and len(self.deps) == 1
+                and len(self.outs) == 1)
 
-    def changed(self):
-        ret = False
+    def _changed_deps(self):
+        if self.locked:
+            return False
 
         if self.is_callback:
-            ret = True
+            logger.warning(
+                "Dvc file '{fname}' is a 'callback' stage (has a command and"
+                " no dependencies) and thus always considered as changed."
+                .format(fname=self.relpath)
+            )
+            return True
 
-        if self.locked:
-            entries = self.outs
-        else:
-            entries = itertools.chain(self.outs, self.deps)
+        for dep in self.deps:
+            if dep.changed():
+                logger.warning("Dependency '{dep}' of '{stage}' changed."
+                               .format(dep=dep, stage=self.relpath))
+                return True
 
-        for entry in entries:
-            if entry.changed():
-                ret = True
+        return False
 
+    def _changed_outs(self):
+        for out in self.outs:
+            if out.changed():
+                logger.warning(
+                    "Output '{out}' of '{stage}' changed."
+                    .format(out=out, stage=self.relpath)
+                )
+                return True
+
+        return False
+
+    def _changed_md5(self):
         if self.changed_md5():
-            ret = True
+            logger.warning("Dvc file '{}' changed.".format(self.relpath))
+            return True
+        return False
+
+    def changed(self):
+        ret = any([self._changed_deps(),
+                   self._changed_outs(),
+                   self._changed_md5()])
 
         if ret:
-            msg = u'Dvc file \'{}\' changed'.format(self.relpath)
+            msg = "Stage '{}' changed.".format(self.relpath)
+            color = 'yellow'
         else:
-            msg = u'Dvc file \'{}\' didn\'t change'.format(self.relpath)
+            msg = "Stage '{}' didn't change.".format(self.relpath)
+            color = 'green'
 
-        self.project.logger.debug(msg)
+        logger.info(logger.colorize(msg, color))
 
         return ret
 
     def remove_outs(self, ignore_remove=False):
+        """
+        Used mainly for `dvc remove --outs`
+        """
         for out in self.outs:
             out.remove(ignore_remove=ignore_remove)
 
+    def unprotect_outs(self):
+        for out in self.outs:
+            if out.scheme != 'local' or not out.exists:
+                continue
+            self.project.unprotect(out.path)
+
     def remove(self):
         self.remove_outs(ignore_remove=True)
         os.unlink(self.path)
@@ -170,49 +243,28 @@ class Stage(object):
             # Removing outputs only if we actually have command to reproduce
             self.remove_outs(ignore_remove=False)
 
-        msg = "Going to reproduce '{}'. Are you sure you want to continue?"
-        msg = msg.format(self.relpath)
-        if interactive and not prompt(msg):
-            raise DvcException('Reproduction aborted by the user')
+        msg = (
+            "Going to reproduce '{stage}'. Are you sure you want to continue?"
+            .format(stage=self.relpath)
+        )
 
-        self.project.logger.info(u'Reproducing \'{}\''.format(self.relpath))
+        if interactive and not prompt.confirm(msg):
+            raise DvcException('reproduction aborted by the user')
+
+        logger.info(u"Reproducing '{stage}'".format(stage=self.relpath))
 
         self.run(dry=dry)
 
-        msg = u'\'{}\' was reproduced'.format(self.relpath)
-        self.project.logger.debug(msg)
+        logger.debug("'{stage}' was reproduced".format(stage=self.relpath))
 
         return self
 
     @staticmethod
-    def validate(d):
+    def validate(d, fname=None):
         try:
             Schema(Stage.SCHEMA).validate(d)
         except SchemaError as exc:
-            Logger.debug(str(exc))
-            raise StageFileFormatError()
-
-    @staticmethod
-    def loadd(project, d, path):
-        Stage.validate(d)
-
-        path = os.path.abspath(path)
-        cwd = os.path.dirname(path)
-        cmd = d.get(Stage.PARAM_CMD, None)
-        md5 = d.get(Stage.PARAM_MD5, None)
-        locked = d.get(Stage.PARAM_LOCKED, False)
-
-        stage = Stage(project=project,
-                      path=path,
-                      cmd=cmd,
-                      cwd=cwd,
-                      md5=md5,
-                      locked=locked)
-
-        stage.deps = dependency.loadd_from(stage, d.get(Stage.PARAM_DEPS, []))
-        stage.outs = output.loadd_from(stage, d.get(Stage.PARAM_OUTS, []))
-
-        return stage
+            raise StageFileFormatError(fname, exc)
 
     @classmethod
     def _stage_fname_cwd(cls, fname, cwd, outs, add):
@@ -223,7 +275,7 @@ class Stage(object):
             return (cls.STAGE_FILE, cwd if cwd else os.getcwd())
 
         out = outs[0]
-        if out.path_info['scheme'] == 'local':
+        if out.scheme == 'local':
             path = os.path
         else:
             path = posixpath
@@ -237,17 +289,58 @@ class Stage(object):
         return (fname, cwd)
 
     @staticmethod
-    def loads(project=None,
-              cmd=None,
-              deps=[],
-              outs=[],
-              outs_no_cache=[],
-              metrics_no_cache=[],
-              fname=None,
-              cwd=os.curdir,
-              locked=False,
-              add=False,
-              overwrite=True):
+    def _check_inside_project(project, cwd):
+        assert project is not None
+        proj_dir = os.path.realpath(project.root_dir)
+        if not os.path.realpath(cwd).startswith(proj_dir):
+            raise StageBadCwdError(cwd)
+
+    @property
+    def is_cached(self):
+        """
+        Checks if this stage has been already ran and stored
+        """
+        from dvc.remote.local import RemoteLOCAL
+        from dvc.remote.s3 import RemoteS3
+
+        old = Stage.load(self.project, self.path)
+        if old._changed_outs():
+            return False
+
+        # NOTE: need to save checksums for deps in order to compare them
+        # with what is written in the old stage.
+        for dep in self.deps:
+            dep.save()
+
+        old_d = old.dumpd()
+        new_d = self.dumpd()
+
+        # NOTE: need to remove checksums from old dict in order to compare
+        # it to the new one, since the new one doesn't have checksums yet.
+        old_d.pop(self.PARAM_MD5, None)
+        new_d.pop(self.PARAM_MD5, None)
+        outs = old_d.get(self.PARAM_OUTS, [])
+        for out in outs:
+            out.pop(RemoteLOCAL.PARAM_CHECKSUM, None)
+            out.pop(RemoteS3.PARAM_CHECKSUM, None)
+
+        return old_d == new_d
+
+    @staticmethod
+    def create(project=None,
+               cmd=None,
+               deps=[],
+               outs=[],
+               outs_no_cache=[],
+               metrics_no_cache=[],
+               fname=None,
+               cwd=os.curdir,
+               locked=False,
+               add=False,
+               overwrite=True,
+               ignore_build_cache=False,
+               remove_outs=False):
+
         stage = Stage(project=project,
                       cwd=cwd,
                       cmd=cmd,
@@ -259,73 +352,129 @@ class Stage(object):
                                         use_cache=False, metric=True)
         stage.deps = dependency.loads_from(stage, deps)
 
+        stage._check_circular_dependency()
+        stage._check_duplicated_arguments()
+
+        if fname is not None and os.path.basename(fname) != fname:
+            raise StageFileBadNameError(
+                "stage file name '{fname}' should not contain subdirectories."
+                " Use '-c|--cwd' to change location of the stage file."
+                .format(fname=fname)
+            )
+
         fname, cwd = Stage._stage_fname_cwd(fname, cwd, stage.outs, add=add)
 
+        Stage._check_inside_project(project, cwd)
+
         cwd = os.path.abspath(cwd)
         path = os.path.join(cwd, fname)
 
-        if os.path.exists(path):
-            relpath = os.path.relpath(path)
-            msg = "'{}' already exists. " \
-                  "Do you wish to run the command and overwrite it?"
-            if not overwrite and not prompt(msg.format(relpath), False):
-                raise DvcException("'{}' already exists".format(relpath))
-
         stage.cwd = cwd
         stage.path = path
 
+        # NOTE: remove outs before we check build cache
+        if remove_outs:
+            stage.remove_outs(ignore_remove=False)
+            logger.warning("Build cache is ignored when using --remove-outs.")
+            ignore_build_cache = True
+        else:
+            stage.unprotect_outs()
+
+        if os.path.exists(path):
+            if not ignore_build_cache and stage.is_cached:
+                logger.info('Stage is cached, skipping.')
+                return None
+
+            msg = "'{}' already exists. Do you wish to run the command and " \
+                  "overwrite it?".format(stage.relpath)
+
+            if not overwrite and not prompt.confirm(msg):
+                raise StageFileAlreadyExistsError(stage.relpath)
+
+            os.unlink(path)
+
         return stage
 
     @staticmethod
-    def _check_dvc_file(fname):
-        sname = fname + Stage.STAGE_FILE_SUFFIX
-        if Stage.is_stage_file(sname):
-            Logger.info("Do you mean '{}'?".format(sname))
+    def _check_dvc_filename(fname):
+        if not Stage.is_valid_filename(fname):
+            raise StageFileBadNameError(
+                "bad stage filename '{}'. Stage files should be named"
+                " 'Dvcfile' or have a '.dvc' suffix(e.g. '{}.dvc')."
+                .format(os.path.relpath(fname), os.path.basename(fname))
+            )
+
+    @staticmethod
+    def _check_file_exists(fname):
+        if not os.path.exists(fname):
+            raise StageFileDoesNotExistError(fname)
 
     @staticmethod
     def load(project, fname):
-        if not os.path.exists(fname):
-            Stage._check_dvc_file(fname)
-            raise StageFileDoesNotExistError(fname)
+        Stage._check_file_exists(fname)
+        Stage._check_dvc_filename(fname)
 
         if not Stage.is_stage_file(fname):
-            Stage._check_dvc_file(fname)
             raise StageFileIsNotDvcFileError(fname)
 
         with open(fname, 'r') as fd:
-            return Stage.loadd(project, yaml.safe_load(fd), fname)
+            d = yaml.safe_load(fd) or {}
+
+        Stage.validate(d, fname=os.path.relpath(fname))
+
+        stage = Stage(project=project,
+                      path=os.path.abspath(fname),
+                      cwd=os.path.dirname(os.path.abspath(fname)),
+                      cmd=d.get(Stage.PARAM_CMD),
+                      md5=d.get(Stage.PARAM_MD5),
+                      locked=d.get(Stage.PARAM_LOCKED, False))
+
+        stage.deps = dependency.loadd_from(stage, d.get(Stage.PARAM_DEPS, []))
+        stage.outs = output.loadd_from(stage, d.get(Stage.PARAM_OUTS, []))
+
+        return stage
 
     def dumpd(self):
-        deps = [x.dumpd() for x in self.deps]
-        outs = [x.dumpd() for x in self.outs]
-
-        ret = {}
-        if self.cmd is not None:
-            ret[Stage.PARAM_CMD] = self.cmd
-
-        if len(deps):
-            ret[Stage.PARAM_DEPS] = deps
-
-        if len(outs):
-            ret[Stage.PARAM_OUTS] = outs
-
-        if self.locked:
-            ret[Stage.PARAM_LOCKED] = self.locked
-
-        ret[Stage.PARAM_MD5] = dict_md5(ret)
-
-        return ret
+        return {
+            key: value
+            for key, value in {
+                Stage.PARAM_MD5: self.md5,
+                Stage.PARAM_CMD: self.cmd,
+                Stage.PARAM_LOCKED: self.locked,
+                Stage.PARAM_DEPS: [d.dumpd() for d in self.deps],
+                Stage.PARAM_OUTS: [o.dumpd() for o in self.outs],
+            }.items()
+            if value
+        }
 
     def dump(self, fname=None):
-        if not fname:
-            fname = self.path
+        fname = fname or self.path
 
-        msg = "Saving information to '{}'.".format(os.path.relpath(fname))
-        Logger.info(msg)
+        self._check_dvc_filename(fname)
+
+        logger.info("Saving information to '{file}'."
+                    .format(file=os.path.relpath(fname)))
 
         with open(fname, 'w') as fd:
             yaml.safe_dump(self.dumpd(), fd, default_flow_style=False)
 
+        self.project.files_to_git_add.append(os.path.relpath(fname))
+
+    def _compute_md5(self):
+        from dvc.output.local import OutputLOCAL
+
+        d = self.dumpd()
+
+        # NOTE: removing md5 manually in order to not affect md5s in deps/outs
+        if self.PARAM_MD5 in d.keys():
+            del d[self.PARAM_MD5]
+
+        # NOTE: excluding parameters that don't affect the state of the
+        # pipeline. Not excluding `OutputLOCAL.PARAM_CACHE`, because if
+        # it has changed, we might not have that output in our cache.
+        return dict_md5(d, exclude=[self.PARAM_LOCKED,
+                                    OutputLOCAL.PARAM_METRIC])
+
     def save(self):
         for dep in self.deps:
             dep.save()
@@ -333,51 +482,109 @@ class Stage(object):
         for out in self.outs:
             out.save()
 
+        self.md5 = self._compute_md5()
+
+    def _check_missing_deps(self):
+        missing = [dep for dep in self.deps if not dep.exists]
+
+        if any(missing):
+            raise MissingDep(missing)
+
+    def _warn_if_fish(self, executable):  # pragma: no cover
+        if executable is None \
+           or os.path.basename(os.path.realpath(executable)) != 'fish':
+            return
+
+        logger.warning(
+            "DVC detected that you are using fish as your default "
+            "shell. Be aware that it might cause problems by overwriting "
+            "your current environment variables with values defined "
+            "in '.fishrc', which might affect your command. See "
+            "https://github.com/iterative/dvc/issues/1307. "
+        )
+
+    def _check_circular_dependency(self):
+        from dvc.exceptions import CircularDependencyError
+
+        circular_dependencies = (
+            set(file.path for file in self.deps) &
+            set(file.path for file in self.outs)
+        )
+
+        if circular_dependencies:
+            raise CircularDependencyError(circular_dependencies.pop())
+
+    def _check_duplicated_arguments(self):
+        from dvc.exceptions import ArgumentDuplicationError
+        from collections import Counter
+
+        count = Counter(file.path for file in self.deps + self.outs)
+
+        for file, occurrence in count.items():
+            if occurrence > 1:
+                raise ArgumentDuplicationError(file)
+
+    def _run(self):
+        self._check_missing_deps()
+        executable = os.getenv('SHELL') if os.name != 'nt' else None
+        self._warn_if_fish(executable)
+
+        p = subprocess.Popen(self.cmd,
+                             cwd=self.cwd,
+                             shell=True,
+                             env=fix_env(os.environ),
+                             executable=executable)
+        p.communicate()
+
+        if p.returncode != 0:
+            raise StageCmdFailedError(self)
+
     def run(self, dry=False):
         if self.locked:
-            msg = u'Verifying outputs in locked stage \'{}\''
-            self.project.logger.info(msg.format(self.relpath))
+            logger.info("Verifying outputs in locked stage '{stage}'"
+                        .format(stage=self.relpath))
             if not dry:
                 self.check_missing_outputs()
+
         elif self.is_import:
-            msg = u'Importing \'{}\' -> \'{}\''
-            self.project.logger.info(msg.format(self.deps[0].path,
-                                                self.outs[0].path))
-
+            logger.info("Importing '{dep}' -> '{out}'"
+                        .format(dep=self.deps[0].path, out=self.outs[0].path))
             if not dry:
-                self.deps[0].download(self.outs[0].path_info)
+                if self._already_cached():
+                    self.outs[0].checkout()
+                else:
+                    self.deps[0].download(self.outs[0].path_info)
+
         elif self.is_data_source:
-            msg = u'Verifying data sources in \'{}\''.format(self.relpath)
-            self.project.logger.info(msg)
+            msg = u"Verifying data sources in '{}'".format(self.relpath)
+            logger.info(msg)
             if not dry:
                 self.check_missing_outputs()
-        else:
-            msg = u'Running command:\n\t{}'.format(self.cmd)
-            self.project.logger.info(msg)
 
+        else:
+            logger.info('Running command:\n\t{}'.format(self.cmd))
             if not dry:
-                p = subprocess.Popen(self.cmd,
-                                     cwd=self.cwd,
-                                     shell=True,
-                                     env=fix_env(os.environ),
-                                     executable=os.getenv('SHELL'))
-                p.communicate()
-                if p.returncode != 0:
-                    raise StageCmdFailedError(self)
+                if self._already_cached():
+                    self.checkout()
+                else:
+                    self._run()
 
         if not dry:
             self.save()
 
     def check_missing_outputs(self):
-        outs = [out for out in self.outs if not out.exists]
-        paths = [out.path if out.path_info['scheme'] != 'local' else
-                 out.rel_path for out in outs]
+        paths = [
+            out.path if out.scheme != 'local' else out.rel_path
+            for out in self.outs
+            if not out.exists
+        ]
+
         if paths:
             raise MissingDataSource(paths)
 
-    def checkout(self):
+    def checkout(self, force=False):
         for out in self.outs:
-            out.checkout()
+            out.checkout(force=force)
 
     def _status(self, entries, name):
         ret = {}
@@ -402,3 +609,11 @@ class Stage(object):
             return {self.relpath: ret}
 
         return {}
+
+    def _already_cached(self):
+        return (not self.changed_md5()
+                and all(not dep.changed() for dep in self.deps)
+                and all(not out.changed_cache()
+                        if out.use_cache
+                        else not out.changed()
+                        for out in self.outs))

dvc/state.py

@@ -1,210 +1,422 @@
-import os
-import json
-import nanotime
-import threading
+"""Manages state database used for checksum caching."""
 
-from dvc.lock import Lock
+import os
+import time
+import sqlite3
+import nanotime
+
+import dvc.logger as logger
+from dvc.config import Config
 from dvc.system import System
 from dvc.utils import file_md5, remove
 from dvc.exceptions import DvcException
-from dvc.signal_handler import SignalHandler
-from dvc.logger import Logger
 
 
-class StateEntry(object):
-    PARAM_MTIME = 'mtime'
-    PARAM_MD5 = 'md5'
-
-    def __init__(self, md5, mtime):
-        self.mtime = mtime
-        self.md5 = md5
-
-    def update(self, md5, mtime):
-        self.mtime = mtime
-        self.md5 = md5
-
-    @staticmethod
-    def loadd(d):
-        mtime = d[StateEntry.PARAM_MTIME]
-        md5 = d[StateEntry.PARAM_MD5]
-        return StateEntry(md5, mtime)
-
-    def dumpd(self):
-        return {
-            self.PARAM_MD5: self.md5,
-            self.PARAM_MTIME: self.mtime,
-        }
+class StateVersionTooNewError(DvcException):
+    """Thrown when dvc version is older than the state database version."""
+    def __init__(self, dvc_version, expected, actual):
+        super(StateVersionTooNewError, self).__init__(
+            "you are using an old version '{dvc_version}' of dvc that is "
+            "using state file version '{expected}' which is not compatible "
+            "with the state file version '{actual}' that is used in this "
+            "projet. Please upgrade right now!"
+            .format(dvc_version=dvc_version,
+                    expected=expected,
+                    actual=actual))
 
 
-class StateDuplicateError(DvcException):
-    pass
+class State(object):  # pylint: disable=too-many-instance-attributes
+    """Class for the state database.
 
+    Args:
+        project (dvc.project.Project): project instance that this state
+            belongs to.
+        config (configobj.ConfigObj): config for the state.
 
-class State(object):
+    Raises:
+        StateVersionTooNewError: thrown when dvc version is older than the
+            state database version.
+    """
+    VERSION = 3
     STATE_FILE = 'state'
-    STATE_LOCK_FILE = 'state.lock'
+    STATE_TABLE = 'state'
+    STATE_TABLE_LAYOUT = ("inode INTEGER PRIMARY KEY, "
+                          "mtime TEXT NOT NULL, "
+                          "size TEXT NOT NULL, "
+                          "md5 TEXT NOT NULL, "
+                          "timestamp TEXT NOT NULL")
 
-    def __init__(self, project):
+    STATE_INFO_TABLE = 'state_info'
+    STATE_INFO_TABLE_LAYOUT = 'count INTEGER'
+    STATE_INFO_ROW = 1
+
+    LINK_STATE_TABLE = 'link_state'
+    LINK_STATE_TABLE_LAYOUT = ("path TEXT PRIMARY KEY, "
+                               "inode INTEGER NOT NULL, "
+                               "mtime TEXT NOT NULL")
+
+    STATE_ROW_LIMIT = 100000000
+    STATE_ROW_CLEANUP_QUOTA = 50
+
+    MAX_INT = 2**63 - 1
+    MAX_UINT = 2**64 - 2
+
+    def __init__(self, project, config):
         self.project = project
         self.dvc_dir = project.dvc_dir
-        self._lock = threading.Lock()
-        if self.dvc_dir:
-            self.state_file = os.path.join(self.dvc_dir, self.STATE_FILE)
-            self._lock_file = Lock(self.dvc_dir, self.STATE_LOCK_FILE)
-        else:
-            self.state_file = None
-            self._lock_file = threading.Lock()
-        self._db = self.load()
+        self.root_dir = project.root_dir
 
-    @staticmethod
-    def init(project):
-        return State(project)
+        self.row_limit = 100
+        self.row_cleanup_quota = 50
+
+        state_config = config.get(Config.SECTION_STATE, {})
+        self.row_limit = state_config.get(Config.SECTION_STATE_ROW_LIMIT,
+                                          self.STATE_ROW_LIMIT)
+        self.row_cleanup_quota = state_config.get(
+            Config.SECTION_STATE_ROW_CLEANUP_QUOTA,
+            self.STATE_ROW_CLEANUP_QUOTA)
+
+        if not self.dvc_dir:
+            self.state_file = None
+            return
+
+        self.state_file = os.path.join(self.dvc_dir, self.STATE_FILE)
+
+        # https://www.sqlite.org/tempfiles.html
+        self.temp_files = [
+            self.state_file + '-journal',
+            self.state_file + '-wal',
+        ]
+
+        self.database = None
+        self.cursor = None
+        self.inserts = 0
+
+    def __enter__(self):
+        self.load()
+
+    def __exit__(self, typ, value, tbck):
+        self.dump()
 
     def _collect(self, path):
         if os.path.isdir(path):
             return self.project.cache.local.collect_dir_cache(path)
-        else:
-            return (file_md5(path)[0], None)
+        return (file_md5(path)[0], None)
 
     def changed(self, path, md5):
+        """Check if file/directory has the expected md5.
+
+        Args:
+            path (str): path to the file/directory to check.
+            md5 (str): expected md5.
+
+        Returns:
+            bool: True if path has the expected md5, False otherwise.
+        """
         actual = self.update(path)
 
+        msg = "File '{}', md5 '{}', actual '{}'"
+        logger.debug(msg.format(path, md5, actual))
+
         if not md5 or not actual:
             return True
 
         return actual.split('.')[0] != md5.split('.')[0]
 
-    def load(self):
-        if not self.state_file or not os.path.isfile(self.state_file):
-            return {}
+    def _execute(self, cmd):
+        logger.debug(cmd)
+        return self.cursor.execute(cmd)
 
-        with open(self.state_file, 'r') as fd:
+    def _fetchall(self):
+        ret = self.cursor.fetchall()
+        logger.debug("fetched: {}".format(ret))
+        return ret
+
+    def _to_sqlite(self, num):
+        assert num >= 0
+        assert num < self.MAX_UINT
+        # NOTE: sqlite stores unit as signed ints, so maximum uint is 2^63-1
+        # see http://jakegoulding.com/blog/2011/02/06/sqlite-64-bit-integers/
+        if num > self.MAX_INT:
+            ret = -(num - self.MAX_INT)
+        else:
+            ret = num
+        assert self._from_sqlite(ret) == num
+        return ret
+
+    def _from_sqlite(self, num):
+        assert abs(num) <= self.MAX_INT
+        if num < 0:
+            return abs(num) + self.MAX_INT
+        assert num < self.MAX_UINT
+        assert num >= 0
+        return num
+
+    def _prepare_db(self, empty=False):
+        from dvc import VERSION
+
+        if not empty:
+            cmd = "PRAGMA user_version;"
+            self._execute(cmd)
+            ret = self._fetchall()
+            assert len(ret) == 1
+            assert len(ret[0]) == 1
+            assert isinstance(ret[0][0], int)
+            version = ret[0][0]
+
+            if version > self.VERSION:
+                raise StateVersionTooNewError(VERSION, self.VERSION, version)
+            elif version < self.VERSION:
+                msg = "State file version '{}' is too old. " \
+                      "Reformatting to the current version '{}'."
+                logger.warning(msg.format(version, self.VERSION))
+                cmd = "DROP TABLE IF EXISTS {};"
+                self._execute(cmd.format(self.STATE_TABLE))
+                self._execute(cmd.format(self.STATE_INFO_TABLE))
+                self._execute(cmd.format(self.LINK_STATE_TABLE))
+
+        # Check that the state file is indeed a database
+        cmd = "CREATE TABLE IF NOT EXISTS {} ({})"
+        self._execute(cmd.format(self.STATE_TABLE,
+                                 self.STATE_TABLE_LAYOUT))
+        self._execute(cmd.format(self.STATE_INFO_TABLE,
+                                 self.STATE_INFO_TABLE_LAYOUT))
+        self._execute(cmd.format(self.LINK_STATE_TABLE,
+                                 self.LINK_STATE_TABLE_LAYOUT))
+
+        cmd = "INSERT OR IGNORE INTO {} (count) SELECT 0 " \
+              "WHERE NOT EXISTS (SELECT * FROM {})"
+        self._execute(cmd.format(self.STATE_INFO_TABLE,
+                                 self.STATE_INFO_TABLE))
+
+        cmd = "PRAGMA user_version = {};"
+        self._execute(cmd.format(self.VERSION))
+
+    def load(self):
+        """Loads state database."""
+        retries = 1
+        while True:
+            assert self.database is None
+            assert self.cursor is None
+            assert self.inserts == 0
+            empty = not os.path.exists(self.state_file)
+            self.database = sqlite3.connect(self.state_file)
+            self.cursor = self.database.cursor()
+
+            # Try loading once to check that the file is indeed a database
+            # and reformat it if it is not.
             try:
-                return json.load(fd)
-            except ValueError as exc:
-                Logger.error('Failed to load \'{}\''.format(self.state_file),
-                             exc)
-                return {}
+                self._prepare_db(empty=empty)
+                return
+            except sqlite3.DatabaseError:
+                self.cursor.close()
+                self.database.close()
+                self.database = None
+                self.cursor = None
+                self.inserts = 0
+                if retries > 0:
+                    os.unlink(self.state_file)
+                    retries -= 1
+                else:
+                    raise
+
+    def _vacuum(self):
+        # NOTE: see https://bugs.python.org/issue28518
+        self.database.isolation_level = None
+        self._execute("VACUUM")
+        self.database.isolation_level = ''
 
     def dump(self):
-        if not self.state_file:
-            return
+        """Saves state database."""
+        assert self.database is not None
 
-        with SignalHandler():
-            with open(self.state_file, 'w+') as fd:
-                json.dump(self._db, fd)
+        cmd = "SELECT count from {} WHERE rowid={}"
+        self._execute(cmd.format(self.STATE_INFO_TABLE,
+                                 self.STATE_INFO_ROW))
+        ret = self._fetchall()
+        assert len(ret) == 1
+        assert len(ret[0]) == 1
+        count = self._from_sqlite(ret[0][0]) + self.inserts
+
+        if count > self.row_limit:
+            msg = "cleaning up state, this might take a while."
+            logger.warning(msg)
+
+            delete = (count - self.row_limit)
+            delete += int(self.row_limit * (self.row_cleanup_quota/100.))
+            cmd = "DELETE FROM {} WHERE timestamp IN (" \
+                  "SELECT timestamp FROM {} ORDER BY timestamp ASC LIMIT {});"
+            self._execute(cmd.format(self.STATE_TABLE,
+                                     self.STATE_TABLE,
+                                     delete))
+
+            self._vacuum()
+
+            cmd = "SELECT COUNT(*) FROM {}"
+
+            self._execute(cmd.format(self.STATE_TABLE))
+            ret = self._fetchall()
+            assert len(ret) == 1
+            assert len(ret[0]) == 1
+            count = ret[0][0]
+
+        cmd = "UPDATE {} SET count = {} WHERE rowid = {}"
+        self._execute(cmd.format(self.STATE_INFO_TABLE,
+                                 self._to_sqlite(count),
+                                 self.STATE_INFO_ROW))
+
+        self.database.commit()
+        self.cursor.close()
+        self.database.close()
+        self.database = None
+        self.cursor = None
+        self.inserts = 0
 
     @staticmethod
-    def mtime(path):
-        return str(int(nanotime.timestamp(os.path.getmtime(path))))
+    def _mtime_and_size(path):
+        size = os.path.getsize(path)
+        mtime = os.path.getmtime(path)
+
+        if os.path.isdir(path):
+            for root, dirs, files in os.walk(path):
+                for name in dirs + files:
+                    entry = os.path.join(root, name)
+                    stat = os.stat(entry)
+                    size += stat.st_size
+                    entry_mtime = stat.st_mtime
+                    if entry_mtime > mtime:
+                        mtime = entry_mtime
+
+        # State of files handled by dvc is stored in db as TEXT.
+        # We cast results to string for later comparisons with stored values.
+        return str(int(nanotime.timestamp(mtime))), str(size)
 
     @staticmethod
-    def inode(path):
-        return str(System.inode(path))
+    def _inode(path):
+        logger.debug('Path {} inode {}'.format(path, System.inode(path)))
+        return System.inode(path)
 
-    def _do_update(self, path, dump=True):
+    def _do_update(self, path):
+        """
+        Make sure the stored info for the given path is up to date.
+        """
         if not os.path.exists(path):
             return (None, None)
 
-        mtime = self.mtime(path)
-        inode = self.inode(path)
+        actual_mtime, actual_size = self._mtime_and_size(path)
+        actual_inode = self._inode(path)
 
-        md5 = self._get(inode, mtime)
-        if md5:
-            return (md5, None)
+        cmd = ('SELECT * from {} WHERE inode={}'
+               .format(self.STATE_TABLE, self._to_sqlite(actual_inode)))
 
-        md5, info = self._collect(path)
-        state = StateEntry(md5, mtime)
-        d = state.dumpd()
-
-        with self._lock:
-            with self._lock_file:
-                self._db[inode] = d
-
-                if dump:
-                    self.dump()
+        self._execute(cmd)
+        ret = self._fetchall()
+        if not ret:
+            md5, info = self._collect(path)
+            cmd = 'INSERT INTO {}(inode, mtime, size, md5, timestamp) ' \
+                  'VALUES ({}, "{}", "{}", "{}", "{}")'
+            self._execute(cmd.format(self.STATE_TABLE,
+                                     self._to_sqlite(actual_inode),
+                                     actual_mtime,
+                                     actual_size,
+                                     md5,
+                                     int(nanotime.timestamp(time.time()))))
+            self.inserts += 1
+        else:
+            assert len(ret) == 1
+            assert len(ret[0]) == 5
+            inode, mtime, size, md5, _ = ret[0]
+            inode = self._from_sqlite(inode)
+            assert inode == actual_inode
+            logger.debug(
+                "Inode '{}', mtime '{}', actual mtime '{}', size '{}', "
+                "actual size '{}'."
+                .format(inode, mtime, actual_mtime, size, actual_size))
+            if actual_mtime != mtime or actual_size != size:
+                md5, info = self._collect(path)
+                cmd = ('UPDATE {} SET '
+                       'mtime = "{}", size = "{}", '
+                       'md5 = "{}", timestamp = "{}" '
+                       'WHERE inode = {}')
+                self._execute(cmd.format(self.STATE_TABLE,
+                                         actual_mtime,
+                                         actual_size,
+                                         md5,
+                                         int(nanotime.timestamp(time.time())),
+                                         self._to_sqlite(actual_inode)))
+            else:
+                info = None
+                cmd = 'UPDATE {} SET timestamp = "{}" WHERE inode = {}'
+                self._execute(cmd.format(self.STATE_TABLE,
+                                         int(nanotime.timestamp(time.time())),
+                                         self._to_sqlite(actual_inode)))
 
         return (md5, info)
 
-    def update(self, path, dump=True):
-        return self._do_update(path, dump=dump)[0]
+    def update(self, path):
+        """Gets the checksum for the specified path. Checksum will be
+        retrieved from the state database if available, otherwise it will be
+        computed and cached in the state database for the further use.
 
-    def update_info(self, path, dump=True):
-        md5, info = self._do_update(path, dump=dump)
+        Args:
+            path (str): path to get the checksum for.
+
+        Returns:
+            str: checksum for the specified path.
+        """
+        return self._do_update(path)[0]
+
+    def update_info(self, path):
+        """Gets the checksum and the directory info (if applicable) for the
+        specified path.
+
+        Args:
+            path (str): path to get the checksum and the directory info for.
+
+        Returns:
+            tuple: checksum for the specified path along with a directory info
+            (list of {relative_path: checksum} entries for each file in the
+            directory) if applicable, otherwise None.
+        """
+        md5, info = self._do_update(path)
         if not info:
             info = self.project.cache.local.load_dir_cache(md5)
         return (md5, info)
 
-    def _get(self, inode, mtime):
-        with self._lock:
-            with self._lock_file:
-                d = self._db.get(inode, None)
+    def update_link(self, path):
+        """Adds the specified path to the list of links created by dvc. This
+        list is later used on `dvc checkout` to cleanup old links.
 
-        if not d:
-            return None
-
-        state = StateEntry.loadd(d)
-        if mtime == state.mtime:
-            return state.md5
-
-        return None
-
-    def get(self, path):
-        mtime = self.mtime(path)
-        inode = self.inode(path)
-
-        return self._get(inode, mtime)
-
-
-class LinkStateEntry(object):
-    PARAM_MTIME = 'mtime'
-    PARAM_INODE = 'inode'
-
-    def __init__(self, inode, mtime):
-        self.mtime = mtime
-        self.inode = inode
-
-    @staticmethod
-    def loadd(d):
-        mtime = d[LinkStateEntry.PARAM_MTIME]
-        inode = d[LinkStateEntry.PARAM_INODE]
-        return LinkStateEntry(inode, mtime)
-
-    def dumpd(self):
-        return {
-            self.PARAM_INODE: self.inode,
-            self.PARAM_MTIME: self.mtime,
-        }
-
-
-class LinkState(State):
-    STATE_FILE = 'link.state'
-    STATE_LOCK_FILE = STATE_FILE + '.lock'
-
-    def __init__(self, project):
-        super(LinkState, self).__init__(project)
-        self.root_dir = project.root_dir
-
-    def update(self, path, dump=True):
+        Args:
+            path (str): path to add to the list of links.
+        """
         if not os.path.exists(path):
             return
 
-        mtime = self.mtime(path)
-        inode = self.inode(path)
+        mtime, _ = self._mtime_and_size(path)
+        inode = self._inode(path)
+        relpath = os.path.relpath(path, self.root_dir)
 
-        with self._lock:
-            state = LinkStateEntry(inode, mtime)
-            d = state.dumpd()
-            self._db[os.path.relpath(path, self.root_dir)] = d
-            if dump:
-                with self._lock_file:
-                    self.dump()
+        cmd = 'REPLACE INTO {}(path, inode, mtime) ' \
+              'VALUES ("{}", {}, "{}")'.format(self.LINK_STATE_TABLE,
+                                               relpath,
+                                               self._to_sqlite(inode),
+                                               mtime)
+        self._execute(cmd)
 
-    def _do_remove_unused(self, used):
-        items = self._db.copy().items()
-        for p, s in items:
-            path = os.path.join(self.root_dir, p)
-            state = LinkStateEntry.loadd(s)
+    def remove_unused_links(self, used):
+        """Removes all saved links except the ones that are used.
+
+        Args:
+            used (list): list of used links that should not be removed.
+        """
+        unused = []
+
+        self._execute('SELECT * FROM {}'.format(self.LINK_STATE_TABLE))
+        for row in self.cursor:
+            relpath, inode, mtime = row
+            inode = self._from_sqlite(inode)
+            path = os.path.join(self.root_dir, relpath)
 
             if path in used:
                 continue
@@ -212,16 +424,14 @@ class LinkState(State):
             if not os.path.exists(path):
                 continue
 
-            inode = self.inode(path)
-            mtime = self.mtime(path)
+            actual_inode = self._inode(path)
+            actual_mtime, _ = self._mtime_and_size(path)
 
-            if inode == state.inode and mtime == state.mtime:
-                Logger.debug('Removing \'{}\' as unused link.'.format(path))
+            if inode == actual_inode and mtime == actual_mtime:
+                logger.debug("Removing '{}' as unused link.".format(path))
                 remove(path)
-                del self._db[p]
+                unused.append(relpath)
 
-    def remove_unused(self, used):
-        with self._lock:
-            with self._lock_file:
-                self._do_remove_unused(used)
-                self.dump()
+        for relpath in unused:
+            cmd = 'DELETE FROM {} WHERE path = "{}"'
+            self._execute(cmd.format(self.LINK_STATE_TABLE, relpath))

dvc/system.py

@@ -1,9 +1,4 @@
 import os
-import ctypes
-import reflink
-
-if os.name == 'nt':
-    import ntfsutils.hardlink as winlink
 
 
 class System(object):
@@ -13,15 +8,38 @@ class System(object):
 
     @staticmethod
     def hardlink(source, link_name):
-        if System.is_unix():
-            return os.link(source, link_name)
+        import ctypes
+        from dvc.exceptions import DvcException
 
-        return winlink.create(source, link_name)
+        if System.is_unix():
+            try:
+                os.link(source, link_name)
+                return
+            except Exception as exc:
+                raise DvcException('link', cause=exc)
+
+        CreateHardLink = ctypes.windll.kernel32.CreateHardLinkW
+        CreateHardLink.argtypes = [ctypes.c_wchar_p,
+                                   ctypes.c_wchar_p,
+                                   ctypes.c_void_p]
+        CreateHardLink.restype = ctypes.wintypes.BOOL
+
+        res = CreateHardLink(link_name, source, None)
+        if res == 0:
+            raise DvcException('CreateHardLinkW', cause=ctypes.WinError())
 
     @staticmethod
     def symlink(source, link_name):
+        import ctypes
+        from dvc.exceptions import DvcException
+
         if System.is_unix():
-            return os.symlink(source, link_name)
+            try:
+                os.symlink(source, link_name)
+                return
+            except Exception as exc:
+                msg = "failed to symlink '{}' -> '{}': {}"
+                raise DvcException(msg.format(source, link_name, str(exc)))
 
         flags = 0
         if source is not None and os.path.isdir(source):
@@ -32,45 +50,217 @@ class System(object):
         func.restype = ctypes.c_ubyte
 
         if func(link_name, source, flags) == 0:
-            raise ctypes.WinError()
+            raise DvcException('CreateSymbolicLinkW', cause=ctypes.WinError())
+
+    @staticmethod
+    def _reflink_darwin(src, dst):
+        import ctypes
+
+        clib = ctypes.CDLL('libc.dylib')
+        if not hasattr(clib, 'clonefile'):
+            return -1
+
+        clonefile = clib.clonefile
+        clonefile.argtypes = [ctypes.c_char_p,
+                              ctypes.c_char_p,
+                              ctypes.c_int]
+        clonefile.restype = ctypes.c_int
+
+        return clonefile(ctypes.c_char_p(src.encode('utf-8')),
+                         ctypes.c_char_p(dst.encode('utf-8')),
+                         ctypes.c_int(0))
+
+    @staticmethod
+    def _reflink_windows(src, dst):
+        return -1
+
+    @staticmethod
+    def _reflink_linux(src, dst):
+        import os
+        import fcntl
+
+        FICLONE = 0x40049409
+
+        s = open(src, 'r')
+        d = open(dst, 'w+')
+
+        try:
+            ret = fcntl.ioctl(d.fileno(), FICLONE, s.fileno())
+        except IOError:
+            s.close()
+            d.close()
+            os.unlink(dst)
+            raise
+
+        s.close()
+        d.close()
+
+        if ret != 0:
+            os.unlink(dst)
+
+        return ret
 
     @staticmethod
     def reflink(source, link_name):
-        return reflink.reflink(source, link_name)
+        import platform
+        from dvc.exceptions import DvcException
+
+        system = platform.system()
+        try:
+            if system == 'Windows':
+                ret = System._reflink_windows(source, link_name)
+            elif system == 'Darwin':
+                ret = System._reflink_darwin(source, link_name)
+            elif system == 'Linux':
+                ret = System._reflink_linux(source, link_name)
+            else:
+                ret = -1
+        except IOError:
+            ret = -1
+
+        if ret != 0:
+            raise DvcException('reflink is not supported')
 
-    # FIXME
-    # Temporary fix while waiting for the PR to be merged and released:
-    # https://github.com/sunshowers/ntfs/pull/11
     @staticmethod
     def getdirinfo(path):
-        from ntfsutils.fs import FILE_FLAG_BACKUP_SEMANTICS, FILE_SHARE_READ
-        from ntfsutils.fs import OPEN_EXISTING, BY_HANDLE_FILE_INFORMATION
-        from ntfsutils.fs import CreateFile, WinError
-        from ntfsutils.fs import GetFileInformationByHandle, CloseHandle
-        flags = FILE_FLAG_BACKUP_SEMANTICS
-        hfile = CreateFile(path,
-                           0,
-                           FILE_SHARE_READ,
-                           None,
-                           OPEN_EXISTING,
-                           flags,
-                           None)
+        import ctypes
+        from ctypes import c_void_p, c_wchar_p, Structure, WinError, POINTER
+        from ctypes.wintypes import DWORD, HANDLE, BOOL
+
+        # NOTE: use this flag to open symlink itself and not the target
+        # See https://docs.microsoft.com/en-us/windows/desktop/api/
+        # fileapi/nf-fileapi-createfilew#symbolic-link-behavior
+        FILE_FLAG_OPEN_REPARSE_POINT = 0x00200000
+
+        FILE_FLAG_BACKUP_SEMANTICS = 0x02000000
+        FILE_SHARE_READ = 0x00000001
+        OPEN_EXISTING = 3
+
+        class FILETIME(Structure):
+            _fields_ = [("dwLowDateTime", DWORD),
+                        ("dwHighDateTime", DWORD)]
+
+        class BY_HANDLE_FILE_INFORMATION(Structure):
+            _fields_ = [("dwFileAttributes", DWORD),
+                        ("ftCreationTime", FILETIME),
+                        ("ftLastAccessTime", FILETIME),
+                        ("ftLastWriteTime", FILETIME),
+                        ("dwVolumeSerialNumber", DWORD),
+                        ("nFileSizeHigh", DWORD),
+                        ("nFileSizeLow", DWORD),
+                        ("nNumberOfLinks", DWORD),
+                        ("nFileIndexHigh", DWORD),
+                        ("nFileIndexLow", DWORD)]
+
+        flags = FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT
+
+        func = ctypes.windll.kernel32.CreateFileW
+        func.argtypes = [c_wchar_p,
+                         DWORD,
+                         DWORD,
+                         c_void_p,
+                         DWORD,
+                         DWORD,
+                         HANDLE]
+        func.restype = HANDLE
+
+        hfile = func(path,
+                     0,
+                     FILE_SHARE_READ,
+                     None,
+                     OPEN_EXISTING,
+                     flags,
+                     None)
         if hfile is None:
             raise WinError()
+
+        func = ctypes.windll.kernel32.GetFileInformationByHandle
+        func.argtypes = [HANDLE, POINTER(BY_HANDLE_FILE_INFORMATION)]
+        func.restype = BOOL
+
         info = BY_HANDLE_FILE_INFORMATION()
-        rv = GetFileInformationByHandle(hfile, info)
-        CloseHandle(hfile)
+        rv = func(hfile, info)
+
+        func = ctypes.windll.kernel32.CloseHandle
+        func.argtypes = [HANDLE]
+        func.restype = BOOL
+
+        func(hfile)
+
         if rv == 0:
             raise WinError()
+
         return info
 
     @staticmethod
     def inode(path):
         if System.is_unix():
-            return os.lstat(path).st_ino
+            import ctypes
+            inode = os.lstat(path).st_ino
+            # NOTE: See https://bugs.python.org/issue29619 and
+            # https://stackoverflow.com/questions/34643289/
+            # pythons-os-stat-is-returning-wrong-inode-value
+            inode = ctypes.c_ulong(inode).value
+        else:
+            # getdirinfo from ntfsutils works on both files and dirs
+            info = System.getdirinfo(path)
+            inode = abs(hash((info.dwVolumeSerialNumber,
+                              info.nFileIndexHigh,
+                              info.nFileIndexLow)))
+        assert inode >= 0
+        assert inode < 2**64
+        return inode
 
-        # getdirinfo from ntfsutils works on both files and dirs
+    @staticmethod
+    def _wait_for_input_windows(timeout):
+        import sys
+        import ctypes
+        import msvcrt
+        from ctypes.wintypes import DWORD, HANDLE
+
+        # https://docs.microsoft.com/en-us/windows/desktop/api/synchapi/nf-synchapi-waitforsingleobject
+        WAIT_OBJECT_0 = 0
+        WAIT_TIMEOUT = 0x00000102
+
+        func = ctypes.windll.kernel32.WaitForSingleObject
+        func.argtypes = [HANDLE, DWORD]
+        func.restype = DWORD
+
+        rc = func(msvcrt.get_osfhandle(sys.stdin.fileno()), timeout * 1000)
+        if rc not in [WAIT_OBJECT_0, WAIT_TIMEOUT]:
+            raise RuntimeError(rc)
+
+    @staticmethod
+    def _wait_for_input_posix(timeout):
+        import sys
+        import select
+        try:
+            select.select([sys.stdin], [], [], timeout)
+        except select.error:
+            pass
+
+    @staticmethod
+    def wait_for_input(timeout):
+        if System.is_unix():
+            return System._wait_for_input_posix(timeout)
+        else:
+            return System._wait_for_input_windows(timeout)
+
+    @staticmethod
+    def is_symlink(path):
+        if System.is_unix():
+            return os.path.islink(path)
+
+        # https://docs.microsoft.com/en-us/windows/desktop/fileio/
+        # file-attribute-constants
+        FILE_ATTRIBUTE_REPARSE_POINT = 0x400
         info = System.getdirinfo(path)
-        return hash((info.dwVolumeSerialNumber,
-                     info.nFileIndexHigh,
-                     info.nFileIndexLow))
+        return info.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT
+
+    @staticmethod
+    def is_hardlink(path):
+        if System.is_unix():
+            return os.stat(path).st_nlink > 1
+
+        info = System.getdirinfo(path)
+        return info.nNumberOfLinks > 1

dvc/updater.py

@@ -1,61 +1,193 @@
+import sys
 import os
 import time
 import requests
+import colorama
 
+import dvc.logger as logger
 from dvc import VERSION_BASE
-from dvc.logger import Logger
+from dvc.lock import Lock, LockError
+from dvc.utils import is_binary
 
 
-class Updater(object):
-    URL = 'https://4ki8820rsf.execute-api.us-east-2.amazonaws.com/' \
-          'prod/latest-version'
+class Updater(object):  # pragma: no cover
+    URL = 'https://updater.dvc.org'
     UPDATER_FILE = 'updater'
-    TIMEOUT = 7 * 24 * 60 * 60  # every week
+    TIMEOUT = 24 * 60 * 60  # every day
     TIMEOUT_GET = 10
 
     def __init__(self, dvc_dir):
         self.dvc_dir = dvc_dir
         self.updater_file = os.path.join(dvc_dir, self.UPDATER_FILE)
+        self.lock = Lock(dvc_dir, self.updater_file + '.lock')
+        self.current = VERSION_BASE
 
-    @staticmethod
-    def init(dvc_dir):
-        return Updater(dvc_dir)
+    def _is_outdated_file(self):
+        ctime = os.path.getmtime(self.updater_file)
+        outdated = (time.time() - ctime >= self.TIMEOUT)
+        if outdated:
+            logger.debug("'{}' is outdated(".format(self.updater_file))
+        return outdated
+
+    def _with_lock(self, func, action):
+        try:
+            with self.lock:
+                func()
+        except LockError:
+            msg = "Failed to acquire '{}' before {} updates"
+            logger.debug(msg.format(self.lock.lock_file, action))
 
     def check(self):
-        current = VERSION_BASE
-
-        if os.getenv('CI'):
+        if os.getenv('CI') or os.getenv('DVC_TEST'):
             return
 
-        if os.path.isfile(self.updater_file):
-            ctime = os.path.getctime(self.updater_file)
-            if time.time() - ctime < self.TIMEOUT:
-                msg = '{} is not old enough to check for updates'
-                Logger.debug(msg.format(self.UPDATER_FILE))
+        self._with_lock(self._check, 'checking')
+
+    def _check(self):
+        if not os.path.exists(self.updater_file) or self._is_outdated_file():
+            self.fetch()
+            return
+
+        with open(self.updater_file, 'r') as fobj:
+            import json
+
+            try:
+                info = json.load(fobj)
+                self.latest = info['version']
+            except Exception as exc:
+                msg = "'{}' is not a valid json: {}"
+                logger.debug(msg.format(self.updater_file, exc))
+                self.fetch()
                 return
 
-            os.unlink(self.updater_file)
+        if self._is_outdated():
+            self._notify()
 
-        Logger.info('Checking for updates...')
+    def fetch(self, detach=True):
+        from dvc.daemon import daemon
+
+        if detach:
+            daemon(['updater'])
+            return
+
+        self._with_lock(self._get_latest_version, 'fetching')
+
+    def _get_latest_version(self):
+        import json
 
         try:
             r = requests.get(self.URL, timeout=self.TIMEOUT_GET)
-            j = r.json()
-            latest = j['version']
-            open(self.updater_file, 'w+').close()
-        except Exception as exc:
-            msg = 'Failed to obtain latest version: {}'.format(str(exc))
-            Logger.debug(msg)
+            info = r.json()
+        except requests.exceptions.RequestException as exc:
+            msg = "Failed to retrieve latest version: {}"
+            logger.debug(msg.format(exc))
             return
 
-        l_major, l_minor, l_patch = [int(x) for x in latest.split('.')]
-        c_major, c_minor, c_patch = [int(x) for x in current.split('.')]
+        with open(self.updater_file, 'w+') as fobj:
+            json.dump(info, fobj)
 
-        if l_major <= c_major and \
-           l_minor <= c_minor and \
-           l_patch <= c_patch:
+    def _is_outdated(self):
+        l_major, l_minor, l_patch = [int(x) for x in self.latest.split('.')]
+        c_major, c_minor, c_patch = [int(x) for x in self.current.split('.')]
+
+        if l_major != c_major:
+            return l_major > c_major
+
+        if l_minor != c_minor:
+            return l_minor > c_minor
+
+        return l_patch > c_patch
+
+    def _notify(self):
+        if not sys.stdout.isatty():
             return
 
-        msg = 'You are using dvc version {}, however version {} is ' \
-              'available. Consider upgrading.'
-        Logger.warn(msg.format(current, latest))
+        message = (
+            'Update available {red}{current}{reset} -> {green}{latest}{reset}'
+            + '\n'
+            + self._get_update_instructions()
+        ).format(red=colorama.Fore.RED,
+                 reset=colorama.Fore.RESET,
+                 green=colorama.Fore.GREEN,
+                 yellow=colorama.Fore.YELLOW,
+                 blue=colorama.Fore.BLUE,
+                 current=self.current,
+                 latest=self.latest)
+
+        logger.box(message, border_color='yellow')
+
+    def _get_update_instructions(self):
+        instructions = {
+            'pip': 'Run {yellow}pip{reset} install dvc {blue}--upgrade{reset}',
+            'yum': 'Run {yellow}yum{reset} update dvc',
+            'yay': 'Run {yellow}yay{reset} {blue}-S{reset} dvc',
+            'formula': 'Run {yellow}brew{reset} upgrade dvc',
+            'cask': 'Run {yellow}brew cask{reset} upgrade dvc',
+            'apt': ('Run {yellow}apt-get{reset} install'
+                    ' {blue}--only-upgrade{reset} dvc'),
+            'binary': ('To upgrade follow this steps:\n'
+                       '1. Uninstall dvc binary\n'
+                       '2. Go to {blue}https://dvc.org{reset}\n'
+                       '3. Download and install new binary'),
+            None: ('Find the latest release at\n{blue}'
+                   'https://github.com/iterative/dvc/releases/latest'
+                   '{reset}'),
+        }
+
+        package_manager = self._get_package_manager()
+
+        return instructions[package_manager]
+
+    def _get_linux(self):
+        import distro
+
+        if not is_binary():
+            return 'pip'
+
+        package_managers = {
+            'rhel':     'yum',
+            'centos':   'yum',
+            'fedora':   'yum',
+            'amazon':   'yum',
+            'opensuse': 'yum',
+            'ubuntu':   'apt',
+            'debian':   'apt',
+        }
+
+        return package_managers.get(distro.id())
+
+    def _get_darwin(self):
+        if not is_binary():
+            if __file__.startswith('/usr/local/Cellar'):
+                return 'formula'
+            else:
+                return 'pip'
+
+        # NOTE: both pkg and cask put dvc binary into /usr/local/bin,
+        # so in order to know which method of installation was used,
+        # we need to actually call `brew cask`
+        ret = os.system('brew cask ls dvc')
+        if ret == 0:
+            return 'cask'
+
+        return None
+
+    def _get_windows(self):
+        return None if is_binary() else 'pip'
+
+    def _get_package_manager(self):
+        import platform
+        from dvc.exceptions import DvcException
+
+        m = {
+            'Windows': self._get_windows,
+            'Darwin': self._get_darwin,
+            'Linux': self._get_linux,
+        }
+
+        system = platform.system()
+        func = m.get(system)
+        if func is None:
+            raise DvcException("not supported system '{}'".format(system))
+
+        return func()

dvc/utils/__init__.py

@@ -3,14 +3,15 @@ Helpers for other modules.
 """
 import os
 import sys
+import stat
 import math
 import json
 import shutil
 import hashlib
 
+import dvc.logger as logger
 from dvc.progress import progress
 from dvc.istextfile import istextfile
-from dvc.logger import Logger
 
 
 LOCAL_CHUNK_SIZE = 1024*1024
@@ -32,7 +33,7 @@ def file_md5(fname):
         if size >= LARGE_FILE_SIZE:
             bar = True
             msg = "Computing md5 for a large file {}. This is only done once."
-            Logger.info(msg.format(os.path.relpath(fname)))
+            logger.info(msg.format(os.path.relpath(fname)))
             name = os.path.relpath(fname)
             total = 0
 
@@ -67,8 +68,31 @@ def bytes_md5(byts):
     return hasher.hexdigest()
 
 
-def dict_md5(d):
-    byts = json.dumps(d, sort_keys=True).encode('utf-8')
+def dict_filter(d, exclude=[]):
+    """
+    Exclude specified keys from a nested dict
+    """
+
+    if isinstance(d, list):
+        ret = []
+        for e in d:
+            ret.append(dict_filter(e, exclude))
+        return ret
+    elif isinstance(d, dict):
+        ret = {}
+        for k, v in d.items():
+            assert isinstance(k, str)
+            if k in exclude:
+                continue
+            ret[k] = dict_filter(v, exclude)
+        return ret
+
+    return d
+
+
+def dict_md5(d, exclude=[]):
+    filtered = dict_filter(d, exclude)
+    byts = json.dumps(filtered, sort_keys=True).encode('utf-8')
     return bytes_md5(byts)
 
 
@@ -102,6 +126,7 @@ def copyfile(src, dest, no_progress_bar=False, name=None):
 
 
 def move(src, dst):
+    dst = os.path.abspath(dst)
     dname = os.path.dirname(dst)
     if not os.path.exists(dname):
         os.makedirs(dname)
@@ -118,11 +143,18 @@ def remove(path):
     if not os.path.exists(path):
         return
 
-    Logger.debug(u'Removing \'{}\''.format(os.path.relpath(path)))
+    logger.debug(u"Removing '{}'".format(os.path.relpath(path)))
+
+    def _chmod(func, p, excinfo):
+        perm = os.stat(p).st_mode
+        perm |= stat.S_IWRITE
+        os.chmod(p, perm)
+        func(p)
+
     if os.path.isfile(path):
-        os.unlink(path)
+        _chmod(os.unlink, path, None)
     else:
-        shutil.rmtree(path)
+        shutil.rmtree(path, onerror=_chmod)
 
 
 def to_chunks(l, jobs):
@@ -137,17 +169,21 @@ def to_chunks(l, jobs):
     return [l[x:x+n] for x in range(0, len(l), n)]
 
 
+# NOTE: Check if we are in a bundle
+# https://pythonhosted.org/PyInstaller/runtime-information.html
+def is_binary():
+    return getattr(sys, 'frozen', False)
+
+
 # NOTE: Fix env variables modified by PyInstaller
 # http://pyinstaller.readthedocs.io/en/stable/runtime-information.html
-def fix_env(env):
+def fix_env(env=None):
     if env is None:
         env = os.environ.copy()
     else:
         env = env.copy()
 
-    # NOTE: Check if we are in a bundle
-    # https://pythonhosted.org/PyInstaller/runtime-information.html
-    if getattr(sys, 'frozen', False):
+    if is_binary():
         lp_key = 'LD_LIBRARY_PATH'
         lp_orig = env.get(lp_key + '_ORIG', None)
         if lp_orig is not None:

dvc/utils/compat.py

@@ -0,0 +1,6 @@
+"""Handle import compatibility between Python 2 and Python 3"""
+
+try:
+    from urlparse import urlparse, urljoin  # noqa: F401
+except ImportError:
+    from urllib.parse import urlparse, urljoin  # noqa: F401

requirements.txt

@@ -4,23 +4,24 @@ ply>=3.9 # See https://github.com/pyinstaller/pyinstaller/issues/1945
 configparser>=3.5.0
 zc.lockfile>=1.2.1
 future>=0.16.0
-google-cloud==0.32.0
+google-cloud-storage==1.13.0
 PyInstaller==3.3.1
 colorama>=0.3.9
 configobj>=5.0.6
-networkx==2.1
+networkx>=2.1
 pyyaml>=3.12
 gitpython>=2.1.8
-ntfsutils>=0.1.4
 setuptools>=34.0.0
 nanotime>=0.5.2
 pyasn1>=0.4.1
 schema>=0.6.7
 paramiko>=2.4.1
 jsonpath-rw==1.4.0
-reflink==0.2.0
 requests>=2.18.4
 wheel>=0.31.1
 futures>=3.2.0; python_version == "2.7"
 grandalf==0.6
-asciicanvas==0.0.3
+pydot>=1.2.4
+asciimatics>=1.10.0
+distro>=1.3.0
+appdirs>=1.4.3

scripts/build_posix.sh

@@ -8,11 +8,14 @@ if [[ "$(uname)" == 'Linux' ]]; then
 	FPM_FLAGS=
 else
 	INSTALL_DIR=usr/local
-	FPM_FLAGS='--osxpkg-identifier-prefix com.dataversioncontrol'
+	FPM_FLAGS='--osxpkg-identifier-prefix com.iterative'
+	FPM_FLAGS+=' --after-install scripts/fpm/after-install.sh'
+	FPM_FLAGS+=' --after-remove scripts/fpm/after-remove.sh'
 fi
 
 BUILD_DIR=build
 BIN_DIR=$BUILD_DIR/$INSTALL_DIR/bin
+LIB_DIR=$BUILD_DIR/$INSTALL_DIR/lib
 
 print_error()
 {
@@ -40,7 +43,13 @@ fpm_build()
 {
 	print_info "Building $1..."
 	VERSION=$(python -c "import dvc; from dvc import VERSION; print(str(VERSION))")
-	fpm -s dir -f -t $1 $FPM_FLAGS -n dvc -v $VERSION -C $BUILD_DIR $INSTALL_DIR
+	fpm -s dir \
+	    -f \
+	    -t $1 \
+	    $FPM_FLAGS \
+	    -n dvc \
+	    -v $VERSION \
+	    -C $BUILD_DIR usr
 }
 
 cleanup()
@@ -66,7 +75,7 @@ install_dependencies()
 		echo "Unable to install fpm dependencies" && exit 1
 	fi
 
-	gem install --no-ri --no-rdoc fpm
+	gem install --no-document fpm
 
 	print_info "Installing requirements..."
 	pip install -r requirements.txt
@@ -78,9 +87,24 @@ install_dependencies()
 build_dvc()
 {
 	print_info "Building dvc binary..."
-	pyinstaller --onefile --additional-hooks-dir $(pwd)/scripts/hooks dvc/__main__.py --name dvc --distpath $BIN_DIR --specpath $BUILD_DIR
+	pyinstaller \
+            --additional-hooks-dir $(pwd)/scripts/hooks dvc/__main__.py \
+            --name dvc \
+            --distpath $LIB_DIR \
+            --specpath $BUILD_DIR
 
-	$BIN_DIR/dvc --help
+	$LIB_DIR/dvc/dvc --help
+
+	# NOTE: in osxpkg fpm replaces symlinks with actual file that it
+	# points to, so we need to use after-install hook. See FPM_FLAGS
+	# above.
+	if [[ "$(uname)" == 'Linux' ]]; then
+		mkdir -p $BIN_DIR
+		pushd $BIN_DIR
+		ln -s ../lib/dvc/dvc dvc
+		popd
+		$BIN_DIR/dvc --help
+	fi
 }
 
 cleanup

scripts/build_windows.cmd

@@ -19,7 +19,7 @@ call pip install -r requirements.txt || goto :error
 call pip install pyinstaller || goto :error
 
 echo ====== Building dvc binary... ======
-call pyinstaller --onefile --additional-hooks-dir scripts\hooks dvc/__main__.py --name dvc --specpath build
+call pyinstaller --additional-hooks-dir scripts\hooks dvc/__main__.py --name dvc --specpath build
 
 echo ====== Copying additional files... ======
 copy scripts\innosetup\addSymLinkPermissions.ps1 dist\ || goto :error

scripts/ci/install.sh

@@ -14,7 +14,7 @@ function retry {
 # NOTE: it is not uncommon for pip to hang on travis for what seems to be
 # networking issues. Thus, let's retry a few times to see if it will eventially
 # work or not.
-retry pip install --upgrade pip
+retry pip install --upgrade pip setuptools wheel
 retry pip install -r requirements.txt
 retry pip install -r test-requirements.txt
 

scripts/ci/script.sh

@@ -6,7 +6,8 @@ set -e
 python -mtests
 
 if [[ "$TRAVIS_PULL_REQUEST" == "false" && \
-      "$TRAVIS_SECURE_ENV_VARS" == "true" ]]; then
+      "$TRAVIS_SECURE_ENV_VARS" == "true" && \
+      "$(python -c 'import sys; print(sys.version_info[0])')" == '2' ]]; then
 	codecov
 fi
 

scripts/completion/dvc.bash

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+
+#----------------------------------------------------------
+# Repository:  https://github.com/iterative/dvc
+#
+# References:
+#   - https://www.gnu.org/software/bash/manual/html_node/Programmable-Completion.html
+#   - https://opensource.com/article/18/3/creating-bash-completion-script
+#----------------------------------------------------------
+
+_dvc_commands='init destroy add import checkout run pull push fetch \
+              status repro remove move gc config remote metrics install \
+              root lock unlock pipeline'
+
+_dvc_global_options="-h --help -q --quiet -V --verbose"
+_dvc_options="-h --help -v --version"
+_dvc_init="--no-scm -f --force"
+_dvc_destroy="-f --force"
+_dvc_add="-R --recursive"
+_dvc_run="--no-exec -f --file -c --cwd -d --deps -o --outs -O --outs-no-cache -M --metrics-no-cache -y --yes"
+_dvc_pull="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps"
+_dvc_push="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps"
+_dvc_fetch="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps"
+_dvc_status="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -c --cloud"
+_dvc_repro="--dry -f --force -s --single-item -c --cwd -m --metrics -i --interactive -p --pipeline -P --all-pipelines"
+_dvc_remove="--dry -o --outs -p --purge"
+_dvc_gc="-a --all-branches -T --all-tags -c --cloud -r --remote"
+_dvc_config="-u --unset --local"
+_dvc_checkout="$(compgen -G '*.dvc')"
+_dvc_move="$(compgen -G '*')"
+_dvc_lock="$(compgen -G '*.dvc')"
+_dvc_unlock="$(compgen -G '*.dvc')"
+_dvc_import=""
+_dvc_remote=""
+_dvc_metrics=""
+_dvc_install=""
+_dvc_root=""
+_dvc_pipeline=""
+
+# Notes:
+#
+# `COMPREPLY` contains what will be rendered after completion is triggered
+#
+# `word` refers to the current typed word
+#
+# `${!var}` is to evaluate the content of `var` and expand its content as a variable
+#
+#       hello="world"
+#       x="hello"
+#       ${!x} ->  ${hello} ->  "world"
+#
+_dvc () {
+  local word="${COMP_WORDS[COMP_CWORD]}"
+
+  COMPREPLY=()
+
+  if [ "${COMP_CWORD}" -eq 1 ]; then
+    case "$word" in
+      -*) COMPREPLY=($(compgen -W "$_dvc_options" -- "$word")) ;;
+      *)  COMPREPLY=($(compgen -W "$_dvc_commands" -- "$word")) ;;
+    esac
+  elif [ "${COMP_CWORD}" -eq 2 ]; then
+    local options_list="_dvc_${COMP_WORDS[1]}"
+
+    COMPREPLY=($(compgen -W "$_dvc_global_options ${!options_list}" -- "$word"))
+  fi
+
+  return 0
+}
+
+complete -F _dvc dvc

scripts/completion/dvc.zsh

@@ -0,0 +1,219 @@
+#compdef dvc
+
+#----------------------------------------------------------
+# Repository:  https://github.com/iterative/dvc
+#
+# References:
+#   - https://github.com/zsh-users/zsh-completions
+#   - http://zsh.sourceforge.net/Doc/Release/Completion-System.html
+#   - https://mads-hartmann.com/2017/08/06/writing-zsh-completion-scripts.html
+#   - http://www.linux-mag.com/id/1106/
+#----------------------------------------------------------
+
+_dvc_commands() {
+  local _commands=(
+    "init:Initialize dvc over a directory (should already be a git dir)."
+    "destroy:Destroy dvc. Will remove all project's information, data files and cache."
+    "add:Add files/directories to dvc."
+    "import:Import files from URL."
+    "checkout:Checkout data files from cache."
+    "run:Generate a stage file from a given command and execute the command."
+    "pull:Pull data files from the cloud."
+    "push:Push data files to the cloud."
+    "fetch:Fetch data files from the cloud."
+    "status:Show the project status."
+    "repro:Reproduce DVC file. Default file name - 'Dvcfile'."
+    "remove:Remove outputs of DVC file."
+    "move:Move output of DVC file."
+    "gc:Collect garbage."
+    "config:Get or set config options."
+    "remote:Manage set of tracked repositories."
+    "metrics:Get metrics from all branches."
+    "install:Install dvc hooks into the repository."
+    "root:Relative path to project's directory."
+    "lock:Lock DVC file."
+    "unlock:Unlock DVC file."
+    "pipeline:Manage pipeline."
+  )
+
+  _describe 'dvc commands' _commands
+}
+
+_dvc_global_options=(
+  "(-)"{-h,--help}"[Show help message related to the command]"
+  "(-)"{-q,--quiet}"[Be quiet.]"
+  "(-)"{-V,--verbose}"[Be verbose.]"
+)
+
+_dvc_options=(
+  "(-)"{-h,--help}"[Show this help message and exit]"
+  "(-)"{-v,--version}"[Show program's version]"
+)
+
+_dvc_init=(
+  "--no-scm[Initiate dvc in directory that is not tracket by any scm tool]"
+  {-f,--force}"[Overwrite '.dvc' if it exists. Will remove all local cache.]"
+)
+
+_dvc_destroy=(
+  {-f,--force}"[Overwrite '.dvc' if it exists. Will remove all local cache.]"
+)
+
+_dvc_add=(
+  {-R,--recursive}"[Recursively add each file under the directory.]"
+  "1:File:_files"
+)
+
+_dvc_import=(
+  "1:URL:"
+  "2:Output:"
+)
+
+_dvc_checkout=(
+  "1:DVC files:_files -g '(*.dvc|Dvcfile)'"
+)
+
+_dvc_run=(
+  "--no-exec[Only create stage file without actually running it.]"
+  {-f,--file}"[Specify name of the stage file.]:File:_files"
+  {-c,--cwd}"[Directory to run your command and place state file in.]:CWD:_files -/"
+  "*"{-d,--deps}"[Declare dependencies for reproducible cmd.]:Dependency:_files"
+  "*"{-o,--outs}"[Declare output data file or data directory.]:Output data:_files"
+  "*"{-O,--outs-no-cache}"[Declare output regular file or directory.]:Output regular:_files"
+  "*"{-M,--metrics-no-cache}"[Declare output metric file or directory]:Metrics (no cache):_files"
+  {-y,--yes}"[Automatic 'yes' answer to all prompts.]"
+)
+
+_dvc_pull=(
+  "*:DVC files:_files -g '(*.dvc|Dvcfile)'"
+  "--show-checksums[Show checksums instead of file names]"
+  {-j,--jobs}"[Number of jobs to run simultaneously]:Number of jobs:"
+  {-r,--remote}"[Remote repository to pull from]:Remote repository:"
+  {-a,--all-branches}"[Fetch cache for all branches]"
+  {-T,--all-tags}"[Fetch cache for all tags]"
+  {-d,--with-deps}"[Fetch cache for all dependencies of the specified target]"
+)
+
+_dvc_push=(
+  "*:DVC files:_files -g '(*.dvc|Dvcfile)'"
+  "--show-checksums[Show checksums instead of file names]"
+  {-j,--jobs}"[Number of jobs to run simultaneously]:Number of jobs:"
+  {-r,--remote}"[Remote repository to pull from]:Remote repository:"
+  {-a,--all-branches}"[Fetch cache for all branches]"
+  {-T,--all-tags}"[Fetch cache for all tags]"
+  {-d,--with-deps}"[Fetch cache for all dependencies of the specified target]"
+)
+
+_dvc_fetch=(
+  "*:DVC files:_files -g '(*.dvc|Dvcfile)'"
+  "--show-checksums[Show checksums instead of file names]"
+  {-j,--jobs}"[Number of jobs to run simultaneously]:Number of jobs:"
+  {-r,--remote}"[Remote repository to pull from]:Remote repository:"
+  {-a,--all-branches}"[Fetch cache for all branches]"
+  {-T,--all-tags}"[Fetch cache for all tags]"
+  {-d,--with-deps}"[Fetch cache for all dependencies of the specified target]"
+)
+
+_dvc_status=(
+  "*:DVC files:_files -g '(*.dvc|Dvcfile)'"
+  "--show-checksums[Show checksums instead of file names]"
+  {-j,--jobs}"[Number of jobs to run simultaneously]:Number of jobs:"
+  {-r,--remote}"[Remote repository to pull from]:Remote repository:"
+  {-a,--all-branches}"[Fetch cache for all branches]"
+  {-T,--all-tags}"[Fetch cache for all tags]"
+  {-d,--with-deps}"[Fetch cache for all dependencies of the specified target]"
+  {-c,--cloud}"[Show status of a local cache compared to a remote repository]"
+)
+
+_dvc_repro=(
+  "*:DVC files:_files -g '(*.dvc|Dvcfile)'"
+  "--dry[Only print the commands that would be executed without actually executing]"
+  {-f,--force}"[Reproduce even if dependencies were not changed.]"
+  {-s,--single-item}"[Reproduce only single data item without recursive dependencies check.]"
+  {-c,--cwd}"[Directory to reproduce from]:CWD:_files -/"
+  {-m,--metrics}"[Show metrics after reproduction.]"
+  {-i,--interactive}"[Ask for confirmation before reproducing each stage.]"
+  {-p,--pipeline}"[Reproduce the whole pipeline that the specified stage file belongs to.]"
+  {-P,--all-pipelines}"[Reproduce all pipelines in the project.]"
+)
+
+_dvc_remove=(
+  "*:DVC files:_files -g '(*.dvc|Dvcfile)'"
+  "--dry[Only print the commands that would be executed without actually executing]"
+  {-o,--outs}"[Only remove DVC file outputs.(default)]"
+  {-p,--purge}"[Remove DVC file and all its outputs]"
+)
+
+_dvc_move=(
+  "1:Source:_files"
+  "2:Destination:"
+)
+
+_dvc_gc=(
+  {-a,--all-branches}"[Collect garbage for all branches]"
+  {-T,--all-tags}"[Collect garbage for all tags]"
+  {-c,--cloud}"[Collect garbage in remote repository]"
+  {-r,--remote}"[Remote repository to collect garbage in]:Remote repository:"
+)
+
+_dvc_config=(
+  {-u,--unset}"[Unset option.]"
+  "--local[Unset option.]"
+)
+
+_dvc_remote=(
+  "1:Task:(add remove modify list)"
+)
+
+_dvc_metrics=(
+  "1:Task:(show add modify remove)"
+)
+
+_dvc_install=()
+
+_dvc_root=()
+
+_dvc_lock=(
+  "*:DVC files:_files -g '(*.dvc|Dvcfile)'"
+)
+
+_dvc_unlock=(
+  "*:DVC files:_files -g '(*.dvc|Dvcfile)'"
+)
+
+_dvc_pipeline=(
+  "1:Task:(show)"
+)
+
+typeset -A opt_args
+local context state line curcontext="$curcontext"
+
+_arguments \
+  $_dvc_options \
+  '1: :_dvc_commands' \
+  '*::args:->args'
+
+case $words[1] in
+  init)     _arguments $_dvc_global_options   $_dvc_init     ;;
+  destroy)  _arguments $_dvc_global_options   $_dvc_destroy  ;;
+  add)      _arguments $_dvc_global_options   $_dvc_add      ;;
+  import)   _arguments $_dvc_global_options   $_dvc_import   ;;
+  checkout) _arguments $_dvc_global_options   $_dvc_checkout ;;
+  run)      _arguments $_dvc_global_options   $_dvc_run      ;;
+  pull)     _arguments $_dvc_global_options   $_dvc_pull     ;;
+  push)     _arguments $_dvc_global_options   $_dvc_push     ;;
+  fetch)    _arguments $_dvc_global_options   $_dvc_fetch    ;;
+  status)   _arguments $_dvc_global_options   $_dvc_status   ;;
+  repro)    _arguments $_dvc_global_options   $_dvc_repro    ;;
+  remove)   _arguments $_dvc_global_options   $_dvc_remove   ;;
+  move)     _arguments $_dvc_global_options   $_dvc_move     ;;
+  gc)       _arguments $_dvc_global_options   $_dvc_gc       ;;
+  config)   _arguments $_dvc_global_options   $_dvc_config   ;;
+  remote)   _arguments $_dvc_global_options   $_dvc_remote   ;;
+  metrics)  _arguments $_dvc_global_options   $_dvc_metrics  ;;
+  install)  _arguments $_dvc_global_options   $_dvc_install  ;;
+  root)     _arguments $_dvc_global_options   $_dvc_root     ;;
+  lock)     _arguments $_dvc_global_options   $_dvc_lock     ;;
+  unlock)   _arguments $_dvc_global_options   $_dvc_unlock   ;;
+  pipeline) _arguments $_dvc_global_options   $_dvc_pipeline ;;
+esac

scripts/fpm/after-install.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+ln -sf /usr/local/lib/dvc/dvc /usr/local/bin/dvc

scripts/fpm/after-remove.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+rm -rf /usr/local/bin/dvc

scripts/innosetup/setup.iss

@@ -2,7 +2,7 @@
 #define MyAppVersion ReadIni(".\scripts\innosetup\config.ini", "Version", "version", "unknown")
 #define MyAppPublisher "Dmitry Petrov"
 #define MyAppURL "https://dataversioncontrol.com/"
-#define MyAppDir "..\..\dist"
+#define MyAppDir "..\..\dist\dvc"
 
 [Setup]
 AppId={{8258CE8A-110E-4E0D-AE60-FEE00B15F041}

setup.py

@@ -12,24 +12,24 @@ install_requires = [
     "future>=0.16.0",
     "colorama>=0.3.9",
     "configobj>=5.0.6",
-    "networkx==2.1",
+    "networkx>=2.1",
     "pyyaml>=3.12",
     "gitpython>=2.1.8",
-    "ntfsutils>=0.1.4",
     "setuptools>=34.0.0",
     "nanotime>=0.5.2",
     "pyasn1>=0.4.1",
     "schema>=0.6.7",
     "jsonpath-rw==1.4.0",
-    "reflink==0.2.0",
     "requests>=2.18.4",
     "grandalf==0.6",
-    "asciicanvas==0.0.3",
+    "asciimatics>=1.10.0",
+    "distro>=1.3.0",
+    "appdirs>=1.4.3",
 ]
 
 # Extra dependencies for remote integrations
 gs = [
-    "google-cloud==0.32.0",
+    "google-cloud-storage==1.13.0",
 ]
 s3 = [
     "boto3==1.7.4",
@@ -67,11 +67,11 @@ setup(
         'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 3',
     ],
-    packages=find_packages(exclude=['bin', 'tests', 'functests']),
+    packages=find_packages(exclude=['tests']),
     include_package_data=True,
     url='http://dataversioncontrol.com',
     entry_points={
         'console_scripts': ['dvc = dvc.main:main']
     },
-    zip_safe=False
+    zip_safe=False,
 )

test-requirements.txt

@@ -1,9 +1,11 @@
 pytest>=3.3.1
 nose>=1.3.7
 mock>=2.0.0
-coverage
-codecov
+coverage>=4.5.1
+codecov>=2.0.15
 xmltodict>=0.11.0
 awscli>=1.14.18
 google-compute-engine
 pywin32; sys_platform == 'win32'
+Pygments
+collective.checkdocs

tests/__main__.py

@@ -4,7 +4,11 @@ from subprocess import check_call
 REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 os.chdir(REPO_ROOT)
 
-os.putenv('PATH', '{}:{}'.format(os.path.join(REPO_ROOT, 'bin'), os.getenv('PATH')))
+os.putenv('PATH', '{}:{}'.format(os.path.join(REPO_ROOT, 'bin'),
+                                 os.getenv('PATH')))
 os.putenv('DVC_HOME', REPO_ROOT)
+os.putenv('DVC_TEST', 'true')
 
-check_call('nosetests -v --processes=-1 --process-timeout=200 --cover-inclusive --cover-erase --cover-package=dvc --with-coverage', shell=True)
+cmd = 'nosetests -v --processes=-1 --process-timeout=500 --cover-inclusive ' \
+      '--cover-erase --cover-package=dvc --with-coverage --all-modules'
+check_call(cmd, shell=True)

tests/basic_env.py

@@ -1,14 +1,17 @@
 import os
-import shutil
+import uuid
 import tempfile
 from git import Repo
 from unittest import TestCase
 
+import dvc.logger as logger
 from dvc.project import Project
 
 
 class TestDir(TestCase):
-    GCP_CREDS_FILE = os.path.abspath(os.path.join('scripts', 'ci', 'gcp-creds.json'))
+    GCP_CREDS_FILE = os.path.abspath(os.path.join('scripts',
+                                                  'ci',
+                                                  'gcp-creds.json'))
     DATA_DIR = 'data_dir'
     DATA_SUB_DIR = os.path.join(DATA_DIR, 'data_sub_dir')
     DATA = os.path.join(DATA_DIR, 'data')
@@ -18,9 +21,19 @@ class TestDir(TestCase):
     FOO = 'foo'
     FOO_CONTENTS = FOO
     BAR = 'bar'
-    BAR_CONTENTS = BAR
+    # NOTE: len(FOO_CONTENTS) must be != len(BAR_CONTENTS)
+    #
+    # Our state database relies on file mtime and file size to determine
+    # that a file has changed. Usually, mtime is enough by itself but on
+    # some filesystems like APFS on macOS mtime resolution is so bad,
+    # that our tests can overwrite a file in that time window without dvc
+    # being able to detect that, thus causing tests to fail. Usually,
+    # in tests, we replace foo with bar, so we need to make sure that when we
+    # modify a file in our tests, its content length changes.
+    BAR_CONTENTS = BAR + 'r'
     CODE = 'code.py'
-    CODE_CONTENTS = 'import sys\nimport shutil\nshutil.copyfile(sys.argv[1], sys.argv[2])'
+    CODE_CONTENTS = ('import sys\nimport shutil\n'
+                     'shutil.copyfile(sys.argv[1], sys.argv[2])')
 
     def _pushd(self, d):
         self._saved_dir = os.path.realpath(os.curdir)
@@ -38,8 +51,15 @@ class TestDir(TestCase):
         with open(name, 'a') as f:
             f.write(contents)
 
+    @staticmethod
+    def mkdtemp():
+        prefix = 'dvc-test.{}.'.format(os.getpid())
+        suffix = '.{}'.format(uuid.uuid4())
+        return tempfile.mkdtemp(prefix=prefix, suffix=suffix)
+
     def setUp(self):
-        self._root_dir = tempfile.mkdtemp()
+        self._root_dir = TestDir.mkdtemp()
+
         self._pushd(self._root_dir)
         self.create(self.FOO, self.FOO_CONTENTS)
         self.create(self.BAR, self.BAR_CONTENTS)
@@ -61,8 +81,17 @@ class TestGit(TestDir):
         self.git.index.commit('add code')
 
 
+class TestGitSubmodule(TestGit):
+    def setUp(self):
+        super(TestGitSubmodule, self).setUp()
+        subrepo = Repo.init()
+        subrepo_path = 'subrepo'
+        self.git.create_submodule(subrepo_path, subrepo_path, subrepo.git_dir)
+        self._pushd(subrepo_path)
+
+
 class TestDvc(TestGit):
     def setUp(self):
         super(TestDvc, self).setUp()
         self.dvc = Project.init(self._root_dir)
-        self.dvc.logger.be_verbose()
+        logger.be_verbose()

tests/test_add.py

@@ -1,16 +1,15 @@
 import os
-import stat
+import yaml
+import time
 import shutil
 import filecmp
-import tempfile
 
 from dvc.main import main
 from dvc.utils import file_md5
 from dvc.stage import Stage
 from dvc.exceptions import DvcException
 from dvc.output.base import OutputAlreadyTrackedError
-from dvc.output.base import OutputDoesNotExistError, OutputIsNotFileOrDirError
-from dvc.command.add import CmdAdd
+from dvc.project import Project
 
 from tests.basic_env import TestDvc
 
@@ -34,7 +33,7 @@ class TestAdd(TestDvc):
 
 class TestAddUnupportedFile(TestDvc):
     def test(self):
-        with self.assertRaises(DvcException) as cx:
+        with self.assertRaises(DvcException):
             self.dvc.add('unsupported://unsupported')
 
 
@@ -80,7 +79,8 @@ class TestAddDirectoryWithForwardSlash(TestDvc):
         self.assertEqual(len(stages), 1)
         stage = stages[0]
         self.assertTrue(stage is not None)
-        self.assertEquals(os.path.abspath('directory.dvc'), stage.path)
+        self.assertEqual(os.path.abspath('directory.dvc'), stage.path)
+
 
 class TestAddTrackedFile(TestDvc):
     def test(self):
@@ -89,7 +89,7 @@ class TestAddTrackedFile(TestDvc):
         self.dvc.scm.add([fname])
         self.dvc.scm.commit('add {}'.format(fname))
 
-        with self.assertRaises(OutputAlreadyTrackedError) as cx:
+        with self.assertRaises(OutputAlreadyTrackedError):
             self.dvc.add(fname)
 
 
@@ -114,6 +114,9 @@ class TestAddModifiedDir(TestDvc):
         self.assertEqual(len(stages), 1)
         self.assertTrue(stages[0] is not None)
         os.unlink(self.DATA)
+
+        time.sleep(2)
+
         stages = self.dvc.add(self.DATA_DIR)
         self.assertEqual(len(stages), 1)
         self.assertTrue(stages[0] is not None)
@@ -132,7 +135,7 @@ class TestAddFileInDir(TestDvc):
 
 class TestAddExternalLocalFile(TestDvc):
     def test(self):
-        dname = tempfile.mkdtemp()
+        dname = TestDvc.mkdtemp()
         fname = os.path.join(dname, 'foo')
         shutil.copyfile(self.FOO, fname)
 
@@ -148,6 +151,36 @@ class TestAddExternalLocalFile(TestDvc):
         self.assertTrue(filecmp.cmp(fname, 'foo', shallow=False))
 
 
+class TestAddLocalRemoteFile(TestDvc):
+    def test(self):
+        """
+        Making sure that 'remote' syntax is handled properly for local outs.
+        """
+        cwd = os.getcwd()
+        remote = 'myremote'
+
+        ret = main(['remote', 'add', remote, cwd])
+        self.assertEqual(ret, 0)
+
+        self.dvc = Project()
+
+        foo = 'remote://{}/{}'.format(remote, self.FOO)
+        ret = main(['add', foo])
+        self.assertEqual(ret, 0)
+
+        with open('foo.dvc', 'r') as fobj:
+            d = yaml.safe_load(fobj)
+            self.assertEqual(d['outs'][0]['path'], foo)
+
+        bar = os.path.join(cwd, self.BAR)
+        ret = main(['add', bar])
+        self.assertEqual(ret, 0)
+
+        with open('bar.dvc', 'r') as fobj:
+            d = yaml.safe_load(fobj)
+            self.assertEqual(d['outs'][0]['path'], bar)
+
+
 class TestCmdAdd(TestDvc):
     def test(self):
         ret = main(['add',

tests/test_analytics.py

@@ -0,0 +1,61 @@
+import os
+import mock
+import requests
+
+from dvc.main import main
+from dvc.analytics import Analytics
+from tests.basic_env import TestDvc, TestGit, TestDir
+
+
+def _clean_getenv(key, default=None):
+    """
+    Remove env vars that affect dvc behavior in tests
+    """
+    if key in ['DVC_TEST', 'CI']:
+        return None
+    return os.environ.get(key, default)
+
+
+class TestAnalytics(TestDir):
+    def test(self):
+        a = Analytics()
+        a.collect()
+        self.assertTrue(isinstance(a.info, dict))
+        self.assertNotEqual(a.info, {})
+        self.assertTrue(a.PARAM_USER_ID in a.info.keys())
+        self.assertTrue(a.PARAM_SYSTEM_INFO in a.info.keys())
+        self.assertNotEqual(a.info[a.PARAM_SYSTEM_INFO], {})
+
+    @mock.patch.object(os, 'getenv', new=_clean_getenv)
+    @mock.patch('requests.post')
+    def test_send(self, mockpost):
+        ret = main(['daemon', 'analytics', Analytics().dump(), '-v'])
+        self.assertEqual(ret, 0)
+
+        self.assertTrue(mockpost.called)
+
+    @mock.patch.object(os, 'getenv', new=_clean_getenv)
+    @mock.patch.object(requests, 'post',
+                       side_effect=requests.exceptions.RequestException())
+    def test_send_failed(self, mockpost):
+        ret = main(['daemon', 'analytics', Analytics().dump(), '-v'])
+        self.assertEqual(ret, 0)
+
+        self.assertTrue(mockpost.called)
+
+
+class TestAnalyticsGit(TestAnalytics, TestGit):
+    pass
+
+
+class TestAnalyticsDvc(TestAnalytics, TestDvc):
+    @mock.patch('requests.post')
+    def test_send_disabled(self, mockpost):
+        ret = main(['config', 'core.analytics', 'false'])
+        self.assertEqual(ret, 0)
+
+        with mock.patch.object(os, 'getenv', new=_clean_getenv):
+            ret = main(['daemon', 'analytics', Analytics().dump(), '-v'])
+        self.assertEqual(ret, 0)
+
+        self.assertFalse(mockpost.called)

tests/test_cache.py

@@ -1,11 +1,9 @@
 import os
 import shutil
-import tempfile
+import configobj
 
 from dvc.cache import Cache
-from dvc.system import System
 from dvc.main import main
-from dvc.remote.local import RemoteLOCAL
 
 from tests.basic_env import TestDvc, TestDir
 
@@ -25,14 +23,14 @@ class TestCache(TestDvc):
         self.create(self.cache2, '2')
 
     def test_all(self):
-        md5_list = Cache(self.dvc).local.all()
-        self.assertEquals(len(md5_list), 2)
-        self.assertTrue(self.cache1_md5 in md5_list)
-        self.assertTrue(self.cache2_md5 in md5_list)
+        md5_list = list(Cache(self.dvc).local.all())
+        self.assertEqual(len(md5_list), 2)
+        self.assertIn(self.cache1_md5, md5_list)
+        self.assertIn(self.cache2_md5, md5_list)
 
     def test_get(self):
         cache = Cache(self.dvc).local.get(self.cache1_md5)
-        self.assertEquals(cache, self.cache1)
+        self.assertEqual(cache, self.cache1)
 
 
 class TestCacheLoadBadDirCache(TestDvc):
@@ -52,7 +50,7 @@ class TestCacheLoadBadDirCache(TestDvc):
 
 class TestExternalCacheDir(TestDvc):
     def test(self):
-        cache_dir = tempfile.mkdtemp()
+        cache_dir = TestDvc.mkdtemp()
 
         ret = main(['config', 'cache.dir', cache_dir])
         self.assertEqual(ret, 0)
@@ -66,7 +64,7 @@ class TestExternalCacheDir(TestDvc):
         self.assertEqual(ret, 0)
 
         self.assertFalse(os.path.exists('.dvc/cache'))
-        self.assertNotEquals(len(os.listdir(cache_dir)), 0)
+        self.assertNotEqual(len(os.listdir(cache_dir)), 0)
 
 
 class TestSharedCacheDir(TestDir):
@@ -98,8 +96,52 @@ class TestSharedCacheDir(TestDir):
         self.assertFalse(os.path.exists(os.path.join('dir1', '.dvc', 'cache')))
         self.assertFalse(os.path.exists(os.path.join('dir2', '.dvc', 'cache')))
 
-        subdirs = list(filter(lambda x: os.path.isdir(os.path.join(cache_dir, x)), os.listdir(cache_dir)))
+        subdirs = list(filter(
+                        lambda x: os.path.isdir(os.path.join(cache_dir, x)),
+                        os.listdir(cache_dir)))
         self.assertEqual(len(subdirs), 3)
-        self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[0]))), 1)
-        self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[1]))), 1)
-        self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[2]))), 1)
+        self.assertEqual(len(os.listdir(os.path.join(cache_dir,
+                                                     subdirs[0]))), 1)
+        self.assertEqual(len(os.listdir(os.path.join(cache_dir,
+                                                     subdirs[1]))), 1)
+        self.assertEqual(len(os.listdir(os.path.join(cache_dir,
+                                                     subdirs[2]))), 1)
+
+
+class TestCacheLinkType(TestDvc):
+    def test(self):
+        ret = main(['config', 'cache.type', 'reflink,copy'])
+        self.assertEqual(ret, 0)
+
+        ret = main(['add', self.FOO])
+        self.assertEqual(ret, 0)
+
+
+class TestCmdCacheDir(TestDvc):
+    def test_abs_path(self):
+        dname = os.path.join(os.path.dirname(self._root_dir), 'dir')
+        ret = main(['cache', 'dir', dname])
+        self.assertEqual(ret, 0)
+
+        config = configobj.ConfigObj(self.dvc.config.config_file)
+        self.assertEqual(config['cache']['dir'], dname)
+
+    def test_relative_path(self):
+        tmpdir = self.mkdtemp()
+        dname = os.path.relpath(tmpdir)
+        ret = main(['cache', 'dir', dname])
+        self.assertEqual(ret, 0)
+
+        # NOTE: we are in the project's root and config is in .dvc/, so
+        # dir path written to config should be just one level above.
+        rel = os.path.join('..', dname)
+        config = configobj.ConfigObj(self.dvc.config.config_file)
+        self.assertEqual(config['cache']['dir'], rel)
+
+        ret = main(['add', self.FOO])
+        self.assertEqual(ret, 0)
+
+        subdirs = os.listdir(tmpdir)
+        self.assertEqual(len(subdirs), 1)
+        files = os.listdir(os.path.join(tmpdir, subdirs[0]))
+        self.assertEqual(len(files), 1)