Merge pull request #1158 from efiop/master

osx: reflink: check if clib has clonefile()

.appveyor.yml

@@ -1,6 +1,7 @@
 branches:
   only:
     - master
+    - devel
 
 platform: x86
 
@@ -12,6 +13,10 @@ environment:
   GOOGLE_APPLICATION_CREDENTIALS: gcp-creds.json
   GCP_CREDS:
     secure: 96fJ3r2i2GohbXHwnSs5N4EplQ7q8YmLpPWM0AC+f4s=
+  CODECOV_TOKEN:
+    secure: XN4jRtmGE5Bqg8pPZkwNs7kn3UEI73Rkldqc0MGsQISZBm5TNJZOPofDMc1QnUsf
+  AZURE_STORAGE_CONTAINER_NAME: appveyor-tests
+  AZURE_STORAGE_CONNECTION_STRING: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;
 
   matrix:
 
@@ -41,11 +46,16 @@ environment:
 #      PYTHON_ARCH: "64"
 
 install:
+  - ps: Install-Product node
+  - npm install -g azurite
+  - ps: $AzuriteProcess = Start-Process azurite-blob -PassThru
   - cinst wget
   - cinst awscli
+  - cinst gcloudsdk
+  - refreshenv
   - cinst gsutil
   - cinst openssl.light
-  - pip install --user --upgrade pip
+  - python -m pip install -U pip setuptools wheel
   - pip install -r requirements.txt
   - python setup.py install
 
@@ -54,15 +64,21 @@ build: false
 before_test:
   - aws configure set aws_access_key_id "%aws_access_key_id%"
   - aws configure set aws_secret_access_key "%aws_secret_access_key%"
+  - aws configure set region us-east-2
   - openssl enc -d -aes-256-cbc -md md5 -k "%GCP_CREDS%" -in scripts\ci\gcp-creds.json.enc -out scripts\ci\gcp-creds.json & exit 0
   - pip install -r test-requirements.txt
 
 test_script:
-  - nosetests --cover-inclusive --cover-erase --cover-package=dvc --with-coverage
+  - python -mtests
 
 after_test:
+  - if "%APPVEYOR_PULL_REQUEST_NUMBER%"=="" codecov
+  - if "%APPVEYOR_REPO_TAG%"=="true" python -m dvc pull
   - if "%APPVEYOR_REPO_TAG%"=="true" .\scripts\build_windows.cmd
 
+on_finish:
+  - ps: Stop-Process $AzuriteProcess
+
 artifacts:
   - path: dvc*.exe
     name: DVC Installer

.codecov.yml

@@ -0,0 +1 @@
+comment: false

.coveragerc

@@ -5,3 +5,7 @@ omit =
 [report]
 exclude_lines =
     if __name__ == .__main__.:
+    pass
+    raise NotImplementedError
+    # pragma: no cover
+    #  pragma: no cover

.dvc/.gitignore

@@ -0,0 +1,8 @@
+link.state
+link.state.lock
+lock
+config.local
+updater
+cache
+state
+state.lock

.dvc/config

@@ -0,0 +1,4 @@
+['remote "dvc"']
+url = s3://dvc-share/dvc/
+[core]
+remote = dvc

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,10 @@
+---
+name: Bug report
+about: Create a report to help us improve
+
+---
+
+
+**Please provide information about your setup**
+DVC version(i.e. `dvc --version`), Platform and method of installation (pip, homebrew, pkg Mac, exe (Windows), DEB(Linux), RPM(Linux))
+

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,6 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+
+---
+

.gitignore

@@ -23,3 +23,6 @@ innosetup/config.ini
 
 .coverage
 
+dvc/version.py
+
+*.swp

.travis.yml

@@ -1,4 +1,5 @@
 language: python
+cache: pip
 sudo: required
 dist: trusty
 group: deprecated-2017Q4
@@ -6,7 +7,7 @@ python:
   - "2.7"
   - "3.4"
   - "3.5"
-#  - "3.6" Not supported by pyinstaller yet
+  - "3.6"
 # Travis doesn't support testing python on Mac yet, so we need
 # to workaround it by installing it directly with brew.
 #
@@ -15,28 +16,75 @@ python:
 # version, that causes fpm to crash.
 matrix:
   include:
+  # 3.7 needs Xenial image because of https://github.com/travis-ci/travis-ci/issues/9069
+  - python: "3.7"
+    dist: xenial
+    sudo: true
   - if: type = cron OR tag is present
     os: osx
     osx_image: xcode7.3
     language: ruby
     rvm: 2.4.0
-    env: PYTHON_VER=2.7.10
+    env:
+    - PYTHON_VER=2.7.10
+    - PYENV_ROOT="$HOME/.pyenv"
+    - PATH="$PYENV_ROOT/bin:$PATH"
+    - CPPFLAGS="-I/usr/local/opt/openssl/include $CPPFLAGS"
+    - LDFLAGS="-L/usr/local/opt/openssl/lib $LDFLAGS"
   - if: type = cron OR tag is present
     os: osx
     osx_image: xcode7.3
     language: ruby
     rvm: 2.4.0
-    env: PYTHON_VER=3.4.3
+    env:
+    - PYTHON_VER=3.4.3
+    - PYENV_ROOT="$HOME/.pyenv"
+    - PATH="$PYENV_ROOT/bin:$PATH"
   - if: type = cron OR tag is present
     os: osx
     osx_image: xcode7.3
     language: ruby
     rvm: 2.4.0
-    env: PYTHON_VER=3.5.0
+    env:
+    - PYTHON_VER=3.5.0
+    - PYENV_ROOT="$HOME/.pyenv"
+    - PATH="$PYENV_ROOT/bin:$PATH"
 before_install:
   - bash ./scripts/ci/before_install.sh
+  - if  [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
+      eval "$(pyenv init -)";
+      PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing --keep --verbose "$PYTHON_VER";
+      pyenv shell "$PYTHON_VER";
+      brew install openssl;
+      brew cask install google-cloud-sdk;
+    fi
+  - which python
+  - which pip
+  - python --version
+  - if [[ "$TRAVIS_OS_NAME" != "osx" ]]; then
+      bash ./scripts/ci/install_hadoop.sh;
+      source ~/.bashrc;
+      hadoop fs -help;
+      pushd ~;
+      sudo mkdir -p /home/hadoop/hadoopinfra/hdfs/namenode/;
+      sudo chmod 777 /home/hadoop/hadoopinfra/hdfs/namenode/;
+      sudo chmod 777 -R /usr/local/hadoop;
+      hdfs namenode -format;
+      start-dfs.sh;
+      jps;
+      hadoop fs -ls /;
+      hadoop fs -ls hdfs://127.0.0.1/;
+      hadoop fs -mkdir hdfs://127.0.0.1/ololo;
+      popd;
+    fi
 install:
   - bash ./scripts/ci/install.sh
+  - pip install flake8
+before_script:
+  # stop the build if there are any readme formatting errors
+  - python setup.py checkdocs
+  # stop the build if there are any flake8 errors
+  - flake8 dvc
 script:
   - bash ./scripts/ci/script.sh
 deploy:
@@ -52,4 +100,4 @@ deploy:
   skip_cleanup: true
   on:
     tags: true
-    repo: dataversioncontrol/dvc
+    repo: iterative/dvc

README.md

@@ -1,79 +0,0 @@
-
-| Linux & Mac OS | Windows | Code quality | Unit-tests |
-|-------------|---------------|--------------|------------|
-|[![Build Status](https://travis-ci.org/dataversioncontrol/dvc.svg?branch=master)](https://travis-ci.org/dataversioncontrol/dvc)|[![Build status](https://ci.appveyor.com/api/projects/status/rnqygb4rp1tsjvhb/branch/master?svg=true)](https://ci.appveyor.com/project/dataversioncontrol/dvc/branch/master)|[![Code Climate](https://codeclimate.com/github/dataversioncontrol/dvc/badges/gpa.svg)](https://codeclimate.com/github/dataversioncontrol/dvc)|[![Test Coverage](https://codeclimate.com/github/dataversioncontrol/dvc/badges/coverage.svg)](https://codeclimate.com/github/dataversioncontrol/dvc)|
-
-Data Version Control or DVC is an open source tool for data science projects. 
-It helps data scientists manage their code and data together in a simple form of Git-like commands.
-
-# Get started
-
-|Step|Command|
-|----|-------|
-|Track code and data together|`$ git add train.py` <br /> `$ dvc add images.zip` |
-|Connect code and data by commands| `$ dvc run -d images.zip -o images/ unzip -q images.zip` <br /> `$ dvc run -d images/ -d train.py -o model.p python train.py` |
-|Make changes and reproduce|`$ vi train.py` <br /> `$ dvc repro model.p.dvc` |
-|Share code|`$ git add .` <br /> `$ git commit -m 'The baseline model'` <br />  `$ git push`|
-|Share data and ML models|`$ dvc config AWS.StoragePath mybucket/image_cnn` <br/> `$ dvc push`|
-
-See more in [tutorial](https://blog.dataversioncontrol.com/data-version-control-tutorial-9146715eda46).
-
-# Installation
-
-## Packages
-
-Operating system dependent packages are the recommended way to install DVC.
-The latest version of the packages can be found at GitHub releases page: https://github.com/dataversioncontrol/dvc/releases
-
-## Python Pip
-
-DVC could be installed via the Python Package Index (PyPI).
-
-```bash
-pip install dvc
-```
-
-## Homebrew (Mac OS)
-
-### Formula:
-
-```bash
-brew install dataversioncontrol/homebrew-dvc/dvc
-```
-
-### Cask:
-
-```bash
-brew cask install dataversioncontrol/homebrew-dvc/dvc
-```
-
-# Links
-
-Website: https://dataversioncontrol.com
-
-Tutorial: https://blog.dataversioncontrol.com/data-version-control-tutorial-9146715eda46
-
-Documentation: http://dataversioncontrol.com/docs/
-
-Discussion: https://discuss.dataversioncontrol.com/
-
-# Related technologies
-
-1. [Git-annex](https://git-annex.branchable.com/) - DVC uses the idea of storing the content of large files (that you don't want to see in your Git repository) in a local key-value store and uses file hardlinks/symlinks instead of the copying actual files.
-2. [Git-LFS](https://git-lfs.github.com/).
-3. Makefile (and it's analogues). DVC tracks dependencies (DAG). 
-4. [Workflow Management Systems](https://en.wikipedia.org/wiki/Workflow_management_system). DVC is workflow management system designed specificaly to manage machine learning experiments. DVC was built on top of Git.
-
-DVC is compatible with Git for storing code and the dependency graph (DAG), but not data files cache.
-Data files caches can be transferred separately - now data cache transfer throught AWS S3 and GCP storge are supported.
-
-# How DVC works
-
-![how_dvc_works](https://s3-us-west-2.amazonaws.com/dvc-share/images/0.9/how_dvc_works.png)
-
-# Copyright
-
-This project is distributed under the Apache license version 2.0 (see the LICENSE file in the project root).
-
-By submitting a pull request for this project, you agree to license your contribution under the Apache license version 2.0 to this project.
-

README.rst

@@ -0,0 +1,179 @@
+.. image:: https://dvc.org/static/img/logo-owl-readme.png
+  :target: https://dvc.org
+  :alt: DVC logo
+
+`Website <https://dvc.org>`_
+• `Docs <https://dvc.org/doc>`_
+• `Twitter <https://twitter.com/iterativeai>`_
+• `Chat (Community & Support) <https://dvc.org/chat>`_
+• `Tutorial <https://dvc.org/doc/tutorial>`_
+• `Mailing List <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_
+
+.. image:: https://travis-ci.org/iterative/dvc.svg?branch=master
+  :target: https://travis-ci.org/iterative/dvc
+  :alt: Travis
+
+.. image:: https://ci.appveyor.com/api/projects/status/github/iterative/dvc?branch=master&svg=true
+  :target: https://ci.appveyor.com/project/iterative/dvc/branch/master
+  :alt: Windows Build
+
+.. image:: https://codeclimate.com/github/iterative/dvc/badges/gpa.svg
+  :target: https://codeclimate.com/github/iterative/dvc
+  :alt: Code Climate
+
+.. image:: https://codecov.io/gh/iterative/dvc/branch/master/graph/badge.svg
+  :target: https://codecov.io/gh/iterative/dvc
+  :alt: Codecov
+
+|
+
+**Data Science Version Control** or **DVC** is an **open-source** tool for data science and 
+machine learning projects. With a simple and flexible Git-like architecture and interface it
+helps data scientists:
+
+#. manage **machine learning models** - versioning, including data sets and transformations (scripts) that were
+   used to generate models;
+
+#. make projects **reproducible**;
+
+#. make projects **shareable**;
+
+#. manage experiments with branching and **metrics** tracking;
+
+It aims to replace tools like Excel and Docs that are being commonly used as a knowledge repo and
+a ledger for the team, ad-hoc scripts to track and move deploy different model versions, ad-hoc
+data file suffixes and prefixes.
+
+.. contents:: **Contents**
+  :backlinks: none
+
+How DVC works
+=============
+
+DVC is compatible with Git for storing code and the dependency graph (DAG), but not data files cache.
+To store and share data files cache DVC supports remotes - any cloud (S3, Azure, Google Cloud, etc) or any on-premise
+network storage (via SSH, for example).
+
+.. image:: https://dvc.org/static/img/flow.gif
+   :target: https://dvc.org/static/img/flow.gif
+   :alt: how_dvc_works
+
+Quick start
+===========
+
+Please read `Get Started <https://dvc.org/doc/get-started>`_ for the full version. Common workflow commands include:
+
++-----------------------------------+-------------------------------------------------------------------+
+| Step                              | Command                                                           |
++===================================+===================================================================+
+| Track code and data together      | | ``$ git add train.py``                                          |
+|                                   | | ``$ dvc add images.zip``                                        |
++-----------------------------------+-------------------------------------------------------------------+
+| Connect code and data by commands | | ``$ dvc run -d images.zip -o images/ unzip -q images.zip``      |
+|                                   | | ``$ dvc run -d images/ -d train.py -o model.p python train.py`` |
++-----------------------------------+-------------------------------------------------------------------+
+| Make changes and reproduce        | | ``$ vi train.py``                                               |
+|                                   | | ``$ dvc repro model.p.dvc``                                     |
++-----------------------------------+-------------------------------------------------------------------+
+| Share code                        | | ``$ git add .``                                                 |
+|                                   | | ``$ git commit -m 'The baseline model'``                        |
+|                                   | | ``$ git push``                                                  |
++-----------------------------------+-------------------------------------------------------------------+
+| Share data and ML models          | | ``$ dvc remote add myremote s3://mybucket/image_cnn``           |
+|                                   | | ``$ dvc core.remote myremote``                                  |
+|                                   | | ``$ dvc push``                                                  |
++-----------------------------------+-------------------------------------------------------------------+
+
+Installation
+============
+
+There are three options to install DVC: ``pip``, Homebrew, or an OS-specific package:
+
+.. code-block:: bash
+
+   pip install dvc
+
+Package
+-------
+
+Self-contained packages for Windows, Linux, Mac are available. The latest version of the packages can be found at
+GitHub `releases page <https://github.com/iterative/dvc/releases>`_.
+
+Homebrew
+--------
+
+.. code-block:: bash
+
+   $ brew install iterative/homebrew-dvc/dvc
+
+or:
+
+.. code-block:: bash
+
+   $ brew cask install iterative/homebrew-dvc/dvc
+
+Related technologies
+====================
+
+#. `Git-annex <https://git-annex.branchable.com/>`_ - DVC uses the idea of storing the content of large files (that you
+   don't want to see in your Git repository) in a local key-value store and uses file hardlinks/symlinks instead of the
+   copying actual files.
+
+#. `Git-LFS <https://git-lfs.github.com/>`_ - DVC is compatible with any remote storage (S3, Google Cloud, Azure, SSH,
+   etc). DVC utilizes reflinks or hardlinks to avoid copy operation on checkouts which makes much more efficient for
+   large data files.
+
+#. *Makefile* (and its analogues). DVC tracks dependencies (DAG).
+
+#. `Workflow Management Systems <https://en.wikipedia.org/wiki/Workflow_management_system>`_. DVC is a workflow
+   management system designed specifically to manage machine learning experiments. DVC is built on top of Git.
+
+Contributing
+============
+Contributions are welcome! Please see our `Contributing Guide <https://dvc.org/doc/user-guide/contributing/>`_ for more
+details.
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/0
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/0
+  :alt: 0
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/1
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/1
+  :alt: 1
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/2
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/2
+  :alt: 2   
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/3
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/3   
+  :alt: 3
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/4
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/4
+  :alt: 4   
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/5
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/5
+  :alt: 5
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/6
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/6
+  :alt: 6   
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/7
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/7
+  :alt: 7
+
+Mailing List
+============
+
+Want to stay up to date? Want to help improve DVC by participating in our ocassional polls? Subscribe to our `mailing list <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_. No spam, really low traffic.
+
+Copyright
+=========
+
+This project is distributed under the Apache license version 2.0 (see the LICENSE file in the project root).
+
+By submitting a pull request for this project, you agree to license your contribution under the Apache license version
+2.0 to this project.

docs/cmd.bat

@@ -1,3 +0,0 @@
-REM pandoc -s doc.rst -o doc_simple.html
-pandoc -s --toc -H header.html -B top.html -A footer.html doc.rst -o doc.html
-

docs/cmd.sh

@@ -1,3 +0,0 @@
-pandoc -s doc.rst -o doc_simple.html
-pandoc -s --toc -H header.html -B top.html -A footer.html doc.rst -o doc.html
-

docs/footer.html

@@ -1,13 +0,0 @@
-</div> <!-- end of content-containder -->
-<footer> 
-  <div class="footer">
-	<div class="inner">
-		<ul>
-			<li><span>© 2017 dataversioncontrol.com</span></li>
-			<li><a href="https://blog.dataversioncontrol.com/">Blog</a></li>
-			<li class="last"><a href="https://twitter.com/FullStackML" class="twitter">
-				<span class="ico"><svg width="16" height="16" viewBox="0 0 25 25"><path d="M21.725 5.338c-.744.47-1.605.804-2.513 1.006a3.978 3.978 0 0 0-2.942-1.293c-2.22 0-4.02 1.81-4.02 4.02 0 .32.034.63.07.94-3.31-.18-6.27-1.78-8.255-4.23a4.544 4.544 0 0 0-.574 2.01c.04 1.43.74 2.66 1.8 3.38-.63-.01-1.25-.19-1.79-.5v.08c0 1.93 1.38 3.56 3.23 3.95-.34.07-.7.12-1.07.14-.25-.02-.5-.04-.72-.07.49 1.58 1.97 2.74 3.74 2.8a8.49 8.49 0 0 1-5.02 1.72c-.3-.03-.62-.04-.93-.07A11.447 11.447 0 0 0 8.88 21c7.386 0 11.43-6.13 11.414-11.414.015-.21.01-.38 0-.578a7.604 7.604 0 0 0 2.01-2.08 7.27 7.27 0 0 1-2.297.645 3.856 3.856 0 0 0 1.72-2.23"></path></svg></span>@FullStackML</a></li>
-		</ul>
-	</div>
-  </div>
-</footer>

docs/header.html

@@ -1,9 +0,0 @@
-<meta charset="utf-8">
-<meta name="description" content="DVC product documnetation">
-<meta name="HandheldFriendly" content="True">
-<meta name="MobileOptimized" content="320">
-<meta name="viewport" content="initial-scale=1.0, minimum-scale=1.0, maximum-scale=1.0, user-scalable=no">
-<title>Data Version Control (DVC) - Product Documentation</title>
-<link href="http://fonts.googleapis.com/css?family=Lora" rel="stylesheet" type="text/css">
-<link rel="stylesheet" href="css/style.css">
-

docs/js/jquery.localscroll-1.2.7.js

@@ -1,133 +0,0 @@
-/**
- * jQuery.LocalScroll
- * Copyright (c) 2007-2009 Ariel Flesler - aflesler(at)gmail(dot)com | http://flesler.blogspot.com
- * Dual licensed under MIT and GPL.
- * Date: 3/11/2009
- *
- * @projectDescription Animated scrolling navigation, using anchors.
- * http://flesler.blogspot.com/2007/10/jquerylocalscroll-10.html
- * @author Ariel Flesler
- * @version 1.2.7
- *
- * @id jQuery.fn.localScroll
- * @param {Object} settings Hash of settings, it is passed in to jQuery.ScrollTo, none is required.
- * @return {jQuery} Returns the same jQuery object, for chaining.
- *
- * @example $('ul.links').localScroll();
- *
- * @example $('ul.links').localScroll({ filter:'.animated', duration:400, axis:'x' });
- *
- * @example $.localScroll({ target:'#pane', axis:'xy', queue:true, event:'mouseover' });
- *
- * Notes:
- *	- The plugin requires jQuery.ScrollTo.
- *	- The hash of settings, is passed to jQuery.ScrollTo, so the settings are valid for that plugin as well.
- *	- jQuery.localScroll can be used if the desired links, are all over the document, it accepts the same settings.
- *  - If the setting 'lazy' is set to true, then the binding will still work for later added anchors.
-  *	- If onBefore returns false, the event is ignored.
- **/
-;(function( $ ){
-	var URI = location.href.replace(/#.*/,''); // local url without hash
-
-	var $localScroll = $.localScroll = function( settings ){
-		$('body').localScroll( settings );
-	};
-
-	// Many of these defaults, belong to jQuery.ScrollTo, check it's demo for an example of each option.
-	// @see http://flesler.demos.com/jquery/scrollTo/
-	// The defaults are public and can be overriden.
-	$localScroll.defaults = {
-		duration:1000, // How long to animate.
-		axis:'y', // Which of top and left should be modified.
-		event:'click', // On which event to react.
-		stop:true, // Avoid queuing animations 
-		target: window, // What to scroll (selector or element). The whole window by default.
-		reset: true // Used by $.localScroll.hash. If true, elements' scroll is resetted before actual scrolling
-		/*
-		lock:false, // ignore events if already animating
-		lazy:false, // if true, links can be added later, and will still work.
-		filter:null, // filter some anchors out of the matched elements.
-		hash: false // if true, the hash of the selected link, will appear on the address bar.
-		*/
-	};
-
-	// If the URL contains a hash, it will scroll to the pointed element
-	$localScroll.hash = function( settings ){
-		if( location.hash ){
-			settings = $.extend( {}, $localScroll.defaults, settings );
-			settings.hash = false; // can't be true
-			
-			if( settings.reset ){
-				var d = settings.duration;
-				delete settings.duration;
-				$(settings.target).scrollTo( 0, settings );
-				settings.duration = d;
-			}
-			scroll( 0, location, settings );
-		}
-	};
-
-	$.fn.localScroll = function( settings ){
-		settings = $.extend( {}, $localScroll.defaults, settings );
-
-		return settings.lazy ?
-			// use event delegation, more links can be added later.		
-			this.bind( settings.event, function( e ){
-				// Could use closest(), but that would leave out jQuery -1.3.x
-				var a = $([e.target, e.target.parentNode]).filter(filter)[0];
-				// if a valid link was clicked
-				if( a )
-					scroll( e, a, settings ); // do scroll.
-			}) :
-			// bind concretely, to each matching link
-			this.find('a,area')
-				.filter( filter ).bind( settings.event, function(e){
-					scroll( e, this, settings );
-				}).end()
-			.end();
-
-		function filter(){// is this a link that points to an anchor and passes a possible filter ? href is checked to avoid a bug in FF.
-			return !!this.href && !!this.hash && this.href.replace(this.hash,'') == URI && (!settings.filter || $(this).is( settings.filter ));
-		};
-	};
-
-	function scroll( e, link, settings ){
-		var id = link.hash.slice(1),
-			elem = document.getElementById(id) || document.getElementsByName(id)[0];
-
-		if ( !elem )
-			return;
-
-		if( e )
-			e.preventDefault();
-
-		var $target = $( settings.target );
-
-		if( settings.lock && $target.is(':animated') ||
-			settings.onBefore && settings.onBefore.call(settings, e, elem, $target) === false ) 
-			return;
-
-		if( settings.stop )
-			$target.stop(true); // remove all its animations
-
-		if( settings.hash ){
-			var attr = elem.id == id ? 'id' : 'name',
-				$a = $('<a> </a>').attr(attr, id).css({
-					position:'absolute',
-					top: $(window).scrollTop(),
-					left: $(window).scrollLeft()
-				});
-
-			elem[attr] = '';
-			$('body').prepend($a);
-			location = link.hash;
-			$a.remove();
-			elem[attr] = id;
-		}
-			
-		$target
-			.scrollTo( elem, settings ) // do scroll
-			.trigger('notify.serialScroll',[elem]); // notify serialScroll about this change
-	};
-
-})( jQuery );

docs/js/jquery.scrollTo-1.4.3.1.js

@@ -1,218 +0,0 @@
-/*!
- * jQuery.ScrollTo
- * Copyright (c) 2007-2012 Ariel Flesler - aflesler(at)gmail(dot)com | http://flesler.blogspot.com
- * Dual licensed under MIT and GPL.
- * Date: 4/09/2012
- *
- * @projectDescription Easy element scrolling using jQuery.
- * http://flesler.blogspot.com/2007/10/jqueryscrollto.html
- * @author Ariel Flesler
- * @version 1.4.3.1
- *
- * @id jQuery.scrollTo
- * @id jQuery.fn.scrollTo
- * @param {String, Number, DOMElement, jQuery, Object} target Where to scroll the matched elements.
- *	  The different options for target are:
- *		- A number position (will be applied to all axes).
- *		- A string position ('44', '100px', '+=90', etc ) will be applied to all axes
- *		- A jQuery/DOM element ( logically, child of the element to scroll )
- *		- A string selector, that will be relative to the element to scroll ( 'li:eq(2)', etc )
- *		- A hash { top:x, left:y }, x and y can be any kind of number/string like above.
- *		- A percentage of the container's dimension/s, for example: 50% to go to the middle.
- *		- The string 'max' for go-to-end. 
- * @param {Number, Function} duration The OVERALL length of the animation, this argument can be the settings object instead.
- * @param {Object,Function} settings Optional set of settings or the onAfter callback.
- *	 @option {String} axis Which axis must be scrolled, use 'x', 'y', 'xy' or 'yx'.
- *	 @option {Number, Function} duration The OVERALL length of the animation.
- *	 @option {String} easing The easing method for the animation.
- *	 @option {Boolean} margin If true, the margin of the target element will be deducted from the final position.
- *	 @option {Object, Number} offset Add/deduct from the end position. One number for both axes or { top:x, left:y }.
- *	 @option {Object, Number} over Add/deduct the height/width multiplied by 'over', can be { top:x, left:y } when using both axes.
- *	 @option {Boolean} queue If true, and both axis are given, the 2nd axis will only be animated after the first one ends.
- *	 @option {Function} onAfter Function to be called after the scrolling ends. 
- *	 @option {Function} onAfterFirst If queuing is activated, this function will be called after the first scrolling ends.
- * @return {jQuery} Returns the same jQuery object, for chaining.
- *
- * @desc Scroll to a fixed position
- * @example $('div').scrollTo( 340 );
- *
- * @desc Scroll relatively to the actual position
- * @example $('div').scrollTo( '+=340px', { axis:'y' } );
- *
- * @desc Scroll using a selector (relative to the scrolled element)
- * @example $('div').scrollTo( 'p.paragraph:eq(2)', 500, { easing:'swing', queue:true, axis:'xy' } );
- *
- * @desc Scroll to a DOM element (same for jQuery object)
- * @example var second_child = document.getElementById('container').firstChild.nextSibling;
- *			$('#container').scrollTo( second_child, { duration:500, axis:'x', onAfter:function(){
- *				alert('scrolled!!');																   
- *			}});
- *
- * @desc Scroll on both axes, to different values
- * @example $('div').scrollTo( { top: 300, left:'+=200' }, { axis:'xy', offset:-20 } );
- */
-
-;(function( $ ){
-	
-	var $scrollTo = $.scrollTo = function( target, duration, settings ){
-		$(window).scrollTo( target, duration, settings );
-	};
-
-	$scrollTo.defaults = {
-		axis:'xy',
-		duration: parseFloat($.fn.jquery) >= 1.3 ? 0 : 1,
-		limit:true
-	};
-
-	// Returns the element that needs to be animated to scroll the window.
-	// Kept for backwards compatibility (specially for localScroll & serialScroll)
-	$scrollTo.window = function( scope ){
-		return $(window)._scrollable();
-	};
-
-	// Hack, hack, hack :)
-	// Returns the real elements to scroll (supports window/iframes, documents and regular nodes)
-	$.fn._scrollable = function(){
-		return this.map(function(){
-			var elem = this,
-				isWin = !elem.nodeName || $.inArray( elem.nodeName.toLowerCase(), ['iframe','#document','html','body'] ) != -1;
-
-				if( !isWin )
-					return elem;
-
-			var doc = (elem.contentWindow || elem).document || elem.ownerDocument || elem;
-			
-			return /webkit/i.test(navigator.userAgent) || doc.compatMode == 'BackCompat' ?
-				doc.body : 
-				doc.documentElement;
-		});
-	};
-
-	$.fn.scrollTo = function( target, duration, settings ){
-		if( typeof duration == 'object' ){
-			settings = duration;
-			duration = 0;
-		}
-		if( typeof settings == 'function' )
-			settings = { onAfter:settings };
-			
-		if( target == 'max' )
-			target = 9e9;
-			
-		settings = $.extend( {}, $scrollTo.defaults, settings );
-		// Speed is still recognized for backwards compatibility
-		duration = duration || settings.duration;
-		// Make sure the settings are given right
-		settings.queue = settings.queue && settings.axis.length > 1;
-		
-		if( settings.queue )
-			// Let's keep the overall duration
-			duration /= 2;
-		settings.offset = both( settings.offset );
-		settings.over = both( settings.over );
-
-		return this._scrollable().each(function(){
-			// Null target yields nothing, just like jQuery does
-			if (target == null) return;
-
-			var elem = this,
-				$elem = $(elem),
-				targ = target, toff, attr = {},
-				win = $elem.is('html,body');
-
-			switch( typeof targ ){
-				// A number will pass the regex
-				case 'number':
-				case 'string':
-					if( /^([+-]=)?\d+(\.\d+)?(px|%)?$/.test(targ) ){
-						targ = both( targ );
-						// We are done
-						break;
-					}
-					// Relative selector, no break!
-					targ = $(targ,this);
-					if (!targ.length) return;
-				case 'object':
-					// DOMElement / jQuery
-					if( targ.is || targ.style )
-						// Get the real position of the target 
-						toff = (targ = $(targ)).offset();
-			}
-			$.each( settings.axis.split(''), function( i, axis ){
-				var Pos	= axis == 'x' ? 'Left' : 'Top',
-					pos = Pos.toLowerCase(),
-					key = 'scroll' + Pos,
-					old = elem[key],
-					max = $scrollTo.max(elem, axis);
-
-				if( toff ){// jQuery / DOMElement
-					attr[key] = toff[pos] + ( win ? 0 : old - $elem.offset()[pos] );
-
-					// If it's a dom element, reduce the margin
-					if( settings.margin ){
-						attr[key] -= parseInt(targ.css('margin'+Pos)) || 0;
-						attr[key] -= parseInt(targ.css('border'+Pos+'Width')) || 0;
-					}
-					
-					attr[key] += settings.offset[pos] || 0;
-					
-					if( settings.over[pos] )
-						// Scroll to a fraction of its width/height
-						attr[key] += targ[axis=='x'?'width':'height']() * settings.over[pos];
-				}else{ 
-					var val = targ[pos];
-					// Handle percentage values
-					attr[key] = val.slice && val.slice(-1) == '%' ? 
-						parseFloat(val) / 100 * max
-						: val;
-				}
-
-				// Number or 'number'
-				if( settings.limit && /^\d+$/.test(attr[key]) )
-					// Check the limits
-					attr[key] = attr[key] <= 0 ? 0 : Math.min( attr[key], max );
-
-				// Queueing axes
-				if( !i && settings.queue ){
-					// Don't waste time animating, if there's no need.
-					if( old != attr[key] )
-						// Intermediate animation
-						animate( settings.onAfterFirst );
-					// Don't animate this axis again in the next iteration.
-					delete attr[key];
-				}
-			});
-
-			animate( settings.onAfter );			
-
-			function animate( callback ){
-				$elem.animate( attr, duration, settings.easing, callback && function(){
-					callback.call(this, target, settings);
-				});
-			};
-
-		}).end();
-	};
-	
-	// Max scrolling position, works on quirks mode
-	// It only fails (not too badly) on IE, quirks mode.
-	$scrollTo.max = function( elem, axis ){
-		var Dim = axis == 'x' ? 'Width' : 'Height',
-			scroll = 'scroll'+Dim;
-		
-		if( !$(elem).is('html,body') )
-			return elem[scroll] - $(elem)[Dim.toLowerCase()]();
-		
-		var size = 'client' + Dim,
-			html = elem.ownerDocument.documentElement,
-			body = elem.ownerDocument.body;
-
-		return Math.max( html[scroll], body[scroll] ) 
-			 - Math.min( html[size]  , body[size]   );
-	};
-
-	function both( val ){
-		return typeof val == 'object' ? val : { top:val, left:val };
-	};
-
-})( jQuery );

docs/js/layout.js

@@ -1,83 +0,0 @@
-$(document).ready(function () {
-    var AFFIX_TOP_LIMIT = 300;
-    var AFFIX_OFFSET = 49;
-	$('#menu-left').localScroll({hash:true, onAfterFirst:function(){$('html, body').scrollTo( {top:'-=25px'}, 'fast' );}});
-    var $menu = $("#menu"),
-		$btn = $("#menu-toggle");
-
-    $("#menu-toggle").on("click", function () {
-        $menu.toggleClass("open");
-        return false;
-    });
-
-
-    $(".docs-nav").each(function () {
-        var $affixNav = $(this),
-			$container = $affixNav.parent(),
-			affixNavfixed = false,
-			originalClassName = this.className,
-			current = null,
-			$links = $affixNav.find("a");
-
-        function getClosestHeader(top) {
-            var last = $links.first();
-
-            if (top < AFFIX_TOP_LIMIT) {
-                return last;
-            }
-
-            for (var i = 0; i < $links.length; i++) {
-                var $link = $links.eq(i),
-					href = $link.attr("href");
-
-                if (href.charAt(0) === "#" && href.length > 1) {
-                    var $anchor = $(href).first();
-
-                    if ($anchor.length > 0) {
-                        var offset = $anchor.offset();
-
-                        if (top < offset.top - AFFIX_OFFSET) {
-                            return last;
-                        }
-
-                        last = $link;
-                    }
-                }
-            }
-            return last;
-        }
-
-
-        $(window).on("scroll", function (evt) {
-            var top = window.scrollY,
-		    	height = $affixNav.outerHeight(),
-		    	max_bottom = $container.offset().top + $container.outerHeight(),
-		    	bottom = top + height + AFFIX_OFFSET;
-
-            if (affixNavfixed) {
-                if (top <= AFFIX_TOP_LIMIT) {
-                    $affixNav.removeClass("fixed");
-                    $affixNav.css("top", 0);
-                    affixNavfixed = false;
-                } else if (bottom > max_bottom) {
-                    $affixNav.css("top", (max_bottom - height) - top);
-                } else {
-                    $affixNav.css("top", AFFIX_OFFSET);
-                }
-            } else if (top > AFFIX_TOP_LIMIT) {
-                $affixNav.addClass("fixed");
-                affixNavfixed = true;
-            }
-
-            var $current = getClosestHeader(top);
-
-            if (current !== $current) {
-                $affixNav.find(".active").removeClass("active");
-                $current.addClass("active");
-                current = $current;
-            }
-        });
-    });
-
-    prettyPrint();
-});

docs/tutorial/branches.sh

@@ -1,117 +0,0 @@
-#!/bin/bash
-
-set -e
-
-mkdir myrepo
-cd myrepo
-
-# 1. Set up environment and get code
-git init
-echo "This is an empty readme" > README.md
-git add README.md
-git commit -m 'add readme'
-
-mkdir code
-wget -nv -P code/ https://s3-us-west-2.amazonaws.com/dvc-share/so/code/featurization.py \
-        https://s3-us-west-2.amazonaws.com/dvc-share/so/code/evaluate.py \
-        https://s3-us-west-2.amazonaws.com/dvc-share/so/code/train_model.py \
-        https://s3-us-west-2.amazonaws.com/dvc-share/so/code/split_train_test.py \
-        https://s3-us-west-2.amazonaws.com/dvc-share/so/code/xml_to_tsv.py \
-        https://s3-us-west-2.amazonaws.com/dvc-share/so/code/requirements.txt
-pip install -U -r code/requirements.txt
-git add code/
-git commit -m 'Download code'
-dvc init
-git commit -m 'Init DVC'
-
-# 2. First steps in 100K dataset
-git checkout -b input_100K
-mkdir data
-wget https://s3-us-west-2.amazonaws.com/dvc-share/so/100K/Posts.xml.tgz -O data/Posts.xml.tgz
-#cp ../stackoverflow_small_xml/100K/Posts.xml.tgz data/
-dvc add data/Posts.xml.tgz
-dvc run -o data/Posts.xml -d data/Posts.xml.tgz tar zxf data/Posts.xml.tgz -C data/
-dvc run -d code/xml_to_tsv.py -d data/Posts.xml -o data/Posts.tsv python code/xml_to_tsv.py data/Posts.xml data/Posts.tsv python
-git add .
-git commit -m 'Extract TSV for 100K dataset'
-
-# 3. Define full pipeline for 25K dataset
-git checkout master
-dvc checkout
-git checkout -b input_25K
-dvc checkout
-mkdir data
-wget https://s3-us-west-2.amazonaws.com/dvc-share/so/25K/Posts.xml.tgz -O data/Posts.xml.tgz
-#cp ../stackoverflow_small_xml/25K/Posts.xml.tgz data/
-dvc add data/Posts.xml.tgz
-dvc run -o data/Posts.xml -d data/Posts.xml.tgz tar zxf data/Posts.xml.tgz -C data/
-dvc run -d code/xml_to_tsv.py -d data/Posts.xml -o data/Posts.tsv python code/xml_to_tsv.py data/Posts.xml data/Posts.tsv python
-
-dvc run -d code/split_train_test.py -d data/Posts.tsv -o data/Posts-train.tsv -o data/Posts-test.tsv python code/split_train_test.py data/Posts.tsv 0.33 20170426 data/Posts-train.tsv data/Posts-test.tsv
-dvc run -d code/featurization.py -d data/Posts-train.tsv -d data/Posts-test.tsv -o data/matrix-train.p -o data/matrix-test.p python code/featurization.py data/Posts-train.tsv data/Posts-test.tsv data/matrix-train.p data/matrix-test.p
-dvc run -d code/train_model.py -d data/matrix-train.p -o data/model.p python code/train_model.py data/matrix-train.p 20170426 data/model.p
-dvc run -d code/evaluate.py -d data/model.p -d data/matrix-test.p -O data/eval_auc.txt python code/evaluate.py data/model.p data/matrix-test.p data/eval_auc.txt
-dvc run -d data/eval_auc.txt
-cat data/eval_auc.txt
-# AUC: 0.596182
-
-git status
-git add .
-git commit -m 'Full pipeline'
-
-# 4. Integrate the 25K dataset pipeline into 100K branch
-git checkout input_100K
-dvc checkout
-git merge -X ours input_25K # You can also merge and resolve conflicts by hands!!!
-
-dvc repro
-git add .
-git commit -m 'Repro'
-
-# 5. Improve the 25K model
-git checkout input_25K
-dvc checkout
-vi code/train_model.py  # Change: estimators=500 and n_jobs=6
-dvc repro
-git commit -am 'estimators=500'
-cat data/eval_auc.txt
-# 0.619262
-
-vi code/featurization.py # Change: add ngram_range=(1, 2) to CountVectorizer
-dvc repro
-git commit -am 'Add bigrams'
-cat data/eval_auc.txt
-# 0.628989
-
-vi code/featurization.py # Change: ngram_range=(1, 3)
-dvc repro
-git commit -am 'Add three-grams'
-cat data/eval_auc.txt
-# 0.630682
-
-vi code/featurization.py  # Change: ngram_range=(1, 4)
-dvc repro
-git commit -am 'Add 4-grams'
-cat data/eval_auc.txt
-# 0.621002
-
-# 6. Revert the last step back
-# Create a branch for unsuccessful commit
-git checkout -b four_grams
-dvc checkout
-# The last successful commit which is the second last humman-created commit
-git log --pretty=oneline | grep -v "DVC repro-run: " | head -n 2 | tail -n 1
-# Move the current HEAD to the last successful commit
-git branch -f input_25K `git log --pretty=oneline | grep -v "DVC repro-run: " | head -n 2 | tail -n 1 | cut -d" " -f1`
-git checkout input_25K
-dvc checkout
-
-# 7. Integrate the 25K model improvements into the 100K model
-git checkout input_100K
-dvc checkout
-git merge -X theirs input_25K
-dvc repro
-git commit -am 'Final model for 100K dataset'
-cat data/eval_auc.txt
-# 0.652154
-

docs/tutorial/cnn_classify.sh

@@ -1,227 +0,0 @@
-#!/bin/bash
-
-set -x
-
-GIT_REMOTE=https://github.com/dmpetrov/cnn_classify.git
-
-mkdir cnn_classify
-cd cnn_classify
-git init
-dvc init
-dvc config AWS.StoragePath dvc-share/dvc_tutorial
-git add .
-git commit -m 'Init DVC'
-
-mkdir data
-cp ../train.zip data/
-dvc add data/train.zip
-git add data/
-git status
-#On branch master
-#Changes to be committed:
-#  (use "git reset HEAD <file>..." to unstage)
-#
-#    new file:   data/.gitignore
-#    new file:   data/train.zip.dvc
-git commit -m 'Raw data'
-
-dvc run -d data/train.zip -o data/train unzip -d data/ data/train.zip
-git status
-#On branch master
-#Changes to be committed:
-#  (use "git reset HEAD <file>..." to unstage)
-#  (use "git checkout -- <file>..." to discard changes in working directory)
-#
-#    modified:   data/.gitignore
-#
-#Untracked files:
-#  (use "git add <file>..." to include in what will be committed)
-#    new file:   train.dvc
-#
-#no changes added to commit (use "git add" and/or "git commit -a")
-
-git add data/.gitignore train.dvc
-git commit -m 'Unzip files'
-
-mkdir code
-wget -P code https://s3-us-west-2.amazonaws.com/dvc-share/tutorial_cnn_code/process_files.py
-wget -P code https://s3-us-west-2.amazonaws.com/dvc-share/tutorial_cnn_code/conf.py
-dvc run -d code/process_files.py -d data/train -o data/cats_and_dogs_small -f process.dvc python code/process_files.py
-git status
-#On branch master
-#Changes not staged for commit:
-#  (use "git add <file>..." to update what will be committed)
-#    (use "git checkout -- <file>..." to discard changes in working directory)
-#
-#        modified:   data/.gitignore
-#
-#Untracked files:
-#  (use "git add <file>..." to include in what will be committed)
-#
-#        code/
-#        process.dvc
-#
-#no changes added to commit (use "git add" and/or "git commit -a")
-
-echo '__pycache__/' >> code/.gitignore
-echo '*.pyc' >> code/.gitignore
-git add .
-git commit -m 'Process raw data'
-
-wget -O code/model.py https://s3-us-west-2.amazonaws.com/dvc-share/tutorial_cnn_code/model_first_cnn.py
-
-dvc run -d code/model.py -d code/conf.py -d data/cats_and_dogs_small -o data/model.h5 -o data/history.p python code/model.py
-git status
-#On branch master
-#Changes not staged for commit:
-#  (use "git add <file>..." to update what will be committed)
-#    (use "git checkout -- <file>..." to discard changes in working directory)
-#
-#        modified:   data/.gitignore
-#
-#Untracked files:
-#  (use "git add <file>..." to include in what will be committed)
-#
-#        code/model.py
-#        model.h5.dvc
-#
-#no changes added to commit (use "git add" and/or "git commit -a")
-
-git add .
-git commit -m 'First model'
-
-wget -P code https://s3-us-west-2.amazonaws.com/dvc-share/tutorial_cnn_code/plot.py
-dvc run -d code/conf.py -d code/plot.py -d data/history.p -o data/plot_loss.jpeg -o data/plot_acc.jpeg python code/plot.py
-
-dvc run -d data/plot_loss.jpeg
-
-git status
-#On branch master
-#Changes not staged for commit:
-#  (use "git add <file>..." to update what will be committed)
-#    (use "git checkout -- <file>..." to discard changes in working directory)
-#
-#        modified:   data/.gitignore
-#
-#Untracked files:
-#  (use "git add <file>..." to include in what will be committed)
-#
-#        Dvcfile
-#        code/plot.py
-#        plot_loss.jpeg.dvc
-#
-#no changes added to commit (use "git add" and/or "git commit -a")
-
-git add .
-git commit -m 'Accuracy and loss plots'
-
-git tag -a v0.1-first-cnn -m 'First CNN model'
-
-
-############################################ v2
-rm code/model.py
-wget -O code/model.py https://s3-us-west-2.amazonaws.com/dvc-share/tutorial_cnn_code/model_augm.py
-# Actual changes: augmentation, Dropout=0.5, batch_size=20-->32, epochs=30-->100
-dvc repro
-
-git status
-#On branch master
-#Changes not staged for commit:
-#  (use "git add <file>..." to update what will be committed)
-#    (use "git checkout -- <file>..." to discard changes in working directory)
-#
-#        modified:   Dvcfile
-#        modified:   code/model.py
-#        modified:   model.h5.dvc
-#        modified:   plot_loss.jpeg.dvc
-#
-#no changes added to commit (use "git add" and/or "git commit -a")
-
-git add .
-git commit -m 'Retrain augmentation model'
-git tag -a v0.2-augm-cnn -m 'CNN model with augmentation'
-
-########################################### v3
-
-git checkout v0.1-first-cnn -b pre_trained
-dvc checkout
-rm code/model.py
-wget -O code/model.py https://s3-us-west-2.amazonaws.com/dvc-share/tutorial_cnn_code/model_vgg16_base.py
-
-dvc repro
-git status
-#On branch pre_trained
-#Changes not staged for commit:
-#  (use "git add <file>..." to update what will be committed)
-#    (use "git checkout -- <file>..." to discard changes in working directory)
-#
-#        modified:   Dvcfile
-#        modified:   code/model.py
-#        modified:   model.h5.dvc
-#        modified:   plot_loss.jpeg.dvc
-#
-#no changes added to commit (use "git add" and/or "git commit -a")
-
-git add .
-git commit -m 'Pretrained VGG16'
-git tag -a v0.3-vgg16_base -m 'Pretrained VGG16 tag'
-
-
-############################################### v4
-
-git checkout v0.2-augm-cnn -b vgg16_augm
-dvc checkout
-rm code/model.py
-wget -O code/model.py https://s3-us-west-2.amazonaws.com/dvc-share/tutorial_cnn_code/model_vgg16_augm.py
-
-dvc repro
-git status
-#On branch vgg16_augm
-#Changes not staged for commit:
-#  (use "git add <file>..." to update what will be committed)
-#    (use "git checkout -- <file>..." to discard changes in working directory)
-#
-#        modified:   Dvcfile
-#        modified:   code/model.py
-#        modified:   model.h5.dvc
-#        modified:   plot_loss.jpeg.dvc
-#
-#no changes added to commit (use "git add" and/or "git commit -a")
-
-git add .
-git commit -m 'VGG16 with augm'
-git tag -a v0.4-vgg16_augm -m 'VGG16 with augm tag'
-
-################################################# v5
-
-rm code/model.py
-wget -O code/model.py https://s3-us-west-2.amazonaws.com/dvc-share/tutorial_cnn_code/model_vgg16_augm_fine.py
-dvc repro
-git status
-#On branch vgg16_augm
-#Changes not staged for commit:
-#  (use "git add <file>..." to update what will be committed)
-#    (use "git checkout -- <file>..." to discard changes in working directory)
-#
-#        modified:   Dvcfile
-#        modified:   code/model.py
-#        modified:   model.h5.dvc
-#        modified:   plot_loss.jpeg.dvc
-#
-#no changes added to commit (use "git add" and/or "git commit -a")
-
-git add .
-git commit -m 'VGG, augmentation and fine-tuning'
-git tag -a v0.5-vgg_augm_fine -m 'VGG, augmentation and fine-tuning tag'
-
-git checkout master
-git merge vgg16_augm
-
-git remote add origin ${GIT_REMOTE}
-git push -u origin master --tags
-
-git push -u origin pre_trained
-git push -u origin vgg16_augm
-
-dvc push
-

dvc/__init__.py

@@ -3,5 +3,57 @@ DVC
 ----
 Make your data science projects reproducible and shareable.
 """
-VERSION = '0.9.1'
-__version__ = VERSION
+import os
+import warnings
+
+
+VERSION_BASE = '0.19.2'
+__version__ = VERSION_BASE
+
+PACKAGEPATH = os.path.abspath(os.path.dirname(__file__))
+HOMEPATH = os.path.dirname(PACKAGEPATH)
+VERSIONPATH = os.path.join(PACKAGEPATH, 'version.py')
+
+if os.path.exists(os.path.join(HOMEPATH, 'setup.py')):
+    # dvc is run directly from source without installation or
+    # __version__ is called from setup.py
+    if os.getenv('APPVEYOR_REPO_TAG', '').lower() != 'true' \
+       and os.getenv('TRAVIS_TAG', '') == '':
+        # Dynamically update version
+        try:
+            import git
+            repo = git.Repo(HOMEPATH)
+            sha = repo.head.object.hexsha
+            short_sha = repo.git.rev_parse(sha, short=6)
+            dirty = '.mod' if repo.is_dirty() else ''
+            __version__ = '{}+{}{}'.format(__version__, short_sha, dirty)
+
+            # Write a helper file, that will be installed with the package
+            # and will provide a true version of the installed dvc
+            with open(VERSIONPATH, 'w+') as fd:
+                fd.write('# AUTOGENERATED by dvc/__init__.py\n')
+                fd.write('version = "{}"\n'.format(__version__))
+        except Exception:
+            pass
+    else:
+        # Remove version.py so that it doesn't get into the release
+        if os.path.exists(VERSIONPATH):
+            os.unlink(VERSIONPATH)
+else:
+    # dvc was installed with pip or something. Hopefully we have our
+    # auto-generated version.py to help us provide a true version
+    try:
+        from dvc.version import version
+        __version__ = version
+    except Exception:
+        pass
+
+VERSION = __version__
+
+
+# Ignore numpy's runtime warnings: https://github.com/numpy/numpy/pull/432.
+# We don't directly import numpy, but our dependency networkx does, causing
+# these warnings in some environments. Luckily these warnings are benign and
+# we can simply ignore them so that they don't show up when you are using dvc.
+warnings.filterwarnings("ignore", message="numpy.dtype size changed")
+warnings.filterwarnings("ignore", message="numpy.ufunc size changed")

dvc/cache.py

@@ -1,48 +1,48 @@
 import os
 
-from dvc.system import System
+from dvc.config import Config
 
 
 class Cache(object):
     CACHE_DIR = 'cache'
 
-    def __init__(self, dvc_dir):
-        self.cache_dir = os.path.abspath(os.path.realpath(os.path.join(dvc_dir, self.CACHE_DIR)))
-        if not os.path.exists(self.cache_dir):
-            os.mkdir(self.cache_dir)
+    def __init__(self, project):
+        from dvc.remote import Remote
 
-    @staticmethod
-    def init(dvc_dir):
-        cache_dir = os.path.join(dvc_dir, Cache.CACHE_DIR)
-        os.mkdir(cache_dir)
-        return Cache(dvc_dir)
+        self.project = project
 
-    def all(self):
-        clist = []
-        for entry in os.listdir(self.cache_dir):
-            subdir = os.path.join(self.cache_dir, entry)
-            if not os.path.isdir(subdir):
-                continue
+        config = project.config._config[Config.SECTION_CACHE]
 
-            for cache in os.listdir(subdir):
-                path = os.path.join(subdir, cache)
-                clist.append(path)
+        local = config.get(Config.SECTION_CACHE_LOCAL, None)
+        if local:
+            name = Config.SECTION_REMOTE_FMT.format(local)
+            sect = project.config._config[name]
+        else:
+            sect = {}
+            cache_dir = config.get(Config.SECTION_CACHE_DIR, self.CACHE_DIR)
+            if not os.path.isabs(cache_dir):
+                cache_dir = os.path.join(project.dvc_dir, cache_dir)
+                cache_dir = os.path.abspath(os.path.realpath(cache_dir))
+            sect[Config.SECTION_REMOTE_URL] = cache_dir
+            t = config.get(Config.SECTION_CACHE_TYPE, None)
+            if t:
+                sect[Config.SECTION_CACHE_TYPE] = t
 
-        return clist
+        self.local = Remote(project, sect)
 
-    def get(self, md5):
-        return os.path.join(self.cache_dir, md5[0:2], md5[2:])
+        self.s3 = self._get_remote(config, Config.SECTION_CACHE_S3)
+        self.gs = self._get_remote(config, Config.SECTION_CACHE_GS)
+        self.ssh = self._get_remote(config, Config.SECTION_CACHE_SSH)
+        self.hdfs = self._get_remote(config, Config.SECTION_CACHE_HDFS)
+        self.azure = self._get_remote(config, Config.SECTION_CACHE_AZURE)
 
-    def path_to_md5(self, path):
-        relpath = os.path.relpath(path, self.cache_dir)
-        return os.path.dirname(relpath) + os.path.basename(relpath)
+    def _get_remote(self, config, name):
+        from dvc.remote import Remote
 
-    def find_cache(self, files):
-        file_set = set(files)
-        cached = {}
-        for cache_file in self.all():
-            cached_files = list(filter(lambda f: System.samefile(cache_file, f), file_set))
-            cached.update(dict((f, os.path.basename(os.path.dirname(cache_file)) + os.path.basename(cache_file)) for f in cached_files))
-            file_set = file_set - set(cached_files)
-        return cached
+        remote = config.get(name, None)
+        if not remote:
+            return None
 
+        name = Config.SECTION_REMOTE_FMT.format(remote)
+        sect = self.project.config._config[name]
+        return Remote(self.project, sect)

dvc/cli.py

@@ -6,19 +6,53 @@ import argparse
 from multiprocessing import cpu_count
 
 from dvc.command.init import CmdInit
+from dvc.command.destroy import CmdDestroy
 from dvc.command.remove import CmdRemove
+from dvc.command.move import CmdMove
 from dvc.command.run import CmdRun
 from dvc.command.repro import CmdRepro
 from dvc.command.data_sync import CmdDataPush, CmdDataPull, CmdDataFetch
 from dvc.command.status import CmdDataStatus
 from dvc.command.gc import CmdGC
 from dvc.command.add import CmdAdd
+from dvc.command.imp import CmdImport
 from dvc.command.config import CmdConfig
 from dvc.command.checkout import CmdCheckout
-from dvc.stage import Stage
+from dvc.command.remote import CmdRemoteAdd, CmdRemoteRemove
+from dvc.command.remote import CmdRemoteModify, CmdRemoteList
+from dvc.command.metrics import CmdMetricsShow, CmdMetricsAdd
+from dvc.command.metrics import CmdMetricsRemove, CmdMetricsModify
+from dvc.command.install import CmdInstall
+from dvc.command.root import CmdRoot
+from dvc.command.lock import CmdLock, CmdUnlock
+from dvc.command.pipeline import CmdPipelineShow
+from dvc.logger import Logger
 from dvc import VERSION
 
 
+def _fix_subparsers(subparsers):
+    # NOTE: Workaround for bug in Python 3
+    # More info at:
+    #  https://bugs.python.org/issue16308
+    #  https://github.com/iterative/dvc/issues/769
+    if sys.version_info[0] == 3:  # pragma: no cover
+        subparsers.required = True
+        subparsers.dest = 'cmd'
+
+
+class DvcParser(argparse.ArgumentParser):
+    def error(self, message):
+        sys.stderr.write('{}{}\n'.format(Logger.error_prefix(), message))
+        self.print_help()
+        sys.exit(2)
+
+
+class VersionAction(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string=None):
+        print(VERSION)
+        sys.exit(0)
+
+
 def parse_args(argv=None):
     # Common args
     parent_parser = argparse.ArgumentParser(add_help=False)
@@ -37,100 +71,199 @@ def parse_args(argv=None):
 
     # Main parser
     desc = 'Data Version Control'
-    parser = argparse.ArgumentParser(
-                        prog='dvc',
-                        description=desc,
-                        parents=[parent_parser],
-                        formatter_class=argparse.RawTextHelpFormatter)
+    parser = DvcParser(prog='dvc',
+                       description=desc,
+                       parents=[parent_parser],
+                       formatter_class=argparse.RawTextHelpFormatter)
 
+    # NOTE: On some python versions action='version' prints to stderr
+    # instead of stdout https://bugs.python.org/issue18920
     parser.add_argument('-V',
                         '--version',
-                        action='version',
-                        version='%(prog)s ' + VERSION,
-                        help='Show program\'s version')
+                        action=VersionAction,
+                        nargs=0,
+                        help='Show program\'s version.')
 
     # Sub commands
     subparsers = parser.add_subparsers(
+                        title='Available Commands',
+                        metavar='COMMAND',
                         dest='cmd',
-                        help='Use dvc CMD --help for command-specific help')
+                        help='Use dvc COMMAND --help for command-specific '
+                             'help.')
 
-    # NOTE: Workaround for bug in Python 3
-    if sys.version_info[0] == 3:
-        subparsers.required = True
-        subparsers.dest = 'cmd'
+    _fix_subparsers(subparsers)
 
     # Init
+    INIT_HELP = 'Initialize dvc over a directory ' \
+                '(should already be a git dir).'
     init_parser = subparsers.add_parser(
                         'init',
                         parents=[parent_parser],
-                        help='Initialize dvc over a directory (should already be a git dir)')
+                        description=INIT_HELP,
+                        help=INIT_HELP)
+    init_parser.add_argument(
+                        '--no-scm',
+                        action='store_true',
+                        default=False,
+                        help="Initiate dvc in directory that is "
+                             "not tracked by any scm tool(e.g. git).")
+    init_parser.add_argument(
+                        '-f',
+                        '--force',
+                        action='store_true',
+                        default=False,
+                        help="Overwrite '.dvc' if it exists. Will remove "
+                             "all local cache.")
     init_parser.set_defaults(func=CmdInit)
 
+    # Destroy
+    DESTROY_HELP = "Destroy dvc. Will remove all project's information, " \
+                   "data files and cache."
+    destroy_parser = subparsers.add_parser(
+                        'destroy',
+                        parents=[parent_parser],
+                        description=DESTROY_HELP,
+                        help=DESTROY_HELP)
+    destroy_parser.add_argument(
+                        '-f',
+                        '--force',
+                        action='store_true',
+                        default=False,
+                        help='Force destruction.')
+    destroy_parser.set_defaults(func=CmdDestroy)
+
     # Add
-    import_parser = subparsers.add_parser(
+    ADD_HELP = 'Add files/directories to dvc.'
+    add_parser = subparsers.add_parser(
                         'add',
                         parents=[parent_parser],
-                        help='Add files/directories to dvc')
-    import_parser.add_argument(
+                        description=ADD_HELP,
+                        help=ADD_HELP)
+    add_parser.add_argument(
+                        '-R',
+                        '--recursive',
+                        action='store_true',
+                        default=False,
+                        help='Recursively add each file under the directory.')
+    add_parser.add_argument(
                         'targets',
                         nargs='+',
-                        help='Input files/directories')
-    import_parser.set_defaults(func=CmdAdd)
+                        help='Input files/directories.')
+    add_parser.set_defaults(func=CmdAdd)
+
+    # Import
+    IMPORT_HELP = 'Import files from URL.'
+    import_parser = subparsers.add_parser(
+                        'import',
+                        parents=[parent_parser],
+                        description=IMPORT_HELP,
+                        help=IMPORT_HELP)
+    import_parser.add_argument(
+                        'url',
+                        help='URL.')
+    import_parser.add_argument(
+                        'out',
+                        help='Output.')
+    import_parser.set_defaults(func=CmdImport)
 
     # Checkout
+    CHECKOUT_HELP = 'Checkout data files from cache.'
     checkout_parser = subparsers.add_parser(
                         'checkout',
                         parents=[parent_parser],
-                        help='Checkout data files from cache')
+                        description=CHECKOUT_HELP,
+                        help=CHECKOUT_HELP)
+    checkout_parser.add_argument(
+                        'targets',
+                        nargs='*',
+                        help='DVC files.')
     checkout_parser.set_defaults(func=CmdCheckout)
 
     # Run
+    RUN_HELP = 'Generate a stage file from a given ' \
+               'command and execute the command.'
     run_parser = subparsers.add_parser(
                         'run',
                         parents=[parent_parser],
-                        help='Generate a stage file from a given command and execute the command')
-    run_parser.add_argument('-d',
+                        description=RUN_HELP,
+                        help=RUN_HELP)
+    run_parser.add_argument(
+                        '-d',
                         '--deps',
                         action='append',
                         default=[],
                         help='Declare dependencies for reproducible cmd.')
-    run_parser.add_argument('-o',
+    run_parser.add_argument(
+                        '-o',
                         '--outs',
                         action='append',
                         default=[],
                         help='Declare output data file or data directory.')
-    run_parser.add_argument('-O',
+    run_parser.add_argument(
+                        '-O',
                         '--outs-no-cache',
                         action='append',
                         default=[],
-                        help='Declare output regular file or directory (sync to Git, not DVC cache).')
-    run_parser.add_argument('-f',
+                        help='Declare output regular file or '
+                             'directory (sync to Git, not DVC cache).')
+    run_parser.add_argument(
+                        '-M',
+                        '--metrics-no-cache',
+                        action='append',
+                        default=[],
+                        help='Declare output metric file or '
+                             'directory (not cached by DVC).')
+    run_parser.add_argument(
+                        '-f',
                         '--file',
-                        help='Specify name of the state file')
-    run_parser.add_argument('-c',
+                        help="Specify name of the stage file. It should be "
+                             "either 'Dvcfile' or have a '.dvc' suffix (e.g. "
+                             "'prepare.dvc', 'clean.dvc', etc) in order for "
+                             "dvc to be able to find it later. By default "
+                             "the first output basename + .dvc is used as "
+                             "a stage filename.")
+    run_parser.add_argument(
+                        '-c',
                         '--cwd',
                         default=os.path.curdir,
-                        help='Directory to run your command and place state file in')
+                        help='Directory to run your command and place state '
+                             'file in.')
     run_parser.add_argument(
                         '--no-exec',
                         action='store_true',
                         default=False,
-                        help="Only create stage file without actually running it")
+                        help="Only create stage file without actually "
+                             "running it.")
+    run_parser.add_argument(
+                        '-y',
+                        '--yes',
+                        action='store_true',
+                        default=False,
+                        help="Automatic 'yes' answer to all prompts. E.g. "
+                             "when '.dvc' file exists and dvc asks if you "
+                             "want to overwrite it.")
     run_parser.add_argument(
                         'command',
                         nargs=argparse.REMAINDER,
-                        help='Command or command file to execute')
+                        help='Command or command file to execute.')
     run_parser.set_defaults(func=CmdRun)
 
     # Parent parser used in pull/push/status
     parent_cache_parser = argparse.ArgumentParser(
                         add_help=False,
                         parents=[parent_parser])
-    parent_cache_parser.add_argument('-j',
+    parent_cache_parser.add_argument(
+                        '-j',
                         '--jobs',
                         type=int,
-                        default=8*cpu_count(),
+                        default=4*cpu_count(),
                         help='Number of jobs to run simultaneously.')
+    parent_cache_parser.add_argument(
+                        '--show-checksums',
+                        action='store_true',
+                        default=False,
+                        help='Show checksums instead of file names.')
     parent_cache_parser.add_argument(
                         'targets',
                         nargs='*',
@@ -138,93 +271,589 @@ def parse_args(argv=None):
                         help='DVC files.')
 
     # Pull
+    PULL_HELP = 'Pull data files from the cloud.'
     pull_parser = subparsers.add_parser(
                         'pull',
                         parents=[parent_cache_parser],
-                        help='Pull data files from the cloud')
+                        description=PULL_HELP,
+                        help=PULL_HELP)
+    pull_parser.add_argument(
+                        '-r',
+                        '--remote',
+                        help='Remote repository to pull from.')
+    pull_parser.add_argument(
+                        '-a',
+                        '--all-branches',
+                        action='store_true',
+                        default=False,
+                        help='Fetch cache for all branches.')
+    pull_parser.add_argument(
+                        '-d',
+                        '--with-deps',
+                        action='store_true',
+                        default=False,
+                        help='Fetch cache for all dependencies of the '
+                             'specified target.')
     pull_parser.set_defaults(func=CmdDataPull)
 
     # Push
+    PUSH_HELP = 'Push data files to the cloud.'
     push_parser = subparsers.add_parser(
                         'push',
                         parents=[parent_cache_parser],
-                        help='Push data files to the cloud')
+                        description=PUSH_HELP,
+                        help=PUSH_HELP)
+    push_parser.add_argument(
+                        '-r',
+                        '--remote',
+                        help='Remote repository to push to.')
+    push_parser.add_argument(
+                        '-a',
+                        '--all-branches',
+                        action='store_true',
+                        default=False,
+                        help='Push cache for all branches.')
+    push_parser.add_argument(
+                        '-d',
+                        '--with-deps',
+                        action='store_true',
+                        default=False,
+                        help='Push cache for all dependencies of the '
+                             'specified target.')
     push_parser.set_defaults(func=CmdDataPush)
 
     # Fetch
+    FETCH_HELP = 'Fetch data files from the cloud.'
     fetch_parser = subparsers.add_parser(
                         'fetch',
                         parents=[parent_cache_parser],
-                        help='Fetch data files from the cloud')
+                        description=FETCH_HELP,
+                        help=FETCH_HELP)
+    fetch_parser.add_argument(
+                        '-r',
+                        '--remote',
+                        help='Remote repository to fetch from.')
+    fetch_parser.add_argument(
+                        '-a',
+                        '--all-branches',
+                        action='store_true',
+                        default=False,
+                        help='Fetch cache for all branches.')
+    fetch_parser.add_argument(
+                        '-d',
+                        '--with-deps',
+                        action='store_true',
+                        default=False,
+                        help='Fetch cache for all dependencies of the '
+                             'specified target.')
     fetch_parser.set_defaults(func=CmdDataFetch)
 
     # Status
+    STATUS_HELP = 'Show the project status.'
     status_parser = subparsers.add_parser(
                         'status',
                         parents=[parent_cache_parser],
-                        help='Show the project status')
-    status_parser.add_argument('-c',
+                        description=STATUS_HELP,
+                        help=STATUS_HELP)
+    status_parser.add_argument(
+                        '-c',
                         '--cloud',
                         action='store_true',
                         default=False,
-                        help='Show status of a local cache compared to a cloud')
+                        help='Show status of a local cache compared to a '
+                             'remote repository.')
+    status_parser.add_argument(
+                        '-r',
+                        '--remote',
+                        help='Remote repository to compare local cache to.')
+    status_parser.add_argument(
+                        '-a',
+                        '--all-branches',
+                        action='store_true',
+                        default=False,
+                        help='Show status of a local cache compared to a '
+                             'remote repository for all branches.')
+    status_parser.add_argument(
+                        '-d',
+                        '--with-deps',
+                        action='store_true',
+                        default=False,
+                        help='Show statusof a local cache compared to a '
+                             'remote repository for all dependencies of the '
+                             'specified target.')
     status_parser.set_defaults(func=CmdDataStatus)
 
     # Repro
+    REPRO_HELP = 'Reproduce DVC file. Default file name - \'Dvcfile\'.'
     repro_parser = subparsers.add_parser(
                         'repro',
                         parents=[parent_parser],
-                        help='Reproduce DVC file. Default file name - \'Dvcfile\'')
+                        description=REPRO_HELP,
+                        help=REPRO_HELP)
     repro_parser.add_argument(
                         'targets',
                         nargs='*',
-                        default=[Stage.STAGE_FILE],
                         help='DVC file to reproduce.')
-    repro_parser.add_argument('-f',
+    repro_parser.add_argument(
+                        '-f',
                         '--force',
                         action='store_true',
                         default=False,
-                        help='Reproduce even if dependencies were not changed.')
-    repro_parser.add_argument('-s',
+                        help='Reproduce even if dependencies were not '
+                             'changed.')
+    repro_parser.add_argument(
+                        '-s',
                         '--single-item',
                         action='store_true',
                         default=False,
-                        help='Reproduce only single data item without recursive dependencies check.')
+                        help='Reproduce only single data item without '
+                             'recursive dependencies check.')
+    repro_parser.add_argument(
+                        '-c',
+                        '--cwd',
+                        default=os.path.curdir,
+                        help='Directory to reproduce from.')
+    repro_parser.add_argument(
+                        '-m',
+                        '--metrics',
+                        action='store_true',
+                        default=False,
+                        help='Show metrics after reproduction.')
+    repro_parser.add_argument(
+                        '--dry',
+                        action='store_true',
+                        default=False,
+                        help='Only print the commands that would be executed '
+                             'without actually executing.')
+    repro_parser.add_argument(
+                        '-i',
+                        '--interactive',
+                        action='store_true',
+                        default=False,
+                        help='Ask for confirmation before reproducing each '
+                             'stage.')
+    repro_parser.add_argument(
+                        '-p',
+                        '--pipeline',
+                        action='store_true',
+                        default=False,
+                        help='Reproduce the whole pipeline that the '
+                             'specified stage file belongs to.')
     repro_parser.set_defaults(func=CmdRepro)
 
     # Remove
+    REMOVE_HELP = 'Remove outputs of DVC file.'
     remove_parser = subparsers.add_parser(
                         'remove',
                         parents=[parent_parser],
-                        help='Remove outputs of DVC file.')
-    remove_parser.add_argument('targets',
+                        description=REMOVE_HELP,
+                        help=REMOVE_HELP)
+    remove_parser_group = remove_parser.add_mutually_exclusive_group()
+    remove_parser_group.add_argument(
+                        '-o',
+                        '--outs',
+                        action='store_true',
+                        default=True,
+                        help='Only remove DVC file outputs.')
+    remove_parser_group.add_argument(
+                        '-p',
+                        '--purge',
+                        action='store_true',
+                        default=False,
+                        help='Remove DVC file and all its outputs.')
+    remove_parser.add_argument(
+                        'targets',
                         nargs='+',
                         help='DVC files.')
     remove_parser.set_defaults(func=CmdRemove)
 
+    # Move
+    MOVE_HELP = 'Move output of DVC file.'
+    move_parser = subparsers.add_parser(
+                        'move',
+                        parents=[parent_parser],
+                        description=MOVE_HELP,
+                        help=MOVE_HELP)
+    move_parser.add_argument(
+                        'src',
+                        help='Source.')
+    move_parser.add_argument(
+                        'dst',
+                        help='Destination.')
+    move_parser.set_defaults(func=CmdMove)
+
     # Garbage collector
+    GC_HELP = 'Collect garbage.'
     gc_parser = subparsers.add_parser(
                         'gc',
                         parents=[parent_parser],
-                        help='Collect garbage')
+                        description=GC_HELP,
+                        help=GC_HELP)
+    gc_parser.add_argument(
+                        '-a',
+                        '--all-branches',
+                        action='store_true',
+                        default=False,
+                        help='Collect garbage for all branches.')
+    gc_parser.add_argument(
+                        '-c',
+                        '--cloud',
+                        action='store_true',
+                        default=False,
+                        help='Collect garbage in remote repository.')
+    gc_parser.add_argument(
+                        '-r',
+                        '--remote',
+                        help='Remote repository to collect garbage in.')
     gc_parser.set_defaults(func=CmdGC)
 
     # Config
+    CONFIG_HELP = 'Get or set config options.'
     config_parser = subparsers.add_parser(
                         'config',
                         parents=[parent_parser],
-                        help='Get or set config options')
-    config_parser.add_argument('-u',
+                        description=CONFIG_HELP,
+                        help=CONFIG_HELP)
+    config_parser.add_argument(
+                        '-u',
                         '--unset',
                         default=False,
                         action='store_true',
-                        help='Unset option')
-    config_parser.add_argument('name',
-                        help='Option name')
-    config_parser.add_argument('value',
+                        help='Unset option.')
+    config_parser.add_argument(
+                        'name',
+                        help='Option name.')
+    config_parser.add_argument(
+                        'value',
                         nargs='?',
                         default=None,
-                        help='Option value')
+                        help='Option value.')
+    config_parser.add_argument(
+                        '--local',
+                        action='store_true',
+                        default=False,
+                        help='Use local config.')
     config_parser.set_defaults(func=CmdConfig)
 
-    return parser.parse_args(argv)
+    # Remote
+    REMOTE_HELP = 'Manage set of tracked repositories.'
+    remote_parser = subparsers.add_parser(
+                        'remote',
+                        parents=[parent_parser],
+                        description=REMOTE_HELP,
+                        help=REMOTE_HELP)
+
+    remote_subparsers = remote_parser.add_subparsers(
+                        dest='cmd',
+                        help='Use dvc remote CMD --help for '
+                             'command-specific help.')
+
+    _fix_subparsers(remote_subparsers)
+
+    REMOTE_ADD_HELP = 'Add remote.'
+    remote_add_parser = remote_subparsers.add_parser(
+                        'add',
+                        parents=[parent_parser],
+                        description=REMOTE_ADD_HELP,
+                        help=REMOTE_ADD_HELP)
+    remote_add_parser.add_argument(
+                        'name',
+                        help='Name.')
+    remote_add_parser.add_argument(
+                        'url',
+                        help='URL.')
+    remote_add_parser.add_argument(
+                        '--local',
+                        action='store_true',
+                        default=False,
+                        help='Use local config.')
+    remote_add_parser.add_argument(
+                        '-d',
+                        '--default',
+                        action='store_true',
+                        default=False,
+                        help='Set as default remote.')
+    remote_add_parser.set_defaults(func=CmdRemoteAdd)
+
+    REMOTE_REMOVE_HELP = 'Remove remote.'
+    remote_remove_parser = remote_subparsers.add_parser(
+                        'remove',
+                        parents=[parent_parser],
+                        description=REMOTE_REMOVE_HELP,
+                        help=REMOTE_REMOVE_HELP)
+    remote_remove_parser.add_argument(
+                        'name',
+                        help='Name')
+    remote_remove_parser.add_argument(
+                        '--local',
+                        action='store_true',
+                        default=False,
+                        help='Use local config.')
+    remote_remove_parser.set_defaults(func=CmdRemoteRemove)
+
+    REMOTE_MODIFY_HELP = 'Modify remote.'
+    remote_modify_parser = remote_subparsers.add_parser(
+                        'modify',
+                        parents=[parent_parser],
+                        description=REMOTE_MODIFY_HELP,
+                        help=REMOTE_MODIFY_HELP)
+    remote_modify_parser.add_argument(
+                        'name',
+                        help='Name.')
+    remote_modify_parser.add_argument(
+                        'option',
+                        help='Option.')
+    remote_modify_parser.add_argument(
+                        'value',
+                        nargs='?',
+                        help='Value.')
+    remote_modify_parser.add_argument(
+                        '-u',
+                        '--unset',
+                        default=False,
+                        action='store_true',
+                        help='Unset option.')
+    remote_modify_parser.add_argument(
+                        '--local',
+                        action='store_true',
+                        default=False,
+                        help='Use local config.')
+    remote_modify_parser.set_defaults(func=CmdRemoteModify)
+
+    REMOTE_LIST_HELP = 'List remotes.'
+    remote_list_parser = remote_subparsers.add_parser(
+                        'list',
+                        parents=[parent_parser],
+                        description=REMOTE_LIST_HELP,
+                        help=REMOTE_LIST_HELP)
+    remote_list_parser.add_argument(
+                        '--local',
+                        action='store_true',
+                        default=False,
+                        help='Use local config.')
+    remote_list_parser.set_defaults(func=CmdRemoteList)
+
+    # Metrics
+    METRICS_HELP = 'Get metrics from all branches.'
+    metrics_parser = subparsers.add_parser(
+                        'metrics',
+                        parents=[parent_parser],
+                        description=METRICS_HELP,
+                        help=METRICS_HELP)
+
+    metrics_subparsers = metrics_parser.add_subparsers(
+                        dest='cmd',
+                        help='Use dvc metrics CMD --help for '
+                             'command-specific help.')
+
+    _fix_subparsers(metrics_subparsers)
+
+    METRICS_SHOW_HELP = 'Show metrics.'
+    metrics_show_parser = metrics_subparsers.add_parser(
+                        'show',
+                        parents=[parent_parser],
+                        description=METRICS_SHOW_HELP,
+                        help=METRICS_SHOW_HELP)
+    metrics_show_parser.add_argument(
+                        'path',
+                        nargs='?',
+                        help='Path to metrics file.')
+    metrics_show_parser.add_argument(
+                        '-t',
+                        '--type',
+                        help='Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV).')
+    metrics_show_parser.add_argument(
+                        '-x',
+                        '--xpath',
+                        help='JSON/TSV/HTSV/CSV/HCSV path.')
+    metrics_show_group = metrics_show_parser.add_mutually_exclusive_group()
+    metrics_show_group.add_argument(
+                        '--json-path',
+                        help='JSON path.')
+    metrics_show_group.add_argument(
+                        '--tsv-path',
+                        help='TSV path \'row,column\'(e.g. \'1,2\').')
+    metrics_show_group.add_argument(
+                        '--htsv-path',
+                        help='Headed TSV path \'row,column\'(e.g. \'Name,3\'.')
+    metrics_show_group.add_argument(
+                        '--csv-path',
+                        help='CSV path \'row,column\'(e.g. \'1,2\').')
+    metrics_show_group.add_argument(
+                        '--hcsv-path',
+                        help='Headed CSV path \'row,column\'(e.g. \'Name,3\'.')
+    metrics_show_parser.add_argument(
+                        '-a',
+                        '--all-branches',
+                        action='store_true',
+                        default=False,
+                        help='Show metrics for all branches.')
+    metrics_show_parser.set_defaults(func=CmdMetricsShow)
+
+    METRICS_ADD_HELP = 'Add metrics.'
+    metrics_add_parser = metrics_subparsers.add_parser(
+                        'add',
+                        parents=[parent_parser],
+                        description=METRICS_ADD_HELP,
+                        help=METRICS_ADD_HELP)
+    metrics_add_parser.add_argument(
+                        '-t',
+                        '--type',
+                        help='Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV).')
+    metrics_add_parser.add_argument(
+                        '-x',
+                        '--xpath',
+                        help='JSON/TSV/HTSV/CSV/HCSV path.')
+    metrics_add_parser.add_argument(
+                        'path',
+                        help='Path to metrics file.')
+    metrics_add_parser.set_defaults(func=CmdMetricsAdd)
+
+    METRICS_MODIFY_HELP = 'Modify metrics.'
+    metrics_modify_parser = metrics_subparsers.add_parser(
+                        'modify',
+                        parents=[parent_parser],
+                        description=METRICS_MODIFY_HELP,
+                        help=METRICS_MODIFY_HELP)
+    metrics_modify_parser.add_argument(
+                        '-t',
+                        '--type',
+                        help='Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV).')
+    metrics_modify_parser.add_argument(
+                        '-x',
+                        '--xpath',
+                        help='JSON/TSV/HTSV/CSV/HCSV path.')
+    metrics_modify_parser.add_argument(
+                        'path',
+                        help='Metrics file.')
+    metrics_modify_parser.set_defaults(func=CmdMetricsModify)
+
+    METRICS_REMOVE_HELP = 'Remove metrics.'
+    metrics_remove_parser = metrics_subparsers.add_parser(
+                        'remove',
+                        parents=[parent_parser],
+                        description=METRICS_REMOVE_HELP,
+                        help=METRICS_REMOVE_HELP)
+    metrics_remove_parser.add_argument(
+                        'path',
+                        help='Path to metrics file.')
+    metrics_remove_parser.set_defaults(func=CmdMetricsRemove)
+
+    # Install
+    INSTALL_HELP = 'Install dvc hooks into the repository.'
+    install_parser = subparsers.add_parser(
+                        'install',
+                        parents=[parent_parser],
+                        description=INSTALL_HELP,
+                        help=INSTALL_HELP)
+    install_parser.set_defaults(func=CmdInstall)
+
+    # Root
+    ROOT_HELP = 'Relative path to project\'s directory.'
+    root_parser = subparsers.add_parser(
+                        'root',
+                        parents=[parent_parser],
+                        description=ROOT_HELP,
+                        help=ROOT_HELP)
+    root_parser.set_defaults(func=CmdRoot)
+
+    # Lock
+    LOCK_HELP = 'Lock DVC file.'
+    lock_parser = subparsers.add_parser(
+                        'lock',
+                        parents=[parent_parser],
+                        description=LOCK_HELP,
+                        help=LOCK_HELP)
+    lock_parser.add_argument(
+                        'targets',
+                        nargs='+',
+                        help='DVC files.')
+    lock_parser.set_defaults(func=CmdLock)
+
+    # Unlock
+    UNLOCK_HELP = 'Unlock DVC file.'
+    unlock_parser = subparsers.add_parser(
+                        'unlock',
+                        parents=[parent_parser],
+                        description=UNLOCK_HELP,
+                        help=UNLOCK_HELP)
+    unlock_parser.add_argument(
+                        'targets',
+                        nargs='+',
+                        help='DVC files.')
+    unlock_parser.set_defaults(func=CmdUnlock)
+
+    # Pipeline
+    PIPELINE_HELP = 'Manage pipeline.'
+    pipeline_parser = subparsers.add_parser(
+                        'pipeline',
+                        parents=[parent_parser],
+                        description=PIPELINE_HELP,
+                        help=PIPELINE_HELP)
+
+    pipeline_subparsers = pipeline_parser.add_subparsers(
+                        dest='cmd',
+                        help='Use dvc pipeline CMD --help '
+                             'for command-specific help.')
+
+    _fix_subparsers(pipeline_subparsers)
+
+    PIPELINE_SHOW_HELP = 'Show pipeline.'
+    pipeline_show_parser = pipeline_subparsers.add_parser(
+                        'show',
+                        parents=[parent_parser],
+                        description=PIPELINE_SHOW_HELP,
+                        help=PIPELINE_SHOW_HELP)
+    pipeline_show_group = pipeline_show_parser.add_mutually_exclusive_group()
+    pipeline_show_group.add_argument(
+                        '-c',
+                        '--commands',
+                        action='store_true',
+                        default=False,
+                        help='Print commands instead of paths to DVC files.')
+    pipeline_show_group.add_argument(
+                        '-o',
+                        '--outs',
+                        action='store_true',
+                        default=False,
+                        help='Print output files instead of '
+                             'paths to DVC files.')
+    pipeline_show_parser.add_argument(
+                        '--ascii',
+                        action='store_true',
+                        default=False,
+                        help='Output DAG as ASCII.')
+    pipeline_show_parser.add_argument(
+                        'targets',
+                        nargs='*',
+                        help="DVC files. 'Dvcfile' by default.")
+    pipeline_show_parser.set_defaults(func=CmdPipelineShow)
+
+    args = parser.parse_args(argv)
+
+    if (issubclass(args.func, CmdRepro)
+        or issubclass(args.func, CmdPipelineShow)) \
+       and hasattr(args, 'targets') \
+       and len(args.targets) == 0:
+        if hasattr(args, 'cwd'):
+            cwd = args.cwd
+        else:
+            cwd = os.curdir
+        path = os.path.join(cwd, 'Dvcfile')
+        if not os.path.exists(path):
+            msg = "default target '{}' does not exist.".format(path)
+            if issubclass(args.func, CmdRepro):
+                repro_parser.error(msg)
+            elif issubclass(args.func, CmdPipelineShow):
+                pipeline_show_parser.error(msg)
+        args.targets = ['Dvcfile']
+
+    if issubclass(args.func, CmdRun) \
+       and len(args.deps) == 0 \
+       and len(args.outs) == 0 \
+       and len(args.outs_no_cache) == 0 \
+       and len(args.command) == 0:
+        run_parser.error("Too few arguments. Specify at least one: "
+                         "'-d', '-o', '-O', 'command'.")
+
+    return args

dvc/cloud/aws.py

@@ -1,295 +0,0 @@
-import os
-import math
-
-from boto.s3.connection import S3Connection
-try:        
-    import httplib        
-except ImportError:       
-    # Python3 workaround for ResumableDownloadHandler.        
-    # See https://github.com/boto/boto/pull/3755.     
-    import sys        
-    import http.client as httplib     
-    sys.modules['httplib'] = httplib
-from boto.s3.resumable_download_handler import ResumableDownloadHandler
-
-from dvc.logger import Logger
-from dvc.progress import progress
-from dvc.cloud.credentials_aws import AWSCredentials
-from dvc.cloud.base import DataCloudError, DataCloudBase
-from dvc.utils import file_md5
-
-
-def sizeof_fmt(num, suffix='B'):
-    """ Convert number of bytes to human-readable string """
-    for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
-        if abs(num) < 1024.0:
-            return "%3.1f%s%s" % (num, unit, suffix)
-        num /= 1024.0
-    return "%.1f%s%s" % (num, 'Y', suffix)
-
-
-def percent_cb(name, part_complete, part_total, offset=0, multipart_total=None):
-    """ Callback for updating target progress """
-    complete = offset + part_complete
-    total = multipart_total if multipart_total != None else part_total
-
-    Logger.debug('{}: {} transferred out of {}'.format(name,
-                                                       sizeof_fmt(complete),
-                                                       sizeof_fmt(total)))
-    progress.update_target(name, complete, total)
-
-
-def create_cb(name, offset=0, multipart_total=None):
-    """ Create callback function for multipart object """
-    return (lambda cur, tot: percent_cb(name, cur, tot, offset, multipart_total))
-
-
-class DataCloudAWS(DataCloudBase):
-    """ DataCloud class for Amazon Web Services """
-    def __init__(self, cloud_settings):
-        super(DataCloudAWS, self).__init__(cloud_settings)
-        self._aws_creds = AWSCredentials(cloud_settings.cloud_config)
-
-    @property
-    def aws_region_host(self):
-        """ get the region host needed for s3 access
-
-        See notes http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
-        """
-
-        region = self._cloud_settings.cloud_config.get('Region', None)
-        if region is None or region == '':
-            return 's3.amazonaws.com'
-        if region == 'us-east-1':
-            return 's3.amazonaws.com'
-        return 's3.%s.amazonaws.com' % region
-
-    def credential_paths(self, default):
-        """
-        Try obtaining path to aws credentials from config file.
-        """
-        paths = []
-        credpath = self._cloud_settings.cloud_config.get('CredentialPath', None)
-        if credpath is not None and len(credpath) > 0:
-            credpath = os.path.expanduser(credpath)
-            if os.path.isfile(credpath):
-                paths.append(credpath)
-            else:
-                Logger.warn('AWS CredentialPath "%s" not found;'
-                            'falling back to default "%s"' % (credpath, default))
-                paths.append(default)
-        else:
-            paths.append(default)
-        return paths
-
-    def connect(self):
-        if all([self._aws_creds.access_key_id,
-                self._aws_creds.secret_access_key,
-                self.aws_region_host]):
-            conn = S3Connection(self._aws_creds.access_key_id,
-                                self._aws_creds.secret_access_key,
-                                host=self.aws_region_host)
-        else:
-            conn = S3Connection()
-        self.bucket = conn.lookup(self.storage_bucket)
-        if self.bucket is None:
-            raise DataCloudError('Storage path {} is not setup correctly'.format(self.storage_bucket))
-
-    @staticmethod
-    def _cmp_checksum(key, fname):
-        """
-        Verify local and remote checksums. Used 'dvc-md5' metadata if supported
-        or falls back to etag.
-        """
-
-        # Calling get_key() once more to avoid empty metadata.
-        # See http://blog.bidiuk.com/2014/02/get-amazon-s3-metadata-in-python-using-boto/
-        key = key.bucket.get_key(key.name)
-        md5_cloud = key.metadata.get('dvc-md5', None)
-        md5_local = file_md5(fname)[0]
-
-        if md5_cloud == None:
-            md5_cloud = key.etag[1:-1]
-
-        if md5_cloud == md5_local:
-            return True
-
-        return False
-
-    @staticmethod
-    def _upload_tracker(fname):
-        """
-        File name for upload tracker.
-        """
-        return fname + '.upload'
-
-    @staticmethod
-    def _download_tracker(fname):
-        """
-        File name for download tracker.
-        """
-        return fname + '.download'
-
-    def _pull_key(self, key, fname, no_progress_bar=False):
-        Logger.debug("Pulling key '{}' from bucket '{}' to file '{}'".format(key.name,
-                                                                             key.bucket.name,
-                                                                             fname))
-        self._makedirs(fname)
-
-        tmp_file = self.tmp_file(fname)
-        name = os.path.relpath(fname, self._cloud_settings.cache.cache_dir)
-
-        if self._cmp_checksum(key, fname):
-            Logger.debug('File "{}" matches with "{}".'.format(fname, key.name))
-            return fname
-
-        Logger.debug('Downloading cache file from S3 "{}/{}" to "{}"'.format(key.bucket.name,
-                                                                             key.name,
-                                                                             fname))
-
-        if no_progress_bar:
-            cb = None
-        else:
-            cb = create_cb(name)
-
-        res_h = ResumableDownloadHandler(tracker_file_name=self._download_tracker(tmp_file),
-                                         num_retries=10)
-        try:
-            key.get_contents_to_filename(tmp_file, cb=cb, res_download_handler=res_h)
-        except Exception as exc:
-            Logger.error('Failed to download "{}": {}'.format(key.name, exc))
-            return None
-
-        os.rename(tmp_file, fname)
-
-        if not no_progress_bar:
-            progress.finish_target(name)
-
-        Logger.debug('Downloading completed')
-
-        return fname
-
-    def _get_key(self, path):
-        key_name = self.cache_file_key(path)
-        return self.bucket.get_key(key_name)
-
-    def _new_key(self, path):
-        key_name = self.cache_file_key(path)
-        return self.bucket.new_key(key_name)
-
-    def _write_upload_tracker(self, fname, mp_id):
-        """
-        Write multipart id to upload tracker.
-        """
-        try:
-            open(self._upload_tracker(fname), 'w+').write(mp_id)
-        except Exception as exc:
-            Logger.debug("Failed to write upload tracker file for {}: {}".format(fname, exc))
-
-    def _unlink_upload_tracker(self, fname):
-        """
-        Remove upload tracker file.
-        """
-        try:
-            os.unlink(self._upload_tracker(fname))
-        except Exception as exc:
-            Logger.debug("Failed to unlink upload tracker file for {}: {}".format(fname, exc))
-
-    def _resume_multipart(self, key, fname):
-        """
-        Try resuming multipart upload.
-        """
-        try:
-            mp_id = open(self._upload_tracker(fname), 'r').read()
-        except Exception as exc:
-            Logger.debug("Failed to read upload tracker file for {}: {}".format(fname, exc))
-            return None
-
-        for part in key.bucket.get_all_multipart_uploads():
-            if part.id != mp_id:
-                continue
-
-            Logger.debug("Found existing multipart {}".format(mp_id))
-            return part
-
-        return None
-
-    def _create_multipart(self, key, fname):
-        """
-        Create multipart upload and save info to tracker file.
-        """
-        # AWS doesn't provide easilly accessible md5 for multipart
-        # objects, so we have to store our own md5 sum to use later.
-        metadata = {'dvc-md5' : str(file_md5(fname)[0])}
-        multipart = key.bucket.initiate_multipart_upload(key.name, metadata=metadata)
-        self._write_upload_tracker(fname, multipart.id)
-        return multipart
-
-    def _get_multipart(self, key, fname):
-        """
-        Try resuming multipart upload if supported.
-        """
-        multipart = self._resume_multipart(key, fname)
-        if multipart != None:
-            return multipart
-
-        return self._create_multipart(key, fname)
-
-    @staticmethod
-    def _skip_part(multipart, part_num, size):
-        """
-        Skip part of multipart upload if it has been already uploaded to the server.
-        """
-        for part in multipart.get_all_parts():
-            if part.part_number == part_num and part.size == size:# and p.etag and p.last_modified
-                Logger.debug("Skipping part #{}".format(str(part_num)))
-                return True
-        return False
-
-    def _push_multipart(self, key, fname):
-        """
-        Upload local file to cloud as a multipart upload.
-        """
-        multipart = self._get_multipart(key, fname)
-
-        source_size = os.stat(fname).st_size
-        chunk_size = 50*1024*1024
-        chunk_count = int(math.ceil(source_size / float(chunk_size)))
-
-        with open(fname, 'rb') as fobj:
-            for i in range(chunk_count):
-                offset = i * chunk_size
-                left = source_size - offset
-                size = min([chunk_size, left])
-                part_num = i + 1
-
-                if self._skip_part(multipart, part_num, size):
-                    continue
-
-                fobj.seek(offset)
-                name = os.path.relpath(fname, self._cloud_settings.cache.cache_dir)
-                cb = create_cb(name, offset, source_size)
-                multipart.upload_part_from_file(fp=fobj,
-                                                replace=False,
-                                                size=size,
-                                                num_cb=100,
-                                                part_num=part_num,
-                                                cb=cb)
-
-        if len(multipart.get_all_parts()) != chunk_count:
-            raise Exception("Couldn't upload all file parts")
-
-        multipart.complete_upload()
-        self._unlink_upload_tracker(fname)
-
-    def _push_key(self, key, path):
-        """ push, aws version """
-        try:
-            self._push_multipart(key, path)
-        except Exception as exc:
-            Logger.error('Failed to upload "{}": {}'.format(path, exc))
-            return None
-
-        progress.finish_target(os.path.relpath(path, self._cloud_settings.cache.cache_dir))
-
-        return path

dvc/cloud/base.py

@@ -1,179 +0,0 @@
-import os
-import tempfile
-from checksumdir import dirhash
-
-from dvc.logger import Logger
-from dvc.exceptions import DvcException
-from dvc.config import ConfigError
-from dvc.stage import Output
-
-
-STATUS_UNKNOWN = 0
-STATUS_OK = 1
-STATUS_MODIFIED = 2
-STATUS_NEW = 3
-STATUS_DELETED = 4
-
-
-STATUS_MAP = {
-    # (local_exists, remote_exists, cmp)
-    (True, True, True)  : STATUS_OK,
-    (True, True, False) : STATUS_MODIFIED,
-    (True, False, None) : STATUS_NEW,
-    (False, True, None) : STATUS_DELETED,
-}
-
-
-class DataCloudError(DvcException):
-    """ Data Cloud exception """
-    def __init__(self, msg):
-        super(DataCloudError, self).__init__('Data sync error: {}'.format(msg))
-
-
-class DataCloudBase(object):
-    """ Base class for DataCloud """
-    def __init__(self, cloud_settings):
-        self._cloud_settings = cloud_settings
-
-    @property
-    def storage_path(self):
-        """ get storage path
-
-        Precedence: Storage, then cloud specific
-        """
-
-        if self._cloud_settings.global_storage_path:
-            return self._cloud_settings.global_storage_path
-
-        path = self._cloud_settings.cloud_config.get('StoragePath', None)
-        if path is None:
-            raise ConfigError('invalid StoragePath: not set for Data or cloud specific')
-
-        return path
-
-    def _storage_path_parts(self):
-        """
-        Split storage path into parts. I.e. 'dvc-test/myrepo' -> ['dvc', 'myrepo']
-        """
-        return self.storage_path.strip('/').split('/', 1)
-
-    @property
-    def storage_bucket(self):
-        """ Data -> StoragePath takes precedence; if doesn't exist, use cloud-specific """
-        return self._storage_path_parts()[0]
-
-    @property
-    def storage_prefix(self):
-        """
-        Prefix within the bucket. I.e. 'myrepo' in 'dvc-test/myrepo'.
-        """
-        parts = self._storage_path_parts()
-        if len(parts) > 1:
-            return parts[1]
-        return ''
-
-    def cache_file_key(self, fname):
-        """ Key of a file within the bucket """
-        relpath = os.path.relpath(fname, self._cloud_settings.cache.cache_dir)
-        relpath.replace('\\', '/')
-        return '{}/{}'.format(self.storage_prefix, relpath).strip('/')
-
-    @staticmethod
-    def tmp_file(fname):
-        """ Temporary name for a partial download """
-        return fname + '.part'
-
-    def sanity_check(self):
-        """
-        Cloud-specific method to check config for basic requirements.
-        """
-        pass
-
-    def _push_key(self, key, path):
-        pass
-
-    def collect(self, arg):
-        path, local = arg
-        ret = [path]
-
-        if not Output.is_dir_cache(path):
-            return ret
-
-        if local:
-            if not os.path.isfile(path):
-                return ret
-            dir_path = path
-        else:
-            key = self._get_key(path)
-            if not key:
-                Logger.debug("File '{}' does not exist in the cloud".format(path))
-                return ret
-            tmp = os.path.join(tempfile.mkdtemp(), os.path.basename(path))
-            self._pull_key(key, tmp, no_progress_bar=True)
-            dir_path = tmp
-
-        for relpath, md5 in Output.get_dir_cache(dir_path).items():
-            cache = self._cloud_settings.cache.get(md5)
-            ret.append(cache)
-
-        return ret
-
-    def push(self, path):
-        key = self._get_key(path)
-        if key:
-            Logger.debug("File '{}' already uploaded to the cloud. Validating checksum...".format(path))
-            if self._cmp_checksum(key, path):
-                Logger.debug('File checksum matches. No uploading is needed.')
-                return []
-            Logger.debug('Checksum mismatch. Reuploading is required.')
-
-        key = self._new_key(path)
-        return self._push_key(key, path)
-
-    def _makedirs(self, fname):
-        dname = os.path.dirname(fname)
-        try:
-            os.makedirs(dname)
-        except OSError as e:
-            if e.errno != os.errno.EEXIST:
-                raise
-
-    def _pull_key(self, key, path, no_progress_bar=False):
-        """ Cloud-specific method of pulling keys """
-        pass
-
-    def _get_key(self, path):
-        """ Cloud-specific method of getting keys """
-        pass
-
-    def pull(self, path):
-        """ Generic method for pulling data from the cloud """
-        key = self._get_key(path)
-        if not key:
-            Logger.error("File '{}' does not exist in the cloud".format(path))
-            return None
-
-        return self._pull_key(key, path)
-
-    def _status(self, key, path):
-        remote_exists = key != None
-        local_exists = os.path.exists(path)
-
-        diff = None
-        if remote_exists and local_exists:
-            diff = self._cmp_checksum(key, path)
-
-        return STATUS_MAP.get((local_exists, remote_exists, diff), STATUS_UNKNOWN)
-
-    def status(self, path):
-        """
-        Generic method for checking data item status.
-        """
-        key = self._get_key(path)
-        if not key:
-            return STATUS_NEW
-
-        return self._status(key, path)
-
-    def connect(self):
-        pass

dvc/cloud/credentials_aws.py

@@ -1,91 +0,0 @@
-import os
-import configparser
-
-from dvc.utils import cached_property
-from dvc.logger import Logger
-
-
-class AWSCredentials(object):
-    def __init__(self, cloud_config):
-        self._conf_credpath = cloud_config.get('CredentialPath', None)
-        self._conf_credsect = cloud_config.get('Profile', 'default')
-
-    @property
-    def access_key_id(self):
-        if self.creds:
-            return self.creds[0]
-        return None
-
-    @property
-    def secret_access_key(self):
-        if self.creds:
-            return self.creds[1]
-        return None
-
-    @cached_property
-    def creds(self):
-        return self._get_credentials()
-
-    def _get_credentials(self):
-        """ gets aws credentials, looking in various places
-
-        Params:
-
-        Searches:
-        1 any override in dvc.conf [AWS] CredentialPath;
-        2 ~/.aws/credentials
-
-
-        Returns:
-            if successfully found, (access_key_id, secret)
-            None otherwise
-        """
-
-        # FIX: It won't work in Windows.
-        default_path = os.path.expanduser('~/.aws/credentials')
-        default_sect = 'default'
-        default_cred_location = (default_path, default_sect)
-
-        cred_locations = self._credential_paths(default_cred_location)
-        for cred_location in cred_locations:
-            try:
-                path = cred_location[0]
-                section = cred_location[1]
-
-                cc = configparser.SafeConfigParser()
-
-                # use readfp(open( ... to aid mocking.
-                cc.readfp(open(path, 'r'))
-
-                if section in cc.keys():
-                    access_key = cc[section].get('aws_access_key_id', None)
-                    secret = cc[section].get('aws_secret_access_key', None)
-
-                    if access_key is not None and secret is not None:
-                        return (access_key, secret)
-                else:
-                    Logger.warn('Unable to find section {} in AWS credential file {}'.format(section, path))
-            except Exception as e:
-                pass
-
-        return None
-
-    def _credential_paths(self, default_cred_location):
-        results = []
-        if self._conf_credpath is not None and len(self._conf_credpath) > 0:
-            credpath = os.path.expanduser(self._conf_credpath)
-            if os.path.isfile(credpath):
-                results.append((credpath, self._conf_credsect))
-            else:
-                msg = 'AWS CredentialPath {} not found; falling back to default file {} and section {}'
-                Logger.warn(msg.format(credpath, default_cred_location[0], default_cred_location[1]))
-                results.append(default_cred_location)
-        else:
-            results.append(default_cred_location)
-        return results
-
-    def sanity_check(self):
-        creds = self._get_credentials()
-        if creds is None:
-            Logger.info("can't find aws credetials, assuming envirment variables or iam role")
-        # self._aws_creds = creds

dvc/cloud/data_cloud.py

@@ -1,127 +0,0 @@
-from multiprocessing.pool import ThreadPool
-
-from dvc.cloud.instance_manager import CloudSettings
-from dvc.logger import Logger
-from dvc.exceptions import DvcException
-from dvc.config import ConfigError
-from dvc.utils import map_progress
-from dvc.config import Config
-
-from dvc.cloud.aws import DataCloudAWS
-from dvc.cloud.gcp import DataCloudGCP
-from dvc.cloud.local import DataCloudLOCAL
-
-
-class DataCloud(object):
-    """ Generic class to do initial config parsing and redirect to proper DataCloud methods """
-
-    CLOUD_MAP = {
-        'AWS'   : DataCloudAWS,
-        'GCP'   : DataCloudGCP,
-        'LOCAL' : DataCloudLOCAL,
-    }
-
-    SCHEME_MAP = {
-        's3'    : 'AWS',
-        'gs'    : 'GCP',
-        ''      : 'LOCAL',
-    }
-
-    def __init__(self, cache, config):
-        self._config = config
-
-        cloud_type = self._config[Config.SECTION_CORE].get('Cloud', '').strip().upper()
-        if cloud_type not in self.CLOUD_MAP.keys():
-            raise ConfigError('Wrong cloud type %s specified' % cloud_type)
-
-        if cloud_type not in self._config.keys():
-            raise ConfigError('Can\'t find cloud section \'[%s]\' in config' % cloud_type)
-
-        cloud_settings = self.get_cloud_settings(cache,
-                                                 self._config,
-                                                 cloud_type)
-
-        self.typ = cloud_type
-        self._cloud = self.CLOUD_MAP[cloud_type](cloud_settings)
-
-        self.sanity_check()
-
-    @staticmethod
-    def get_cloud_settings(cache, config, cloud_type):
-        """
-        Obtain cloud settings from config.
-        """
-        if cloud_type not in config.keys():
-            cloud_config = None
-        else:
-            cloud_config = config[cloud_type]
-        global_storage_path = config[Config.SECTION_CORE].get('StoragePath', None)
-        cloud_settings = CloudSettings(cache, global_storage_path, cloud_config)
-        return cloud_settings
-
-    def sanity_check(self):
-        """ sanity check a config
-
-        check that we have a cloud and storagePath
-        if aws, check can read credentials
-        if google, check ProjectName
-
-        Returns:
-            (T,) if good
-            (F, issues) if bad
-        """
-        key = 'Cloud'
-        core = self._config[Config.SECTION_CORE]
-        if key.lower() not in [k.lower() for k in core.keys()] or len(core[key]) < 1:
-            raise ConfigError('Please set {} in section {} in config file'.format(key, Config.SECTION_CORE))
-
-        # now that a cloud is chosen, can check StoragePath
-        storage_path = self._cloud.storage_path
-        if storage_path is None or len(storage_path) == 0:
-            raise ConfigError('Please set StoragePath = bucket/{optional path} '
-                              'in config file in a cloud specific section')
-
-        self._cloud.sanity_check()
-
-    def _collect(self, targets, jobs, local):
-        collected = set()
-        pool = ThreadPool(processes=jobs)
-        args = zip(targets, [local]*len(targets))
-        ret = pool.map(self._cloud.collect, args)
-
-        for r in ret:
-            collected |= set(r)
-
-        return collected
-
-    def _map_targets(self, func, targets, jobs, collect_local=False, collect_cloud=False):
-        """
-        Process targets as data items in parallel.
-        """
-        self._cloud.connect()
-
-        collected = set()
-        if collect_local:
-            collected |= self._collect(targets, jobs, True)
-        if collect_cloud:
-            collected |= self._collect(targets, jobs, False)
-
-        return map_progress(func, list(collected), jobs)
-
-    def push(self, targets, jobs=1):
-        """
-        Push data items in a cloud-agnostic way.
-        """
-        return self._map_targets(self._cloud.push, targets, jobs, collect_local=True)
-
-    def pull(self, targets, jobs=1):
-        """
-        Pull data items in a cloud-agnostic way.
-        """
-        return self._map_targets(self._cloud.pull, targets, jobs, collect_cloud=True)
-
-    def status(self, targets, jobs=1):
-        """
-        Check status of data items in a cloud-agnostic way.
-        """
-        return self._map_targets(self._cloud.status, targets, jobs, True, True)

dvc/cloud/gcp.py

@@ -1,99 +0,0 @@
-import base64
-import os
-
-from google.cloud import storage as gc
-
-from dvc.logger import Logger
-from dvc.config import ConfigError
-from dvc.cloud.base import DataCloudError, DataCloudBase
-from dvc.utils import file_md5
-from dvc.progress import progress
-
-
-class DataCloudGCP(DataCloudBase):
-    """ DataCloud class for Google Cloud Platform """
-    @property
-    def gc_project_name(self):
-        """
-        Get project name from config.
-        """
-        return self._cloud_settings.cloud_config.get('ProjectName', None)
-
-    def sanity_check(self):
-        project = self.gc_project_name
-        if project is None or len(project) < 1:
-            raise ConfigError('can\'t read google cloud project name. '
-                              'Please set ProjectName in section GC.')
-
-    def connect(self):
-        client = gc.Client(project=self.gc_project_name)
-        self.bucket = client.bucket(self.storage_bucket)
-        if not self.bucket.exists():
-            raise DataCloudError('sync up: google cloud bucket {} '
-                                 'doesn\'t exist'.format(self.storage_bucket))
-
-    @staticmethod
-    def _cmp_checksum(blob, fname):
-        """
-        Verify local and remote checksums.
-        """
-        md5 = file_md5(fname)[1]
-        b64_encoded_md5 = base64.b64encode(md5).decode() if md5 else None
-
-        if blob.md5_hash == b64_encoded_md5:
-            return True
-
-        return False
-
-    def _pull_key(self, key, path, no_progress_bar=False):
-        self._makedirs(path)
-
-        name = os.path.relpath(path, self._cloud_settings.cache.cache_dir)
-        tmp_file = self.tmp_file(path)
-
-        if self._cmp_checksum(key, path):
-            Logger.debug('File "{}" matches with "{}".'.format(path, key.name))
-            return path
-
-        Logger.debug('Downloading cache file from gc "{}/{}"'.format(key.bucket.name, key.name))
-
-        if not no_progress_bar:
-            # percent_cb is not available for download_to_filename, so
-            # lets at least update progress at keypoints(start, finish)
-            progress.update_target(name, 0, None)
-
-        try:
-            key.download_to_filename(tmp_file)
-        except Exception as exc:
-            Logger.error('Failed to download "{}": {}'.format(key.name, exc))
-            return None
-
-        os.rename(tmp_file, path)
-
-        if not no_progress_bar:
-            progress.finish_target(name)
-
-        Logger.debug('Downloading completed')
-
-        return path
-
-    def _get_key(self, path):
-        key_name = self.cache_file_key(path)
-        return self.bucket.get_blob(key_name)
-
-    def _new_key(self, path):
-        key_name = self.cache_file_key(path)
-        return self.bucket.blob(key_name)
-
-    def _push_key(self, key, path):
-        """ push, gcp version """
-        name = os.path.relpath(path, self._cloud_settings.cache.cache_dir)
-
-        progress.update_target(name, 0, None)
-
-        key.upload_from_filename(path)
-
-        progress.finish_target(name)
-        Logger.debug('uploading %s completed' % path)
-
-        return path

dvc/cloud/instance_manager.py

@@ -1,5 +0,0 @@
-class CloudSettings(object):
-    def __init__(self, cache, global_storage_path, cloud_config):
-        self.cache = cache
-        self.cloud_config = cloud_config
-        self.global_storage_path = global_storage_path

dvc/cloud/local.py

@@ -1,59 +0,0 @@
-import os
-import filecmp
-
-from dvc.logger import Logger
-from dvc.utils import copyfile, file_md5
-from dvc.cloud.base import DataCloudError, DataCloudBase
-
-
-class LocalKey(object):
-    def __init__(self, bucket, name):
-        self.name = name
-        self.bucket = bucket
-
-    @property
-    def path(self):
-        return os.path.join(self.bucket, self.name)
-
-
-class DataCloudLOCAL(DataCloudBase):
-    """
-    Driver for local storage.
-    """
-    def cache_file_key(self, path):
-        return os.path.relpath(os.path.abspath(path), self._cloud_settings.cache.cache_dir)
-
-    def _cmp_checksum(self, key, path):
-        return filecmp.cmp(path, key.path)
-
-    def _get_key(self, path):
-        key_name = self.cache_file_key(path)
-        key = LocalKey(self.storage_path, key_name)
-        if os.path.exists(key.path) and os.path.isfile(key.path):
-            return key
-        return None
-
-    def _new_key(self, path):
-        key_name = self.cache_file_key(path)
-        key = LocalKey(self.storage_path, key_name)
-        self._makedirs(key.path)
-        return key
-
-    def _push_key(self, key, path):
-        self._makedirs(key.path)
-        copyfile(path, key.path)
-        return path
-
-    def _pull_key(self, key, path, no_progress_bar=False):
-        self._makedirs(path)
-
-        tmp_file = self.tmp_file(path)
-        try:
-            copyfile(key.path, tmp_file, no_progress_bar=no_progress_bar)
-        except Exception as exc:
-            Logger.error('Failed to copy "{}": {}'.format(key.path, exc))
-            return None
-
-        os.rename(tmp_file, path)
-
-        return path

dvc/command/add.py

@@ -1,13 +1,14 @@
 from dvc.exceptions import DvcException
-from dvc.command.common.base import CmdBase
+from dvc.command.base import CmdBase
+from dvc.logger import Logger
 
 
 class CmdAdd(CmdBase):
     def run(self):
         for target in self.args.targets:
             try:
-                self.project.add(target)
+                self.project.add(target, recursive=self.args.recursive)
             except DvcException as ex:
-                self.project.logger.error('Failed to add {}', ex)
+                Logger.error('Failed to add \'{}\''.format(target), ex)
                 return 1
         return 0

dvc/command/base.py

@@ -1,6 +1,5 @@
 import os
 
-from dvc.project import Project
 from dvc.logger import Logger
 from dvc.exceptions import DvcException
 from dvc.lock import LockError
@@ -8,6 +7,8 @@ from dvc.lock import LockError
 
 class CmdBase(object):
     def __init__(self, args):
+        from dvc.project import Project
+
         self.project = Project(self._find_root())
         self.args = args
         self._set_loglevel(args)
@@ -20,11 +21,15 @@ class CmdBase(object):
             Logger.be_verbose()
 
     def _find_root(self):
+        from dvc.project import Project
+
         root = os.getcwd()
-        while not os.path.ismount(root):
+        while True:
             dvc_dir = os.path.join(root, Project.DVC_DIR)
             if os.path.isdir(dvc_dir):
                 return root
+            if os.path.ismount(root):
+                break
             root = os.path.dirname(root)
         msg = "Not a dvc repository (checked up to mount point {})"
         raise DvcException(msg.format(root))
@@ -34,7 +39,7 @@ class CmdBase(object):
             with self.project.lock:
                 return self.run()
         except LockError as ex:
-            self.project.logger.error('Failed to lock before running a command', ex) 
+            Logger.error('Failed to lock before running a command', ex)
             return 1
 
     # Abstract methods that have to be implemented by any inheritance class

dvc/command/checkout.py

@@ -1,12 +1,11 @@
-import os
-import stat
-
-from dvc.system import System
-from dvc.command.common.base import CmdBase
-from dvc.logger import Logger
+from dvc.command.base import CmdBase
 
 
 class CmdCheckout(CmdBase):
     def run(self):
-        self.project.checkout()
+        if not self.args.targets:
+            self.project.checkout()
+        else:
+            for target in self.args.targets:
+                self.project.checkout(target=target)
         return 0

dvc/command/common/common_error.py

@@ -1,6 +0,0 @@
-from dvc.exceptions import DvcException
-
-
-class CmdCommonError(DvcException):
-    def __init__(self, msg):
-        super(CmdCommonError, self).__init__('{}'.format(msg))

dvc/command/config.py

@@ -1,18 +1,25 @@
 import os
 import configobj
 
-from dvc.command.common.base import CmdBase
+from dvc.command.base import CmdBase
 from dvc.logger import Logger
 from dvc.config import Config
-from dvc.project import Project
 
 
 class CmdConfig(CmdBase):
     def __init__(self, args):
+        from dvc.project import Project
+
         self.args = args
         root_dir = self._find_root()
-        self.config_file = os.path.join(root_dir, Project.DVC_DIR, Config.CONFIG)
-        self._set_loglevel(args)
+        if args.local:
+            config = Config.CONFIG_LOCAL
+        else:
+            config = Config.CONFIG
+        self.config_file = os.path.join(root_dir, Project.DVC_DIR, config)
+        # Using configobj because it doesn't
+        # drop comments like configparser does.
+        self.configobj = configobj.ConfigObj(self.config_file)
 
     def run_cmd(self):
         return self.run()
@@ -28,35 +35,43 @@ class CmdConfig(CmdBase):
 
         return None
 
-    def unset(self):
+    def save(self):
         try:
-            del self.configobj[self.section][self.opt]
             self.configobj.write()
         except Exception as exc:
-            Logger.error('Failed to unset \'{}\': {}'.format(self.args.name, exc))
+            msg = "Failed to write config '{}'".format(self.configobj.filename)
+            Logger.error(msg, exc)
+            return 1
+        return 0
+
+    def unset(self, section, opt=None):
+        if section not in self.configobj.keys():
+            Logger.error("Section '{}' doesn't exist".format(section))
             return 1
 
-        return 0
+        if opt in self.configobj[section].keys():
+            del self.configobj[section][opt]
+
+        if len(self.configobj[section]) == 0 or opt is None:
+            del self.configobj[section]
+
+        return self.save()
 
     def show(self):
         Logger.info(self.configobj[self.section][self.opt])
         return 0
 
-    def set(self):
-        try:
-            self.configobj[self.section][self.opt] = self.args.value
-            self.configobj.write()
-        except Exception as exc:
-            Logger.error('Failed to set \'{}\' to \'{}\': {}'.format(self.args.name,
-                                                                     self.args.value,
-                                                                     exc))
-            return 1
+    def set(self, section, opt, value):
+        if section not in self.configobj.keys():
+            self.configobj[section] = {}
 
-        return 0
+        self.configobj[section][opt] = value
+
+        return self.save()
 
     def check_opt(self):
         _section, _opt = self.args.name.strip().split('.', 1)
-        add = (self.args.value != None and self.args.unset == False)
+        add = (self.args.value is not None and self.args.unset is False)
 
         section = self._get_key(self.configobj, _section, add)
 
@@ -75,17 +90,13 @@ class CmdConfig(CmdBase):
         return 0
 
     def run(self):
-        # Using configobj because it doesn't
-        # drop comments like configparser does.
-        self.configobj = configobj.ConfigObj(self.config_file, write_empty_values=True)
-
         if self.check_opt() != 0:
             return 1
 
         if self.args.unset:
-            return self.unset()
+            return self.unset(self.section, self.opt)
 
         if self.args.value is None:
             return self.show()
 
-        return self.set()
+        return self.set(self.section, self.opt, self.args.value)

dvc/command/data_sync.py

@@ -1,8 +1,4 @@
-import os
-
-import dvc.cloud.base as cloud
-
-from dvc.command.common.base import CmdBase
+from dvc.command.base import CmdBase
 
 
 class CmdDataBase(CmdBase):
@@ -23,9 +19,15 @@ class CmdDataBase(CmdBase):
 class CmdDataPull(CmdDataBase):
     def do_run(self, target=None):
         try:
-            self.project.pull(target=target, jobs=self.args.jobs)
+            self.project.pull(target=target,
+                              jobs=self.args.jobs,
+                              remote=self.args.remote,
+                              show_checksums=self.args.show_checksums,
+                              all_branches=self.args.all_branches,
+                              with_deps=self.args.with_deps)
         except Exception as exc:
-            self.project.logger.error('Failed to pull data from the cloud', exc)
+            self.project.logger.error('Failed to pull data from the cloud',
+                                      exc)
             return 1
         return 0
 
@@ -33,7 +35,12 @@ class CmdDataPull(CmdDataBase):
 class CmdDataPush(CmdDataBase):
     def do_run(self, target=None):
         try:
-            self.project.push(target=target, jobs=self.args.jobs)
+            self.project.push(target=target,
+                              jobs=self.args.jobs,
+                              remote=self.args.remote,
+                              show_checksums=self.args.show_checksums,
+                              all_branches=self.args.all_branches,
+                              with_deps=self.args.with_deps)
         except Exception as exc:
             self.project.logger.error('Failed to push data to the cloud', exc)
             return 1
@@ -43,8 +50,14 @@ class CmdDataPush(CmdDataBase):
 class CmdDataFetch(CmdDataBase):
     def do_run(self, target=None):
         try:
-            self.project.fetch(target=target, jobs=self.args.jobs)
+            self.project.fetch(target=target,
+                               jobs=self.args.jobs,
+                               remote=self.args.remote,
+                               show_checksums=self.args.show_checksums,
+                               all_branches=self.args.all_branches,
+                               with_deps=self.args.with_deps)
         except Exception as exc:
-            self.project.logger.error('Failed to fetch data from the cloud', exc)
+            self.project.logger.error('Failed to fetch data from the cloud',
+                                      exc)
             return 1
         return 0

dvc/command/destroy.py

@@ -0,0 +1,22 @@
+from dvc.command.base import CmdBase
+from dvc.exceptions import DvcException
+
+
+class CmdDestroy(CmdBase):
+    def run_cmd(self):
+        try:
+            msg = u'This will destroy all information about your pipelines, ' \
+                  u'all data files, as well as cache in .dvc/cache.\n' \
+                  u'Are you sure you want to continue?'
+
+            if not self.args.force \
+               and not self.project.prompt.prompt(msg, False):
+                msg = u'Cannot destroy without a confirmation from the ' \
+                      u'user. Use \'-f\' to force.'
+                raise DvcException(msg)
+
+            self.project.destroy()
+        except Exception as exc:
+            self.project.logger.error('Failed to destroy DVC', exc)
+            return 1
+        return 0

dvc/command/gc.py

@@ -1,7 +1,9 @@
-from dvc.command.common.base import CmdBase
+from dvc.command.base import CmdBase
 
 
 class CmdGC(CmdBase):
     def run(self):
-        self.project.gc()
+        self.project.gc(all_branches=self.args.all_branches,
+                        cloud=self.args.cloud,
+                        remote=self.args.remote)
         return 0

dvc/command/imp.py

@@ -0,0 +1,13 @@
+from dvc.exceptions import DvcException
+from dvc.command.base import CmdBase
+from dvc.logger import Logger
+
+
+class CmdImport(CmdBase):
+    def run(self):
+        try:
+            self.project.imp(self.args.url, self.args.out)
+        except DvcException as ex:
+            Logger.error('Failed to import {}'.format(self.args.url), ex)
+            return 1
+        return 0

dvc/command/init.py

@@ -1,4 +1,4 @@
-from dvc.project import Project
+from dvc.logger import Logger
 
 
 class CmdInit(object):
@@ -6,5 +6,11 @@ class CmdInit(object):
         self.args = args
 
     def run_cmd(self):
-        Project.init('.')
+        from dvc.project import Project, InitError
+
+        try:
+            Project.init('.', no_scm=self.args.no_scm, force=self.args.force)
+        except InitError as e:
+            Logger.error('Failed to initiate dvc', e)
+            return 1
         return 0

dvc/command/install.py

@@ -0,0 +1,12 @@
+from dvc.logger import Logger
+from dvc.command.base import CmdBase
+
+
+class CmdInstall(CmdBase):
+    def run_cmd(self):
+        try:
+            self.project.install()
+        except Exception as e:
+            Logger.error('Failed to install dvc hooks', e)
+            return 1
+        return 0

dvc/command/lock.py

@@ -0,0 +1,25 @@
+from dvc.exceptions import DvcException
+from dvc.command.base import CmdBase
+
+
+class CmdLockBase(CmdBase):
+    def run(self, unlock=False):
+        for target in self.args.targets:
+            try:
+                self.project.lock_stage(target, unlock=unlock)
+            except DvcException as ex:
+                msg = 'Failed to {}lock \'{}\''
+                msg.format('un' if unlock else '', target)
+                self.project.logger.error(msg, ex)
+                return 1
+        return 0
+
+
+class CmdLock(CmdLockBase):
+    def run(self):
+        return super(CmdLock, self).run(False)
+
+
+class CmdUnlock(CmdLockBase):
+    def run(self):
+        return super(CmdUnlock, self).run(True)

dvc/command/metrics.py

@@ -0,0 +1,75 @@
+from dvc.command.base import CmdBase
+from dvc.exceptions import DvcException
+
+
+class CmdMetricsShow(CmdBase):
+    def run(self):
+        try:
+            # backward compatibility
+            if self.args.json_path:
+                typ = 'json'
+                xpath = self.args.json_path
+            elif self.args.tsv_path:
+                typ = 'tsv'
+                xpath = self.args.tsv_path
+            elif self.args.htsv_path:
+                typ = 'htsv'
+                xpath = self.args.htsv_path
+            elif self.args.csv_path:
+                typ = 'csv'
+                xpath = self.args.csv_path
+            elif self.args.hcsv_path:
+                typ = 'hcsv'
+                xpath = self.args.hcsv_path
+            else:
+                typ = self.args.type
+                xpath = self.args.xpath
+
+            self.project.metrics_show(self.args.path,
+                                      typ=typ,
+                                      xpath=xpath,
+                                      all_branches=self.args.all_branches)
+        except DvcException as exc:
+            self.project.logger.error('Failed to show metrics', exc)
+            return 1
+
+        return 0
+
+
+class CmdMetricsModify(CmdBase):
+    def run(self):
+        try:
+            self.project.metrics_modify(self.args.path,
+                                        typ=self.args.type,
+                                        xpath=self.args.xpath)
+        except DvcException as exc:
+            self.project.logger.error('Failed to modify metrics', exc)
+            return 1
+
+        return 0
+
+
+class CmdMetricsAdd(CmdBase):
+    def run(self):
+        try:
+            self.project.metrics_add(self.args.path,
+                                     self.args.type,
+                                     self.args.xpath)
+        except DvcException as exc:
+            msg = 'Failed to add metric file \'{}\''.format(self.args.path)
+            self.project.logger.error(msg, exc)
+            return 1
+
+        return 0
+
+
+class CmdMetricsRemove(CmdBase):
+    def run(self):
+        try:
+            self.project.metrics_remove(self.args.path)
+        except DvcException as exc:
+            msg = 'Failed to remove metric file \'{}\''.format(self.args.path)
+            self.project.logger.error(msg, exc)
+            return 1
+
+        return 0

dvc/command/move.py

@@ -0,0 +1,14 @@
+from dvc.exceptions import DvcException
+from dvc.command.base import CmdBase
+
+
+class CmdMove(CmdBase):
+    def run(self):
+        try:
+            self.project.move(self.args.src, self.args.dst)
+        except DvcException as ex:
+            msg = 'Failed to move \'{}\' -> \'{}\''.format(self.args.src,
+                                                           self.args.dst)
+            self.project.logger.error(msg, ex)
+            return 1
+        return 0

dvc/command/pipeline.py

@@ -0,0 +1,91 @@
+import os
+
+from dvc.exceptions import DvcException
+from dvc.command.base import CmdBase
+from dvc.dagascii import Dagascii
+
+
+class CmdPipelineShow(CmdBase):
+    def _show(self, target, commands, outs):
+        import networkx
+        from dvc.stage import Stage
+
+        stage = Stage.load(self.project, target)
+        G = self.project.graph()[0]
+        stages = networkx.get_node_attributes(G, 'stage')
+        node = os.path.relpath(stage.path, self.project.root_dir)
+
+        for n in networkx.dfs_postorder_nodes(G, node):
+            if commands:
+                self.project.logger.info(stages[n].cmd)
+            elif outs:
+                for out in stages[n].outs:
+                    self.project.logger.info(str(out))
+            else:
+                self.project.logger.info(n)
+
+    def _show_ascii(self, target, commands, outs):
+        import networkx
+        from dvc.stage import Stage
+
+        stage = Stage.load(self.project, target)
+        node = os.path.relpath(stage.path, self.project.root_dir)
+
+        pipelines = list(filter(lambda g: node in g.nodes(),
+                                self.project.pipelines()))
+
+        assert len(pipelines) == 1
+        G = pipelines[0]
+        stages = networkx.get_node_attributes(G, 'stage')
+
+        nodes = []
+        for n in G.nodes():
+            stage = stages[n]
+            if commands:
+                if stage.cmd is None:
+                    continue
+                nodes.append(stage.cmd)
+            elif outs:
+                for out in stage.outs:
+                    nodes.append(str(out))
+            else:
+                nodes.append(stage.relpath)
+
+        if len(nodes) == 0:
+            return
+
+        edges = []
+        for e in G.edges():
+            from_stage = stages[e[0]]
+            to_stage = stages[e[1]]
+            if commands:
+                if to_stage.cmd is None:
+                    continue
+                edges.append((from_stage.cmd, to_stage.cmd))
+            elif outs:
+                for from_out in from_stage.outs:
+                    for to_out in to_stage.outs:
+                        edges.append((str(from_out),
+                                      str(to_out)))
+            else:
+                edges.append((from_stage.relpath, to_stage.relpath))
+
+        d = Dagascii(nodes, edges)
+        d.draw()
+
+    def run(self, unlock=False):
+        for target in self.args.targets:
+            try:
+                if self.args.ascii:
+                    self._show_ascii(target,
+                                     self.args.commands,
+                                     self.args.outs)
+                else:
+                    self._show(target,
+                               self.args.commands,
+                               self.args.outs)
+            except DvcException as ex:
+                msg = 'Failed to show pipeline for \'{}\''.format(target)
+                self.project.logger.error(msg, ex)
+                return 1
+        return 0

dvc/command/remote.py

@@ -0,0 +1,75 @@
+import os
+import re
+import configobj
+
+from dvc.config import Config
+from dvc.command.config import CmdConfig
+from dvc.logger import Logger
+
+
+class CmdRemoteAdd(CmdConfig):
+    def run(self):
+        section = Config.SECTION_REMOTE_FMT.format(self.args.name)
+        ret = self.set(section, Config.SECTION_REMOTE_URL, self.args.url)
+        if ret != 0:
+            return ret
+
+        if self.args.default:
+            msg = 'Setting \'{}\' as a default remote.'.format(self.args.name)
+            Logger.info(msg)
+            ret = self.set(Config.SECTION_CORE,
+                           Config.SECTION_CORE_REMOTE,
+                           self.args.name)
+
+        return ret
+
+
+class CmdRemoteRemove(CmdConfig):
+    def _remove_default(self, config_file, remote):
+        path = os.path.join(os.path.dirname(self.config_file),
+                            config_file)
+        config = configobj.ConfigObj(path)
+
+        core = config.get(Config.SECTION_CORE, None)
+        if core is None:
+            return
+
+        default = core.get(Config.SECTION_CORE_REMOTE, None)
+        if default is None:
+            return
+
+        if default == remote:
+            del config[Config.SECTION_CORE][Config.SECTION_CORE_REMOTE]
+            if len(config[Config.SECTION_CORE]) == 0:
+                del config[Config.SECTION_CORE]
+
+        config.write()
+
+    def run(self):
+        section = Config.SECTION_REMOTE_FMT.format(self.args.name)
+        ret = self.unset(section)
+        if ret != 0:
+            return ret
+
+        self._remove_default(Config.CONFIG, self.args.name)
+        self._remove_default(Config.CONFIG_LOCAL, self.args.name)
+        return 0
+
+
+class CmdRemoteModify(CmdConfig):
+    def run(self):
+        section = Config.SECTION_REMOTE_FMT.format(self.args.name)
+        self.args.name = '{}.{}'.format(section, self.args.option)
+        return super(CmdRemoteModify, self).run()
+
+
+class CmdRemoteList(CmdConfig):
+    def run(self):
+        for section in self.configobj.keys():
+            r = re.match(Config.SECTION_REMOTE_REGEX, section)
+            if r:
+                name = r.group('name')
+                url = self.configobj[section].get(Config.SECTION_REMOTE_URL,
+                                                  '')
+                Logger.info('{}\t{}'.format(name, url))
+        return 0

dvc/command/remove.py

@@ -1,13 +1,15 @@
 from dvc.exceptions import DvcException
-from dvc.command.common.base import CmdBase
+from dvc.command.base import CmdBase
+from dvc.logger import Logger
 
 
 class CmdRemove(CmdBase):
     def run(self):
+        outs_only = not self.args.purge
         for target in self.args.targets:
             try:
-                self.project.remove(target)
+                self.project.remove(target, outs_only=outs_only)
             except DvcException as ex:
-                self.project.logger.error('Failed to remove {}', ex)
+                Logger.error('Failed to remove {}'.format(target), ex)
                 return 1
         return 0

dvc/command/repro.py

@@ -1,21 +1,38 @@
-from dvc.command.common.base import CmdBase
-from dvc.project import ReproductionError
+import os
+
+from dvc.command.base import CmdBase
+from dvc.command.status import CmdDataStatus
+from dvc.exceptions import DvcException
 
 
 class CmdRepro(CmdBase):
     def run(self):
         recursive = not self.args.single_item
+        saved_dir = os.path.realpath(os.curdir)
+        if self.args.cwd:
+            os.chdir(self.args.cwd)
+
+        ret = 0
         for target in self.args.targets:
             try:
-                self.project.reproduce(target,
+                stages = self.project.reproduce(
+                                       target,
                                        recursive=recursive,
-                                       force=self.args.force)
-            except ReproductionError as ex:
+                                       force=self.args.force,
+                                       dry=self.args.dry,
+                                       interactive=self.args.interactive,
+                                       pipeline=self.args.pipeline)
+
+                if len(stages) == 0:
+                    self.project.logger.info(CmdDataStatus.UP_TO_DATE_MSG)
+
+                if self.args.metrics:
+                    self.project.metrics_show()
+            except DvcException as ex:
                 msg = 'Failed to reproduce \'{}\''.format(target)
                 self.project.logger.error(msg, ex)
-                return 1
-            except Exception as ex:
-                msg = 'Failed to reproduce \'{}\' - unexpected error'.format(target)
-                self.project.logger.error(msg, ex)
-                return 1
-        return 0
+                ret = 1
+                break
+
+        os.chdir(saved_dir)
+        return ret

dvc/command/root.py

@@ -0,0 +1,9 @@
+import os
+
+from dvc.command.base import CmdBase
+
+
+class CmdRoot(CmdBase):
+    def run(self):
+        self.project.logger.info(os.path.relpath(self.project.root_dir))
+        return 0

dvc/command/run.py

@@ -1,21 +1,19 @@
-import os
-
-from dvc.command.common.base import CmdBase
-from dvc.logger import Logger
-from dvc.stage import Stage
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
 
 
 class CmdRun(CmdBase):
     def run(self):
-        fname = self.stage_file_name(self.args.file, self.args.outs, self.args.outs_no_cache)
-
         try:
+            if self.args.yes:
+                self.project.prompt.default = True
+
             self.project.run(cmd=' '.join(self.args.command),
                              outs=self.args.outs,
                              outs_no_cache=self.args.outs_no_cache,
+                             metrics_no_cache=self.args.metrics_no_cache,
                              deps=self.args.deps,
-                             fname=fname,
+                             fname=self.args.file,
                              cwd=self.args.cwd,
                              no_exec=self.args.no_exec)
         except DvcException as ex:
@@ -23,24 +21,3 @@ class CmdRun(CmdBase):
             return 1
 
         return 0
-
-    @staticmethod
-    def stage_file_name(args_file, args_outs, args_outs_no_cache):
-        if args_file:
-            return args_file
-
-        if args_outs:
-            result = CmdRun._stage_file_basename(args_outs[0])
-        elif args_outs_no_cache:
-            result = CmdRun._stage_file_basename(args_outs_no_cache[0])
-        else:
-            result = Stage.STAGE_FILE
-
-        Logger.info(u'Using \'{}\' as a stage file'.format(result))
-        return result
-
-    @staticmethod
-    def _stage_file_basename(fname):
-        result = os.path.basename(fname)
-        result += Stage.STAGE_FILE_SUFFIX
-        return result

dvc/command/status.py

@@ -1,18 +1,14 @@
-import os
-
-import dvc.cloud.base as cloud
-
 from dvc.command.data_sync import CmdDataBase
 
 
 class CmdDataStatus(CmdDataBase):
     STATUS_LEN = 10
     STATUS_INDENT = '\t'
+    UP_TO_DATE_MSG = "Pipeline is up to date. Nothing to reproduce."
 
     def _normalize(self, s):
-        s+=':'
-        if len(s) >= self.STATUS_LEN:
-            return s
+        s += ':'
+        assert len(s) < self.STATUS_LEN
         return s + (self.STATUS_LEN - len(s))*' '
 
     def _show(self, status, indent=0):
@@ -23,13 +19,24 @@ class CmdDataStatus(CmdDataBase):
                 self.project.logger.info('{}{}'.format(ind, key))
                 self._show(value, indent+1)
             else:
-                self.project.logger.info('{}{}{}'.format(ind, self._normalize(value), key))
+                msg = '{}{}{}'.format(ind, self._normalize(value), key)
+                self.project.logger.info(msg)
 
     def do_run(self, target=None):
         indent = 1 if self.args.cloud else 0
         try:
-            status = self.project.status(target=target, jobs=self.args.jobs, cloud=self.args.cloud)
-            self._show(status, indent)
+            st = self.project.status(target=target,
+                                     jobs=self.args.jobs,
+                                     cloud=self.args.cloud,
+                                     show_checksums=self.args.show_checksums,
+                                     remote=self.args.remote,
+                                     all_branches=self.args.all_branches,
+                                     with_deps=self.args.with_deps)
+            if st:
+                self._show(st, indent)
+            else:
+                self.project.logger.info(self.UP_TO_DATE_MSG)
+
         except Exception as exc:
             self.project.logger.error('Failed to obtain data status', exc)
             return 1

dvc/config.py

@@ -2,56 +2,232 @@
 DVC config objects.
 """
 import os
-import configparser
+import configobj
+from schema import Schema, Optional, And, Use, Regex
 
 from dvc.exceptions import DvcException
 
 
 class ConfigError(DvcException):
     """ DVC config exception """
-    def __init__(self, msg):
-        DvcException.__init__(self, 'Config file error: {}'.format(msg))
+    def __init__(self, msg, ex=None):
+        super(ConfigError, self).__init__('Config file error: {}'.format(msg),
+                                          ex)
+
+
+def supported_url(url):
+    from dvc.remote import supported_url as supported
+    return supported(url)
+
+
+def supported_cache_type(types):
+    if isinstance(types, str):
+        types = [t.strip() for t in types.split(',')]
+    for t in types:
+        if t not in ['reflink', 'hardlink', 'symlink', 'copy']:
+            return False
+    return True
+
+
+def supported_loglevel(level):
+    return level in ['info', 'debug', 'warning', 'error']
+
+
+def supported_cloud(cloud):
+    return cloud in ['aws', 'gcp', 'local', '']
+
+
+def is_bool(val):
+    return val.lower() in ['true', 'false']
+
+
+def to_bool(val):
+    return val.lower() == 'true'
+
+
+def is_whole(val):
+    return int(val) >= 0
+
+
+def is_percent(val):
+    return int(val) >= 0 and int(val) <= 100
 
 
 class Config(object):
     CONFIG = 'config'
-    SECTION_CORE = 'Core'
-    CONFIG_TEMPLATE = '''
-[Core]
-# Supported clouds: AWS, GCP
-Cloud = AWS
+    CONFIG_LOCAL = 'config.local'
 
-# Log levels: Debug, Info, Warning and Error
-LogLevel = Info
+    SECTION_CORE = 'core'
+    SECTION_CORE_LOGLEVEL = 'loglevel'
+    SECTION_CORE_LOGLEVEL_SCHEMA = And(Use(str.lower), supported_loglevel)
+    SECTION_CORE_REMOTE = 'remote'
+    SECTION_CORE_INTERACTIVE_SCHEMA = And(str, is_bool, Use(to_bool))
+    SECTION_CORE_INTERACTIVE = 'interactive'
 
-[AWS]
-CredentialPath = ~/.aws/credentials
-Profile = default
+    SECTION_CACHE = 'cache'
+    SECTION_CACHE_DIR = 'dir'
+    SECTION_CACHE_TYPE = 'type'
+    SECTION_CACHE_TYPE_SCHEMA = supported_cache_type
+    SECTION_CACHE_LOCAL = 'local'
+    SECTION_CACHE_S3 = 's3'
+    SECTION_CACHE_GS = 'gs'
+    SECTION_CACHE_SSH = 'ssh'
+    SECTION_CACHE_HDFS = 'hdfs'
+    SECTION_CACHE_AZURE = 'azure'
+    SECTION_CACHE_SCHEMA = {
+        Optional(SECTION_CACHE_LOCAL): str,
+        Optional(SECTION_CACHE_S3): str,
+        Optional(SECTION_CACHE_GS): str,
+        Optional(SECTION_CACHE_HDFS): str,
+        Optional(SECTION_CACHE_SSH): str,
+        Optional(SECTION_CACHE_AZURE): str,
 
-StoragePath = dvc/tutorial
+        # backward compatibility
+        Optional(SECTION_CACHE_DIR, default='cache'): str,
+        Optional(SECTION_CACHE_TYPE, default=None): SECTION_CACHE_TYPE_SCHEMA,
+    }
 
-[GCP]
-StoragePath = 
-ProjectName = 
+    # backward compatibility
+    SECTION_CORE_CLOUD = 'cloud'
+    SECTION_CORE_CLOUD_SCHEMA = And(Use(str.lower), supported_cloud)
+    SECTION_CORE_STORAGEPATH = 'storagepath'
 
-'''
+    SECTION_CORE_SCHEMA = {
+        Optional(SECTION_CORE_LOGLEVEL,
+                 default='info'): And(str, Use(str.lower),
+                                      SECTION_CORE_LOGLEVEL_SCHEMA),
+        Optional(SECTION_CORE_REMOTE, default=''): And(str, Use(str.lower)),
+        Optional(SECTION_CORE_INTERACTIVE,
+                 default=False): SECTION_CORE_INTERACTIVE_SCHEMA,
+
+        # backward compatibility
+        Optional(SECTION_CORE_CLOUD, default=''): SECTION_CORE_CLOUD_SCHEMA,
+        Optional(SECTION_CORE_STORAGEPATH, default=''): str,
+    }
+
+    # backward compatibility
+    SECTION_AWS = 'aws'
+    SECTION_AWS_STORAGEPATH = 'storagepath'
+    SECTION_AWS_CREDENTIALPATH = 'credentialpath'
+    SECTION_AWS_ENDPOINT_URL = 'endpointurl'
+    SECTION_AWS_REGION = 'region'
+    SECTION_AWS_PROFILE = 'profile'
+    SECTION_AWS_SCHEMA = {
+        SECTION_AWS_STORAGEPATH: str,
+        Optional(SECTION_AWS_REGION): str,
+        Optional(SECTION_AWS_PROFILE, default='default'): str,
+        Optional(SECTION_AWS_CREDENTIALPATH, default=''): str,
+        Optional(SECTION_AWS_ENDPOINT_URL, default=None): str,
+    }
+
+    # backward compatibility
+    SECTION_GCP = 'gcp'
+    SECTION_GCP_STORAGEPATH = SECTION_AWS_STORAGEPATH
+    SECTION_GCP_PROJECTNAME = 'projectname'
+    SECTION_GCP_SCHEMA = {
+        SECTION_GCP_STORAGEPATH: str,
+        Optional(SECTION_GCP_PROJECTNAME): str,
+    }
+
+    # backward compatibility
+    SECTION_LOCAL = 'local'
+    SECTION_LOCAL_STORAGEPATH = SECTION_AWS_STORAGEPATH
+    SECTION_LOCAL_SCHEMA = {
+        SECTION_LOCAL_STORAGEPATH: str,
+    }
+
+    SECTION_REMOTE_REGEX = r'^\s*remote\s*"(?P<name>.*)"\s*$'
+    SECTION_REMOTE_FMT = 'remote "{}"'
+    SECTION_REMOTE_URL = 'url'
+    SECTION_REMOTE_USER = 'user'
+    SECTION_REMOTE_PORT = 'port'
+    SECTION_REMOTE_KEY_FILE = 'keyfile'
+    SECTION_REMOTE_TIMEOUT = 'timeout'
+    SECTION_REMOTE_PASSWORD = 'password'
+    SECTION_REMOTE_ASK_PASSWORD = 'ask_password'
+    SECTION_REMOTE_SCHEMA = {
+        SECTION_REMOTE_URL: And(supported_url, error="Unsupported URL"),
+        Optional(SECTION_AWS_REGION): str,
+        Optional(SECTION_AWS_PROFILE, default='default'): str,
+        Optional(SECTION_AWS_CREDENTIALPATH, default=''): str,
+        Optional(SECTION_AWS_ENDPOINT_URL, default=None): str,
+        Optional(SECTION_GCP_PROJECTNAME): str,
+        Optional(SECTION_CACHE_TYPE): SECTION_CACHE_TYPE_SCHEMA,
+        Optional(SECTION_REMOTE_USER): str,
+        Optional(SECTION_REMOTE_PORT): Use(int),
+        Optional(SECTION_REMOTE_KEY_FILE): str,
+        Optional(SECTION_REMOTE_TIMEOUT): Use(int),
+        Optional(SECTION_REMOTE_PASSWORD): str,
+        Optional(SECTION_REMOTE_ASK_PASSWORD): And(str, is_bool, Use(to_bool)),
+    }
+
+    SECTION_STATE = 'state'
+    SECTION_STATE_ROW_LIMIT = 'row_limit'
+    SECTION_STATE_ROW_CLEANUP_QUOTA = 'row_cleanup_quota'
+    SECTION_STATE_SCHEMA = {
+        Optional(SECTION_STATE_ROW_LIMIT): And(Use(int), is_whole),
+        Optional(SECTION_STATE_ROW_CLEANUP_QUOTA): And(Use(int), is_percent),
+    }
+
+    SCHEMA = {
+        Optional(SECTION_CORE, default={}): SECTION_CORE_SCHEMA,
+        Optional(Regex(SECTION_REMOTE_REGEX)): SECTION_REMOTE_SCHEMA,
+        Optional(SECTION_CACHE, default={}): SECTION_CACHE_SCHEMA,
+        Optional(SECTION_STATE, default={}): SECTION_STATE_SCHEMA,
+
+        # backward compatibility
+        Optional(SECTION_AWS, default={}): SECTION_AWS_SCHEMA,
+        Optional(SECTION_GCP, default={}): SECTION_GCP_SCHEMA,
+        Optional(SECTION_LOCAL, default={}): SECTION_LOCAL_SCHEMA,
+    }
 
     def __init__(self, dvc_dir):
         self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
         self.config_file = os.path.join(dvc_dir, self.CONFIG)
-
-        self._config = configparser.SafeConfigParser()
+        self.config_local_file = os.path.join(dvc_dir, self.CONFIG_LOCAL)
 
         try:
-            self._config.read(self.config_file)
-        except configparser.Error as ex:
-            raise ConfigError(ex.message)
+            self._config = configobj.ConfigObj(self.config_file)
+            local = configobj.ConfigObj(self.config_local_file)
 
-        if not self._config.has_section(self.SECTION_CORE):
-            raise ConfigError(u'section \'{}\' was not found'.format(self.SECTION_CORE))
+            # NOTE: schema doesn't support ConfigObj.Section validation, so we
+            # need to convert our config to dict before passing it to
+            self._config = self._lower(self._config)
+            local = self._lower(local)
+            self._config = self._merge(self._config, local)
+
+            self._config = Schema(self.SCHEMA).validate(self._config)
+
+            # NOTE: now converting back to ConfigObj
+            self._config = configobj.ConfigObj(self._config,
+                                               write_empty_values=True)
+            self._config.filename = self.config_file
+        except Exception as ex:
+            raise ConfigError(ex)
+
+    @staticmethod
+    def _merge(first, second):
+        res = {}
+        sections = list(first.keys()) + list(second.keys())
+        for section in sections:
+            f = first.get(section, {}).copy()
+            s = second.get(section, {}).copy()
+            f.update(s)
+            res[section] = f
+        return res
+
+    @staticmethod
+    def _lower(config):
+        new_config = {}
+        for s_key, s_value in config.items():
+            new_s = {}
+            for key, value in s_value.items():
+                new_s[key.lower()] = value
+            new_config[s_key.lower()] = new_s
+        return new_config
 
     @staticmethod
     def init(dvc_dir):
         config_file = os.path.join(dvc_dir, Config.CONFIG)
-        open(config_file, 'w').write(Config.CONFIG_TEMPLATE)
+        open(config_file, 'w+').close()
         return Config(dvc_dir)

dvc/dagascii.py

@@ -0,0 +1,121 @@
+import math
+
+from asciicanvas.asciicanvas import AsciiCanvas
+from asciicanvas.style import Style
+
+from grandalf.graphs import Vertex, Edge, Graph
+from grandalf.layouts import SugiyamaLayout
+from grandalf.routing import route_with_lines, EdgeViewer
+
+
+class Dagascii(object):
+    def __init__(self, vertexes, edges):
+        #
+        # Just a reminder about naming conventions:
+        # +------------X
+        # |
+        # |
+        # |
+        # |
+        # Y
+        #
+
+        V = {v: Vertex(" {} ".format(v)) for v in vertexes}
+        # NOTE: reverting edges to correctly orientate the graph
+        E = [Edge(V[e], V[s]) for s, e in edges]
+        V = V.values()
+        g = Graph(V, E)
+
+        class VertexViewer(object):
+            h = 3  # top and bottom box edges + text
+
+            def __init__(self, name):
+                self.w = len(name) + 2  # right and left bottom edges + text
+
+        for v in V:
+            v.view = VertexViewer(v.data)
+
+        # NOTE: determine min box length to create the best layout
+        minw = min([v.view.w for v in V])
+
+        for e in E:
+            e.view = EdgeViewer()
+
+        sug = SugiyamaLayout(g.C[0])
+        gr = g.C[0]
+        r = list(filter(lambda x: len(x.e_in()) == 0, gr.sV))
+
+        sug.init_all(roots=r, optimize=True)
+
+        sug.yspace = VertexViewer.h
+        sug.xspace = minw
+        sug.route_edge = route_with_lines
+
+        sug.draw()
+
+        self.sug = sug
+
+    def draw(self):
+        # NOTE: coordinates might me negative, so we need to shift
+        # everything to the positive plane before we actually draw it.
+        Xs = []
+        Ys = []
+
+        for v in self.sug.g.sV:
+            # NOTE: moving boxes w/2 to the left
+            Xs.append(v.view.xy[0] - v.view.w/2.0)
+            Xs.append(v.view.xy[0] + v.view.w/2.0)
+            Ys.append(v.view.xy[1])
+            Ys.append(v.view.xy[1] + v.view.h)
+
+        for e in self.sug.g.sE:
+            for xy in e.view._pts:
+                Xs.append(xy[0])
+                Ys.append(xy[1])
+
+        minx = min(Xs)
+        miny = min(Ys)
+        maxx = max(Xs)
+        maxy = max(Ys)
+
+        canvas_cols = int(math.ceil(math.ceil(maxx) - math.floor(minx))) + 1
+        canvas_lines = int(round(maxy - miny))
+
+        canvas = AsciiCanvas(canvas_cols, canvas_lines)
+
+        # NOTE: first draw edges so that node boxes could overwrite them
+        for e in self.sug.g.sE:
+            assert len(e.view._pts) > 1
+            for index in range(1, len(e.view._pts)):
+                start = e.view._pts[index - 1]
+                end = e.view._pts[index]
+
+                start_x = int(round(start[0] - minx))
+                start_y = int(round(start[1] - miny))
+                end_x = int(round(end[0] - minx))
+                end_y = int(round(end[1] - miny))
+
+                assert start_x >= 0
+                assert start_y >= 0
+                assert end_x >= 0
+                assert end_y >= 0
+
+                canvas.add_line(start_x, start_y,
+                                end_x, end_y,
+                                style=Style('*'))
+
+        for v in self.sug.g.sV:
+            # NOTE: moving boxes w/2 to the left
+            x = v.view.xy[0] - v.view.w/2.0
+            y = v.view.xy[1]
+
+            canvas.add_nine_patch_rect(int(round(x - minx)),
+                                       int(round(y - miny)),
+                                       v.view.w,
+                                       v.view.h)
+
+            canvas.add_text(int(round(x - minx)) + 1,
+                            int(round(y - miny)) + 1,
+                            v.data)
+
+        canvas.print_out()

dvc/data_cloud.py

@@ -0,0 +1,128 @@
+from dvc.logger import Logger
+from dvc.config import Config, ConfigError
+
+from dvc.remote.s3 import RemoteS3
+from dvc.remote.gs import RemoteGS
+from dvc.remote.azure import RemoteAzure
+from dvc.remote.ssh import RemoteSSH
+from dvc.remote.hdfs import RemoteHDFS
+from dvc.remote.local import RemoteLOCAL
+
+
+class DataCloud(object):
+    CLOUD_MAP = {
+        'aws': RemoteS3,
+        'gcp': RemoteGS,
+        'azure': RemoteAzure,
+        'ssh': RemoteSSH,
+        'hdfs': RemoteHDFS,
+        'local': RemoteLOCAL,
+    }
+
+    def __init__(self, project, config=None):
+        self.project = project
+        self._config = config
+        self._core = self._config[Config.SECTION_CORE]
+
+        remote = self._core.get(Config.SECTION_CORE_REMOTE, '')
+        if remote == '':
+            if self._core.get(Config.SECTION_CORE_CLOUD, None):
+                # backward compatibility
+                msg = 'Using obsoleted config format. Consider updating.'
+                Logger.warn(msg)
+                self._cloud = self.__init__compat()
+            else:
+                self._cloud = None
+            return
+
+        self._cloud = self._init_remote(remote)
+
+    @staticmethod
+    def supported(config):
+        for cloud in DataCloud.CLOUD_MAP.values():
+            if cloud.supported(config):
+                return cloud
+        return None
+
+    def _init_remote(self, remote):
+        section = Config.SECTION_REMOTE_FMT.format(remote)
+        cloud_config = self._config.get(section, None)
+        if not cloud_config:
+            msg = "Can't find remote section '{}' in config"
+            raise ConfigError(msg.format(section))
+
+        cloud_type = self.supported(cloud_config)
+        if not cloud_type:
+            raise ConfigError("Unsupported cloud '{}'".format(cloud_config))
+
+        return self._init_cloud(cloud_config, cloud_type)
+
+    def __init__compat(self):
+        name = self._core.get(Config.SECTION_CORE_CLOUD, '').strip().lower()
+        if name == '':
+            self._cloud = None
+            return
+
+        cloud_type = self.CLOUD_MAP.get(name, None)
+        if not cloud_type:
+            msg = "Wrong cloud type {} specified"
+            raise ConfigError(msg.format(name))
+
+        cloud_config = self._config.get(name, None)
+        if not cloud_config:
+            msg = "Can't find cloud section '{}' in config"
+            raise ConfigError(msg.format(name))
+
+        return self._init_cloud(cloud_config, cloud_type)
+
+    def _init_cloud(self, cloud_config, cloud_type):
+        global_storage_path = self._core.get(Config.SECTION_CORE_STORAGEPATH)
+        if global_storage_path:
+            Logger.warn('Using obsoleted config format. Consider updating.')
+
+        cloud = cloud_type(self.project, cloud_config)
+        return cloud
+
+    def _get_cloud(self, remote, cmd):
+        if remote:
+            return self._init_remote(remote)
+
+        if self._cloud:
+            return self._cloud
+
+        msg = "No remote repository specified. Setup default " \
+              "repository with\n"\
+              "    dvc config core.remote <name>\n" \
+              "or use:\n" \
+              "    dvc {} -r <name>".format(cmd)
+        raise ConfigError(msg)
+
+    def push(self, targets, jobs=1, remote=None, show_checksums=False):
+        """
+        Push data items in a cloud-agnostic way.
+        """
+        return self.project.cache.local.push(targets,
+                                             jobs=jobs,
+                                             remote=self._get_cloud(remote,
+                                                                    'push'),
+                                             show_checksums=show_checksums)
+
+    def pull(self, targets, jobs=1, remote=None, show_checksums=False):
+        """
+        Pull data items in a cloud-agnostic way.
+        """
+        return self.project.cache.local.pull(targets,
+                                             jobs=jobs,
+                                             remote=self._get_cloud(remote,
+                                                                    'pull'),
+                                             show_checksums=show_checksums)
+
+    def status(self, targets, jobs=1, remote=None, show_checksums=False):
+        """
+        Check status of data items in a cloud-agnostic way.
+        """
+        cloud = self._get_cloud(remote, 'status')
+        return self.project.cache.local.status(targets,
+                                               jobs=jobs,
+                                               remote=cloud,
+                                               show_checksums=show_checksums)

dvc/dependency/__init__.py

@@ -0,0 +1,69 @@
+import schema
+
+try:
+    from urlparse import urlparse
+except ImportError:
+    from urllib.parse import urlparse
+
+from dvc.exceptions import DvcException
+from dvc.config import Config
+
+from dvc.dependency.base import DependencyBase
+from dvc.dependency.s3 import DependencyS3
+from dvc.dependency.gs import DependencyGS
+from dvc.dependency.local import DependencyLOCAL
+from dvc.dependency.hdfs import DependencyHDFS
+from dvc.dependency.ssh import DependencySSH
+
+from dvc.remote import Remote
+from dvc.remote.local import RemoteLOCAL
+from dvc.remote.s3 import RemoteS3
+from dvc.remote.hdfs import RemoteHDFS
+
+DEPS = [DependencyHDFS,
+        DependencyS3,
+        DependencyGS,
+        DependencySSH,
+        DependencyLOCAL]
+
+DEP_MAP = {'': DependencyLOCAL,
+           'ssh': DependencySSH,
+           's3': DependencyS3,
+           'gs': DependencyGS,
+           'hdfs': DependencyHDFS}
+
+SCHEMA = {
+    DependencyBase.PARAM_PATH: str,
+    schema.Optional(RemoteLOCAL.PARAM_MD5): schema.Or(str, None),
+    schema.Optional(RemoteS3.PARAM_ETAG): schema.Or(str, None),
+    schema.Optional(RemoteHDFS.PARAM_CHECKSUM): schema.Or(str, None),
+}
+
+
+def _get(stage, p, info):
+    parsed = urlparse(p)
+    if parsed.scheme == 'remote':
+        name = Config.SECTION_REMOTE_FMT.format(parsed.netloc)
+        sect = stage.project.config._config[name]
+        remote = Remote(stage.project, sect)
+        return DEP_MAP[remote.scheme](stage, p, info, remote=remote)
+
+    for d in DEPS:
+        if d.supported(p):
+            return d(stage, p, info)
+    raise DvcException('Dependency \'{}\' is not supported'.format(p))
+
+
+def loadd_from(stage, d_list):
+    ret = []
+    for d in d_list:
+        p = d.pop(DependencyBase.PARAM_PATH)
+        ret.append(_get(stage, p, d))
+    return ret
+
+
+def loads_from(stage, s_list):
+    ret = []
+    for s in s_list:
+        ret.append(_get(stage, s, {}))
+    return ret

dvc/dependency/base.py

@@ -0,0 +1,79 @@
+import re
+
+from dvc.exceptions import DvcException
+
+
+class DependencyError(DvcException):
+    def __init__(self, path, msg):
+        msg = 'Dependency \'{}\' error: {}'
+        super(DependencyError, self).__init__(msg.format(path, msg))
+
+
+class DependencyDoesNotExistError(DependencyError):
+    def __init__(self, path):
+        msg = 'does not exist'
+        super(DependencyDoesNotExistError, self).__init__(path, msg)
+
+
+class DependencyIsNotFileOrDirError(DependencyError):
+    def __init__(self, path):
+        msg = 'not a file or directory'
+        super(DependencyIsNotFileOrDirError, self).__init__(path, msg)
+
+
+class DependencyBase(object):
+    REGEX = None
+
+    PARAM_PATH = 'path'
+
+    def __init__(self, stage, path):
+        self.stage = stage
+        self.project = stage.project
+        self.path = path
+
+    def __str__(self):
+        return self.path
+
+    @classmethod
+    def match(cls, url):
+        return re.match(cls.REGEX, url)
+
+    def group(self, name):
+        match = self.match(self.path)
+        if not match:
+            return None
+        return match.group(name)
+
+    @classmethod
+    def supported(cls, url):
+        return cls.match(url) is not None
+
+    @property
+    def sep(self):
+        return '/'
+
+    @property
+    def exists(self):
+        return self.remote.exists([self.path_info])[0]
+
+    def changed(self):
+        raise NotImplementedError
+
+    def status(self):
+        if self.changed():
+            # FIXME better msgs
+            if self.path_info['scheme'] == 'local':
+                p = self.rel_path
+            else:
+                p = self.path
+            return {p: 'changed'}
+        return {}
+
+    def save(self):
+        raise NotImplementedError
+
+    def dumpd(self):
+        return {self.PARAM_PATH: self.path}
+
+    def download(self, to_info):
+        self.remote.download([self.path_info], [to_info])

dvc/dependency/gs.py

@@ -0,0 +1,25 @@
+import posixpath
+
+try:
+    from urlparse import urlparse
+except ImportError:
+    from urllib.parse import urlparse
+
+from dvc.dependency.s3 import DependencyS3
+from dvc.remote.gs import RemoteGS
+
+
+class DependencyGS(DependencyS3):
+    REGEX = RemoteGS.REGEX
+
+    def __init__(self, stage, path, info=None, remote=None):
+        super(DependencyGS, self).__init__(stage, path)
+        self.info = info
+        self.remote = remote if remote else RemoteGS(stage.project, {})
+        bucket = remote.bucket if remote else urlparse(path).netloc
+        key = urlparse(path).path.lstrip('/')
+        if remote:
+            key = posixpath.join(remote.prefix, key)
+        self.path_info = {'scheme': 'gs',
+                          'bucket': bucket,
+                          'key': key}

dvc/dependency/hdfs.py

@@ -0,0 +1,35 @@
+import posixpath
+
+try:
+    from urlparse import urlparse
+except ImportError:
+    from urllib.parse import urlparse
+
+from dvc.dependency.base import DependencyBase
+from dvc.remote.hdfs import RemoteHDFS
+
+
+class DependencyHDFS(DependencyBase):
+    REGEX = RemoteHDFS.REGEX
+
+    def __init__(self, stage, path, info=None, remote=None):
+        super(DependencyHDFS, self).__init__(stage, path)
+        self.info = info
+        self.remote = remote if remote else RemoteHDFS(stage.project, {})
+        if remote:
+            path = posixpath.join(remote.url, urlparse(path).path.lstrip('/'))
+        user = remote.user if remote else self.group('user')
+        self.path_info = {'scheme': 'hdfs',
+                          'user': user,
+                          'url': path}
+
+    def changed(self):
+        return self.info != self.remote.save_info(self.path_info)
+
+    def save(self):
+        self.info = self.remote.save_info(self.path_info)
+
+    def dumpd(self):
+        ret = self.info
+        ret[self.PARAM_PATH] = self.path
+        return ret

dvc/dependency/local.py

@@ -0,0 +1,88 @@
+import os
+
+try:
+    from urlparse import urlparse
+except ImportError:
+    from urllib.parse import urlparse
+
+from dvc.dependency.base import DependencyBase
+from dvc.dependency.base import DependencyDoesNotExistError
+from dvc.dependency.base import DependencyIsNotFileOrDirError
+from dvc.logger import Logger
+from dvc.remote.local import RemoteLOCAL
+
+
+class DependencyLOCAL(DependencyBase):
+    REGEX = r'^(?P<path>(/+|.:\\+)?[^:]*)$'
+
+    DoesNotExistError = DependencyDoesNotExistError
+    IsNotFileOrDirError = DependencyIsNotFileOrDirError
+
+    def __init__(self, stage, path, info=None, remote=None):
+        self.stage = stage
+        self.project = stage.project
+        self.info = info
+        if remote is not None:
+            self.remote = remote
+        else:
+            self.remote = RemoteLOCAL(stage.project, {})
+
+        if remote:
+            path = os.path.join(remote.prefix, urlparse(path).path.lstrip('/'))
+
+        if not os.path.isabs(path):
+            path = self.remote.ospath(path)
+            path = os.path.join(stage.cwd, path)
+        self.path = os.path.abspath(os.path.normpath(path))
+
+        self.path_info = {'scheme': 'local',
+                          'path': self.path}
+
+    def __str__(self):
+        return self.rel_path
+
+    @property
+    def is_local(self):
+        assert os.path.isabs(self.path)
+        assert os.path.isabs(self.project.root_dir)
+        return self.path.startswith(self.project.root_dir)
+
+    @property
+    def sep(self):
+        return os.sep
+
+    @property
+    def rel_path(self):
+        return os.path.relpath(self.path)
+
+    def changed(self):
+        if not self.exists:
+            return True
+
+        info = self.remote.save_info(self.path_info)
+
+        return self.info != info
+
+    def save(self):
+        if not self.exists:
+            raise self.DoesNotExistError(self.rel_path)
+
+        if not os.path.isfile(self.path) and not os.path.isdir(self.path):
+            raise self.IsNotFileOrDirError(self.rel_path)
+
+        if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \
+           (os.path.isdir(self.path) and len(os.listdir(self.path)) == 0):
+            Logger.warn("File/directory '{}' is empty.".format(self.rel_path))
+
+        self.info = self.remote.save_info(self.path_info)
+
+    def dumpd(self):
+        if self.is_local:
+            path = self.remote.unixpath(os.path.relpath(self.path,
+                                                        self.stage.cwd))
+        else:
+            path = self.path
+
+        info = self.info.copy()
+        info[self.PARAM_PATH] = path
+        return info

dvc/dependency/s3.py

@@ -0,0 +1,37 @@
+import posixpath
+
+try:
+    from urlparse import urlparse
+except ImportError:
+    from urllib.parse import urlparse
+
+from dvc.dependency.base import DependencyBase
+from dvc.remote.s3 import RemoteS3
+
+
+class DependencyS3(DependencyBase):
+    REGEX = RemoteS3.REGEX
+
+    def __init__(self, stage, path, info=None, remote=None):
+        super(DependencyS3, self).__init__(stage, path)
+        self.info = info
+        self.remote = remote if remote else RemoteS3(stage.project, {})
+
+        bucket = remote.bucket if remote else urlparse(path).netloc
+        key = urlparse(path).path.lstrip('/')
+        if remote:
+            key = posixpath.join(remote.prefix, key)
+        self.path_info = {'scheme': 's3',
+                          'bucket': bucket,
+                          'key': key}
+
+    def changed(self):
+        return self.info != self.remote.save_info(self.path_info)
+
+    def save(self):
+        self.info = self.remote.save_info(self.path_info)
+
+    def dumpd(self):
+        ret = self.info
+        ret[self.PARAM_PATH] = self.path
+        return ret

dvc/dependency/ssh.py

@@ -0,0 +1,44 @@
+import posixpath
+
+try:
+    from urlparse import urlparse
+except ImportError:
+    from urllib.parse import urlparse
+
+from dvc.dependency.base import DependencyBase
+from dvc.remote.ssh import RemoteSSH
+
+
+class DependencySSH(DependencyBase):
+    REGEX = RemoteSSH.REGEX
+
+    def __init__(self, stage, path, info=None, remote=None):
+        super(DependencySSH, self).__init__(stage, path)
+        self.info = info
+        self.remote = remote if remote else RemoteSSH(stage.project, {})
+
+        host = remote.host if remote else self.group('host')
+        port = remote.port if remote else RemoteSSH.DEFAULT_PORT
+        user = remote.user if remote else self.group('user')
+        if remote:
+            path = posixpath.join(remote.prefix,
+                                  urlparse(path).path.lstrip('/'))
+        else:
+            path = self.match(self.path).group('path')
+
+        self.path_info = {'scheme': 'ssh',
+                          'host': host,
+                          'port': port,
+                          'user': user,
+                          'path': path}
+
+    def changed(self):
+        return self.info != self.remote.save_info(self.path_info)
+
+    def save(self):
+        self.info = self.remote.save_info(self.path_info)
+
+    def dumpd(self):
+        ret = self.info
+        ret[self.PARAM_PATH] = self.path
+        return ret

dvc/exceptions.py

@@ -4,5 +4,28 @@ import traceback
 class DvcException(Exception):
     def __init__(self, msg, cause=None):
         self.cause = cause
-        self.cause_tb = traceback.format_exc() if cause else None
+        self.case_tb = None
+        if cause:
+            try:
+                self.cause_tb = traceback.format_exc()
+            except Exception:
+                pass
         super(DvcException, self).__init__(msg)
+
+
+class UnsupportedRemoteError(DvcException):
+    def __init__(self, config):
+        msg = "Remote '{}' is not supported.".format(config)
+        super(UnsupportedRemoteError, self).__init__(msg)
+
+
+class OutputDuplicationError(DvcException):
+    def __init__(self, output, stages):
+        assert isinstance(output, str)
+        assert isinstance(stages, list)
+        assert all(isinstance(stage, str) for stage in stages)
+        msg = "File '{}' is specified as an output in more than one stage:{}"
+        s = ""
+        for stage in stages:
+            s += "\n    " + stage
+        super(OutputDuplicationError, self).__init__(msg.format(output, s))

dvc/istextfile.py

@@ -0,0 +1,38 @@
+# Based on https://eli.thegreenplace.net/2011/10/19/
+# perls-guess-if-file-is-text-or-binary-implemented-in-python
+
+import sys
+PY3 = sys.version_info[0] == 3
+
+# A function that takes an integer in the 8-bit range and returns
+# a single-character byte object in py3 / a single-character string
+# in py2.
+#
+int2byte = (lambda x: bytes((x,))) if PY3 else chr
+
+
+_text_characters = (
+        b''.join(int2byte(i) for i in range(32, 127)) +
+        b'\n\r\t\f\b')
+
+
+def istextfile(fname, blocksize=512):
+    """ Uses heuristics to guess whether the given file is text or binary,
+        by reading a single block of bytes from the file.
+        If more than 30% of the chars in the block are non-text, or there
+        are NUL ('\x00') bytes in the block, assume this is a binary file.
+    """
+    with open(fname, 'rb') as fd:
+        block = fd.read(blocksize)
+
+    if b'\x00' in block:
+        # Files with null bytes are binary
+        return False
+    elif not block:
+        # An empty file is considered a valid text file
+        return True
+
+    # Use translate's 'deletechars' argument to efficiently remove all
+    # occurrences of _text_characters from the block
+    nontext = block.translate(None, _text_characters)
+    return float(len(nontext)) / len(block) <= 0.30

dvc/lock.py

@@ -13,19 +13,16 @@ class Lock(object):
     LOCK_FILE = 'lock'
     TIMEOUT = 5
 
-    def __init__(self, dvc_dir):
-        self.lock_file = os.path.join(dvc_dir, self.LOCK_FILE)
+    def __init__(self, dvc_dir, name=LOCK_FILE):
+        self.lock_file = os.path.join(dvc_dir, name)
         self._lock = None
 
-    @staticmethod
-    def init(dvc_dir):
-        return Lock(dvc_dir)
-
     def _do_lock(self):
         try:
             self._lock = zc.lockfile.LockFile(self.lock_file)
         except zc.lockfile.LockError:
-            raise LockError('Cannot perform the cmd since DVC is busy and locked. Please retry the cmd later.')
+            raise LockError('Cannot perform the cmd since DVC is busy and '
+                            'locked. Please retry the cmd later.')
 
     def lock(self):
         try:

dvc/logger.py

@@ -3,7 +3,6 @@ import logging
 import colorama
 import traceback
 
-from dvc.config import Config
 
 colorama.init()
 
@@ -25,13 +24,12 @@ class Logger(object):
         'error': colorama.Fore.RED
     }
 
-    def __init__(self, config=None):
-        if config:
-            level = config[Config.SECTION_CORE].get('LogLevel', None)
-            Logger.set_level(level)
+    def __init__(self, loglevel=None):
+        if loglevel:
+            Logger.set_level(loglevel)
 
     @staticmethod
-    def init(config=None):
+    def init():
         sh = logging.StreamHandler(sys.stdout)
         sh.setFormatter(logging.Formatter(Logger.FMT))
         sh.setLevel(logging.DEBUG)
@@ -64,7 +62,7 @@ class Logger(object):
         header = ''
         footer = ''
 
-        if sys.stdout.isatty():
+        if sys.stdout.isatty():  # pragma: no cover
             header = Logger.COLOR_MAP.get(typ.lower(), '')
             footer = colorama.Style.RESET_ALL
 
@@ -76,29 +74,47 @@ class Logger(object):
         str_exc = ': {}'.format(str(exc)) if exc else ""
 
         if exc and hasattr(exc, 'cause') and exc.cause:
-            cause_str_exc, cause_str_tb = Logger.parse_exc(exc.cause, exc.cause_tb)
+            cause_tb = exc.cause_tb if hasattr(exc, 'cause_tb') else None
+            cause_str_exc, cause_str_tb = Logger.parse_exc(exc.cause, cause_tb)
 
             str_tb = cause_str_tb
             str_exc = '{}{}'.format(str_exc, cause_str_exc)
 
         return (str_exc, str_tb)
 
+    @staticmethod
+    def _prefix(msg, typ):
+        return Logger.colorize('{}: '.format(msg), typ)
+
+    @staticmethod
+    def error_prefix():
+        return Logger._prefix('Error', 'error')
+
+    @staticmethod
+    def warning_prefix():
+        return Logger._prefix('Warning', 'warn')
+
+    @staticmethod
+    def debug_prefix():
+        return Logger._prefix('Debug', 'debug')
+
     @staticmethod
     def error(msg, exc=None):
+        prefix = Logger.error_prefix()
         str_exc, str_tb = Logger.parse_exc(exc)
         if Logger.logger().getEffectiveLevel() == logging.DEBUG and exc:
             str_tb = str_tb if str_tb else traceback.format_exc()
-            Logger.logger().error(str_tb)
-        return Logger.logger().error(Logger.colorize(msg + str_exc, 'error'))
+            Logger.logger().error(prefix + str_tb)
+        return Logger.logger().error(prefix + msg + str_exc)
 
     @staticmethod
     def warn(msg):
-        return Logger.logger().warn(Logger.colorize(msg, 'warn'))
+        return Logger.logger().warn(Logger.warning_prefix() + msg)
 
     @staticmethod
     def debug(msg):
-        return Logger.logger().debug(Logger.colorize(msg, 'debug'))
+        return Logger.logger().debug(Logger.debug_prefix() + msg)
 
     @staticmethod
     def info(msg):
-        return Logger.logger().info(Logger.colorize(msg, 'info'))
+        return Logger.logger().info(msg)

dvc/main.py

@@ -1,11 +1,17 @@
 from dvc.logger import Logger
 from dvc.cli import parse_args
+from dvc.command.base import CmdBase
+
 
 def main(argv=None):
     Logger.init()
 
     args = parse_args(argv)
 
+    # Init loglevel early in case we'll run
+    # into errors before setting it properly
+    CmdBase._set_loglevel(args)
+
     try:
         cmd = args.func(args)
     except Exception as ex:

dvc/output.py

@@ -1,393 +0,0 @@
-import os
-import stat
-import json
-import shutil
-import schema
-from checksumdir import dirhash
-
-from dvc.system import System
-from dvc.utils import file_md5
-from dvc.exceptions import DvcException
-from dvc.logger import Logger
-
-
-class OutputError(DvcException):
-    pass
-
-
-class CmdOutputError(DvcException):
-    def __init__(self, path, msg):
-        super(CmdOutputError, self).__init__('Output file \'{}\' error: {}'.format(path, msg))
-
-
-class CmdOutputNoCacheError(CmdOutputError):
-    def __init__(self, path):
-        super(CmdOutputNoCacheError, self).__init__(path, 'no cache')
-
-
-class CmdOutputOutsideOfRepoError(CmdOutputError):
-    def __init__(self, path):
-        super(CmdOutputOutsideOfRepoError, self).__init__(path, 'outside of repository')
-
-
-class CmdOutputDoesNotExistError(CmdOutputError):
-    def __init__(self, path):
-        super(CmdOutputDoesNotExistError, self).__init__(path, 'does not exist')
-
-
-class CmdOutputIsNotFileOrDirError(CmdOutputError):
-    def __init__(self, path):
-        super(CmdOutputIsNotFileOrDirError, self).__init__(path, 'not a file or directory')
-
-
-class CmdOutputAlreadyTrackedError(CmdOutputError):
-    def __init__(self, path):
-        super(CmdOutputAlreadyTrackedError, self).__init__(path, 'already tracked by scm(e.g. git)')
-
-
-class Dependency(object):
-    PARAM_RELPATH = 'relpath'
-    PARAM_PATH = 'path'
-    PARAM_MD5 = 'md5'
-    MD5_DIR_SUFFIX = '.dir'
-
-    SCHEMA = {
-        PARAM_PATH: str,
-        schema.Optional(PARAM_MD5): schema.Or(str, None),
-    }
-
-    def __init__(self, project, path, md5=None):
-        self.project = project
-        self.path = os.path.abspath(os.path.realpath(path))
-
-        if not self.path.startswith(self.project.root_dir):
-            raise CmdOutputOutsideOfRepoError(self.rel_path)
-
-        self.md5 = md5
-
-    @property
-    def rel_path(self):
-        return os.path.relpath(self.path)
-
-    def _changed_md5(self):
-        if not os.path.exists(self.path):
-            return True
-
-        return self.project.state.changed(self.path, self.md5)
-
-    def changed(self):
-        return self._changed_md5()
-
-    def status(self):
-        if self.changed():
-            #FIXME better msgs
-            return {self.rel_path: 'changed'}
-        return {}
-
-    @staticmethod
-    def is_dir_cache(cache):
-        return cache.endswith(Output.MD5_DIR_SUFFIX)
-
-    def save(self):
-        if not os.path.exists(self.path):
-            raise CmdOutputDoesNotExistError(self.rel_path)
-
-        if not os.path.isfile(self.path) and not os.path.isdir(self.path):
-            raise CmdOutputIsNotFileOrDirError(self.rel_path)
-
-        self.md5 = self.project.state.update(self.path)
-
-    def dumpd(self, cwd):
-        return {
-            Output.PARAM_PATH: os.path.relpath(self.path, cwd),
-            Output.PARAM_MD5: self.md5,
-        }
-
-    @classmethod
-    def loadd(cls, project, d, cwd=os.curdir):
-        path = os.path.join(cwd, d[Output.PARAM_PATH])
-        md5 = d.get(Output.PARAM_MD5, None)
-        return cls(project, path, md5=md5)
-
-    @classmethod
-    def loadd_from(cls, project, d_list, cwd=os.curdir):
-        return [cls.loadd(project, x, cwd=cwd) for x in d_list]
-
-    @classmethod
-    def loads(cls, project, s, cwd=os.curdir):
-        return cls(project, os.path.join(cwd, s), md5=None)
-
-    @classmethod
-    def loads_from(cls, project, s_list, cwd=os.curdir):
-        return [cls.loads(project, x, cwd=cwd) for x in s_list]
-
-    def stage(self):
-        for stage in self.project.stages():
-            for out in stage.outs:
-                if self.path == out.path:
-                    return stage
-        return None
-
-
-class Output(Dependency):
-    PARAM_CACHE = 'cache'
-
-    SCHEMA = Dependency.SCHEMA
-    SCHEMA[schema.Optional(PARAM_CACHE)] = bool
-
-    def __init__(self, project, path, md5=None, use_cache=True):
-        super(Output, self).__init__(project, path, md5=md5)
-        self.use_cache = use_cache
-
-    @property
-    def cache(self):
-        if not self.md5:
-            return None
-
-        return self.project.cache.get(self.md5)
-
-    @property
-    def rel_cache(self):
-        if not self.cache:
-            return None
-
-        return os.path.relpath(self.cache)
-
-    def dumpd(self, cwd):
-        ret = super(Output, self).dumpd(cwd)
-        ret[Output.PARAM_CACHE] = self.use_cache
-        return ret
-
-    @classmethod
-    def loadd(cls, project, d, cwd=os.curdir):
-        ret = super(Output, cls).loadd(project, d, cwd=cwd)
-        ret.use_cache = d.get(Output.PARAM_CACHE, True)
-        return ret
-
-    @classmethod
-    def loads(cls, project, s, use_cache=True, cwd=os.curdir):
-        ret = super(Output, cls).loads(project, s, cwd=cwd)
-        ret.use_cache = use_cache
-        return ret
-
-    @classmethod
-    def loads_from(cls, project, s_list, use_cache=False, cwd=os.curdir):
-        return [cls.loads(project, x, use_cache=use_cache, cwd=cwd) for x in s_list]
-
-    def _changed_cache(self, cache):
-        md5 = self.project.state.update(cache)
-        if md5 != self.project.cache.path_to_md5(cache):
-            self.project.logger.warn('Corrupted cache file {}'.format(os.path.relpath(cache)))
-            os.unlink(cache)
-            return True
-
-        return False
-
-    def _changed_file(self, path, cache):
-        if os.path.isfile(path) and \
-           os.path.isfile(cache) and \
-           System.samefile(path, cache) and \
-           not self._changed_cache(cache):
-            return False
-
-        return True
-
-    def _changed_dir(self):
-        if not os.path.isdir(self.path) or not os.path.isfile(self.cache):
-            return True
-
-        dir_info = self._collect_dir() # slow!
-        dir_info_cached = self.load_dir_cache(self.cache) # slow. why?
-
-        if not self.are_dir_info_equal(dir_info, dir_info_cached):
-            return True
-
-        return False
-
-    @staticmethod
-    def are_dir_info_equal(dir_info1, dir_info2):
-        return Output.dir_info_dict(dir_info1) == Output.dir_info_dict(dir_info2)
-
-    @staticmethod
-    def dir_info_dict(dir_info):
-        return {i['relpath']: i['md5'] for i in dir_info}
-
-    def changed(self):
-        if not self.use_cache:
-            ret = super(Output, self).changed()
-        elif not self.cache:
-            ret = True
-        elif self.is_dir_cache(self.cache):
-            ret = self._changed_dir()
-        else:
-            ret = self._changed_file(self.path, self.cache)
-
-        msg = u'Data file or dir \'{}\' with cache \'{}\' '
-        if ret:
-            msg += 'changed'
-        else:
-            msg += 'didn\'t change'
-        self.project.logger.debug(msg.format(self.rel_path, self.rel_cache))
-
-        return ret
-
-    def hardlink(self, src, link):
-        rel_src = os.path.relpath(src)
-        rel_link = os.path.relpath(link)
-        self.project.logger.debug(u'creating hardlink {} -> {}'.format(rel_src, rel_link))
-
-        dname = os.path.dirname(link)
-        if not os.path.exists(dname):
-            os.makedirs(dname)
-
-        System.hardlink(src, link)
-
-    @staticmethod
-    def load_dir_cache(path):
-        if os.path.isabs(path):
-            relpath = os.path.relpath(path)
-        else:
-            relpath = path
-
-        try:
-            with open(path, 'r') as fd:
-                d = json.load(fd)
-        except Exception as exc:
-            msg = u'Failed to load dir cache \'{}\''
-            Logger.error(msg.format(relpath), exc)
-            return []
-
-        if not isinstance(d, list):
-            msg = u'Dir cache file format error \'{}\': skipping the file'
-            Logger.error(msg.format(relpath))
-            return []
-
-        return d
-
-    @staticmethod
-    def get_dir_cache(path):
-        res = {}
-        d = Output.load_dir_cache(path)
-
-        for entry in d:
-            res[entry[Output.PARAM_RELPATH]] = entry[Output.PARAM_MD5]
-
-        return res
-
-    def dir_cache(self):
-        res = {}
-        dir_cache = self.get_dir_cache(self.cache)
-
-        for relpath, md5 in dir_cache.items():
-            res[relpath] = self.project.cache.get(md5)
-
-        return res
-
-    def checkout(self):
-        if not self.use_cache:
-            return
-
-        msg = u'Checking out \'{}\' with cache \'{}\''
-        self.project.logger.debug(msg.format(self.rel_path, self.rel_cache))
-
-        if not self.changed():
-            msg = u'Data file \'{}\' with cache \'{}\' didn\'t change, skipping checkout.'
-            self.project.logger.debug(msg.format(self.rel_path, self.rel_cache))
-            return
-
-        if not self.cache or not os.path.exists(self.cache):
-            if self.cache:
-                self.project.logger.warn(u'\'{}({})\': cache file not found'.format(self.rel_cache, self.rel_path))
-            self.remove()
-            return
-
-        if os.path.exists(self.path):
-            msg = u'Data file \'{}\' exists. Removing before checkout'
-            self.project.logger.debug(msg.format(self.rel_path))
-            self.remove()
-
-        if not self.is_dir_cache(self.cache):
-            self.hardlink(self.cache, self.path)
-            return
-
-        for relpath, cache in self.dir_cache().items():
-            path = os.path.join(self.path, relpath)
-            self.hardlink(cache, path)
-
-    def _collect_dir(self):
-        dir_info = []
-
-        for root, dirs, files in os.walk(self.path):
-            for fname in files:
-                path = os.path.join(root, fname)
-                relpath = os.path.relpath(path, self.path)
-
-                md5 = self.project.state.update(path, dump=False)
-                dir_info.append({self.PARAM_RELPATH: relpath, self.PARAM_MD5: md5})
-
-        self.project.state.dump()
-
-        return dir_info
-
-    def _save_dir(self):
-        dname = os.path.dirname(self.cache)
-        dir_info = self._collect_dir()
-
-        for entry in dir_info:
-            md5 = entry[self.PARAM_MD5]
-            relpath = entry[self.PARAM_RELPATH]
-            path = os.path.join(self.path, relpath)
-            cache = self.project.cache.get(md5)
-
-            if os.path.exists(cache):
-                self._remove(path)
-                self.hardlink(cache, path)
-            else:
-                self.hardlink(path, cache)
-
-        if not os.path.isdir(dname):
-            os.makedirs(dname)
-
-        with open(self.cache, 'w+') as fd:
-            json.dump(dir_info, fd)
-
-    def save(self):
-        super(Output, self).save()
-
-        if not self.use_cache:
-            return
-
-        self.project.logger.debug(u'Saving \'{}\' to \'{}\''.format(self.rel_path, self.rel_cache))
-
-        if self.project.scm.is_tracked(self.path):
-            raise CmdOutputAlreadyTrackedError(self.rel_path)
-
-        if not self.changed():
-             return
-
-        if os.path.exists(self.cache):
-            # This means that we already have cache for this data.
-            # We remove data and link it to existing cache to save
-            # some space.
-            msg = u'Cache \'{}\' already exists, performing checkout for \'{}\''
-            self.project.logger.debug(msg.format(self.rel_cache, self.rel_path))
-            self.checkout()
-            return
-
-        if os.path.isdir(self.path):
-            self._save_dir()
-        else:
-            self.hardlink(self.path, self.cache)
-
-    def _remove(self, path):
-        if not os.path.exists(path):
-            return
-
-        self.project.logger.debug(u'Removing \'{}\''.format(os.path.relpath(path)))
-        if os.path.isfile(path):
-            os.unlink(path)
-        else:
-            shutil.rmtree(path)
-
-    def remove(self):
-        self._remove(self.path)

dvc/output/__init__.py

@@ -0,0 +1,62 @@
+import schema
+
+from dvc.exceptions import DvcException
+from dvc.config import Config
+
+from dvc.dependency import SCHEMA, urlparse
+from dvc.dependency.base import DependencyBase
+from dvc.output.s3 import OutputS3
+from dvc.output.gs import OutputGS
+from dvc.output.local import OutputLOCAL
+from dvc.output.hdfs import OutputHDFS
+from dvc.output.ssh import OutputSSH
+
+from dvc.remote import Remote
+
+
+OUTS = [OutputHDFS, OutputS3, OutputGS, OutputSSH, OutputLOCAL]
+
+OUTS_MAP = {'hdfs': OutputHDFS,
+            's3': OutputS3,
+            'gs': OutputGS,
+            'ssh': OutputSSH,
+            '': OutputLOCAL}
+
+SCHEMA[schema.Optional(OutputLOCAL.PARAM_CACHE)] = bool
+SCHEMA[schema.Optional(OutputLOCAL.PARAM_METRIC)] = OutputLOCAL.METRIC_SCHEMA
+
+
+def _get(stage, p, info, cache, metric):
+    parsed = urlparse(p)
+    if parsed.scheme == 'remote':
+        name = Config.SECTION_REMOTE_FMT.format(parsed.netloc)
+        sect = stage.project.config._config[name]
+        remote = Remote(stage.project, sect)
+        return OUTS_MAP[remote.scheme](stage,
+                                       p,
+                                       info,
+                                       cache=cache,
+                                       remote=remote,
+                                       metric=metric)
+
+    for o in OUTS:
+        if o.supported(p):
+            return o(stage, p, info, cache=cache, remote=None, metric=metric)
+    raise DvcException('Output \'{}\' is not supported'.format(p))
+
+
+def loadd_from(stage, d_list):
+    ret = []
+    for d in d_list:
+        p = d.pop(DependencyBase.PARAM_PATH)
+        cache = d.pop(OutputLOCAL.PARAM_CACHE, True)
+        metric = d.pop(OutputLOCAL.PARAM_METRIC, False)
+        ret.append(_get(stage, p, info=d, cache=cache, metric=metric))
+    return ret
+
+
+def loads_from(stage, s_list, use_cache=True, metric=False):
+    ret = []
+    for s in s_list:
+        ret.append(_get(stage, s, info={}, cache=use_cache, metric=metric))
+    return ret

dvc/output/base.py

@@ -0,0 +1,24 @@
+from dvc.exceptions import DvcException
+
+
+class OutputError(DvcException):
+    def __init__(self, path, msg):
+        msg = 'Output \'{}\' error: {}'.format(path, msg)
+        super(OutputError, self).__init__(msg)
+
+
+class OutputDoesNotExistError(OutputError):
+    def __init__(self, path):
+        super(OutputDoesNotExistError, self).__init__(path, 'does not exist')
+
+
+class OutputIsNotFileOrDirError(OutputError):
+    def __init__(self, path):
+        msg = 'not a file or directory'
+        super(OutputIsNotFileOrDirError, self).__init__(path, msg)
+
+
+class OutputAlreadyTrackedError(OutputError):
+    def __init__(self, path):
+        msg = 'already tracked by scm(e.g. git)'
+        super(OutputAlreadyTrackedError, self).__init__(path, msg)

dvc/output/gs.py

@@ -0,0 +1,53 @@
+from dvc.dependency.gs import DependencyGS
+from dvc.exceptions import DvcException
+
+
+class OutputGS(DependencyGS):
+    PARAM_CACHE = 'cache'
+    PARAM_METRIC = 'metric'
+
+    def __init__(self,
+                 stage,
+                 path,
+                 info=None,
+                 remote=None,
+                 cache=True,
+                 metric=False):
+        super(OutputGS, self).__init__(stage, path, info, remote=remote)
+        self.use_cache = cache
+        self.metric = metric
+        if cache and self.project.cache.gs is None:
+            raise DvcException("No cache location setup for \'gs\' outputs.")
+
+    def dumpd(self):
+        ret = super(OutputGS, self).dumpd()
+        ret[self.PARAM_CACHE] = self.use_cache
+        ret[self.PARAM_METRIC] = self.metric
+        return ret
+
+    def changed(self):
+        if super(OutputGS, self).changed():
+            return True
+
+        if self.use_cache \
+           and self.project.cache.gs.changed(self.path_info, self.info):
+            return True
+
+        return False
+
+    def checkout(self):
+        if not self.use_cache:
+            return
+
+        self.project.cache.gs.checkout(self.path_info, self.info)
+
+    def save(self):
+        super(OutputGS, self).save()
+
+        if not self.use_cache:
+            return
+
+        self.info = self.project.cache.gs.save(self.path_info)
+
+    def remove(self, ignore_remove=False):
+        self.remote.remove(self.path_info)

dvc/output/hdfs.py

@@ -0,0 +1,53 @@
+from dvc.dependency.hdfs import DependencyHDFS
+from dvc.exceptions import DvcException
+
+
+class OutputHDFS(DependencyHDFS):
+    PARAM_CACHE = 'cache'
+    PARAM_METRIC = 'metric'
+
+    def __init__(self,
+                 stage,
+                 path,
+                 info=None,
+                 remote=None,
+                 cache=True,
+                 metric=False):
+        super(OutputHDFS, self).__init__(stage, path, info, remote=remote)
+        self.use_cache = cache
+        self.metric = metric
+        if cache and self.project.cache.hdfs is None:
+            raise DvcException("No cache location setup for \'hdfs\' outputs.")
+
+    def dumpd(self):
+        ret = super(OutputHDFS, self).dumpd()
+        ret[self.PARAM_CACHE] = self.use_cache
+        ret[self.PARAM_METRIC] = self.metric
+        return ret
+
+    def changed(self):
+        if super(OutputHDFS, self).changed():
+            return True
+
+        if self.use_cache \
+           and self.project.cache.hdfs.changed(self.path_info, self.info):
+            return True
+
+        return False
+
+    def checkout(self):
+        if not self.use_cache:
+            return
+
+        self.project.cache.hdfs.checkout(self.path_info, self.info)
+
+    def save(self):
+        super(OutputHDFS, self).save()
+
+        if not self.use_cache:
+            return
+
+        self.info = self.project.cache.hdfs.save(self.path_info)
+
+    def remove(self, ignore_remove=False):
+        self.remote.remove(self.path_info)

dvc/output/local.py

@@ -0,0 +1,129 @@
+import os
+from schema import Optional, Or
+
+from dvc.output.base import OutputDoesNotExistError, OutputIsNotFileOrDirError
+from dvc.output.base import OutputAlreadyTrackedError
+from dvc.dependency.local import DependencyLOCAL
+from dvc.exceptions import DvcException
+from dvc.istextfile import istextfile
+
+
+class OutputLOCAL(DependencyLOCAL):
+    PARAM_CACHE = 'cache'
+    PARAM_METRIC = 'metric'
+    PARAM_METRIC_TYPE = 'type'
+    PARAM_METRIC_XPATH = 'xpath'
+
+    METRIC_SCHEMA = Or(None, bool,
+                       {Optional(PARAM_METRIC_TYPE): Or(str, None),
+                        Optional(PARAM_METRIC_XPATH): Or(str, None)})
+
+    DoesNotExistError = OutputDoesNotExistError
+    IsNotFileOrDirError = OutputIsNotFileOrDirError
+
+    def __init__(self,
+                 stage,
+                 path,
+                 info=None,
+                 remote=None,
+                 cache=True,
+                 metric=False):
+        super(OutputLOCAL, self).__init__(stage, path, info, remote=remote)
+        self.use_cache = cache
+        self.metric = metric
+
+    @property
+    def md5(self):
+        return self.info.get(self.project.cache.local.PARAM_MD5, None)
+
+    @property
+    def cache(self):
+        return self.project.cache.local.get(self.md5)
+
+    def dumpd(self):
+        ret = super(OutputLOCAL, self).dumpd()
+        ret[self.PARAM_CACHE] = self.use_cache
+
+        if isinstance(self.metric, dict):
+            if self.PARAM_METRIC_XPATH in self.metric and \
+               not self.metric[self.PARAM_METRIC_XPATH]:
+                del self.metric[self.PARAM_METRIC_XPATH]
+
+        if self.metric:
+            ret[self.PARAM_METRIC] = self.metric
+
+        return ret
+
+    def changed(self):
+        if not self.use_cache:
+            return super(OutputLOCAL, self).changed()
+
+        return self.project.cache.local.changed(self.path_info, self.info)
+
+    def checkout(self):
+        if not self.use_cache:
+            return
+
+        self.project.cache.local.checkout(self.path_info, self.info)
+
+    def _verify_metric(self):
+        if not self.metric:
+            return
+
+        if os.path.isdir(self.path):
+            msg = 'Directory \'{}\' cannot be used as metrics.'
+            raise DvcException(msg.format(self.rel_path))
+
+        if not istextfile(self.path):
+            msg = 'Binary file \'{}\' cannot be used as metrics.'
+            raise DvcException(msg.format(self.rel_path))
+
+    def save(self):
+        if not self.use_cache:
+            super(OutputLOCAL, self).save()
+            self._verify_metric()
+            msg = 'Output \'{}\' doesn\'t use cache. Skipping saving.'
+            self.project.logger.debug(msg.format(self.rel_path))
+            return
+
+        if not os.path.exists(self.path):
+            raise self.DoesNotExistError(self.rel_path)
+
+        if not os.path.isfile(self.path) and not os.path.isdir(self.path):
+            raise self.IsNotFileOrDirError(self.rel_path)
+
+        if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \
+           (os.path.isdir(self.path) and len(os.listdir(self.path)) == 0):
+            msg = "File/directory '{}' is empty.".format(self.rel_path)
+            self.project.logger.warn(msg)
+
+        if not self.changed():
+            msg = 'Output \'{}\' didn\'t change. Skipping saving.'
+            self.project.logger.debug(msg.format(self.rel_path))
+            return
+
+        if self.is_local:
+            if self.project.scm.is_tracked(self.path):
+                raise OutputAlreadyTrackedError(self.rel_path)
+
+            if self.use_cache:
+                self.project.scm.ignore(self.path)
+
+        self.info = self.project.cache.local.save(self.path_info)
+
+    def remove(self, ignore_remove=False):
+        self.remote.remove(self.path_info)
+        if ignore_remove and self.use_cache and self.is_local:
+            self.project.scm.ignore_remove(self.path)
+
+    def move(self, out):
+        if self.use_cache and self.is_local:
+            self.project.scm.ignore_remove(self.path)
+
+        self.remote.move(self.path_info, out.path_info)
+        self.path = out.path
+        self.path_info = out.path_info
+        self.save()
+
+        if self.use_cache and self.is_local:
+            self.project.scm.ignore(self.path)

dvc/output/s3.py

@@ -0,0 +1,53 @@
+from dvc.dependency.s3 import DependencyS3
+from dvc.exceptions import DvcException
+
+
+class OutputS3(DependencyS3):
+    PARAM_CACHE = 'cache'
+    PARAM_METRIC = 'metric'
+
+    def __init__(self,
+                 stage,
+                 path,
+                 info=None,
+                 remote=None,
+                 cache=True,
+                 metric=False):
+        super(OutputS3, self).__init__(stage, path, info, remote=remote)
+        self.use_cache = cache
+        self.metric = metric
+        if cache and self.project.cache.s3 is None:
+            raise DvcException("No cache location setup for \'s3\' outputs.")
+
+    def dumpd(self):
+        ret = super(OutputS3, self).dumpd()
+        ret[self.PARAM_CACHE] = self.use_cache
+        ret[self.PARAM_METRIC] = self.metric
+        return ret
+
+    def changed(self):
+        if super(OutputS3, self).changed():
+            return True
+
+        if self.use_cache \
+           and self.project.cache.s3.changed(self.path_info, self.info):
+            return True
+
+        return False
+
+    def checkout(self):
+        if not self.use_cache:
+            return
+
+        self.project.cache.s3.checkout(self.path_info, self.info)
+
+    def save(self):
+        super(OutputS3, self).save()
+
+        if not self.use_cache:
+            return
+
+        self.info = self.project.cache.s3.save(self.path_info)
+
+    def remove(self, ignore_remove=False):
+        self.remote.remove(self.path_info)

dvc/output/ssh.py

@@ -0,0 +1,53 @@
+from dvc.dependency.ssh import DependencySSH
+from dvc.exceptions import DvcException
+
+
+class OutputSSH(DependencySSH):
+    PARAM_CACHE = 'cache'
+    PARAM_METRIC = 'metric'
+
+    def __init__(self,
+                 stage,
+                 path,
+                 info=None,
+                 remote=None,
+                 cache=True,
+                 metric=False):
+        super(OutputSSH, self).__init__(stage, path, info, remote=remote)
+        self.use_cache = cache
+        self.metric = metric
+        if cache and self.project.cache.ssh is None:
+            raise DvcException("No cache location setup for \'ssh\' outputs.")
+
+    def dumpd(self):
+        ret = super(OutputSSH, self).dumpd()
+        ret[self.PARAM_CACHE] = self.use_cache
+        ret[self.PARAM_METRIC] = self.metric
+        return ret
+
+    def changed(self):
+        if super(OutputSSH, self).changed():
+            return True
+
+        if self.use_cache \
+           and self.project.cache.ssh.changed(self.path_info, self.info):
+            return True
+
+        return False
+
+    def checkout(self):
+        if not self.use_cache:
+            return
+
+        self.project.cache.ssh.checkout(self.path_info, self.info)
+
+    def save(self):
+        super(OutputSSH, self).save()
+
+        if not self.use_cache:
+            return
+
+        self.info = self.project.cache.ssh.save(self.path_info)
+
+    def remove(self, ignore_remove=False):
+        self.remote.remove(self.path_info)

dvc/progress.py

@@ -1,6 +1,7 @@
 from __future__ import print_function
 import sys
-import threading 
+import threading
+
 
 class Progress(object):
     """
@@ -15,6 +16,10 @@ class Progress(object):
         self._n_total = total
         self._n_finished = 0
 
+    @property
+    def is_finished(self):
+        return self._n_total == self._n_finished
+
     def _clearln(self):
         print('\r\x1b[K', end='')
 
@@ -47,19 +52,14 @@ class Progress(object):
 
             self._n_finished += 1
 
-    def finish(self):
-        if sys.stdout.isatty():
-            print()
-
     def _bar(self, target_name, current, total):
         """
         Make a progress bar out of info, which looks like:
         (1/2): [########################################] 100% master.zip
         """
-        total_len = 100
         bar_len = 30
 
-        if total == None:
+        if total is None:
             progress = 0
             percent = "?% "
         else:
@@ -67,15 +67,16 @@ class Progress(object):
             progress = int((100 * current)/total) if current < total else 100
             percent = str(progress) + "% "
 
-        num = "({}/{}): ".format(self._n_finished + 1, self._n_total)
+        if self._n_total > 1:
+            num = "({}/{}): ".format(self._n_finished + 1, self._n_total)
+        else:
+            num = ""
 
         n_sh = int((progress * bar_len)/100)
         n_sp = bar_len - n_sh
         bar = "[" + '#'*n_sh + ' '*n_sp + "] "
 
-        name_len = total_len - len(num + bar + percent)
-        name = target_name[:name_len] if len(target_name) > name_len else target_name
+        return num + bar + percent + target_name
 
-        return num + bar + percent + name
 
 progress = Progress()

dvc/prompt.py

@@ -0,0 +1,40 @@
+import sys
+
+from dvc.progress import progress
+
+try:
+    # NOTE: in Python3 raw_input() was renamed to input()
+    input = raw_input
+except NameError:
+    pass
+
+
+class Prompt(object):
+    def __init__(self):
+        self.default = None
+
+    def prompt(self, msg, default=False):  # pragma: no cover
+        if self.default is not None:
+            return self.default
+
+        if not sys.stdout.isatty():
+            return default
+
+        answer = input(msg + u' (y/n)\n').lower()
+        while answer not in ['yes', 'no', 'y', 'n']:
+            answer = input('Enter \'yes\' or \'no\'.\n').lower()
+
+        return answer[0] == "y"
+
+    def prompt_password(self, msg):  # pragma: no cover
+        import getpass
+
+        if not sys.stdout.isatty():
+            return None
+
+        msg = 'Enter password for {}:\n'.format(msg)
+
+        if not progress.is_finished:
+            msg = u'\n' + msg
+
+        return getpass.getpass(msg)

dvc/remote/__init__.py

@@ -0,0 +1,27 @@
+from dvc.remote.local import RemoteLOCAL
+from dvc.remote.s3 import RemoteS3
+from dvc.remote.gs import RemoteGS
+from dvc.remote.hdfs import RemoteHDFS
+from dvc.remote.ssh import RemoteSSH
+from dvc.remote.azure import RemoteAzure
+
+from dvc.config import Config
+from dvc.exceptions import UnsupportedRemoteError
+
+
+REMOTES = [RemoteHDFS, RemoteSSH, RemoteS3, RemoteGS, RemoteAzure, RemoteLOCAL]
+
+
+def supported_url(url):
+    config = {Config.SECTION_REMOTE_URL: url}
+    for r in REMOTES:
+        if r.supported(config):
+            return True
+    return False
+
+
+def Remote(project, config):
+    for r in REMOTES:
+        if r.supported(config):
+            return r(project, config)
+    raise UnsupportedRemoteError(str(config))

dvc/remote/azure.py

@@ -0,0 +1,244 @@
+from __future__ import absolute_import
+import os
+import re
+
+try:
+    from azure.storage.blob import BlockBlobService
+except ImportError:
+    BlockBlobService = None
+
+from dvc.logger import Logger
+from dvc.progress import progress
+from dvc.config import Config
+from dvc.remote.base import RemoteBase
+
+
+class Callback(object):
+    def __init__(self, name):
+        self.name = name
+
+    def __call__(self, current, total):
+        progress.update_target(self.name, current, total)
+
+
+class RemoteAzure(RemoteBase):
+    scheme = 'azure'
+    REGEX = (r'^azure://'
+             r'(ContainerName=(?P<container_name>[^;]+);?)?'
+             r'(?P<connection_string>.+)?$')
+    REQUIRES = {'azure-storage-blob': BlockBlobService}
+    PARAM_ETAG = 'etag'
+    COPY_POLL_SECONDS = 5
+
+    def __init__(self, project, config):
+        super(RemoteAzure, self).__init__(project, config)
+        self.project = project
+
+        self.url = config.get(Config.SECTION_REMOTE_URL)
+        match = re.match(self.REGEX, self.url)
+
+        self.bucket = (
+            match.group('container_name')
+            or os.getenv('AZURE_STORAGE_CONTAINER_NAME'))
+
+        self.connection_string = (
+            match.group('connection_string')
+            or os.getenv('AZURE_STORAGE_CONNECTION_STRING'))
+
+        if not self.bucket:
+            raise ValueError('Azure Storage container name missing')
+
+        if not self.connection_string:
+            raise ValueError('Azure Storage connection string missing')
+
+        self.__blob_service = None
+
+    @property
+    def blob_service(self):
+        if self.__blob_service is None:
+            self.__blob_service = BlockBlobService(
+                connection_string=self.connection_string)
+            self.__blob_service.create_container(self.bucket)
+        return self.__blob_service
+
+# FIXME: temporarily disabled because of the lack of test for external azure
+# dependencies/outputs/cache.
+#
+#    def _get_etag(self, bucket, key):
+#        try:
+#            blob = self.blob_service.get_blob_properties(bucket, key)
+#            return blob.properties.etag
+#        except Exception:
+#            return None
+#
+#    def save_info(self, path_info):
+#        if path_info['scheme'] != self.scheme:
+#            raise NotImplementedError
+#
+#        return {self.PARAM_ETAG: self._get_etag(
+#            path_info['bucket'], path_info['key'])}
+#
+#    def save(self, path_info):
+#        if path_info['scheme'] != self.scheme:
+#            raise NotImplementedError
+#
+#        etag = self._get_etag(path_info['bucket'], path_info['key'])
+#        dest_key = '{}/{}'.format(etag[0:2], etag[2:])
+#
+#        self._copy_blob(to_bucket=self.bucket,
+#                        to_key=dest_key,
+#                        from_bucket=path_info['bucket'],
+#                        from_key=path_info['key'])
+#
+#        return {self.PARAM_ETAG: etag}
+#
+#    def changed(self, path_info, checksum_info):
+#        return checksum_info != self.save_info(path_info)
+#
+#    def _copy_blob(self, to_bucket, to_key, from_bucket, from_key):
+#        source = self.blob_service.make_blob_url(from_bucket, from_key)
+#
+#        copy = self.blob_service.copy_blob(to_bucket, to_key, source)
+#
+#        if self.COPY_POLL_SECONDS <= 0:
+#            return
+#
+#        while copy.status != 'success':
+#            time.sleep(self.COPY_POLL_SECONDS)
+#            copy = self.blob_service.get_blob_properties(
+#                to_bucket, to_key).properties.copy
+#
+#    def checkout(self, path_info, checksum_info):
+#        if path_info['scheme'] != self.scheme:
+#            raise NotImplementedError
+#
+#        etag = checksum_info.get(self.PARAM_ETAG, None)
+#        if not etag:
+#            return
+#
+#        self._copy_blob(to_bucket=path_info['bucket'],
+#                        to_key=path_info['key'],
+#                        from_bucket=self.bucket,
+#                        from_key='{}/{}'.format(etag[0:2], etag[2:]))
+#
+#    def remove(self, path_info):
+#        if path_info['scheme'] != self.scheme:
+#            raise NotImplementedError
+#
+#        Logger.debug('Removing azure://{}/{}'.format(path_info['bucket'],
+#                                                     path_info['key']))
+#
+#        self.blob_service.delete_blob(path_info['bucket'], path_info['key'])
+
+    def md5s_to_path_infos(self, md5s):
+        return [{
+            'scheme': self.scheme,
+            'bucket': self.bucket,
+            'key': '{}/{}'.format(md5[0:2], md5[2:])
+        } for md5 in md5s]
+
+    def exists(self, path_infos):
+        keys = {blob.name
+                for blob in self.blob_service.list_blobs(self.bucket)}
+
+        ret = []
+        for path_info in path_infos:
+            if path_info['scheme'] != self.scheme:
+                raise NotImplementedError
+
+            key = path_info['key']
+            ret.append(key in keys)
+
+        return ret
+
+    def upload(self, from_infos, to_infos, names=None):
+        names = self._verify_path_args(to_infos, from_infos, names)
+
+        for from_info, to_info, name in zip(from_infos, to_infos, names):
+            if to_info['scheme'] != self.scheme:
+                raise NotImplementedError
+
+            if from_info['scheme'] != 'local':
+                raise NotImplementedError
+
+            bucket = to_info['bucket']
+            key = to_info['key']
+
+            Logger.debug("Uploading '{}' to '{}/{}'".format(
+                from_info['path'], bucket, key))
+
+            if not name:
+                name = os.path.basename(from_info['path'])
+
+            cb = Callback(name)
+
+            try:
+                self.blob_service.create_blob_from_path(
+                    bucket, key, from_info['path'], progress_callback=cb)
+            except Exception as ex:
+                Logger.error("Failed to upload '{}'".format(from_info['path']),
+                             ex)
+            else:
+                progress.finish_target(name)
+
+    def download(self,
+                 from_infos,
+                 to_infos,
+                 no_progress_bar=False,
+                 names=None):
+        names = self._verify_path_args(from_infos, to_infos, names)
+
+        for to_info, from_info, name in zip(to_infos, from_infos, names):
+            if from_info['scheme'] != self.scheme:
+                raise NotImplementedError
+
+            if to_info['scheme'] != 'local':
+                raise NotImplementedError
+
+            bucket = from_info['bucket']
+            key = from_info['key']
+
+            Logger.debug("Downloading '{}/{}' to '{}'".format(
+                bucket, key, to_info['path']))
+
+            tmp_file = self.tmp_file(to_info['path'])
+            if not name:
+                name = os.path.basename(to_info['path'])
+
+            cb = None if no_progress_bar else Callback(name)
+
+            self._makedirs(to_info['path'])
+
+            try:
+                self.blob_service.get_blob_to_path(
+                    bucket, key, tmp_file, progress_callback=cb)
+            except Exception as exc:
+                Logger.error("Failed to download '{}/{}'".format(
+                    bucket, key), exc)
+            else:
+                os.rename(tmp_file, to_info['path'])
+
+                if not no_progress_bar:
+                    progress.finish_target(name)
+
+# FIXME: temporarily disabled because of the lack of test for external azure
+# dependencies/outputs/cache.
+#
+#    def gc(self, cinfos):
+#        used = [info[self.PARAM_ETAG] for info in cinfos['azure']]
+#        used += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
+#
+#        all_blobs = self.blob_service.list_blobs(self.bucket)
+#
+#        removed = False
+#        for blob in all_blobs:
+#            etag = blob.properties.etag
+#            if etag in used:
+#                continue
+#            path_info = {'scheme': self.scheme,
+#                         'key': blob.name,
+#                         'bucket': self.bucket}
+#            self.remove(path_info)
+#            removed = True
+#
+#        return removed

dvc/remote/base.py

@@ -0,0 +1,132 @@
+import os
+import re
+import errno
+
+from dvc.config import Config
+from dvc.logger import Logger
+from dvc.exceptions import DvcException
+
+
+STATUS_OK = 1
+STATUS_NEW = 3
+STATUS_DELETED = 4
+
+
+STATUS_MAP = {
+    # (local_exists, remote_exists)
+    (True, True): STATUS_OK,
+    (False, False): STATUS_OK,
+    (True, False): STATUS_NEW,
+    (False, True): STATUS_DELETED,
+}
+
+
+class DataCloudError(DvcException):
+    """ Data Cloud exception """
+    def __init__(self, msg):
+        super(DataCloudError, self).__init__('Data sync error: {}'.format(msg))
+
+
+class RemoteBase(object):
+    REGEX = None
+    REQUIRES = {}
+
+    def __init__(self, project, config):
+        pass
+
+    @classmethod
+    def supported(cls, config):
+        url = config[Config.SECTION_REMOTE_URL]
+        url_ok = cls.match(url)
+        deps_ok = all(cls.REQUIRES.values())
+        if url_ok and not deps_ok:
+            missing = [k for k, v in cls.REQUIRES.items() if v is None]
+            msg = "URL \'{}\' is supported but requires these missing " \
+                  "dependencies: {}. If you have installed dvc using pip, " \
+                  "choose one of these options to proceed: \n" \
+                  "\n" \
+                  "    1) Install specific missing dependencies:\n" \
+                  "        pip install {}\n" \
+                  "    2) Install dvc package that includes those missing " \
+                  "dependencies: \n" \
+                  "        pip install dvc[{}]\n" \
+                  "    3) Install dvc package with all possible " \
+                  "dependencies included: \n" \
+                  "        pip install dvc[all]\n" \
+                  "\n" \
+                  "If you have installed dvc from a binary package and you " \
+                  "are still seeing this message, please report it to us " \
+                  "using https://github.com/iterative/dvc/issues. Thank you!"
+            msg = msg.format(url, missing, " ".join(missing), cls.scheme)
+            Logger.warn(msg)
+
+        return url_ok and deps_ok
+
+    @classmethod
+    def match(cls, url):
+        return re.match(cls.REGEX, url)
+
+    def group(self, name):
+        m = self.match(self.url)
+        if not m:
+            return None
+        return m.group(name)
+
+    @staticmethod
+    def tmp_file(fname):
+        """ Temporary name for a partial download """
+        # FIXME probably better use uuid()
+        return fname + '.part'
+
+    def save_info(self, path_info):
+        raise NotImplementedError
+
+    def changed(self, path_info, checksum_info):
+        raise NotImplementedError
+
+    def save(self, path_info):
+        raise NotImplementedError
+
+    def checkout(self, path_info, checksum_info):
+        raise NotImplementedError
+
+    def download(self, from_infos, to_infos, no_progress_bar=False, name=None):
+        raise NotImplementedError
+
+    def upload(self, from_infos, to_infos, path_info, name=None):
+        raise NotImplementedError
+
+    def remove(self, path_info):
+        raise NotImplementedError
+
+    def move(self, path_info):
+        raise NotImplementedError
+
+    def _makedirs(self, fname):
+        dname = os.path.dirname(fname)
+
+        try:
+            os.makedirs(dname)
+        except OSError as e:
+            if e.errno != errno.EEXIST:
+                raise
+
+    def md5s_to_path_infos(self, md5s):
+        raise NotImplementedError
+
+    def exists(self, path_infos):
+        raise NotImplementedError
+
+    @classmethod
+    def _verify_path_args(cls, from_infos, to_infos, names=None):
+        assert isinstance(from_infos, list)
+        assert isinstance(to_infos, list)
+        assert len(from_infos) == len(to_infos)
+
+        if not names:
+            names = len(to_infos) * [None]
+        else:
+            assert isinstance(names, list)
+            assert len(names) == len(to_infos)
+
+        return names

dvc/remote/gs.py

@@ -0,0 +1,293 @@
+import os
+import posixpath
+
+try:
+    from google.cloud import storage
+except ImportError:
+    storage = None
+
+try:
+    from urlparse import urlparse
+except ImportError:
+    from urllib.parse import urlparse
+
+from dvc.logger import Logger
+from dvc.remote.base import RemoteBase
+from dvc.remote.local import RemoteLOCAL
+from dvc.config import Config
+from dvc.progress import progress
+from dvc.exceptions import DvcException
+
+
+class RemoteGS(RemoteBase):
+    scheme = 'gs'
+    REGEX = r'^gs://(?P<path>.*)$'
+    REQUIRES = {'google.cloud.storage': storage}
+    PARAM_ETAG = 'etag'
+
+    def __init__(self, project, config):
+        self.project = project
+        storagepath = 'gs://'
+        storagepath += config.get(Config.SECTION_AWS_STORAGEPATH, '/')
+        storagepath.lstrip('/')
+        self.url = config.get(Config.SECTION_REMOTE_URL, storagepath)
+        self.projectname = config.get(Config.SECTION_GCP_PROJECTNAME, None)
+
+    @property
+    def bucket(self):
+        return urlparse(self.url).netloc
+
+    @property
+    def prefix(self):
+        return urlparse(self.url).path.lstrip('/')
+
+    @property
+    def gs(self):
+        return storage.Client()
+
+    def get_etag(self, bucket, key):
+        blob = self.gs.bucket(bucket).get_blob(key)
+        if not blob:
+            return None
+
+        return blob.etag
+
+    def save_info(self, path_info):
+        if path_info['scheme'] != 'gs':
+            raise NotImplementedError
+
+        return {self.PARAM_ETAG: self.get_etag(path_info['bucket'],
+                                               path_info['key'])}
+
+    @staticmethod
+    def to_string(path_info):
+        return "{}://{}/{}".format(path_info['scheme'],
+                                   path_info['bucket'],
+                                   path_info['key'])
+
+    def changed_cache(self, etag):
+        key = posixpath.join(self.prefix, etag[0:2], etag[2:])
+        cache = {'scheme': 'gs', 'bucket': self.bucket, 'key': key}
+
+        if {self.PARAM_ETAG: etag} != self.save_info(cache):
+            if self.exists([cache])[0]:
+                msg = 'Corrupted cache file {}'
+                Logger.warn(msg.format(self.to_string(cache)))
+                self.remove(cache)
+            return True
+        return False
+
+    def changed(self, path_info, checksum_info):
+        if not self.exists([path_info])[0]:
+            return True
+
+        etag = checksum_info.get(self.PARAM_ETAG, None)
+        if etag is None:
+            return True
+
+        if self.changed_cache(etag):
+            return True
+
+        return checksum_info != self.save_info(path_info)
+
+    def _copy(self, from_info, to_info, gs=None):
+        gs = gs if gs else self.gs
+
+        blob = gs.bucket(from_info['bucket']).get_blob(from_info['key'])
+        if not blob:
+            msg = '{} doesn\'t exist in the cloud'
+            raise DvcException(msg.format(from_info['key']))
+
+        bucket = self.gs.bucket(to_info['bucket'])
+        bucket.copy_blob(blob,
+                         self.gs.bucket(to_info['bucket']),
+                         new_name=to_info['key'])
+
+    def save(self, path_info):
+        if path_info['scheme'] != 'gs':
+            raise NotImplementedError
+
+        etag = self.get_etag(path_info['bucket'], path_info['key'])
+        key = posixpath.join(self.prefix, etag[0:2], etag[2:])
+        to_info = {'scheme': 'gs', 'bucket': self.bucket, 'key': key}
+
+        self._copy(path_info, to_info)
+
+        return {self.PARAM_ETAG: etag}
+
+    def checkout(self, path_info, checksum_info):
+        if path_info['scheme'] != 'gs':
+            raise NotImplementedError
+
+        etag = checksum_info.get(self.PARAM_ETAG, None)
+        if not etag:
+            return
+
+        if not self.changed(path_info, checksum_info):
+            msg = "Data '{}' didn't change."
+            Logger.info(msg.format(self.to_string(path_info)))
+            return
+
+        if self.changed_cache(etag):
+            msg = "Cache '{}' not found. File '{}' won't be created."
+            Logger.warn(msg.format(etag, self.to_string(path_info)))
+            return
+
+        if self.exists([path_info])[0]:
+            msg = "Data '{}' exists. Removing before checkout."
+            Logger.warn(msg.format(self.to_string(path_info)))
+            self.remove(path_info)
+            return
+
+        msg = "Checking out '{}' with cache '{}'."
+        Logger.info(msg.format(self.to_string(path_info), etag))
+
+        key = posixpath.join(self.prefix, etag[0:2], etag[2:])
+        from_info = {'scheme': 'gs', 'bucket': self.bucket, 'key': key}
+
+        self._copy(from_info, path_info)
+
+    def remove(self, path_info):
+        if path_info['scheme'] != 'gs':
+            raise NotImplementedError
+
+        Logger.debug("Removing gs://{}/{}".format(path_info['bucket'],
+                                                  path_info['key']))
+
+        blob = self.gs.bucket(path_info['bucket']).get_blob(path_info['key'])
+        if not blob:
+            return
+
+        blob.delete()
+
+    def md5s_to_path_infos(self, md5s):
+        return [{'scheme': 'gs',
+                 'bucket': self.bucket,
+                 'key': posixpath.join(self.prefix,
+                                       md5[0:2], md5[2:])} for md5 in md5s]
+
+    def exists(self, path_infos):
+        ret = []
+        gs = self.gs
+
+        bucket = gs.bucket(self.bucket)
+        keys = [blob.name for blob in bucket.list_blobs(prefix=self.prefix)]
+
+        for path_info in path_infos:
+            exists = False
+            if path_info['key'] in keys:
+                exists = True
+            ret.append(exists)
+        return ret
+
+    def upload(self, from_infos, to_infos, names=None):
+        names = self._verify_path_args(to_infos, from_infos, names)
+
+        gs = self.gs
+
+        for from_info, to_info, name in zip(from_infos, to_infos, names):
+            if to_info['scheme'] != 'gs':
+                raise NotImplementedError
+
+            if from_info['scheme'] != 'local':
+                raise NotImplementedError
+
+            Logger.debug("Uploading '{}' to '{}/{}'".format(from_info['path'],
+                                                            to_info['bucket'],
+                                                            to_info['key']))
+
+            if not name:
+                name = os.path.basename(from_info['path'])
+
+            progress.update_target(name, 0, None)
+
+            try:
+                bucket = gs.bucket(to_info['bucket'])
+                blob = bucket.blob(to_info['key'])
+                blob.upload_from_filename(from_info['path'])
+            except Exception as exc:
+                msg = "Failed to upload '{}' to '{}/{}'"
+                Logger.error(msg.format(from_info['path'],
+                                        to_info['bucket'],
+                                        to_info['key']), exc)
+                continue
+
+            progress.finish_target(name)
+
+    def download(self,
+                 from_infos,
+                 to_infos,
+                 no_progress_bar=False,
+                 names=None):
+        names = self._verify_path_args(from_infos, to_infos, names)
+
+        gs = self.gs
+
+        for to_info, from_info, name in zip(to_infos, from_infos, names):
+            if from_info['scheme'] != 'gs':
+                raise NotImplementedError
+
+            if to_info['scheme'] == 'gs':
+                self._copy(from_info, to_info, gs=gs)
+                continue
+
+            if to_info['scheme'] != 'local':
+                raise NotImplementedError
+
+            msg = "Downloading '{}/{}' to '{}'".format(from_info['bucket'],
+                                                       from_info['key'],
+                                                       to_info['path'])
+            Logger.debug(msg)
+
+            tmp_file = self.tmp_file(to_info['path'])
+            if not name:
+                name = os.path.basename(to_info['path'])
+
+            if not no_progress_bar:
+                # percent_cb is not available for download_to_filename, so
+                # lets at least update progress at keypoints(start, finish)
+                progress.update_target(name, 0, None)
+
+            self._makedirs(to_info['path'])
+
+            try:
+                bucket = gs.bucket(from_info['bucket'])
+                blob = bucket.get_blob(from_info['key'])
+                blob.download_to_filename(tmp_file)
+            except Exception as exc:
+                msg = "Failed to download '{}/{}' to '{}'"
+                Logger.error(msg.format(from_info['bucket'],
+                                        from_info['key'],
+                                        to_info['path']), exc)
+                continue
+
+            os.rename(tmp_file, to_info['path'])
+
+            if not no_progress_bar:
+                progress.finish_target(name)
+
+    def _path_to_etag(self, path):
+        relpath = posixpath.relpath(path, self.prefix)
+        return posixpath.dirname(relpath) + posixpath.basename(relpath)
+
+    def _all_etags(self):
+        blobs = self.gs.bucket(self.bucket).list_blobs(prefix=self.prefix)
+        blobs = list(blobs)
+        return [self._path_to_etag(blob.name) for blob in blobs]
+
+    def gc(self, cinfos):
+        used = [info[self.PARAM_ETAG] for info in cinfos['gs']]
+        used += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
+
+        removed = False
+        for etag in self._all_etags():
+            if etag in used:
+                continue
+            path_info = {'scheme': 'gs',
+                         'bucket': self.bucket,
+                         'key': posixpath.join(self.prefix,
+                                               etag[0:2], etag[2:])}
+            self.remove(path_info)
+            removed = True
+
+        return removed

dvc/remote/hdfs.py

@@ -0,0 +1,265 @@
+import os
+import re
+import getpass
+import posixpath
+from subprocess import Popen, PIPE
+
+from dvc.config import Config
+from dvc.remote.base import RemoteBase
+from dvc.remote.local import RemoteLOCAL
+from dvc.exceptions import DvcException
+from dvc.logger import Logger
+from dvc.utils import fix_env
+
+
+class RemoteHDFS(RemoteBase):
+    scheme = 'hdfs'
+    REGEX = r'^hdfs://((?P<user>.*)@)?.*$'
+    PARAM_CHECKSUM = 'checksum'
+
+    def __init__(self, project, config):
+        self.project = project
+        self.url = config.get(Config.SECTION_REMOTE_URL, '/')
+        self.user = self.group('user')
+        if not self.user:
+            self.user = config.get(Config.SECTION_REMOTE_USER,
+                                   getpass.getuser())
+
+    def hadoop_fs(self, cmd, user=None):
+        cmd = 'hadoop fs -' + cmd
+        if user:
+            cmd = 'HADOOP_USER_NAME={} '.format(user) + cmd
+        p = Popen(cmd,
+                  shell=True,
+                  close_fds=True,
+                  executable=os.getenv('SHELL'),
+                  env=fix_env(os.environ),
+                  stdin=PIPE,
+                  stdout=PIPE,
+                  stderr=PIPE)
+        out, err = p.communicate()
+        if p.returncode != 0:
+            raise DvcException('HDFS command failed: {}: {}'.format(cmd, err))
+        return out.decode('utf-8')
+
+    @staticmethod
+    def _group(regex, s, gname):
+        match = re.match(regex, s)
+        assert match is not None
+        return match.group(gname)
+
+    def checksum(self, path_info):
+        regex = r'.*\t.*\t(?P<checksum>.*)'
+        stdout = self.hadoop_fs('checksum {}'.format(path_info['url']),
+                                user=path_info['user'])
+        return self._group(regex, stdout, 'checksum')
+
+    def cp(self, from_info, to_info):
+        self.hadoop_fs('mkdir -p {}'.format(posixpath.dirname(to_info['url'])),
+                       user=to_info['user'])
+        self.hadoop_fs('cp -f {} {}'.format(from_info['url'], to_info['url']),
+                       user=to_info['user'])
+
+    def rm(self, path_info):
+        self.hadoop_fs('rm {}'.format(path_info['url']),
+                       user=path_info['user'])
+
+    def save_info(self, path_info):
+        if path_info['scheme'] != 'hdfs':
+            raise NotImplementedError
+
+        assert path_info.get('url')
+
+        return {self.PARAM_CHECKSUM: self.checksum(path_info)}
+
+    @staticmethod
+    def to_string(path_info):
+        return "{}://{}".format(path_info['scheme'],
+                                path_info['url'])
+
+    def changed_cache(self, checksum):
+        cache = {}
+        cache['scheme'] = 'hdfs'
+        cache['user'] = self.user
+        cache['url'] = posixpath.join(self.url, checksum[0:2], checksum[2:])
+
+        if {self.PARAM_CHECKSUM: checksum} != self.save_info(cache):
+            if self.exists([cache])[0]:
+                msg = 'Corrupted cache file {}'
+                Logger.warn(msg.format(self.to_string(cache)))
+                self.remove(cache)
+            return True
+
+        return False
+
+    def changed(self, path_info, checksum_info):
+        if not self.exists([path_info])[0]:
+            return True
+
+        checksum = checksum_info.get(self.PARAM_CHECKSUM, None)
+        if checksum is None:
+            return True
+
+        if self.changed_cache(self, checksum):
+            return True
+
+        return checksum_info != self.save_info(path_info)
+
+    def save(self, path_info):
+        if path_info['scheme'] != 'hdfs':
+            raise NotImplementedError
+
+        assert path_info.get('url')
+
+        checksum = self.checksum(path_info)
+        dest = path_info.copy()
+        dest['url'] = posixpath.join(self.url, checksum[0:2], checksum[2:])
+
+        self.cp(path_info, dest)
+
+        return {self.PARAM_CHECKSUM: checksum}
+
+    def checkout(self, path_info, checksum_info):
+        if path_info['scheme'] != 'hdfs':
+            raise NotImplementedError
+
+        assert path_info.get('url')
+
+        checksum = checksum_info.get(self.PARAM_CHECKSUM, None)
+        if not checksum:
+            return
+
+        if not self.changed(path_info, checksum_info):
+            msg = "Data '{}' didn't change."
+            Logger.info(msg.format(self.to_string(path_info)))
+            return
+
+        if self.changed_cache(checksum):
+            msg = "Cache '{}' not found. File '{}' won't be created."
+            Logger.warn(msg.format(checksum, self.to_string(path_info)))
+            return
+
+        if self.exists([path_info])[0]:
+            msg = "Data '{}' exists. Removing before checkout."
+            Logger.warn(msg.format(self.to_string(path_info)))
+            self.remove(path_info)
+            return
+
+        msg = "Checking out '{}' with cache '{}'."
+        Logger.info(msg.format(self.to_string(path_info), checksum))
+
+        src = path_info.copy()
+        src['url'] = posixpath.join(self.url, checksum[0:2], checksum[2:])
+
+        self.cp(src, path_info)
+
+    def remove(self, path_info):
+        if path_info['scheme'] != 'hdfs':
+            raise NotImplementedError
+
+        assert path_info.get('url')
+
+        Logger.debug('Removing {}'.format(path_info['url']))
+
+        self.rm(path_info)
+
+    def md5s_to_path_infos(self, md5s):
+        return [{'scheme': 'hdfs',
+                 'user': self.user,
+                 'url': posixpath.join(self.url,
+                                       md5[0:2], md5[2:])} for md5 in md5s]
+
+    def exists(self, path_infos):
+        try:
+            stdout = self.hadoop_fs('ls -R {}'.format(self.url))
+        except DvcException:
+            return len(path_infos) * [False]
+
+        lines = stdout.split('\n')
+        lurl = []
+        for line in lines:
+            if not line.startswith('-'):
+                continue
+            lurl.append(line.split()[-1])
+
+        ret = []
+        for path_info in path_infos:
+            exists = False
+            if path_info['url'] in lurl:
+                exists = True
+            ret.append(exists)
+
+        return ret
+
+    def upload(self, from_infos, to_infos, names=None):
+        names = self._verify_path_args(to_infos, from_infos, names)
+
+        for from_info, to_info, name in zip(from_infos, to_infos, names):
+            if to_info['scheme'] != 'hdfs':
+                raise NotImplementedError
+
+            if from_info['scheme'] != 'local':
+                raise NotImplementedError
+
+            cmd = 'mkdir -p {}'.format(posixpath.dirname(to_info['url']))
+            self.hadoop_fs(cmd, user=to_info['user'])
+
+            cmd = 'copyFromLocal {} {}'.format(from_info['path'],
+                                               to_info['url'])
+            self.hadoop_fs(cmd, user=to_info['user'])
+
+    def download(self,
+                 from_infos,
+                 to_infos,
+                 no_progress_bar=False,
+                 names=None):
+        names = self._verify_path_args(from_infos, to_infos, names)
+
+        for to_info, from_info, name in zip(to_infos, from_infos, names):
+            if from_info['scheme'] != 'hdfs':
+                raise NotImplementedError
+
+            if to_info['scheme'] == 'hdfs':
+                self.cp(from_info, to_info)
+                continue
+
+            if to_info['scheme'] != 'local':
+                raise NotImplementedError
+
+            dname = os.path.dirname(to_info['path'])
+            if not os.path.exists(dname):
+                os.makedirs(dname)
+
+            cmd = 'copyToLocal {} {}'.format(from_info['url'], to_info['path'])
+            self.hadoop_fs(cmd, user=from_info['user'])
+
+    def _path_to_checksum(self, path):
+        relpath = posixpath.relpath(path, self.url)
+        return posixpath.dirname(relpath) + posixpath.basename(relpath)
+
+    def _all_checksums(self):
+        stdout = self.hadoop_fs('ls -R {}'.format(self.url))
+        lines = stdout.split('\n')
+        flist = []
+        for line in lines:
+            if not line.startswith('-'):
+                continue
+            flist.append(line.split()[-1])
+        return [self._path_to_checksum(path) for path in flist]
+
+    def gc(self, cinfos):
+        used = [info[self.PARAM_CHECKSUM] for info in cinfos['hdfs']]
+        used += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
+
+        removed = False
+        for checksum in self._all_checksums():
+            if checksum in used:
+                continue
+            path_info = {'scheme': 'hdfs',
+                         'user': self.user,
+                         'url': posixpath.join(self.url,
+                                               checksum[0:2], checksum[2:])}
+            self.remove(path_info)
+            removed = True
+
+        return removed

dvc/remote/local.py

@@ -0,0 +1,664 @@
+import os
+import uuid
+import json
+import ntpath
+import shutil
+import posixpath
+from operator import itemgetter
+
+from dvc.system import System
+from dvc.remote.base import RemoteBase, STATUS_MAP
+from dvc.logger import Logger
+from dvc.utils import remove, move, copyfile, dict_md5, to_chunks
+from dvc.utils import LARGE_DIR_SIZE
+from dvc.config import Config
+from dvc.exceptions import DvcException
+from dvc.progress import progress
+from concurrent.futures import ThreadPoolExecutor
+
+
+class RemoteLOCAL(RemoteBase):
+    scheme = ''
+    REGEX = r'^(?P<path>(/+|.:\\+).*)$'
+    PARAM_MD5 = 'md5'
+    PARAM_PATH = 'path'
+    PARAM_RELPATH = 'relpath'
+    MD5_DIR_SUFFIX = '.dir'
+
+    CACHE_TYPES = ['reflink', 'hardlink', 'symlink', 'copy']
+    CACHE_TYPE_MAP = {
+        'copy': shutil.copyfile,
+        'symlink': System.symlink,
+        'hardlink': System.hardlink,
+        'reflink': System.reflink,
+    }
+
+    def __init__(self, project, config):
+        self.project = project
+        self.state = self.project.state
+        storagepath = config.get(Config.SECTION_AWS_STORAGEPATH, None)
+        self.cache_dir = config.get(Config.SECTION_REMOTE_URL, storagepath)
+
+        types = config.get(Config.SECTION_CACHE_TYPE, None)
+        if types:
+            if isinstance(types, str):
+                types = [t.strip() for t in types.split(',')]
+            self.cache_types = types
+        else:
+            self.cache_types = self.CACHE_TYPES
+
+        if self.cache_dir is not None and not os.path.exists(self.cache_dir):
+            os.mkdir(self.cache_dir)
+
+    @property
+    def url(self):
+        return self.cache_dir
+
+    @property
+    def prefix(self):
+        return self.cache_dir
+
+    def all(self):
+        clist = []
+        for entry in os.listdir(self.cache_dir):
+            subdir = os.path.join(self.cache_dir, entry)
+            if not os.path.isdir(subdir):
+                continue
+
+            for cache in os.listdir(subdir):
+                path = os.path.join(subdir, cache)
+                clist.append(self.path_to_md5(path))
+
+        return clist
+
+    def get(self, md5):
+        if not md5:
+            return None
+
+        return os.path.join(self.cache_dir, md5[0:2], md5[2:])
+
+    def path_to_md5(self, path):
+        relpath = os.path.relpath(path, self.cache_dir)
+        return os.path.dirname(relpath) + os.path.basename(relpath)
+
+    def changed_cache(self, md5):
+        cache = self.get(md5)
+        if self.state.changed(cache, md5=md5):
+            if os.path.exists(cache):
+                msg = 'Corrupted cache file {}'
+                Logger.warn(msg.format(os.path.relpath(cache)))
+                remove(cache)
+            return True
+
+        return False
+
+    def link(self, cache, path):
+        assert os.path.isfile(cache)
+
+        dname = os.path.dirname(path)
+        if not os.path.exists(dname):
+            os.makedirs(dname)
+
+        i = len(self.cache_types)
+        while i > 0:
+            try:
+                self.CACHE_TYPE_MAP[self.cache_types[0]](cache, path)
+                msg = "Created '{}': {} -> {}"
+                Logger.info(msg.format(self.cache_types[0],
+                                       os.path.relpath(cache),
+                                       os.path.relpath(path)))
+                return
+            except DvcException as exc:
+                msg = 'Cache type \'{}\' is not supported: {}'
+                Logger.debug(msg.format(self.cache_types[0], str(exc)))
+                del self.cache_types[0]
+                i -= 1
+
+        raise DvcException('No possible cache types left to try out.')
+
+    @classmethod
+    def ospath(cls, path):
+        if os.name == 'nt':
+            return cls.ntpath(path)
+        return cls.unixpath(path)
+
+    @staticmethod
+    def unixpath(path):
+        assert not ntpath.isabs(path)
+        assert not posixpath.isabs(path)
+        return path.replace('\\', '/')
+
+    @staticmethod
+    def ntpath(path):
+        assert not ntpath.isabs(path)
+        assert not posixpath.isabs(path)
+        return path.replace('/', '\\')
+
+    def collect_dir_cache(self, dname):
+        dir_info = []
+
+        bar = False
+        for root, dirs, files in os.walk(dname):
+            if len(files) > LARGE_DIR_SIZE:
+                msg = "Computing md5 for a large directory {}. " \
+                      "This is only done once."
+                Logger.info(msg.format(os.path.relpath(dname)))
+                bar = True
+                title = os.path.relpath(dname)
+                processed = 0
+                total = len(files)
+                progress.update_target(title, 0, total)
+
+            for fname in files:
+                path = os.path.join(root, fname)
+                relpath = self.unixpath(os.path.relpath(path, dname))
+
+                if bar:
+                    progress.update_target(title, processed, total)
+                    processed += 1
+
+                md5 = self.state.update(path)
+                dir_info.append({self.PARAM_RELPATH: relpath,
+                                 self.PARAM_MD5: md5})
+
+        if bar:
+            progress.finish_target(title)
+
+        # NOTE: sorting the list by path to ensure reproducibility
+        dir_info = sorted(dir_info, key=itemgetter(self.PARAM_RELPATH))
+
+        md5 = dict_md5(dir_info) + self.MD5_DIR_SUFFIX
+        if self.changed_cache(md5):
+            self.dump_dir_cache(md5, dir_info)
+
+        return (md5, dir_info)
+
+    def load_dir_cache(self, md5):
+        path = self.get(md5)
+
+        assert self.is_dir_cache(path)
+
+        try:
+            with open(path, 'r') as fd:
+                d = json.load(fd)
+        except Exception as exc:
+            msg = u'Failed to load dir cache \'{}\''
+            Logger.error(msg.format(os.path.relpath(path)), exc)
+            return []
+
+        if not isinstance(d, list):
+            msg = u'Dir cache file format error \'{}\': skipping the file'
+            Logger.error(msg.format(os.path.relpath(path)))
+            return []
+
+        for info in d:
+            info['relpath'] = self.ospath(info['relpath'])
+
+        return d
+
+    def dump_dir_cache(self, md5, dir_info):
+        path = self.get(md5)
+        dname = os.path.dirname(path)
+
+        assert self.is_dir_cache(path)
+        assert isinstance(dir_info, list)
+
+        if not os.path.isdir(dname):
+            os.makedirs(dname)
+
+        # NOTE: Writing first and renaming after that
+        # to make sure that the operation is atomic.
+        tmp = '{}.{}'.format(path, str(uuid.uuid4()))
+        with open(tmp, 'w+') as fd:
+            json.dump(dir_info, fd, sort_keys=True)
+        move(tmp, path)
+
+    @classmethod
+    def is_dir_cache(cls, cache):
+        return cache.endswith(cls.MD5_DIR_SUFFIX)
+
+    def checkout(self, path_info, checksum_info):
+        path = path_info['path']
+        md5 = checksum_info.get(self.PARAM_MD5, None)
+        cache = self.get(md5)
+
+        if not cache:
+            msg = 'No cache info for \'{}\'. Skipping checkout.'
+            Logger.warn(msg.format(os.path.relpath(path)))
+            return
+
+        if not self.changed(path_info, checksum_info):
+            msg = "Data '{}' didn't change."
+            Logger.info(msg.format(os.path.relpath(path)))
+            return
+
+        if self.changed_cache(md5):
+            msg = u'Cache \'{}\' not found. File \'{}\' won\'t be created.'
+            Logger.warn(msg.format(md5, os.path.relpath(path)))
+            remove(path)
+            return
+
+        if os.path.exists(path):
+            msg = u'Data \'{}\' exists. Removing before checkout.'
+            Logger.warn(msg.format(os.path.relpath(path)))
+            remove(path)
+
+        msg = u'Checking out \'{}\' with cache \'{}\'.'
+        Logger.info(msg.format(os.path.relpath(path), md5))
+
+        if not self.is_dir_cache(cache):
+            self.link(cache, path)
+            self.state.update_link(path)
+            return
+
+        # Create dir separately so that dir is created
+        # even if there are no files in it
+        if not os.path.exists(path):
+            os.makedirs(path)
+
+        for entry in self.load_dir_cache(md5):
+            md5 = entry[self.PARAM_MD5]
+            c = self.get(md5)
+            relpath = entry[self.PARAM_RELPATH]
+            p = os.path.join(path, relpath)
+            self.link(c, p)
+        self.state.update_link(path)
+
+    def _move(self, inp, outp):
+        # moving in two stages to make last the move atomic in
+        # case inp and outp are in different filesystems
+        tmp = '{}.{}'.format(outp, str(uuid.uuid4()))
+        move(inp, tmp)
+        move(tmp, outp)
+
+    def _save_file(self, path_info):
+        path = path_info['path']
+        md5 = self.state.update(path)
+        assert md5 is not None
+
+        cache = self.get(md5)
+
+        if self.changed_cache(md5):
+            self._move(path, cache)
+        else:
+            remove(path)
+
+        self.link(cache, path)
+        self.state.update_link(path)
+
+        return {self.PARAM_MD5: md5}
+
+    def _save_dir(self, path_info):
+        path = path_info['path']
+        md5, dir_info = self.state.update_info(path)
+
+        for entry in dir_info:
+            relpath = entry[self.PARAM_RELPATH]
+            m = entry[self.PARAM_MD5]
+            p = os.path.join(path, relpath)
+            c = self.get(m)
+
+            if self.changed_cache(m):
+                self._move(p, c)
+            else:
+                remove(p)
+
+            self.link(c, p)
+
+        self.state.update_link(path)
+
+        return {self.PARAM_MD5: md5}
+
+    def save(self, path_info):
+        if path_info['scheme'] != 'local':
+            raise NotImplementedError
+
+        path = path_info['path']
+
+        msg = "Saving '{}' to cache '{}'."
+        Logger.info(msg.format(os.path.relpath(path),
+                               os.path.relpath(self.cache_dir)))
+
+        if os.path.isdir(path):
+            return self._save_dir(path_info)
+        else:
+            return self._save_file(path_info)
+
+    def save_info(self, path_info):
+        if path_info['scheme'] != 'local':
+            raise NotImplementedError
+
+        return {self.PARAM_MD5: self.state.update(path_info['path'])}
+
+    def changed(self, path_info, checksum_info):
+        if not self.exists([path_info])[0]:
+            return True
+
+        md5 = checksum_info.get(self.PARAM_MD5, None)
+        if md5 is None:
+            return True
+
+        if self.changed_cache(md5):
+            return True
+
+        return checksum_info != self.save_info(path_info)
+
+    def remove(self, path_info):
+        if path_info['scheme'] != 'local':
+            raise NotImplementedError
+
+        remove(path_info['path'])
+
+    def move(self, from_info, to_info):
+        if from_info['scheme'] != 'local' or to_info['scheme'] != 'local':
+            raise NotImplementedError
+
+        move(from_info['path'], to_info['path'])
+
+    def md5s_to_path_infos(self, md5s):
+        return [{'scheme': 'local',
+                 'path': os.path.join(self.prefix,
+                                      md5[0:2], md5[2:])} for md5 in md5s]
+
+    def exists(self, path_infos):
+        ret = []
+        for path_info in path_infos:
+            ret.append(os.path.exists(path_info['path']))
+        return ret
+
+    def upload(self, from_infos, to_infos, names=None):
+        names = self._verify_path_args(to_infos, from_infos, names)
+
+        for from_info, to_info, name in zip(from_infos, to_infos, names):
+            if to_info['scheme'] != 'local':
+                raise NotImplementedError
+
+            if from_info['scheme'] != 'local':
+                raise NotImplementedError
+
+            Logger.debug("Uploading '{}' to '{}'".format(from_info['path'],
+                                                         to_info['path']))
+
+            if not name:
+                name = os.path.basename(from_info['path'])
+
+            self._makedirs(to_info['path'])
+
+            try:
+                copyfile(from_info['path'], to_info['path'], name=name)
+            except Exception as exc:
+                msg = "Failed to upload '{}' tp '{}'"
+                Logger.error(msg.format(from_info['path'],
+                                        to_info['path']), exc)
+
+    def download(self,
+                 from_infos,
+                 to_infos,
+                 no_progress_bar=False,
+                 names=None):
+        names = self._verify_path_args(from_infos, to_infos, names)
+
+        for to_info, from_info, name in zip(to_infos, from_infos, names):
+            if from_info['scheme'] != 'local':
+                raise NotImplementedError
+
+            if to_info['scheme'] != 'local':
+                raise NotImplementedError
+
+            Logger.debug("Downloading '{}' to '{}'".format(from_info['path'],
+                                                           to_info['path']))
+
+            if not name:
+                name = os.path.basename(to_info['path'])
+
+            self._makedirs(to_info['path'])
+            tmp_file = self.tmp_file(to_info['path'])
+            try:
+                copyfile(from_info['path'],
+                         tmp_file,
+                         no_progress_bar=no_progress_bar,
+                         name=name)
+            except Exception as exc:
+                msg = "Failed to download '{}' to '{}'"
+                Logger.error(msg.format(from_info['path'],
+                                        to_info['path']), exc)
+                continue
+
+            os.rename(tmp_file, to_info['path'])
+
+    def _collect(self, checksum_infos, push=False):
+        missing = []
+        collected = []
+        for info in checksum_infos:
+            md5 = info[self.PARAM_MD5]
+            cache = self.get(md5)
+
+            if not self.is_dir_cache(info[self.PARAM_MD5]):
+                continue
+
+            if not os.path.exists(cache):
+                missing.append(info)
+                continue
+
+            for i in self.load_dir_cache(md5):
+                if info.get('branch'):
+                    i['branch'] = info['branch']
+                i[self.PARAM_PATH] = os.path.join(info[self.PARAM_PATH],
+                                                  i[self.PARAM_RELPATH])
+                collected.append(i.copy())
+
+        collected.extend(checksum_infos)
+        return collected, missing
+
+    def _group(self, checksum_infos, show_checksums=False):
+        by_md5 = {}
+
+        for info in checksum_infos:
+            md5 = info[self.PARAM_MD5]
+
+            if show_checksums:
+                by_md5[md5] = md5
+                continue
+
+            name = info[self.PARAM_PATH]
+            branch = info.get('branch')
+            if branch:
+                name += '({})'.format(branch)
+
+            if md5 not in by_md5.keys():
+                by_md5[md5] = ''
+            else:
+                by_md5[md5] += ' '
+
+            by_md5[md5] += name
+
+        return list(by_md5.keys()), list(by_md5.values())
+
+    def gc(self, checksum_infos):
+        checksum_infos = self._collect(checksum_infos['local'])[0]
+        used_md5s = [info[self.PARAM_MD5] for info in checksum_infos]
+
+        removed = False
+        for md5 in self.all():
+            if md5 in used_md5s:
+                continue
+            remove(self.get(md5))
+            removed = True
+
+        return removed
+
+    def status(self, checksum_infos, remote, jobs=1, show_checksums=False):
+        Logger.info("Preparing to pull data from {}".format(remote.url))
+        title = "Collecting information"
+
+        progress.set_n_total(1)
+        progress.update_target(title, 0, 100)
+
+        checksum_infos, missing = self._collect(checksum_infos)
+        checksum_infos += missing
+
+        progress.update_target(title, 10, 100)
+
+        md5s, names = self._group(checksum_infos,
+                                  show_checksums=show_checksums)
+
+        progress.update_target(title, 20, 100)
+
+        path_infos = remote.md5s_to_path_infos(md5s)
+
+        progress.update_target(title, 30, 100)
+
+        remote_exists = remote.exists(path_infos)
+
+        progress.update_target(title, 90, 100)
+
+        local_exists = [not self.changed_cache(md5) for md5 in md5s]
+
+        progress.finish_target(title)
+
+        return [(name, STATUS_MAP[l, r]) for name, l, r in zip(names,
+                                                               local_exists,
+                                                               remote_exists)]
+
+    def _do_pull(self,
+                 checksum_infos,
+                 remote,
+                 jobs=1,
+                 show_checksums=False):
+        title = "Collecting information"
+
+        progress.set_n_total(1)
+        progress.update_target(title, 0, 100)
+
+        grouped = zip(*self._group(checksum_infos,
+                                   show_checksums=show_checksums))
+
+        progress.update_target(title, 10, 100)
+
+        md5s = []
+        names = []
+        # NOTE: filter files that are not corrupted
+        for md5, name in grouped:
+            if self.changed_cache(md5):
+                md5s.append(md5)
+                names.append(name)
+
+        progress.update_target(title, 30, 100)
+
+        cache = [{'scheme': 'local', 'path': self.get(md5)} for md5 in md5s]
+
+        progress.update_target(title, 50, 100)
+
+        path_infos = remote.md5s_to_path_infos(md5s)
+
+        progress.update_target(title, 60, 100)
+
+        assert len(path_infos) == len(cache) == len(md5s) == len(names)
+
+        chunks = list(zip(to_chunks(path_infos, jobs),
+                          to_chunks(cache, jobs),
+                          to_chunks(names, jobs)))
+
+        progress.finish_target(title)
+
+        progress.set_n_total(len(names))
+
+        if len(chunks) == 0:
+            return
+
+        futures = []
+        with ThreadPoolExecutor(max_workers=len(chunks)) as executor:
+            for from_infos, to_infos, names in chunks:
+                res = executor.submit(remote.download,
+                                      from_infos,
+                                      to_infos,
+                                      names=names)
+                futures.append(res)
+
+        for f in futures:
+            f.result()
+
+    def pull(self, checksum_infos, remote, jobs=1, show_checksums=False):
+        Logger.info("Preparing to pull data from {}".format(remote.url))
+
+        # NOTE: try fetching missing dir info
+        checksum_infos, missing = self._collect(checksum_infos)
+        if len(missing) > 0:
+            self._do_pull(missing,
+                          remote,
+                          jobs,
+                          show_checksums=show_checksums)
+            checksum_infos += self._collect(missing)[0]
+
+        self._do_pull(checksum_infos,
+                      remote,
+                      jobs,
+                      show_checksums=show_checksums)
+
+    def push(self, checksum_infos, remote, jobs=1, show_checksums=False):
+        Logger.info("Preparing to push data to {}".format(remote.url))
+        title = "Collecting information"
+
+        progress.set_n_total(1)
+        progress.update_target(title, 0, 100)
+
+        checksum_infos = self._collect(checksum_infos)[0]
+
+        progress.update_target(title, 10, 100)
+
+        # NOTE: verifying that our cache is not corrupted
+        def func(info):
+            return not self.changed_cache(info[self.PARAM_MD5])
+        checksum_infos = list(filter(func, checksum_infos))
+
+        progress.update_target(title, 20, 100)
+
+        # NOTE: filter files that are already uploaded
+        md5s = [i[self.PARAM_MD5] for i in checksum_infos]
+        exists = remote.exists(remote.md5s_to_path_infos(md5s))
+
+        progress.update_target(title, 30, 100)
+
+        def func(entry):
+            return not entry[0]
+
+        assert len(exists) == len(checksum_infos)
+        infos_exist = list(filter(func, zip(exists, checksum_infos)))
+        checksum_infos = [i for e, i in infos_exist]
+
+        progress.update_target(title, 70, 100)
+
+        md5s, names = self._group(checksum_infos,
+                                  show_checksums=show_checksums)
+        cache = [{'scheme': 'local', 'path': self.get(md5)} for md5 in md5s]
+
+        progress.update_target(title, 80, 100)
+
+        path_infos = remote.md5s_to_path_infos(md5s)
+
+        assert len(path_infos) == len(cache) == len(md5s) == len(names)
+
+        progress.update_target(title, 90, 100)
+
+        chunks = list(zip(to_chunks(path_infos, jobs),
+                          to_chunks(cache, jobs),
+                          to_chunks(names, jobs)))
+
+        progress.finish_target(title)
+
+        progress.set_n_total(len(names))
+
+        if len(chunks) == 0:
+            return
+
+        futures = []
+        with ThreadPoolExecutor(max_workers=len(chunks)) as executor:
+            for to_infos, from_infos, names in chunks:
+                res = executor.submit(remote.upload,
+                                      from_infos,
+                                      to_infos,
+                                      names=names)
+                futures.append(res)
+
+        for f in futures:
+            f.result()

dvc/remote/s3.py

@@ -0,0 +1,347 @@
+import os
+import threading
+import posixpath
+
+try:
+    import boto3
+except ImportError:
+    boto3 = None
+
+try:
+    from urlparse import urlparse
+except ImportError:
+    from urllib.parse import urlparse
+
+from dvc.logger import Logger
+from dvc.progress import progress
+from dvc.config import Config
+from dvc.remote.base import RemoteBase
+from dvc.remote.local import RemoteLOCAL
+from dvc.exceptions import DvcException
+
+
+class Callback(object):
+    def __init__(self, name, total):
+        self.name = name
+        self.total = total
+        self.current = 0
+        self.lock = threading.Lock()
+
+    def __call__(self, byts):
+        with self.lock:
+            self.current += byts
+            progress.update_target(self.name, self.current, self.total)
+
+
+class RemoteS3(RemoteBase):
+    scheme = 's3'
+    REGEX = r'^s3://(?P<path>.*)$'
+    REQUIRES = {'boto3': boto3}
+    PARAM_ETAG = 'etag'
+
+    def __init__(self, project, config):
+        import configobj
+
+        self.project = project
+        storagepath = 's3://' + config.get(Config.SECTION_AWS_STORAGEPATH,
+                                           '').lstrip('/')
+        self.url = config.get(Config.SECTION_REMOTE_URL, storagepath)
+        self.region = config.get(Config.SECTION_AWS_REGION, None)
+        self.profile = config.get(Config.SECTION_AWS_PROFILE, 'default')
+        self.endpoint_url = config.get(Config.SECTION_AWS_ENDPOINT_URL, None)
+
+        credentialpath = config.get(Config.SECTION_AWS_CREDENTIALPATH, None)
+        if credentialpath:
+            creds_conf = configobj.ConfigObj(credentialpath)
+            creds = creds_conf.get(self.profile, {})
+        else:
+            creds = {}
+
+        self.creds = creds
+        self.region = creds.get('region', self.region)
+        self.aws_access_key_id = creds.get('aws_access_key_id', None)
+        self.aws_secret_access_key = creds.get('aws_secret_access_key', None)
+
+    @property
+    def bucket(self):
+        return urlparse(self.url).netloc
+
+    @property
+    def prefix(self):
+        return urlparse(self.url).path.lstrip('/')
+
+    @property
+    def s3(self):
+        if not self.creds:
+            session = boto3.session.Session(profile_name=self.profile)
+        else:
+            session = boto3.session.Session(
+                             aws_access_key_id=self.aws_access_key_id,
+                             aws_secret_access_key=self.aws_secret_access_key,
+                             region_name=self.region)
+        return session.client('s3', endpoint_url=self.endpoint_url)
+
+    def get_etag(self, bucket, key):
+        try:
+            obj = self.s3.head_object(Bucket=bucket, Key=key)
+        except Exception:
+            raise DvcException('s3://{}/{} does not exist'.format(bucket, key))
+
+        return obj['ETag'].strip('"')
+
+    def save_info(self, path_info):
+        if path_info['scheme'] != 's3':
+            raise NotImplementedError
+
+        return {self.PARAM_ETAG: self.get_etag(path_info['bucket'],
+                                               path_info['key'])}
+
+    def changed(self, path_info, checksum_info):
+        if not self.exists([path_info])[0]:
+            return True
+
+        etag = checksum_info.get(self.PARAM_ETAG, None)
+        if etag is None:
+            return True
+
+        if self.changed_cache(etag):
+            return True
+
+        return checksum_info != self.save_info(path_info)
+
+    def _copy(self, from_info, to_info, s3=None):
+        s3 = s3 if s3 else self.s3
+
+        source = {'Bucket': from_info['bucket'],
+                  'Key': from_info['key']}
+        self.s3.copy(source, to_info['bucket'], to_info['key'])
+
+    def save(self, path_info):
+        if path_info['scheme'] != 's3':
+            raise NotImplementedError
+
+        etag = self.get_etag(path_info['bucket'], path_info['key'])
+        key = posixpath.join(self.prefix, etag[0:2], etag[2:])
+        to_info = {'scheme': 's3', 'bucket': self.bucket, 'key': key}
+
+        self._copy(path_info, to_info)
+
+        return {self.PARAM_ETAG: etag}
+
+    @staticmethod
+    def to_string(path_info):
+        return "s3://{}/{}".format(path_info['bucket'], path_info['key'])
+
+    def changed_cache(self, etag):
+        key = posixpath.join(self.prefix, etag[0:2], etag[2:])
+        cache = {'scheme': 's3', 'bucket': self.bucket, 'key': key}
+
+        if {self.PARAM_ETAG: etag} != self.save_info(cache):
+            if self.exists([cache])[0]:
+                msg = 'Corrupted cache file {}'
+                Logger.warn(msg.format(self.to_string(cache)))
+                self.remove(cache)
+            return True
+
+        return False
+
+    def checkout(self, path_info, checksum_info):
+        if path_info['scheme'] != 's3':
+            raise NotImplementedError
+
+        etag = checksum_info.get(self.PARAM_ETAG, None)
+        if not etag:
+            return
+
+        if not self.changed(path_info, checksum_info):
+            msg = "Data '{}' didn't change."
+            Logger.info(msg.format(self.to_string(path_info)))
+            return
+
+        if self.changed_cache(etag):
+            msg = "Cache '{}' not found. File '{}' won't be created."
+            Logger.warn(msg.format(etag, self.to_string(path_info)))
+            return
+
+        if self.exists([path_info])[0]:
+            msg = "Data '{}' exists. Removing before checkout."
+            Logger.warn(msg.format(self.to_string(path_info)))
+            self.remove(path_info)
+            return
+
+        msg = "Checking out '{}' with cache '{}'."
+        Logger.info(msg.format(self.to_string(path_info), etag))
+
+        key = posixpath.join(self.prefix, etag[0:2], etag[2:])
+        from_info = {'scheme': 's3', 'bucket': self.bucket, 'key': key}
+
+        self._copy(from_info, path_info)
+
+    def remove(self, path_info):
+        if path_info['scheme'] != 's3':
+            raise NotImplementedError
+
+        Logger.debug('Removing s3://{}/{}'.format(path_info['bucket'],
+                                                  path_info['key']))
+
+        self.s3.delete_object(Bucket=path_info['bucket'],
+                              Key=path_info['key'])
+
+    def md5s_to_path_infos(self, md5s):
+        return [{'scheme': self.scheme,
+                 'bucket': self.bucket,
+                 'key': posixpath.join(self.prefix,
+                                       md5[0:2], md5[2:])} for md5 in md5s]
+
+    def _all_keys(self):
+        s3 = self.s3
+
+        keys = []
+        kwargs = {'Bucket': self.bucket,
+                  'Prefix': self.prefix}
+        while True:
+            resp = s3.list_objects_v2(**kwargs)
+            contents = resp.get('Contents', None)
+            if not contents:
+                break
+
+            for obj in contents:
+                keys.append(obj['Key'])
+
+            token = resp.get('NextContinuationToken', None)
+            if not token:
+                break
+
+            kwargs['ContinuationToken'] = token
+
+        return keys
+
+    def exists(self, path_infos):
+        # NOTE: We mostly use exists() method when filtering a bulk of cache
+        # files to decide if we need to download/upload them and in s3
+        # list_objects_v2() is much-much faster than trying to check keys
+        # one-by-one.
+        ret = []
+
+        keys = self._all_keys()
+
+        for path_info in path_infos:
+            exists = False
+            if path_info['key'] in keys:
+                exists = True
+            ret.append(exists)
+
+        return ret
+
+    def upload(self, from_infos, to_infos, names=None):
+        names = self._verify_path_args(to_infos, from_infos, names)
+
+        s3 = self.s3
+
+        for from_info, to_info, name in zip(from_infos, to_infos, names):
+            if to_info['scheme'] != 's3':
+                raise NotImplementedError
+
+            if from_info['scheme'] != 'local':
+                raise NotImplementedError
+
+            Logger.debug("Uploading '{}' to '{}/{}'".format(from_info['path'],
+                                                            to_info['bucket'],
+                                                            to_info['key']))
+
+            if not name:
+                name = os.path.basename(from_info['path'])
+
+            total = os.path.getsize(from_info['path'])
+            cb = Callback(name, total)
+
+            try:
+                s3.upload_file(from_info['path'],
+                               to_info['bucket'],
+                               to_info['key'],
+                               Callback=cb)
+            except Exception as exc:
+                msg = "Failed to upload '{}'".format(from_info['path'])
+                Logger.error(msg, exc)
+                continue
+
+            progress.finish_target(name)
+
+    def download(self,
+                 from_infos,
+                 to_infos,
+                 no_progress_bar=False,
+                 names=None):
+        names = self._verify_path_args(from_infos, to_infos, names)
+
+        s3 = self.s3
+
+        for to_info, from_info, name in zip(to_infos, from_infos, names):
+            if from_info['scheme'] != 's3':
+                raise NotImplementedError
+
+            if to_info['scheme'] == 's3':
+                self._copy(from_info, to_info, s3=s3)
+                continue
+
+            if to_info['scheme'] != 'local':
+                raise NotImplementedError
+
+            msg = "Downloading '{}/{}' to '{}'".format(from_info['bucket'],
+                                                       from_info['key'],
+                                                       to_info['path'])
+            Logger.debug(msg)
+
+            tmp_file = self.tmp_file(to_info['path'])
+            if not name:
+                name = os.path.basename(to_info['path'])
+
+            if no_progress_bar:
+                cb = None
+            else:
+                total = s3.head_object(Bucket=from_info['bucket'],
+                                       Key=from_info['key'])['ContentLength']
+                cb = Callback(name, total)
+
+            self._makedirs(to_info['path'])
+
+            try:
+                s3.download_file(from_info['bucket'],
+                                 from_info['key'],
+                                 tmp_file,
+                                 Callback=cb)
+            except Exception as exc:
+                msg = "Failed to download '{}/{}'".format(from_info['bucket'],
+                                                          from_info['key'])
+                Logger.error(msg, exc)
+                return
+
+            os.rename(tmp_file, to_info['path'])
+
+            if not no_progress_bar:
+                progress.finish_target(name)
+
+    def _path_to_etag(self, path):
+        relpath = posixpath.relpath(path, self.prefix)
+        return posixpath.dirname(relpath) + posixpath.basename(relpath)
+
+    def _all(self):
+        keys = self._all_keys()
+        return [self._path_to_etag(key) for key in keys]
+
+    def gc(self, cinfos):
+        used_etags = [info[self.PARAM_ETAG] for info in cinfos['s3']]
+        used_etags += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
+
+        removed = False
+        for etag in self._all():
+            if etag in used_etags:
+                continue
+            path_info = {'scheme': 's3',
+                         'key': posixpath.join(self.prefix,
+                                               etag[0:2], etag[2:]),
+                         'bucket': self.bucket}
+            self.remove(path_info)
+            removed = True
+
+        return removed

dvc/remote/ssh.py

@@ -0,0 +1,432 @@
+import os
+import getpass
+import posixpath
+
+try:
+    import paramiko
+except ImportError:
+    paramiko = None
+
+from dvc.logger import Logger
+from dvc.progress import progress
+from dvc.remote.base import RemoteBase
+from dvc.remote.local import RemoteLOCAL
+from dvc.config import Config
+from dvc.exceptions import DvcException
+
+
+def sizeof_fmt(num, suffix='B'):
+    """ Convert number of bytes to human-readable string """
+    for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
+        if abs(num) < 1024.0:
+            return "%3.1f%s%s" % (num, unit, suffix)
+        num /= 1024.0
+    return "%.1f%s%s" % (num, 'Y', suffix)
+
+
+def percent_cb(name, complete, total):
+    """ Callback for updating target progress """
+    Logger.debug('{}: {} transferred out of {}'.format(name,
+                                                       sizeof_fmt(complete),
+                                                       sizeof_fmt(total)))
+    progress.update_target(name, complete, total)
+
+
+def create_cb(name):
+    """ Create callback function for multipart object """
+    return (lambda cur, tot: percent_cb(name, cur, tot))
+
+
+class RemoteSSH(RemoteBase):
+    scheme = 'ssh'
+
+    # NOTE: temporarily only absolute paths are allowed
+    REGEX = r'^ssh://((?P<user>.*)@)?(?P<host>[^/]*):(?P<path>/.*)$'
+
+    REQUIRES = {'paramiko': paramiko}
+    PARAM_MD5 = 'md5'
+
+    DEFAULT_PORT = 22
+    TIMEOUT = 1800
+
+    def __init__(self, project, config):
+        self.project = project
+        self.url = config.get(Config.SECTION_REMOTE_URL, '/')
+        self.host = self.group('host')
+        self.user = self.group('user')
+        if not self.user:
+            self.user = config.get(Config.SECTION_REMOTE_USER,
+                                   getpass.getuser())
+        self.prefix = self.group('path')
+        self.port = config.get(Config.SECTION_REMOTE_PORT, self.DEFAULT_PORT)
+        self.keyfile = config.get(Config.SECTION_REMOTE_KEY_FILE, None)
+        self.timeout = config.get(Config.SECTION_REMOTE_TIMEOUT, self.TIMEOUT)
+        self.password = config.get(Config.SECTION_REMOTE_PASSWORD, None)
+        self.ask_password = config.get(Config.SECTION_REMOTE_ASK_PASSWORD,
+                                       False)
+
+    def md5s_to_path_infos(self, md5s):
+        return [{'scheme': 'ssh',
+                 'host': self.host,
+                 'user': self.user,
+                 'port': self.port,
+                 'path': posixpath.join(self.prefix,
+                                        md5[0:2], md5[2:])} for md5 in md5s]
+
+    def ssh(self, host=None, user=None, port=None):
+        msg = "Establishing ssh connection with '{}' " \
+              "through port '{}' as user '{}'"
+        Logger.debug(msg.format(host, port, user))
+
+        ssh = paramiko.SSHClient()
+
+        ssh.load_system_host_keys()
+        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+        if self.ask_password and self.password is None:
+            msg = "host '{}' port '{}' user '{}'".format(host, port, user)
+            self.password = self.project.prompt.prompt_password(msg)
+
+        ssh.connect(host,
+                    username=user,
+                    port=port,
+                    key_filename=self.keyfile,
+                    timeout=self.timeout,
+                    password=self.password)
+
+        return ssh
+
+    def exists(self, path_infos):
+        ret = []
+        ssh = self.ssh(host=self.host,
+                       user=self.user,
+                       port=self.port)
+        cmd = 'find {} -type f -follow -print'.format(self.prefix)
+        stdout = self._exec(ssh, cmd)
+        plist = stdout.split()
+        ssh.close()
+
+        for path_info in path_infos:
+            exists = False
+            if path_info['path'] in plist:
+                exists = True
+            ret.append(exists)
+
+        return ret
+
+    def _exec(self, ssh, cmd):
+        stdin, stdout, stderr = ssh.exec_command(cmd)
+        channel = stdout.channel
+
+        stdin.close()
+        channel.shutdown_write()
+
+        stdout_chunks = []
+        stderr_chunks = []
+        while (not channel.closed
+               or channel.recv_ready()
+               or channel.recv_stderr_ready()):
+            import select
+            got_chunk = False
+            readq, _, _ = select.select([stdout.channel], [], [], self.timeout)
+            for c in readq:
+                if c.recv_ready():
+                    stdout_chunks.append(stdout.channel.recv(len(c.in_buffer)))
+                    got_chunk = True
+
+                if c.recv_stderr_ready():
+                    stderr_len = len(c.in_stderr_buffer)
+                    s = stderr.channel.recv_stderr(stderr_len)
+                    stderr_chunks.append(s)
+                    got_chunk = True
+
+            if not got_chunk \
+               and stdout.channel.exit_status_ready() \
+               and not stderr.channel.recv_stderr_ready() \
+               and not stdout.channel.recv_ready():
+                stdout.channel.shutdown_read()
+                stdout.channel.close()
+                break
+
+        stdout.close()
+        stderr.close()
+
+        if stdout.channel.recv_exit_status() != 0:
+            err = b''.join(stderr_chunks).decode('utf-8')
+            msg = 'SSH command \'{}\' failed: {}'.format(cmd, err)
+            raise DvcException(msg)
+
+        return b''.join(stdout_chunks).decode('utf-8')
+
+    def md5(self, path_info):
+        if path_info['scheme'] != 'ssh':
+            raise NotImplementedError
+
+        ssh = self.ssh(host=path_info['host'],
+                       user=path_info['user'],
+                       port=path_info['port'])
+
+        # Use different md5 commands depending on os
+        stdout = self._exec(ssh, 'uname').strip()
+        if stdout == 'Darwin':
+            md5cmd = 'md5'
+            index = -1
+        elif stdout == 'Linux':
+            md5cmd = 'md5sum'
+            index = 0
+        else:
+            msg = '\'{}\' is not supported as a remote'.format(stdout)
+            raise DvcException(msg)
+
+        stdout = self._exec(ssh, '{} {}'.format(md5cmd, path_info['path']))
+        md5 = stdout.split()[index]
+        ssh.close()
+
+        assert len(md5) == 32
+
+        return md5
+
+    def cp(self, from_info, to_info, ssh=None):
+        if from_info['scheme'] != 'ssh' or to_info['scheme'] != 'ssh':
+            raise NotImplementedError
+
+        assert from_info['host'] == to_info['host']
+        assert from_info['user'] == to_info['user']
+
+        s = ssh if ssh else self.ssh(host=from_info['host'],
+                                     user=from_info['user'],
+                                     port=from_info['port'])
+
+        dname = posixpath.dirname(to_info['path'])
+        self._exec(s, 'mkdir -p {}'.format(dname))
+        self._exec(s, 'cp {} {}'.format(from_info['path'], to_info['path']))
+
+        if not ssh:
+            s.close()
+
+    def save_info(self, path_info):
+        if path_info['scheme'] != 'ssh':
+            raise NotImplementedError
+
+        return {self.PARAM_MD5: self.md5(path_info)}
+
+    @staticmethod
+    def to_string(path_info):
+        return "{}://{}@{}:{}".format(path_info['scheme'],
+                                      path_info['user'],
+                                      path_info['host'],
+                                      path_info['path'])
+
+    def changed_cache(self, md5):
+        cache = {}
+        cache['scheme'] = 'ssh'
+        cache['host'] = self.host
+        cache['port'] = self.port
+        cache['user'] = self.user
+        cache['path'] = posixpath.join(self.prefix, md5[0:2], md5[2:])
+
+        if {self.PARAM_MD5: md5} != self.save_info(cache):
+            if self.exists([cache])[0]:
+                msg = 'Corrupted cache file {}'
+                Logger.warn(msg.format(self.to_string(cache)))
+                self.remove(cache)
+            return True
+
+        return False
+
+    def changed(self, path_info, checksum_info):
+        if not self.exists([path_info])[0]:
+            return True
+
+        md5 = checksum_info.get(self.PARAM_MD5, None)
+        if md5 is None:
+            return True
+
+        if self.changed_code(md5):
+            return True
+
+        return checksum_info != self.save_info(path_info)
+
+    def save(self, path_info):
+        if path_info['scheme'] != 'ssh':
+            raise NotImplementedError
+
+        md5 = self.md5(path_info)
+        dest = path_info.copy()
+        dest['path'] = posixpath.join(self.prefix, md5[0:2], md5[2:])
+
+        self.cp(path_info, dest)
+
+        return {self.PARAM_MD5: md5}
+
+    def checkout(self, path_info, checksum_info):
+        if path_info['scheme'] != 'ssh':
+            raise NotImplementedError
+
+        md5 = checksum_info.get(self.PARAM_MD5, None)
+        if not md5:
+            return
+
+        if not self.changed(path_info, checksum_info):
+            msg = "Data '{}' didn't change."
+            Logger.info(msg.format(self.to_string(path_info)))
+            return
+
+        if self.changed_cache(md5):
+            msg = "Cache '{}' not found. File '{}' won't be created."
+            Logger.warn(msg.format(md5, self.to_string(path_info)))
+            return
+
+        if self.exists([path_info])[0]:
+            msg = "Data '{}' exists. Removing before checkout."
+            Logger.warn(msg.format(self.to_string(path_info)))
+            self.remove(path_info)
+            return
+
+        msg = "Checking out '{}' with cache '{}'."
+        Logger.info(msg.format(self.to_string(path_info), md5))
+
+        src = path_info.copy()
+        src['path'] = posixpath.join(self.prefix, md5[0:2], md5[2:])
+
+        self.cp(src, path_info)
+
+    def remove(self, path_info):
+        if path_info['scheme'] != 'ssh':
+            raise NotImplementedError
+
+        Logger.debug('Removing ssh://{}@{}/{}'.format(path_info['user'],
+                                                      path_info['host'],
+                                                      path_info['path']))
+
+        ssh = self.ssh(host=path_info['host'],
+                       user=path_info['user'],
+                       port=path_info['port'])
+        ssh.open_sftp().remove(path_info['path'])
+        ssh.close()
+
+    def download(self,
+                 from_infos,
+                 to_infos,
+                 no_progress_bar=False,
+                 names=None):
+        names = self._verify_path_args(from_infos, to_infos, names)
+
+        ssh = self.ssh(host=from_infos[0]['host'],
+                       user=from_infos[0]['user'],
+                       port=from_infos[0]['port'])
+
+        for to_info, from_info, name in zip(to_infos, from_infos, names):
+            if from_info['scheme'] != 'ssh':
+                raise NotImplementedError
+
+            if to_info['scheme'] == 'ssh':
+                assert from_info['host'] == to_info['host']
+                assert from_info['port'] == to_info['port']
+                assert from_info['user'] == to_info['user']
+                self.cp(from_info, to_info, ssh=ssh)
+                continue
+
+            if to_info['scheme'] != 'local':
+                raise NotImplementedError
+
+            msg = "Downloading '{}/{}' to '{}'".format(from_info['host'],
+                                                       from_info['path'],
+                                                       to_info['path'])
+            Logger.debug(msg)
+
+            if not name:
+                name = os.path.basename(to_info['path'])
+
+            self._makedirs(to_info['path'])
+            tmp_file = self.tmp_file(to_info['path'])
+            try:
+                ssh.open_sftp().get(from_info['path'],
+                                    tmp_file,
+                                    callback=create_cb(name))
+            except Exception as exc:
+                msg = "Failed to download '{}/{}' to '{}'"
+                Logger.error(msg.format(from_info['host'],
+                                        from_info['path'],
+                                        to_info['path']), exc)
+                continue
+
+            os.rename(tmp_file, to_info['path'])
+            progress.finish_target(name)
+
+        ssh.close()
+
+    def upload(self, from_infos, to_infos, names=None):
+        names = self._verify_path_args(to_infos, from_infos, names)
+
+        ssh = self.ssh(host=to_infos[0]['host'],
+                       user=to_infos[0]['user'],
+                       port=to_infos[0]['port'])
+        sftp = ssh.open_sftp()
+
+        for from_info, to_info, name in zip(from_infos, to_infos, names):
+            if to_info['scheme'] != 'ssh':
+                raise NotImplementedError
+
+            if from_info['scheme'] != 'local':
+                raise NotImplementedError
+
+            Logger.debug("Uploading '{}' to '{}/{}'".format(from_info['path'],
+                                                            to_info['host'],
+                                                            to_info['path']))
+
+            if not name:
+                name = os.path.basename(from_info['path'])
+
+            dname = posixpath.dirname(to_info['path'])
+            self._exec(ssh, 'mkdir -p {}'.format(dname))
+
+            try:
+                sftp.put(from_info['path'],
+                         to_info['path'],
+                         callback=create_cb(name))
+            except Exception as exc:
+                msg = "Failed to upload '{}' to '{}/{}'"
+                Logger.error(msg.format(from_info['path'],
+                                        to_info['host'],
+                                        to_info['path'], exc))
+                continue
+
+            progress.finish_target(name)
+
+        sftp.close()
+        ssh.close()
+
+    def _path_to_md5(self, path):
+        relpath = posixpath.relpath(path, self.prefix)
+        return posixpath.dirname(relpath) + posixpath.basename(relpath)
+
+    def _all_md5s(self):
+        ssh = self.ssh(host=self.host,
+                       user=self.user,
+                       port=self.port)
+        cmd = 'find {} -type f -follow -print'.format(self.prefix)
+        stdout = self._exec(ssh, cmd)
+        flist = stdout.split()
+        ssh.close()
+
+        return [self._path_to_md5(path) for path in flist]
+
+    def gc(self, cinfos):
+        used = [info[self.PARAM_MD5] for info in cinfos['ssh']]
+        used += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
+
+        removed = False
+        for md5 in self._all_md5s():
+            if md5 in used:
+                continue
+            path_info = {'scheme': 'ssh',
+                         'user': self.user,
+                         'host': self.host,
+                         'port': self.port,
+                         'path': posixpath.join(self.prefix,
+                                                md5[0:2], md5[2:])}
+            self.remove(path_info)
+            removed = True
+
+        return removed

dvc/scm.py

@@ -1,6 +1,8 @@
 import os
 
 from dvc.exceptions import DvcException
+from dvc.logger import Logger
+from dvc.utils import fix_env
 
 
 class SCMError(DvcException):
@@ -12,13 +14,18 @@ class FileNotInRepoError(DvcException):
 
 
 class Base(object):
-    def __init__(self, root_dir=os.curdir):
-        self.root_dir = os.path.abspath(os.path.realpath(root_dir))
+    def __init__(self, root_dir=os.curdir, project=None):
+        self.project = project
+        self.root_dir = root_dir
 
     @staticmethod
     def is_repo(root_dir):
         return True
 
+    @staticmethod
+    def is_submodule(root_dir):
+        return True
+
     def ignore(self, path):
         pass
 
@@ -31,9 +38,6 @@ class Base(object):
     def ignore_list(self, p_list):
         return [self.ignore(path) for path in p_list]
 
-    def ignore_file(self):
-        return self.GITIGNORE
-
     def add(self, paths):
         pass
 
@@ -46,39 +50,82 @@ class Base(object):
     def branch(self, branch):
         pass
 
+    def brancher(self, branches=None, all_branches=False):
+        if not branches and not all_branches:
+            yield ''
+            return
+
+        saved = self.active_branch()
+        if not branches:
+            branches = self.list_branches() if all_branches else [saved]
+        for branch in branches:
+            self.checkout(branch)
+            yield branch
+        self.checkout(saved)
+
     def untracked_files(self):
         pass
 
     def is_tracked(self, path):
         pass
 
+    def active_branch(self):
+        pass
+
+    def list_branches(self):
+        pass
+
+    def install(self):
+        pass
+
 
 class Git(Base):
     GITIGNORE = '.gitignore'
     GIT_DIR = '.git'
 
-    def __init__(self, root_dir=os.curdir):
-        super(Git, self).__init__(root_dir)
+    def __init__(self, root_dir=os.curdir, project=None):
+        super(Git, self).__init__(root_dir, project=project)
 
         import git
+        from git.exc import InvalidGitRepositoryError
         try:
             self.repo = git.Repo(root_dir)
         except InvalidGitRepositoryError:
-            raise SCMError('{} is not a git repository'.format(root_dir))
+            msg = '{} is not a git repository'
+            raise SCMError(msg.format(root_dir))
+
+        # NOTE: fixing LD_LIBRARY_PATH for binary built by PyInstaller.
+        # http://pyinstaller.readthedocs.io/en/stable/runtime-information.html
+        env = fix_env(None)
+        lp = env.get('LD_LIBRARY_PATH', None)
+        self.repo.git.update_environment(LD_LIBRARY_PATH=lp)
 
     @staticmethod
     def is_repo(root_dir):
-        git_dir = os.path.join(root_dir, Git.GIT_DIR)
-        return os.path.isdir(git_dir)
+        return os.path.isdir(Git._get_git_dir(root_dir))
+
+    @staticmethod
+    def is_submodule(root_dir):
+        return os.path.isfile(Git._get_git_dir(root_dir))
+
+    @staticmethod
+    def _get_git_dir(root_dir):
+        return os.path.join(root_dir, Git.GIT_DIR)
+
+    @property
+    def dir(self):
+        return self.repo.git_dir
+
+    def ignore_file(self):
+        return self.GITIGNORE
 
     def _get_gitignore(self, path):
+        assert os.path.isabs(path)
         entry = os.path.basename(path)
-        gitignore = os.path.join(os.path.abspath(os.path.dirname(path)),
-                                 self.GITIGNORE)
-        gitignore = os.path.abspath(os.path.realpath(gitignore))
+        gitignore = os.path.join(os.path.dirname(path), self.GITIGNORE)
 
         if not gitignore.startswith(self.root_dir):
-            raise FileNotInRepoError()
+            raise FileNotInRepoError(path)
 
         return entry, gitignore
 
@@ -88,15 +135,24 @@ class Git(Base):
         ignore_list = []
         if os.path.exists(gitignore):
             ignore_list = open(gitignore, 'r').readlines()
-            filtered = list(filter(lambda x: x.strip() == entry.strip(), ignore_list))
+            filtered = list(filter(lambda x: x.strip() == entry.strip(),
+                                   ignore_list))
             if len(filtered) != 0:
                 return
 
+        msg = "Adding '{}' to '{}'.".format(os.path.relpath(path),
+                                            os.path.relpath(gitignore))
+        Logger.info(msg)
+
         content = entry
         if len(ignore_list) > 0:
             content = '\n' + content
 
-        open(gitignore, 'a').write(content)
+        with open(gitignore, 'a') as fd:
+            fd.write(content)
+
+        if self.project is not None:
+            self.project._files_to_git_add.append(os.path.relpath(gitignore))
 
     def ignore_remove(self, path):
         entry, gitignore = self._get_gitignore(path)
@@ -112,8 +168,20 @@ class Git(Base):
         with open(gitignore, 'w') as fd:
             fd.writelines(filtered)
 
+        if self.project is not None:
+            self.project._files_to_git_add.append(os.path.relpath(gitignore))
+
     def add(self, paths):
-        self.repo.index.add(paths)
+        # NOTE: GitPython is not currently able to handle index version >= 3.
+        # See https://github.com/iterative/dvc/issues/610 for more details.
+        try:
+            self.repo.index.add(paths)
+        except AssertionError as exc:
+            msg = 'Failed to add \'{}\' to git. You can add those files '
+            msg += 'manually using \'git add\'. '
+            msg += 'See \'https://github.com/iterative/dvc/issues/610\' '
+            msg += 'for more details.'
+            Logger.error(msg.format(str(paths)), exc)
 
     def commit(self, msg):
         self.repo.index.commit(msg)
@@ -128,14 +196,33 @@ class Git(Base):
         self.repo.git.branch(branch)
 
     def untracked_files(self):
-        return self.repo.untracked_files
+        files = self.repo.untracked_files
+        return [os.path.join(self.repo.working_dir, fname) for fname in files]
 
     def is_tracked(self, path):
         return len(self.repo.git.ls_files(path)) != 0
 
+    def active_branch(self):
+        return self.repo.active_branch.name
 
-def SCM(root_dir=os.curdir):
-    if Git.is_repo(root_dir):
-        return Git(root_dir)
+    def list_branches(self):
+        return [h.name for h in self.repo.heads]
 
-    return Base(root_dir)
+    def install(self):
+        hook = os.path.join(self.root_dir,
+                            self.GIT_DIR,
+                            'hooks',
+                            'post-checkout')
+        if os.path.isfile(hook):
+            msg = 'Git hook \'{}\' already exists.'
+            raise SCMError(msg.format(os.path.relpath(hook)))
+        with open(hook, 'w+') as fd:
+            fd.write('#!/bin/sh\nexec dvc checkout\n')
+        os.chmod(hook, 0o777)
+
+
+def SCM(root_dir, no_scm=False, project=None):
+    if Git.is_repo(root_dir) or Git.is_submodule(root_dir):
+        return Git(root_dir, project=project)
+
+    return Base(root_dir, project=project)

dvc/signal_handler.py

@@ -0,0 +1,17 @@
+import signal
+import threading
+
+from dvc.logger import Logger
+
+
+class SignalHandler(object):
+    def __enter__(self):
+        if isinstance(threading.current_thread(), threading._MainThread):
+            self.old_handler = signal.signal(signal.SIGINT, self.handler)
+
+    def handler(self, sig, frame):
+        Logger.debug('Ignoring SIGINT during critical parts of code...')
+
+    def __exit__(self, type, value, traceback):
+        if isinstance(threading.current_thread(), threading._MainThread):
+            signal.signal(signal.SIGINT, self.old_handler)

dvc/stage.py

@@ -1,12 +1,15 @@
 import os
 import yaml
 import itertools
+import posixpath
 import subprocess
-import schema
+from schema import Schema, SchemaError, Optional, Or, And
 
+import dvc.dependency as dependency
+import dvc.output as output
 from dvc.exceptions import DvcException
-from dvc.output import Output, Dependency, OutputError
 from dvc.logger import Logger
+from dvc.utils import dict_md5, fix_env
 
 
 class StageCmdFailedError(DvcException):
@@ -20,7 +23,37 @@ class StageFileFormatError(DvcException):
         super(StageFileFormatError, self).__init__('Stage file format error')
 
 
-class MissingDataSource(OutputError):
+class StageFileDoesNotExistError(DvcException):
+    def __init__(self, fname):
+        msg = "'{}' does not exist.".format(fname)
+        super(StageFileDoesNotExistError, self).__init__(msg)
+
+
+class StageFileIsNotDvcFileError(DvcException):
+    def __init__(self, fname):
+        msg = "'{}' is not a dvc file".format(fname)
+        super(StageFileIsNotDvcFileError, self).__init__(msg)
+
+
+class StageFileBadNameError(DvcException):
+    def __init__(self, msg):
+        super(StageFileBadNameError, self).__init__(msg)
+
+
+class MissingDep(DvcException):
+    def __init__(self, deps):
+        assert len(deps) > 0
+
+        if len(deps) > 1:
+            dep = 'dependencies'
+        else:
+            dep = 'dependency'
+
+        msg = u'missing {}: {}'.format(dep, ', '.join(map(str, deps)))
+        super(MissingDep, self).__init__(msg)
+
+
+class MissingDataSource(DvcException):
     def __init__(self, missing_files):
         assert len(missing_files) > 0
 
@@ -36,23 +69,37 @@ class Stage(object):
     STAGE_FILE = 'Dvcfile'
     STAGE_FILE_SUFFIX = '.dvc'
 
+    PARAM_MD5 = 'md5'
     PARAM_CMD = 'cmd'
     PARAM_DEPS = 'deps'
     PARAM_OUTS = 'outs'
+    PARAM_LOCKED = 'locked'
 
     SCHEMA = {
-        schema.Optional(PARAM_CMD): schema.Or(str, None),
-        schema.Optional(PARAM_DEPS): schema.Or(schema.And(list, schema.Schema([Dependency.SCHEMA])), None),
-        schema.Optional(PARAM_OUTS): schema.Or(schema.And(list, schema.Schema([Output.SCHEMA])), None),
+        Optional(PARAM_MD5): Or(str, None),
+        Optional(PARAM_CMD): Or(str, None),
+        Optional(PARAM_DEPS): Or(And(list, Schema([dependency.SCHEMA])), None),
+        Optional(PARAM_OUTS): Or(And(list, Schema([output.SCHEMA])), None),
+        Optional(PARAM_LOCKED): bool,
     }
 
-    def __init__(self, project, path=None, cmd=None, cwd=None, deps=[], outs=[]):
+    def __init__(self,
+                 project,
+                 path=None,
+                 cmd=None,
+                 cwd=os.curdir,
+                 deps=[],
+                 outs=[],
+                 md5=None,
+                 locked=False):
         self.project = project
         self.path = path
         self.cmd = cmd
         self.cwd = cwd
         self.outs = outs
         self.deps = deps
+        self.md5 = md5
+        self.locked = locked
 
     @property
     def relpath(self):
@@ -63,55 +110,107 @@ class Stage(object):
         return self.cmd is None
 
     @staticmethod
-    def is_stage_file(path):
-        if not os.path.isfile(path):
-            return False
-
-        if not path.endswith(Stage.STAGE_FILE_SUFFIX) and os.path.basename(path) != Stage.STAGE_FILE:
+    def is_stage_filename(path):
+        if not path.endswith(Stage.STAGE_FILE_SUFFIX) \
+           and os.path.basename(path) != Stage.STAGE_FILE:
             return False
 
         return True
 
+    @staticmethod
+    def is_stage_file(path):
+        if not os.path.isfile(path):
+            return False
+
+        return Stage.is_stage_filename(path)
+
+    def changed_md5(self):
+        md5 = self.dumpd().get(self.PARAM_MD5, None)
+
+        # backward compatibility
+        if self.md5 is None:
+            return False
+
+        if self.md5 and md5 and self.md5 == md5:
+            return False
+
+        msg = "Dvc file '{}' md5 changed(expected '{}', actual '{}')"
+        self.project.logger.debug(msg.format(self.relpath, self.md5, md5))
+        return True
+
+    @property
+    def is_callback(self):
+        return not self.is_data_source and len(self.deps) == 0
+
+    @property
+    def is_import(self):
+        return not self.cmd and \
+               len(self.deps) == 1 and \
+               len(self.outs) == 1
+
     def changed(self):
         ret = False
-        for entry in itertools.chain(self.outs, self.deps):
+
+        if self.is_callback:
+            ret = True
+
+        if self.locked:
+            entries = self.outs
+        else:
+            entries = itertools.chain(self.outs, self.deps)
+
+        for entry in entries:
             if entry.changed():
                 ret = True
 
+        if self.changed_md5():
+            ret = True
+
         if ret:
-            self.project.logger.debug(u'Dvc file \'{}\' changed'.format(self.relpath))
+            msg = u'Dvc file \'{}\' changed'.format(self.relpath)
         else:
-            self.project.logger.debug(u'Dvc file \'{}\' didn\'t change'.format(self.relpath))
+            msg = u'Dvc file \'{}\' didn\'t change'.format(self.relpath)
+
+        self.project.logger.debug(msg)
 
         return ret
 
-    def remove_outs(self):
+    def remove_outs(self, ignore_remove=False):
         for out in self.outs:
-            out.remove()
-            if out.use_cache:
-                self.project.scm.ignore_remove(out.path)
+            out.remove(ignore_remove=ignore_remove)
 
     def remove(self):
-        self.remove_outs()
+        self.remove_outs(ignore_remove=True)
         os.unlink(self.path)
 
-    def reproduce(self, force=False):
+    def reproduce(self, force=False, dry=False, interactive=False):
         if not self.changed() and not force:
             return None
 
-        if self.cmd:
+        if (self.cmd or self.is_import) and not self.locked and not dry:
             # Removing outputs only if we actually have command to reproduce
-            self.remove_outs()
+            self.remove_outs(ignore_remove=False)
 
-        self.run()
+        msg = "Going to reproduce '{}'. Are you sure you want to continue?"
+        msg = msg.format(self.relpath)
+        if interactive \
+           and not self.project.prompt.prompt(msg):
+            raise DvcException('Reproduction aborted by the user')
+
+        self.project.logger.info(u'Reproducing \'{}\''.format(self.relpath))
+
+        self.run(dry=dry)
+
+        msg = u'\'{}\' was reproduced'.format(self.relpath)
+        self.project.logger.debug(msg)
 
         return self
 
     @staticmethod
     def validate(d):
         try:
-            schema.Schema(Stage.SCHEMA).validate(d)
-        except schema.SchemaError as exc:
+            Schema(Stage.SCHEMA).validate(d)
+        except SchemaError as exc:
             Logger.debug(str(exc))
             raise StageFileFormatError()
 
@@ -122,88 +221,211 @@ class Stage(object):
         path = os.path.abspath(path)
         cwd = os.path.dirname(path)
         cmd = d.get(Stage.PARAM_CMD, None)
-        deps = Dependency.loadd_from(project, d.get(Stage.PARAM_DEPS, []), cwd=cwd)
-        outs = Output.loadd_from(project, d.get(Stage.PARAM_OUTS, []), cwd=cwd)
+        md5 = d.get(Stage.PARAM_MD5, None)
+        locked = d.get(Stage.PARAM_LOCKED, False)
+
+        stage = Stage(project=project,
+                      path=path,
+                      cmd=cmd,
+                      cwd=cwd,
+                      md5=md5,
+                      locked=locked)
+
+        stage.deps = dependency.loadd_from(stage, d.get(Stage.PARAM_DEPS, []))
+        stage.outs = output.loadd_from(stage, d.get(Stage.PARAM_OUTS, []))
+
+        return stage
+
+    @classmethod
+    def _stage_fname_cwd(cls, fname, cwd, outs, add):
+        if fname and cwd:
+            return (fname, cwd)
+
+        if not outs:
+            return (cls.STAGE_FILE, cwd if cwd else os.getcwd())
+
+        out = outs[0]
+        if out.path_info['scheme'] == 'local':
+            path = os.path
+        else:
+            path = posixpath
+
+        if not fname:
+            fname = path.basename(out.path) + cls.STAGE_FILE_SUFFIX
+
+        if not cwd or (add and out.is_local):
+            cwd = path.dirname(out.path)
+
+        return (fname, cwd)
 
-        return Stage(project=project,
-                     path=path,
-                     cmd=cmd,
-                     cwd=cwd,
-                     deps=deps,
-                     outs=outs)
     @staticmethod
     def loads(project=None,
               cmd=None,
               deps=[],
               outs=[],
               outs_no_cache=[],
+              metrics_no_cache=[],
               fname=None,
-              cwd=os.curdir):
+              cwd=os.curdir,
+              locked=False,
+              add=False,
+              overwrite=True):
+        stage = Stage(project=project,
+                      cwd=cwd,
+                      cmd=cmd,
+                      locked=locked)
+
+        stage.outs = output.loads_from(stage, outs, use_cache=True)
+        stage.outs += output.loads_from(stage, outs_no_cache, use_cache=False)
+        stage.outs += output.loads_from(stage, metrics_no_cache,
+                                        use_cache=False, metric=True)
+        stage.deps = dependency.loads_from(stage, deps)
+
+        if fname is not None and os.path.basename(fname) != fname:
+            msg = "Stage file name '{}' should not contain subdirectories. " \
+                  "Use '-c|--cwd' to change location of the stage file."
+            raise StageFileBadNameError(msg.format(fname))
+
+        fname, cwd = Stage._stage_fname_cwd(fname, cwd, stage.outs, add=add)
+
         cwd = os.path.abspath(cwd)
         path = os.path.join(cwd, fname)
-        outputs = Output.loads_from(project, outs, use_cache=True, cwd=cwd)
-        outputs += Output.loads_from(project, outs_no_cache, use_cache=False, cwd=cwd)
-        dependencies = Dependency.loads_from(project, deps, cwd=cwd)
 
-        return Stage(project=project,
-                     path=path,
-                     cmd=cmd,
-                     cwd=cwd,
-                     outs=outputs,
-                     deps=dependencies)
+        if os.path.exists(path):
+            relpath = os.path.relpath(path)
+            msg = "'{}' already exists. " \
+                  "Do you wish to run the command and overwrite it?"
+            if not overwrite \
+               and not project.prompt.prompt(msg.format(relpath), False):
+                raise DvcException("'{}' already exists".format(relpath))
+
+        stage.cwd = cwd
+        stage.path = path
+
+        return stage
+
+    @staticmethod
+    def _check_dvc_filename(fname):
+        if not Stage.is_stage_filename(fname):
+            msg = "Bad stage filename '{}'. Stage files should be named " \
+                  "'Dvcfile' or have a '.dvc' suffix(e.g. '{}.dvc')."
+            raise StageFileBadNameError(msg.format(os.path.relpath(fname),
+                                                   os.path.basename(fname)))
+
+    @staticmethod
+    def _check_dvc_file(fname):
+        sname = fname + Stage.STAGE_FILE_SUFFIX
+        if Stage.is_stage_file(sname):
+            Logger.info("Do you mean '{}'?".format(sname))
 
     @staticmethod
     def load(project, fname):
+        if not os.path.exists(fname):
+            Stage._check_dvc_file(fname)
+            raise StageFileDoesNotExistError(fname)
+
+        Stage._check_dvc_filename(fname)
+
+        if not Stage.is_stage_file(fname):
+            Stage._check_dvc_file(fname)
+            raise StageFileIsNotDvcFileError(fname)
+
         with open(fname, 'r') as fd:
             return Stage.loadd(project, yaml.safe_load(fd), fname)
 
     def dumpd(self):
-        deps = [x.dumpd(self.cwd) for x in self.deps]
-        outs = [x.dumpd(self.cwd) for x in self.outs]
+        deps = [x.dumpd() for x in self.deps]
+        outs = [x.dumpd() for x in self.outs]
 
-        return {
-            Stage.PARAM_CMD: self.cmd,
-            Stage.PARAM_DEPS: deps,
-            Stage.PARAM_OUTS: outs,
-        }
+        ret = {}
+        if self.cmd is not None:
+            ret[Stage.PARAM_CMD] = self.cmd
+
+        if len(deps):
+            ret[Stage.PARAM_DEPS] = deps
+
+        if len(outs):
+            ret[Stage.PARAM_OUTS] = outs
+
+        if self.locked:
+            ret[Stage.PARAM_LOCKED] = self.locked
+
+        ret[Stage.PARAM_MD5] = dict_md5(ret)
+
+        return ret
 
     def dump(self, fname=None):
         if not fname:
             fname = self.path
 
+        self._check_dvc_filename(fname)
+
+        msg = "Saving information to '{}'.".format(os.path.relpath(fname))
+        Logger.info(msg)
+
         with open(fname, 'w') as fd:
             yaml.safe_dump(self.dumpd(), fd, default_flow_style=False)
 
+        self.project._files_to_git_add.append(os.path.relpath(fname))
+
     def save(self):
         for dep in self.deps:
             dep.save()
 
         for out in self.outs:
             out.save()
-            if out.use_cache:
-                self.project.scm.ignore(out.path)
 
-    def run(self):
-        if not self.is_data_source:
-            self.project.logger.info(u'Reproducing \'{}\':\n\t{}'.format(self.relpath, self.cmd))
+    def _check_missing_deps(self):
+        missing = []
+        for dep in self.deps:
+            if not dep.exists:
+                missing.append(dep)
 
-            p = subprocess.Popen(self.cmd, cwd=self.cwd, shell=True)
-            p.communicate()
-            if p.returncode != 0:
-               raise StageCmdFailedError(self)
- 
-            self.save()
+        if len(missing) > 0:
+            raise MissingDep(missing)
 
-            self.project.logger.debug(u'\'{}\' was reproduced'.format(self.relpath))
+    def run(self, dry=False):
+        if self.locked:
+            msg = u'Verifying outputs in locked stage \'{}\''
+            self.project.logger.info(msg.format(self.relpath))
+            if not dry:
+                self.check_missing_outputs()
+        elif self.is_import:
+            msg = u'Importing \'{}\' -> \'{}\''
+            self.project.logger.info(msg.format(self.deps[0].path,
+                                                self.outs[0].path))
+
+            if not dry:
+                self.deps[0].download(self.outs[0].path_info)
+        elif self.is_data_source:
+            msg = u'Verifying data sources in \'{}\''.format(self.relpath)
+            self.project.logger.info(msg)
+            if not dry:
+                self.check_missing_outputs()
         else:
-            self.project.logger.info(u'Verifying data sources in \'{}\''.format(self.relpath))
-            self.check_missing_outputs()
+            msg = u'Running command:\n\t{}'.format(self.cmd)
+            self.project.logger.info(msg)
+
+            if not dry:
+                self._check_missing_deps()
+                p = subprocess.Popen(self.cmd,
+                                     cwd=self.cwd,
+                                     shell=True,
+                                     env=fix_env(os.environ),
+                                     executable=os.getenv('SHELL'))
+                p.communicate()
+                if p.returncode != 0:
+                    raise StageCmdFailedError(self)
+
+        if not dry:
             self.save()
 
     def check_missing_outputs(self):
-        missing_outs = [out.rel_path for out in self.outs if not os.path.exists(out.rel_path)]
-        if missing_outs:
-            raise MissingDataSource(missing_outs)
+        outs = [out for out in self.outs if not out.exists]
+        paths = [out.path if out.path_info['scheme'] != 'local' else
+                 out.rel_path for out in outs]
+        if paths:
+            raise MissingDataSource(paths)
 
     def checkout(self):
         for out in self.outs:
@@ -222,10 +444,13 @@ class Stage(object):
 
     def status(self):
         ret = {}
-        ret.update(self._status(self.deps, 'deps'))
+
+        if not self.locked:
+            ret.update(self._status(self.deps, 'deps'))
+
         ret.update(self._status(self.outs, 'outs'))
 
-        if ret:
+        if ret or self.changed_md5() or self.is_callback:
             return {self.relpath: ret}
 
         return {}

dvc/state.py

@@ -1,37 +1,13 @@
 import os
-import json
+import time
+import sqlite3
 import nanotime
-from checksumdir import dirhash
 
+from dvc.config import Config
 from dvc.system import System
-from dvc.output import Output
-from dvc.utils import file_md5
+from dvc.utils import file_md5, remove
 from dvc.exceptions import DvcException
-
-
-class StateEntry(object):
-    PARAM_MTIME = 'mtime'
-    PARAM_MD5 = 'md5'
-
-    def __init__(self, md5, mtime):
-        self.mtime = mtime
-        self.md5 = md5
-
-    def update(self, md5, mtime):
-        self.mtime = mtime
-        self.md5 = md5
-
-    @staticmethod
-    def loadd(d):
-        mtime = d[StateEntry.PARAM_MTIME]
-        md5 = d[StateEntry.PARAM_MD5]
-        return StateEntry(md5, mtime)
-
-    def dumpd(self):
-        return {
-            self.PARAM_MD5: self.md5,
-            self.PARAM_MTIME: self.mtime,
-        }
+from dvc.logger import Logger
 
 
 class StateDuplicateError(DvcException):
@@ -40,76 +16,264 @@ class StateDuplicateError(DvcException):
 
 class State(object):
     STATE_FILE = 'state'
+    STATE_TABLE = 'state'
+    STATE_TABLE_LAYOUT = "inode INTEGER PRIMARY KEY, " \
+                         "mtime TEXT NOT NULL, " \
+                         "md5 TEXT NOT NULL, " \
+                         "timestamp TEXT NOT NULL"
 
-    def __init__(self, root_dir, dvc_dir):
-        self.root_dir = root_dir
-        self.dvc_dir = dvc_dir
-        self.state_file = os.path.join(dvc_dir, self.STATE_FILE)
-        self._db = self.load()
+    STATE_INFO_TABLE = 'state_info'
+    STATE_INFO_TABLE_LAYOUT = 'count INTEGER'
+    STATE_INFO_ROW = 1
+
+    LINK_STATE_TABLE = 'link_state'
+    LINK_STATE_TABLE_LAYOUT = "path TEXT PRIMARY KEY, " \
+                              "inode INTEGER NOT NULL, " \
+                              "mtime TEXT NOT NULL"
+
+    STATE_ROW_LIMIT = 10000000
+    STATE_ROW_CLEANUP_QUOTA = 50
+
+    def __init__(self, project, config):
+        self.project = project
+        self.dvc_dir = project.dvc_dir
+        self.root_dir = project.root_dir
+
+        self.row_limit = 100
+        self.row_cleanup_quota = 50
+
+        c = config.get(Config.SECTION_STATE, {})
+        self.row_limit = c.get(Config.SECTION_STATE_ROW_LIMIT,
+                               self.STATE_ROW_LIMIT)
+        self.row_cleanup_quota = c.get(Config.SECTION_STATE_ROW_CLEANUP_QUOTA,
+                                       self.STATE_ROW_CLEANUP_QUOTA)
+
+        if not self.dvc_dir:
+            self.state_file = None
+            return
+
+        self.state_file = os.path.join(self.dvc_dir, self.STATE_FILE)
+        self.db = None
+        self.c = None
+        self.inserts = 0
+
+    def __enter__(self):
+        self.load()
+
+    def __exit__(self, type, value, tb):
+        self.dump()
 
     @staticmethod
-    def init(root_dir, dvc_dir):
-        return State(root_dir, dvc_dir)
+    def init(project):
+        return State(project)
 
-    def compute_md5(self, path):
+    def _collect(self, path):
         if os.path.isdir(path):
-            return dirhash(path, hashfunc='md5') + Output.MD5_DIR_SUFFIX
+            return self.project.cache.local.collect_dir_cache(path)
         else:
-            return file_md5(path)[0]
+            return (file_md5(path)[0], None)
 
     def changed(self, path, md5):
-        return self.update(path) != md5
+        actual = self.update(path)
+
+        if not md5 or not actual:
+            return True
+
+        return actual.split('.')[0] != md5.split('.')[0]
 
     def load(self):
-        if not os.path.isfile(self.state_file):
-            return {}
+        retries = 1
+        while True:
+            assert self.db is None
+            assert self.c is None
+            assert self.inserts == 0
+            self.db = sqlite3.connect(self.state_file)
+            self.c = self.db.cursor()
 
-        with open(self.state_file, 'r') as fd:
-            return json.load(fd)
+            # Try loading once to check that the file is indeed a database
+            # and reformat it if it is not.
+            try:
+                # Check that the state file is indeed a database
+                cmd = "CREATE TABLE IF NOT EXISTS {} ({})"
+                self.c.execute(cmd.format(self.STATE_TABLE,
+                                          self.STATE_TABLE_LAYOUT))
+                self.c.execute(cmd.format(self.STATE_INFO_TABLE,
+                                          self.STATE_INFO_TABLE_LAYOUT))
+                self.c.execute(cmd.format(self.LINK_STATE_TABLE,
+                                          self.LINK_STATE_TABLE_LAYOUT))
+
+                cmd = "INSERT OR IGNORE INTO {} (count) SELECT 0 " \
+                      "WHERE NOT EXISTS (SELECT * FROM {})"
+                self.c.execute(cmd.format(self.STATE_INFO_TABLE,
+                                          self.STATE_INFO_TABLE))
+
+                return
+            except sqlite3.DatabaseError:
+                self.c.close()
+                self.db.close()
+                self.db = None
+                self.c = None
+                self.inserts = 0
+                if retries > 0:
+                    os.unlink(self.state_file)
+                    retries -= 1
+                else:
+                    raise
 
     def dump(self):
-        with open(self.state_file, 'w+') as fd:
-            json.dump(self._db, fd)
+        assert self.db is not None
+
+        cmd = "SELECT count from {} WHERE rowid={}"
+        self.c.execute(cmd.format(self.STATE_INFO_TABLE,
+                                  self.STATE_INFO_ROW))
+        ret = self.c.fetchall()
+        assert len(ret) == 1
+        assert len(ret[0]) == 1
+        count = ret[0][0] + self.inserts
+
+        if count > self.row_limit:
+            msg = "Cleaning up state. This might take a while."
+            self.project.logger.warn(msg)
+
+            delete = (count - self.row_limit)
+            delete += int(self.row_limit * (self.row_cleanup_quota/100.))
+            cmd = "DELETE FROM {} WHERE timestamp IN (" \
+                  "SELECT timestamp FROM {} ORDER BY timestamp ASC LIMIT {});"
+            self.c.execute(cmd.format(self.STATE_TABLE,
+                                      self.STATE_TABLE,
+                                      delete))
+
+            self.c.execute("VACUUM")
+
+            cmd = "SELECT COUNT(*) FROM {}"
+
+            self.c.execute(cmd.format(self.STATE_TABLE))
+            ret = self.c.fetchall()
+            assert len(ret) == 1
+            assert len(ret[0]) == 1
+            count = ret[0][0]
+
+        cmd = "UPDATE {} SET count = {} WHERE rowid = {}"
+        self.c.execute(cmd.format(self.STATE_INFO_TABLE,
+                                  count,
+                                  self.STATE_INFO_ROW))
+
+        self.db.commit()
+        self.c.close()
+        self.db.close()
+        self.db = None
+        self.c = None
+        self.inserts = 0
 
     @staticmethod
     def mtime(path):
-        return str(int(nanotime.timestamp(os.path.getmtime(path))))
+        mtime = os.path.getmtime(path)
+        if os.path.isdir(path):
+            for root, dirs, files in os.walk(path):
+                for entry in list(dirs) + list(files):
+                    p = os.path.join(root, entry)
+                    m = os.path.getmtime(p)
+                    if m > mtime:
+                        mtime = m
+
+        return str(int(nanotime.timestamp(mtime)))
 
     @staticmethod
     def inode(path):
-        return str(System.inode(path))
+        return System.inode(path)
+
+    def _do_update(self, path):
+        if not os.path.exists(path):
+            return (None, None)
 
-    def update(self, path, dump=True):
         mtime = self.mtime(path)
         inode = self.inode(path)
 
-        md5 = self._get(inode, mtime)
-        if md5:
-            return md5
+        cmd = 'SELECT * from {} WHERE inode={}'.format(self.STATE_TABLE,
+                                                       inode)
 
-        md5 = self.compute_md5(path)
-        state = StateEntry(md5, mtime)
-        d = state.dumpd()
-        self._db[inode] = d
+        self.c.execute(cmd)
+        ret = self.c.fetchall()
+        if len(ret) == 0:
+            md5, info = self._collect(path)
+            cmd = 'INSERT INTO {}(inode, mtime, md5, timestamp) ' \
+                  'VALUES ({}, "{}", "{}", "{}")'
+            self.c.execute(cmd.format(self.STATE_TABLE,
+                                      inode,
+                                      mtime,
+                                      md5,
+                                      int(nanotime.timestamp(time.time()))))
+            self.inserts += 1
+        else:
+            assert len(ret) == 1
+            assert len(ret[0]) == 4
+            i, m, md5, timestamp = ret[0]
+            assert i == inode
+            if mtime != m:
+                md5, info = self._collect(path)
+                cmd = 'UPDATE {} SET ' \
+                      'mtime = "{}", md5 = "{}", timestamp = "{}" ' \
+                      'WHERE inode = {}'
+                self.c.execute(cmd.format(self.STATE_TABLE,
+                                          mtime,
+                                          md5,
+                                          int(nanotime.timestamp(time.time())),
+                                          inode))
+            else:
+                info = None
+                cmd = 'UPDATE {} SET timestamp = "{}" WHERE inode = {}'
+                self.c.execute(cmd.format(self.STATE_TABLE,
+                                          int(nanotime.timestamp(time.time())),
+                                          inode))
 
-        if dump:
-            self.dump()
+        return (md5, info)
 
-        return md5
+    def update(self, path):
+        return self._do_update(path)[0]
 
-    def _get(self, inode, mtime):
-        d = self._db.get(inode, None)
-        if not d:
-            return None
+    def update_info(self, path):
+        md5, info = self._do_update(path)
+        if not info:
+            info = self.project.cache.local.load_dir_cache(md5)
+        return (md5, info)
 
-        state = StateEntry.loadd(d)
-        if mtime == state.mtime:
-            return state.md5
+    def update_link(self, path):
+        if not os.path.exists(path):
+            return
 
-        return None
-
-    def get(self, path):
         mtime = self.mtime(path)
         inode = self.inode(path)
+        relpath = os.path.relpath(path, self.root_dir)
 
-        return self._get(inode, mtime)
+        cmd = 'REPLACE INTO {}(path, inode, mtime) ' \
+              'VALUES ("{}", {}, "{}")'.format(self.LINK_STATE_TABLE,
+                                               relpath,
+                                               inode,
+                                               mtime)
+        self.c.execute(cmd)
+
+    def remove_unused_links(self, used):
+        unused = []
+
+        self.c.execute('SELECT * FROM {}'.format(self.LINK_STATE_TABLE))
+        for row in self.c:
+            p, i, m = row
+            path = os.path.join(self.root_dir, p)
+
+            if path in used:
+                continue
+
+            if not os.path.exists(path):
+                continue
+
+            inode = self.inode(path)
+            mtime = self.mtime(path)
+
+            if i == inode and m == mtime:
+                Logger.debug('Removing \'{}\' as unused link.'.format(path))
+                remove(path)
+                unused.append(p)
+
+        for p in unused:
+            cmd = 'DELETE FROM {} WHERE path = "{}"'
+            self.c.execute(cmd.format(self.LINK_STATE_TABLE, p))

dvc/system.py

@@ -1,18 +1,10 @@
-import ctypes
 import os
-import re
-import subprocess
-from builtins import str
 
 if os.name == 'nt':
     import ntfsutils.hardlink as winlink
-    from ntfsutils.fs import *
 
 
 class System(object):
-    SYMLINK_OUTPUT = '<SYMLINK>'
-    LONG_PATH_BUFFER_SIZE = 1024
-
     @staticmethod
     def is_unix():
         return os.name != 'nt'
@@ -25,11 +17,108 @@ class System(object):
         return winlink.create(source, link_name)
 
     @staticmethod
-    def _getfileinfo(path):
-        # FIXME: current implementation from ntfsutils doesn't support dirs,
-        # and until https://github.com/sid0/ntfs/pull/10 is not merged and
-        # pip package not updated, we have to use our own version.
-        hfile = CreateFile(path, GENERIC_READ, FILE_SHARE_READ, None, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, None)
+    def symlink(source, link_name):
+        import ctypes
+        if System.is_unix():
+            return os.symlink(source, link_name)
+
+        flags = 0
+        if source is not None and os.path.isdir(source):
+            flags = 1
+
+        func = ctypes.windll.kernel32.CreateSymbolicLinkW
+        func.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_uint32)
+        func.restype = ctypes.c_ubyte
+
+        if func(link_name, source, flags) == 0:
+            raise ctypes.WinError()
+
+    @staticmethod
+    def _reflink_darwin(src, dst):
+        import ctypes
+
+        clib = ctypes.CDLL('libc.dylib')
+        if not hasattr(clib, 'clonefile'):
+            return -1
+
+        clonefile = clib.clonefile
+        clonefile.argtypes = [ctypes.c_char_p,
+                              ctypes.c_char_p,
+                              ctypes.c_int]
+        clonefile.restype = ctypes.c_int
+
+        return clonefile(ctypes.c_char_p(src.encode('utf-8')),
+                         ctypes.c_char_p(src.encode('utf-8')),
+                         ctypes.c_int(0))
+
+    @staticmethod
+    def _reflink_windows(src, dst):
+        return -1
+
+    @staticmethod
+    def _reflink_linux(src, dst):
+        import os
+        import fcntl
+
+        FICLONE = 0x40049409
+
+        s = open(src, 'r')
+        d = open(dst, 'w+')
+
+        try:
+            ret = fcntl.ioctl(d.fileno(), FICLONE, s.fileno())
+        except IOError:
+            s.close()
+            d.close()
+            os.unlink(dst)
+            raise
+
+        s.close()
+        d.close()
+
+        if ret != 0:
+            os.unlink(dst)
+
+        return ret
+
+    @staticmethod
+    def reflink(source, link_name):
+        import platform
+        from dvc.exceptions import DvcException
+
+        system = platform.system()
+        try:
+            if system == 'Windows':
+                ret = System._reflink_windows(source, link_name)
+            elif system == 'Darwin':
+                ret = System._reflink_darwin(source, link_name)
+            elif system == 'Linux':
+                ret = System._reflink_linux(source, link_name)
+            else:
+                ret = -1
+        except IOError:
+            ret = -1
+
+        if ret != 0:
+            raise DvcException('Reflink is not supported')
+
+    # FIXME
+    # Temporary fix while waiting for the PR to be merged and released:
+    # https://github.com/sunshowers/ntfs/pull/11
+    @staticmethod
+    def getdirinfo(path):
+        from ntfsutils.fs import FILE_FLAG_BACKUP_SEMANTICS, FILE_SHARE_READ
+        from ntfsutils.fs import OPEN_EXISTING, BY_HANDLE_FILE_INFORMATION
+        from ntfsutils.fs import CreateFile, WinError
+        from ntfsutils.fs import GetFileInformationByHandle, CloseHandle
+        flags = FILE_FLAG_BACKUP_SEMANTICS
+        hfile = CreateFile(path,
+                           0,
+                           FILE_SHARE_READ,
+                           None,
+                           OPEN_EXISTING,
+                           flags,
+                           None)
         if hfile is None:
             raise WinError()
         info = BY_HANDLE_FILE_INFORMATION()
@@ -42,17 +131,10 @@ class System(object):
     @staticmethod
     def inode(path):
         if System.is_unix():
-            return os.stat(path).st_ino
+            return os.lstat(path).st_ino
 
-        info = System._getfileinfo(path)
-        return hash((info.dwVolumeSerialNumber, info.nFileIndexHigh, info.nFileIndexLow))
-
-    @staticmethod
-    def samefile(path1, path2):
-        if not os.path.exists(path1) or not os.path.exists(path2):
-            return False
-
-        if System.is_unix():
-            return os.path.samefile(path1, path2)
-
-        return winlink.samefile(path1, path2)
+        # getdirinfo from ntfsutils works on both files and dirs
+        info = System.getdirinfo(path)
+        return hash((info.dwVolumeSerialNumber,
+                     info.nFileIndexHigh,
+                     info.nFileIndexLow))

dvc/updater.py

@@ -0,0 +1,61 @@
+import os
+import time
+import requests
+
+from dvc import VERSION_BASE
+from dvc.logger import Logger
+
+
+class Updater(object):
+    URL = 'https://4ki8820rsf.execute-api.us-east-2.amazonaws.com/' \
+          'prod/latest-version'
+    UPDATER_FILE = 'updater'
+    TIMEOUT = 7 * 24 * 60 * 60  # every week
+    TIMEOUT_GET = 10
+
+    def __init__(self, dvc_dir):
+        self.dvc_dir = dvc_dir
+        self.updater_file = os.path.join(dvc_dir, self.UPDATER_FILE)
+
+    @staticmethod
+    def init(dvc_dir):
+        return Updater(dvc_dir)
+
+    def check(self):
+        current = VERSION_BASE
+
+        if os.getenv('CI'):
+            return
+
+        if os.path.isfile(self.updater_file):
+            ctime = os.path.getctime(self.updater_file)
+            if time.time() - ctime < self.TIMEOUT:
+                msg = '{} is not old enough to check for updates'
+                Logger.debug(msg.format(self.UPDATER_FILE))
+                return
+
+            os.unlink(self.updater_file)
+
+        Logger.info('Checking for updates...')
+
+        try:
+            r = requests.get(self.URL, timeout=self.TIMEOUT_GET)
+            j = r.json()
+            latest = j['version']
+            open(self.updater_file, 'w+').close()
+        except Exception as exc:
+            msg = 'Failed to obtain latest version: {}'.format(str(exc))
+            Logger.debug(msg)
+            return
+
+        l_major, l_minor, l_patch = [int(x) for x in latest.split('.')]
+        c_major, c_minor, c_patch = [int(x) for x in current.split('.')]
+
+        if l_major <= c_major and \
+           l_minor <= c_minor and \
+           l_patch <= c_patch:
+            return
+
+        msg = 'You are using dvc version {}, however version {} is ' \
+              'available. Consider upgrading.'
+        Logger.warn(msg.format(current, latest))

dvc/utils.py

@@ -2,59 +2,86 @@
 Helpers for other modules.
 """
 import os
-import re
-import stat
+import sys
+import math
+import json
 import shutil
 import hashlib
 
-from multiprocessing.pool import ThreadPool
-
 from dvc.progress import progress
+from dvc.istextfile import istextfile
 from dvc.logger import Logger
 
 
 LOCAL_CHUNK_SIZE = 1024*1024
+LARGE_FILE_SIZE = 1024*1024*1024
+LARGE_DIR_SIZE = 100
+
+
+def dos2unix(data):
+    return data.replace(b'\r\n', b'\n')
 
 
 def file_md5(fname):
     """ get the (md5 hexdigest, md5 digest) of a file """
     if os.path.exists(fname):
         hash_md5 = hashlib.md5()
-        with open(fname, "rb") as fobj:
-            for chunk in iter(lambda: fobj.read(1024*1000), b""):
+        binary = not istextfile(fname)
+        size = os.path.getsize(fname)
+        bar = False
+        if size >= LARGE_FILE_SIZE:
+            bar = True
+            msg = "Computing md5 for a large file {}. This is only done once."
+            Logger.info(msg.format(os.path.relpath(fname)))
+            name = os.path.relpath(fname)
+            total = 0
+
+        with open(fname, 'rb') as fobj:
+            while True:
+                data = fobj.read(LOCAL_CHUNK_SIZE)
+                if not data:
+                    break
+
+                if bar:
+                    total += len(data)
+                    progress.update_target(name, total, size)
+
+                if binary:
+                    chunk = data
+                else:
+                    chunk = dos2unix(data)
+
                 hash_md5.update(chunk)
+
+        if bar:
+            progress.finish_target(name)
+
         return (hash_md5.hexdigest(), hash_md5.digest())
     else:
         return (None, None)
 
 
-def cached_property(func):
-    '''A decorator for caching properties in classes.'''
-    def get(self):
-        '''Try obtaining cache'''
-        try:
-            return self._property_cache[func]
-        except AttributeError:
-            self._property_cache = {}
-            ret = self._property_cache[func] = func(self)
-            return ret
-        except KeyError:
-            ret = self._property_cache[func] = func(self)
-            return ret
-
-    return property(get)
+def bytes_md5(byts):
+    hasher = hashlib.md5()
+    hasher.update(byts)
+    return hasher.hexdigest()
 
 
-def copyfile(src, dest, no_progress_bar=False):
+def dict_md5(d):
+    byts = json.dumps(d, sort_keys=True).encode('utf-8')
+    return bytes_md5(byts)
+
+
+def copyfile(src, dest, no_progress_bar=False, name=None):
     '''Copy file with progress bar'''
     copied = 0
-    name = os.path.basename(src)
+    name = name if name else os.path.basename(dest)
     total = os.stat(src).st_size
 
     fsrc = open(src, 'rb')
 
     if os.path.isdir(dest):
-        fdest = open(dest + '/' + name, 'wb+')
+        fdest = open(os.path.join(dest, os.path.basename(src)), 'wb+')
     else:
         fdest = open(dest, 'wb+')
 
@@ -74,19 +101,58 @@ def copyfile(src, dest, no_progress_bar=False):
     fdest.close()
 
 
-def map_progress(func, targets, n_threads):
-    """
-    Process targets in multi-threaded mode with progress bar
-    """
-    progress.set_n_total(len(targets))
-    pool = ThreadPool(processes=n_threads)
-    ret = []
+def move(src, dst):
+    dname = os.path.dirname(dst)
+    if not os.path.exists(dname):
+        os.makedirs(dname)
 
-    try:
-        ret = pool.map(func, targets)
-    except Exception as exc:
-        raise
-    finally:
-        progress.finish()
+    if os.path.islink(src):
+        shutil.copy(os.readlink(src), dst)
+        os.unlink(src)
+        return
 
-    return list(zip(targets, ret))
+    shutil.move(src, dst)
+
+
+def remove(path):
+    if not os.path.exists(path):
+        return
+
+    Logger.debug(u'Removing \'{}\''.format(os.path.relpath(path)))
+    if os.path.isfile(path):
+        os.unlink(path)
+    else:
+        shutil.rmtree(path)
+
+
+def to_chunks(l, jobs):
+    n = int(math.ceil(len(l) / jobs))
+
+    if len(l) == 1:
+        return [l]
+
+    if n == 0:
+        n = 1
+
+    return [l[x:x+n] for x in range(0, len(l), n)]
+
+
+# NOTE: Fix env variables modified by PyInstaller
+# http://pyinstaller.readthedocs.io/en/stable/runtime-information.html
+def fix_env(env):
+    if env is None:
+        env = os.environ.copy()
+    else:
+        env = env.copy()
+
+    # NOTE: Check if we are in a bundle
+    # https://pythonhosted.org/PyInstaller/runtime-information.html
+    if getattr(sys, 'frozen', False):
+        lp_key = 'LD_LIBRARY_PATH'
+        lp_orig = env.get(lp_key + '_ORIG', None)
+        if lp_orig is not None:
+            env[lp_key] = lp_orig
+        else:
+            env.pop(lp_key, None)
+
+    return env