test: don't forget to sleep()

Remote fixes

.appveyor.yml

@@ -54,7 +54,7 @@ install:
   - cinst gcloudsdk
   - refreshenv
   - cinst gsutil
-  - cinst openssl.light
+  - cinst openssl.light --version 1.1.1
   - python -m pip install -U pip setuptools wheel
   - pip install -r requirements.txt
   - python setup.py install

.codeclimate.yml

@@ -1,14 +1,5 @@
 languages:
     Python: true
 
-checks:
-    argument-count:
-        enabled: false
-    method-complexity:
-        enabled: false
-    method-lines:
-        config:
-            threshold: 50
-
 exclude_paths:
     - tests/

README.rst

@@ -27,7 +27,7 @@
 
 |
 
-**Data Science Version Control** or **DVC** is an **open-source** tool for data science and 
+**Data Science Version Control** or **DVC** is an **open-source** tool for data science and
 machine learning projects. With a simple and flexible Git-like architecture and interface it
 helps data scientists:
 
@@ -80,7 +80,7 @@ Please read `Get Started <https://dvc.org/doc/get-started>`_ for the full versio
 |                                   | | ``$ git push``                                                  |
 +-----------------------------------+-------------------------------------------------------------------+
 | Share data and ML models          | | ``$ dvc remote add myremote s3://mybucket/image_cnn``           |
-|                                   | | ``$ dvc core.remote myremote``                                  |
+|                                   | | ``$ dvc config core.remote myremote``                           |
 |                                   | | ``$ dvc push``                                                  |
 +-----------------------------------+-------------------------------------------------------------------+
 
@@ -154,15 +154,15 @@ details.
 
 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/2
   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/2
-  :alt: 2   
+  :alt: 2
 
 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/3
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/3   
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/3
   :alt: 3
 
 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/4
   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/4
-  :alt: 4   
+  :alt: 4
 
 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/5
   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/5
@@ -170,7 +170,7 @@ details.
 
 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/6
   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/6
-  :alt: 6   
+  :alt: 6
 
 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/7
   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/7

dvc/__init__.py

@@ -7,7 +7,7 @@ import os
 import warnings
 
 
-VERSION_BASE = '0.19.12'
+VERSION_BASE = '0.20.1'
 __version__ = VERSION_BASE
 
 PACKAGEPATH = os.path.abspath(os.path.dirname(__file__))

dvc/cache.py

@@ -27,6 +27,9 @@ class Cache(object):
             t = config.get(Config.SECTION_CACHE_TYPE, None)
             if t:
                 sect[Config.SECTION_CACHE_TYPE] = t
+            protected = config.get(Config.SECTION_CACHE_PROTECTED, None)
+            if protected is not None:
+                sect[Config.SECTION_CACHE_PROTECTED] = protected
 
         self.local = Remote(project, sect)
 

dvc/cli.py

@@ -8,6 +8,7 @@ from dvc.command.init import CmdInit
 from dvc.command.destroy import CmdDestroy
 from dvc.command.remove import CmdRemove
 from dvc.command.move import CmdMove
+from dvc.command.unprotect import CmdUnprotect
 from dvc.command.run import CmdRun
 from dvc.command.repro import CmdRepro
 from dvc.command.data_sync import CmdDataPush, CmdDataPull, CmdDataFetch
@@ -55,13 +56,15 @@ class VersionAction(argparse.Action):  # pragma: no cover
 def parse_args(argv=None):
     # Common args
     parent_parser = argparse.ArgumentParser(add_help=False)
-    parent_parser.add_argument(
+
+    log_level_group = parent_parser.add_mutually_exclusive_group()
+    log_level_group.add_argument(
                         '-q',
                         '--quiet',
                         action='store_true',
                         default=False,
                         help='Be quiet.')
-    parent_parser.add_argument(
+    log_level_group.add_argument(
                         '-v',
                         '--verbose',
                         action='store_true',
@@ -181,6 +184,13 @@ def parse_args(argv=None):
                         parents=[parent_parser],
                         description=CHECKOUT_HELP,
                         help=CHECKOUT_HELP)
+    checkout_parser.add_argument(
+                        '-d',
+                        '--with-deps',
+                        action='store_true',
+                        default=False,
+                        help='Checkout all dependencies of the '
+                             'specified target.')
     checkout_parser.add_argument(
                         'targets',
                         nargs='*',
@@ -234,8 +244,8 @@ def parse_args(argv=None):
                         '-c',
                         '--cwd',
                         default=os.path.curdir,
-                        help='Directory to run your command and place state '
-                             'file in.')
+                        help='Directory within your project to run your '
+                             'command and place stage file in.')
     run_parser.add_argument(
                         '--no-exec',
                         action='store_true',
@@ -444,7 +454,8 @@ def parse_args(argv=None):
                         '-c',
                         '--cwd',
                         default=os.path.curdir,
-                        help='Directory to reproduce from.')
+                        help='Directory within your project to '
+                             'reroduce from.')
     repro_parser.add_argument(
                         '-m',
                         '--metrics',
@@ -499,6 +510,12 @@ def parse_args(argv=None):
                         action='store_true',
                         default=False,
                         help='Remove DVC file and all its outputs.')
+    remove_parser.add_argument(
+                        '-f',
+                        '--force',
+                        action='store_true',
+                        default=False,
+                        help='Force purge.')
     remove_parser.add_argument(
                         'targets',
                         nargs='+',
@@ -520,6 +537,19 @@ def parse_args(argv=None):
                         help='Destination.')
     move_parser.set_defaults(func=CmdMove)
 
+    # Unprotect
+    UNPROTECT_HELP = 'Unprotect data file/directory.'
+    unprotect_parser = subparsers.add_parser(
+                        'unprotect',
+                        parents=[parent_parser],
+                        description=UNPROTECT_HELP,
+                        help=UNPROTECT_HELP)
+    unprotect_parser.add_argument(
+                        'targets',
+                        nargs='+',
+                        help='Data files/directory.')
+    unprotect_parser.set_defaults(func=CmdUnprotect)
+
     # Garbage collector
     GC_HELP = 'Collect garbage.'
     gc_parser = subparsers.add_parser(
@@ -555,6 +585,12 @@ def parse_args(argv=None):
                         action='store_true',
                         default=False,
                         help='Force garbage collection.')
+    gc_parser.add_argument(
+                        '-j',
+                        '--jobs',
+                        type=int,
+                        default=None,
+                        help='Number of jobs to run simultaneously.')
     gc_parser.set_defaults(func=CmdGC)
 
     # Config

dvc/command/base.py

@@ -15,9 +15,10 @@ class CmdBase(object):
 
     @staticmethod
     def _set_loglevel(args):
-        if args.quiet and not args.verbose:
+        if args.quiet:
             Logger.be_quiet()
-        elif not args.quiet and args.verbose:
+
+        elif args.verbose:
             Logger.be_verbose()
 
     def _find_root(self):

dvc/command/checkout.py

@@ -7,5 +7,6 @@ class CmdCheckout(CmdBase):
             self.project.checkout()
         else:
             for target in self.args.targets:
-                self.project.checkout(target=target)
+                self.project.checkout(target=target,
+                                      with_deps=self.args.with_deps)
         return 0

dvc/command/gc.py

@@ -22,5 +22,7 @@ class CmdGC(CmdBase):
         self.project.gc(all_branches=self.args.all_branches,
                         all_tags=self.args.all_tags,
                         cloud=self.args.cloud,
-                        remote=self.args.remote)
+                        remote=self.args.remote,
+                        force=self.args.force,
+                        jobs=self.args.jobs)
         return 0

dvc/command/remove.py

@@ -4,10 +4,32 @@ from dvc.logger import Logger
 
 
 class CmdRemove(CmdBase):
+    def _is_outs_only(self, target):
+        if not self.args.purge:
+            return True
+
+        if self.args.force:
+            return False
+
+        msg = (
+            u'Are you sure you want to remove {} with its outputs?'
+            .format(target)
+        )
+
+        confirmed = self.project.prompt.prompt(msg)
+
+        if confirmed:
+            return False
+
+        raise DvcException(
+            u'Cannot purge without a confirmation from the user.'
+            u" Use '-f' to force."
+        )
+
     def run(self):
-        outs_only = not self.args.purge
         for target in self.args.targets:
             try:
+                outs_only = self._is_outs_only(target)
                 self.project.remove(target, outs_only=outs_only)
             except DvcException as ex:
                 Logger.error('Failed to remove {}'.format(target), ex)

dvc/command/root.py

@@ -4,6 +4,9 @@ from dvc.command.base import CmdBase
 
 
 class CmdRoot(CmdBase):
+    def run_cmd(self):
+        return self.run()
+
     def run(self):
         self.project.logger.info(os.path.relpath(self.project.root_dir))
         return 0

dvc/command/unprotect.py

@@ -0,0 +1,14 @@
+from dvc.exceptions import DvcException
+from dvc.command.base import CmdBase
+
+
+class CmdUnprotect(CmdBase):
+    def run(self):
+        for target in self.args.targets:
+            try:
+                self.project.unprotect(target)
+            except DvcException as ex:
+                msg = "Failed to unprotect '{}'".format(target)
+                self.project.logger.error(msg, ex)
+                return 1
+        return 0

dvc/config.py

@@ -68,6 +68,7 @@ class Config(object):
     SECTION_CACHE_DIR = 'dir'
     SECTION_CACHE_TYPE = 'type'
     SECTION_CACHE_TYPE_SCHEMA = supported_cache_type
+    SECTION_CACHE_PROTECTED = 'protected'
     SECTION_CACHE_LOCAL = 'local'
     SECTION_CACHE_S3 = 's3'
     SECTION_CACHE_GS = 'gs'
@@ -82,9 +83,10 @@ class Config(object):
         Optional(SECTION_CACHE_SSH): str,
         Optional(SECTION_CACHE_AZURE): str,
 
-        # backward compatibility
         Optional(SECTION_CACHE_DIR, default='cache'): str,
         Optional(SECTION_CACHE_TYPE, default=None): SECTION_CACHE_TYPE_SCHEMA,
+        Optional(SECTION_CACHE_PROTECTED,
+                 default=False): And(str, is_bool, Use(to_bool)),
     }
 
     # backward compatibility
@@ -153,6 +155,8 @@ class Config(object):
         Optional(SECTION_AWS_ENDPOINT_URL, default=None): str,
         Optional(SECTION_GCP_PROJECTNAME): str,
         Optional(SECTION_CACHE_TYPE): SECTION_CACHE_TYPE_SCHEMA,
+        Optional(SECTION_CACHE_PROTECTED,
+                 default=False): And(str, is_bool, Use(to_bool)),
         Optional(SECTION_REMOTE_USER): str,
         Optional(SECTION_REMOTE_PORT): Use(int),
         Optional(SECTION_REMOTE_KEY_FILE): str,

dvc/data_cloud.py

@@ -73,6 +73,11 @@ class DataCloud(object):
             msg = "Can't find cloud section '{}' in config"
             raise ConfigError(msg.format(name))
 
+        # NOTE: check if the class itself has everything needed for operation.
+        # E.g. all the imported packages.
+        if not cloud_type.supported(cloud_type.compat_config(cloud_config)):
+            raise ConfigError("Unsupported cloud '{}'".format(name))
+
         return self._init_cloud(cloud_config, cloud_type)
 
     def _init_cloud(self, cloud_config, cloud_type):

dvc/exceptions.py

@@ -31,6 +31,13 @@ class OutputDuplicationError(DvcException):
         super(OutputDuplicationError, self).__init__(msg.format(output, s))
 
 
+class CircularDependencyError(DvcException):
+    def __init__(self, dependency):
+        assert isinstance(dependency, str)
+        msg = "File '{}' is specified as an output and as a dependency."
+        super(CircularDependencyError, self).__init__(msg.format(dependency))
+
+
 class MoveNotDataSourceError(DvcException):
     def __init__(self, path):
         msg = "Move is not permitted for stages that are not data sources. " \

dvc/logger.py

@@ -2,11 +2,28 @@ import sys
 import logging
 import colorama
 import traceback
+import re
 
 
 colorama.init()
 
 
+def visual_width(line):
+    """ Get the the number of columns required to display a string """
+
+    return len(re.sub(colorama.ansitowin32.AnsiToWin32.ANSI_CSI_RE, '', line))
+
+
+def visual_center(line, width):
+    """ Center align string according to it's visual width """
+
+    spaces = max(width - visual_width(line), 0)
+    left_padding = int(spaces / 2)
+    right_padding = (spaces - left_padding)
+
+    return (left_padding * ' ') + line + (right_padding * ' ')
+
+
 class Logger(object):
     FMT = '%(message)s'
     DEFAULT_LEVEL = logging.INFO
@@ -37,11 +54,22 @@ class Logger(object):
 
     @staticmethod
     def init():
-        sh = logging.StreamHandler(sys.stdout)
-        sh.setFormatter(logging.Formatter(Logger.FMT))
-        sh.setLevel(logging.DEBUG)
 
-        Logger.logger().addHandler(sh)
+        class LogLevelFilter(logging.Filter):
+            def filter(self, record):
+                return record.levelno <= logging.WARNING
+
+        sh_out = logging.StreamHandler(sys.stdout)
+        sh_out.setFormatter(logging.Formatter(Logger.FMT))
+        sh_out.setLevel(logging.DEBUG)
+        sh_out.addFilter(LogLevelFilter())
+
+        sh_err = logging.StreamHandler(sys.stderr)
+        sh_err.setFormatter(logging.Formatter(Logger.FMT))
+        sh_err.setLevel(logging.ERROR)
+
+        Logger.logger().addHandler(sh_out)
+        Logger.logger().addHandler(sh_err)
         Logger.set_level()
 
     @staticmethod
@@ -107,24 +135,94 @@ class Logger(object):
         return Logger._prefix('Debug', 'debug')
 
     @staticmethod
-    def error(msg, exc=None):
+    def _with_progress(func, msg):
+        from dvc.progress import progress
+        with progress:
+            func(msg)
+
+    @staticmethod
+    def _error_exc(exc):
+        if exc is None:
+            return
+
+        if Logger.logger().getEffectiveLevel() != logging.DEBUG:
+            return
+
         prefix = Logger.error_prefix()
         str_exc, str_tb = Logger.parse_exc(exc)
-        if Logger.logger().getEffectiveLevel() == logging.DEBUG and exc:
-            str_tb = str_tb if str_tb else traceback.format_exc()
-            Logger.logger().error(prefix + str_tb)
+        str_tb = str_tb if str_tb else traceback.format_exc()
+        Logger._with_progress(Logger.logger().error, prefix + str_tb)
+
+    @staticmethod
+    def _with_exc(func, msg, suffix="", exc=None):
+        Logger._error_exc(exc)
+        msg = msg + Logger.parse_exc(exc)[0] + suffix
+        Logger._with_progress(func, msg)
+
+    @staticmethod
+    def error(msg, exc=None):
         chat = "\n\nHaving any troubles? Hit us up at dvc.org/support, " \
                "we are always happy to help!"
-        return Logger.logger().error(prefix + msg + str_exc + chat)
+        Logger._with_exc(Logger.logger().error,
+                         Logger.error_prefix() + msg,
+                         suffix=chat,
+                         exc=exc)
 
-    @staticmethod
-    def warn(msg):
-        return Logger.logger().warn(Logger.warning_prefix() + msg)
+    @classmethod
+    def warn(cls, msg, exc=None):
+        cls._with_exc(cls.logger().warning,
+                      cls.warning_prefix() + msg,
+                      exc=exc)
 
-    @staticmethod
-    def debug(msg):
-        return Logger.logger().debug(Logger.debug_prefix() + msg)
+    @classmethod
+    def debug(cls, msg, exc=None):
+        cls._with_exc(cls.logger().debug,
+                      cls.debug_prefix() + msg,
+                      exc=exc)
 
     @staticmethod
     def info(msg):
-        return Logger.logger().info(msg)
+        Logger._with_progress(Logger.logger().info, msg)
+
+    @classmethod
+    def box(cls, msg, border_color=''):
+        lines = msg.split('\n')
+        max_width = max(visual_width(line) for line in lines)
+
+        padding_horizontal = 5
+        padding_vertical = 1
+
+        box_size_horizontal = (max_width + (padding_horizontal * 2))
+
+        chars = {
+            'corner':     '+',
+            'horizontal': '-',
+            'vertical':   '|',
+            'empty':      ' ',
+        }
+
+        margin = "{corner}{line}{corner}\n".format(
+            corner=chars['corner'],
+            line=chars['horizontal'] * box_size_horizontal,
+        )
+
+        padding_lines = [
+            "{border}{space}{border}\n".format(
+                border=cls.colorize(chars['vertical'], border_color),
+                space=chars['empty'] * box_size_horizontal,
+            ) * padding_vertical
+        ]
+
+        content_lines = [
+            "{border}{space}{content}{space}{border}\n".format(
+                border=cls.colorize(chars['vertical'], border_color),
+                space=chars['empty'] * padding_horizontal,
+                content=visual_center(line, max_width),
+            ) for line in lines
+        ]
+
+        return "{margin}{padding}{content}{padding}{margin}".format(
+            margin=cls.colorize(margin, border_color),
+            padding=''.join(padding_lines),
+            content=''.join(content_lines),
+        )

dvc/progress.py

@@ -11,6 +11,7 @@ class Progress(object):
         self._n_total = 0
         self._n_finished = 0
         self._lock = threading.Lock()
+        self._line = None
 
     def set_n_total(self, total):
         self._n_total = total
@@ -28,18 +29,23 @@ class Progress(object):
         print(line, end='')
         sys.stdout.flush()
 
-    def update_target(self, name, current, total):
+    def refresh(self, line=None):
         # Just go away if it is locked. Will update next time
         if not self._lock.acquire(False):
             return
 
-        bar = self._bar(name, current, total)
+        if line is None:
+            line = self._line
 
-        if sys.stdout.isatty():
-            self._writeln(bar)
+        if sys.stdout.isatty() and line is not None:
+            self._writeln(line)
+            self._line = line
 
         self._lock.release()
 
+    def update_target(self, name, current, total):
+        self.refresh(self._bar(name, current, total))
+
     def finish_target(self, name):
         # We have to write a msg about finished target
         with self._lock:
@@ -51,6 +57,7 @@ class Progress(object):
             print(bar)
 
             self._n_finished += 1
+            self._line = None
 
     def _bar(self, target_name, current, total):
         """
@@ -78,5 +85,15 @@ class Progress(object):
 
         return num + bar + percent + target_name
 
+    def __enter__(self):
+        self._lock.acquire(True)
+        if self._line is not None:
+            self._clearln()
+
+    def __exit__(self, type, value, tb):
+        if self._line is not None:
+            self.refresh()
+        self._lock.release()
+
 
 progress = Progress()

dvc/project.py

@@ -143,6 +143,17 @@ class Project(object):
                         stages = [o.stage.relpath, out.stage.relpath]
                         raise OutputDuplicationError(o.path, stages)
 
+    def _check_circular_dependency(self, deps, outs):
+        from dvc.exceptions import CircularDependencyError
+
+        circular_dependencies = (
+            set(file.path for file in deps) &
+            set(file.path for file in outs)
+        )
+
+        if circular_dependencies:
+            raise CircularDependencyError(circular_dependencies.pop())
+
     def add(self, fname, recursive=False):
         fnames = []
         if recursive and os.path.isdir(fname):
@@ -237,6 +248,38 @@ class Project(object):
             msg = 'Unable to find dvcfile with output \'{}\''
             raise DvcException(msg.format(from_path))
 
+    def _unprotect_file(self, path):
+        import stat
+        import uuid
+        from dvc.utils import copyfile, move, remove
+
+        self.logger.debug("Unprotecting '{}'".format(path))
+
+        tmp = os.path.join(os.path.dirname(path), '.' + str(uuid.uuid4()))
+        move(path, tmp)
+
+        copyfile(tmp, path)
+
+        remove(tmp)
+
+        os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
+
+    def _unprotect_dir(self, path):
+        for root, dirs, files in os.walk(path):
+            for f in files:
+                path = os.path.join(root, f)
+                self._unprotect_file(path)
+
+    def unprotect(self, path):
+        if not os.path.exists(path):
+            raise DvcException("Can't unprotect non-existing "
+                               "data '{}'".format(path))
+
+        if os.path.isdir(path):
+            self._unprotect_dir(path)
+        else:
+            self._unprotect_file(path)
+
     def run(self,
             cmd=None,
             deps=[],
@@ -258,6 +301,7 @@ class Project(object):
                             overwrite=overwrite)
 
         self._check_output_duplication(stage.outs)
+        self._check_circular_dependency(stage.deps, stage.outs)
 
         self._files_to_git_add = []
         with self.state:
@@ -405,11 +449,12 @@ class Project(object):
                 used.append(out.path)
         self.state.remove_unused_links(used)
 
-    def checkout(self, target=None):
+    def checkout(self, target=None, with_deps=False):
         all_stages = self.active_stages()
 
         if target:
-            stages = [Stage.load(self, target)]
+            stages = self._collect(target,
+                                   with_deps=with_deps)
         else:
             stages = all_stages
 
@@ -447,18 +492,89 @@ class Project(object):
 
         return ret
 
+    def _collect_dir_cache(self,
+                           out,
+                           branch=None,
+                           remote=None,
+                           force=False,
+                           jobs=None):
+        info = out.dumpd()
+        ret = [info]
+        r = out.remote
+        md5 = info[r.PARAM_MD5]
+
+        if self.cache.local.changed_cache_file(md5):
+            try:
+                self.cloud.pull(ret,
+                                jobs=jobs,
+                                remote=remote,
+                                show_checksums=False)
+            except DvcException as exc:
+                msg = "Failed to pull cache for '{}': {}"
+                self.logger.debug(msg.format(out, exc))
+
+        if self.cache.local.changed_cache_file(md5):
+            msg = "Missing cache for directory '{}'. " \
+                  "Cache for files inside will be lost. " \
+                  "Would you like to continue? Use '-f' to force. "
+            if not (force or self.prompt.prompt(msg, False)):
+                raise DvcException("Unable to fully collect "
+                                   "used cache without cache "
+                                   "for directory '{}'".format(out))
+            else:
+                return ret
+
+        for i in self.cache.local.load_dir_cache(md5):
+            i['branch'] = branch
+            i[r.PARAM_PATH] = os.path.join(info[r.PARAM_PATH],
+                                           i[r.PARAM_RELPATH])
+            ret.append(i)
+
+        return ret
+
+    def _collect_used_cache(self,
+                            out,
+                            branch=None,
+                            remote=None,
+                            force=False,
+                            jobs=None):
+        if not out.use_cache or not out.info:
+            return []
+
+        info = out.dumpd()
+        info['branch'] = branch
+        ret = [info]
+
+        if out.path_info['scheme'] != 'local':
+            return ret
+
+        md5 = info[out.remote.PARAM_MD5]
+        cache = self.cache.local.get(md5)
+        if not out.remote.is_dir_cache(cache):
+            return ret
+
+        return self._collect_dir_cache(out,
+                                       branch=branch,
+                                       remote=remote,
+                                       force=force,
+                                       jobs=jobs)
+
     def _used_cache(self,
                     target=None,
                     all_branches=False,
                     active=True,
                     with_deps=False,
-                    all_tags=False):
+                    all_tags=False,
+                    remote=None,
+                    force=False,
+                    jobs=None):
         cache = {}
         cache['local'] = []
         cache['s3'] = []
         cache['gs'] = []
         cache['hdfs'] = []
         cache['ssh'] = []
+        cache['azure'] = []
 
         for branch in self.scm.brancher(all_branches=all_branches,
                                         all_tags=all_tags):
@@ -477,13 +593,12 @@ class Project(object):
                     self.logger.warn(msg.format(stage.relpath))
 
                 for out in stage.outs:
-                    if not out.use_cache or not out.info:
-                        continue
-
-                    info = out.dumpd()
-                    info['branch'] = branch
-
-                    cache[out.path_info['scheme']] += [info]
+                    scheme = out.path_info['scheme']
+                    cache[scheme] += self._collect_used_cache(out,
+                                                              branch=branch,
+                                                              remote=remote,
+                                                              force=force,
+                                                              jobs=jobs)
 
         return cache
 
@@ -497,13 +612,18 @@ class Project(object):
            cloud=False,
            remote=None,
            with_deps=False,
-           all_tags=False):
+           all_tags=False,
+           force=False,
+           jobs=None):
         with self.state:
             clist = self._used_cache(target=None,
                                      all_branches=all_branches,
                                      active=False,
                                      with_deps=with_deps,
-                                     all_tags=all_tags)
+                                     all_tags=all_tags,
+                                     remote=remote,
+                                     force=force,
+                                     jobs=jobs)
             self._do_gc('local', self.cache.local.gc, clist)
 
             if self.cache.s3:
@@ -537,7 +657,10 @@ class Project(object):
             used = self._used_cache(target,
                                     all_branches=all_branches,
                                     all_tags=all_tags,
-                                    with_deps=with_deps)['local']
+                                    with_deps=with_deps,
+                                    force=True,
+                                    remote=remote,
+                                    jobs=jobs)['local']
             self.cloud.push(used,
                             jobs,
                             remote=remote,
@@ -555,7 +678,10 @@ class Project(object):
             used = self._used_cache(target,
                                     all_branches=all_branches,
                                     all_tags=all_tags,
-                                    with_deps=with_deps)['local']
+                                    with_deps=with_deps,
+                                    force=True,
+                                    remote=remote,
+                                    jobs=jobs)['local']
             self.cloud.pull(used,
                             jobs,
                             remote=remote,
@@ -576,7 +702,7 @@ class Project(object):
                    all_tags=all_tags,
                    show_checksums=show_checksums,
                    with_deps=with_deps)
-        self.checkout(target=target)
+        self.checkout(target=target, with_deps=with_deps)
 
     def _local_status(self, target=None):
         status = {}
@@ -609,7 +735,10 @@ class Project(object):
         used = self._used_cache(target,
                                 all_branches=all_branches,
                                 all_tags=all_tags,
-                                with_deps=with_deps)['local']
+                                with_deps=with_deps,
+                                force=True,
+                                remote=remote,
+                                jobs=jobs)['local']
 
         status = {}
         for md5, ret in self.cloud.status(used,

dvc/remote/azure.py

@@ -1,6 +1,7 @@
 from __future__ import absolute_import
 import os
 import re
+import posixpath
 
 try:
     from azure.storage.blob import BlockBlobService
@@ -121,14 +122,14 @@ class RemoteAzure(RemoteBase):
 #                        from_bucket=self.bucket,
 #                        from_key='{}/{}'.format(etag[0:2], etag[2:]))
 #
-#    def remove(self, path_info):
-#        if path_info['scheme'] != self.scheme:
-#            raise NotImplementedError
-#
-#        Logger.debug('Removing azure://{}/{}'.format(path_info['bucket'],
-#                                                     path_info['key']))
-#
-#        self.blob_service.delete_blob(path_info['bucket'], path_info['key'])
+    def remove(self, path_info):
+        if path_info['scheme'] != self.scheme:
+            raise NotImplementedError
+
+        Logger.debug('Removing azure://{}/{}'.format(path_info['bucket'],
+                                                     path_info['key']))
+
+        self.blob_service.delete_blob(path_info['bucket'], path_info['key'])
 
     def md5s_to_path_infos(self, md5s):
         return [{
@@ -137,10 +138,12 @@ class RemoteAzure(RemoteBase):
             'key': '{}/{}'.format(md5[0:2], md5[2:])
         } for md5 in md5s]
 
-    def exists(self, path_infos):
-        keys = {blob.name
+    def _all_keys(self):
+        return {blob.name
                 for blob in self.blob_service.list_blobs(self.bucket)}
 
+    def exists(self, path_infos):
+        keys = self._all_keys()
         ret = []
         for path_info in path_infos:
             if path_info['scheme'] != self.scheme:
@@ -176,8 +179,8 @@ class RemoteAzure(RemoteBase):
                 self.blob_service.create_blob_from_path(
                     bucket, key, from_info['path'], progress_callback=cb)
             except Exception as ex:
-                Logger.error("Failed to upload '{}'".format(from_info['path']),
-                             ex)
+                msg = "Failed to upload '{}'".format(from_info['path'])
+                Logger.warn(msg, ex)
             else:
                 progress.finish_target(name)
 
@@ -213,32 +216,36 @@ class RemoteAzure(RemoteBase):
                 self.blob_service.get_blob_to_path(
                     bucket, key, tmp_file, progress_callback=cb)
             except Exception as exc:
-                Logger.error("Failed to download '{}/{}'".format(
-                    bucket, key), exc)
+                msg = "Failed to download '{}/{}'".format(bucket, key)
+                Logger.warn(msg, exc)
             else:
                 os.rename(tmp_file, to_info['path'])
 
                 if not no_progress_bar:
                     progress.finish_target(name)
 
-# FIXME: temporarily disabled because of the lack of test for external azure
-# dependencies/outputs/cache.
-#
-#    def gc(self, cinfos):
-#        used = [info[self.PARAM_ETAG] for info in cinfos['azure']]
-#        used += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
-#
-#        all_blobs = self.blob_service.list_blobs(self.bucket)
-#
-#        removed = False
-#        for blob in all_blobs:
-#            etag = blob.properties.etag
-#            if etag in used:
-#                continue
-#            path_info = {'scheme': self.scheme,
-#                         'key': blob.name,
-#                         'bucket': self.bucket}
-#            self.remove(path_info)
-#            removed = True
-#
-#        return removed
+    def _path_to_etag(self, path):
+        assert len(path.split('/')) == 2
+        return posixpath.dirname(path) + posixpath.basename(path)
+
+    def _all(self):
+        keys = self._all_keys()
+        return [self._path_to_etag(key) for key in keys]
+
+    def gc(self, cinfos):
+        from dvc.remote.local import RemoteLOCAL
+
+        used = [info[self.PARAM_ETAG] for info in cinfos['azure']]
+        used += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
+
+        removed = False
+        for etag in self._all():
+            if etag in used:
+                continue
+            path_info = {'scheme': self.scheme,
+                         'key': posixpath.join(etag[0:2], etag[2:]),
+                         'bucket': self.bucket}
+            self.remove(path_info)
+            removed = True
+
+        return removed

dvc/remote/base.py

@@ -36,10 +36,13 @@ class RemoteBase(object):
     def __init__(self, project, config):
         pass
 
+    def compat_config(config):
+        return config.copy()
+
     @classmethod
     def supported(cls, config):
         url = config[Config.SECTION_REMOTE_URL]
-        url_ok = cls.match(url)
+        url_ok = cls.match(url) is not None
         deps_ok = all(cls.REQUIRES.values())
         if url_ok and not deps_ok:
             missing = [k for k, v in cls.REQUIRES.items() if v is None]

dvc/remote/gs.py

@@ -33,6 +33,13 @@ class RemoteGS(RemoteBase):
         self.url = config.get(Config.SECTION_REMOTE_URL, storagepath)
         self.projectname = config.get(Config.SECTION_GCP_PROJECTNAME, None)
 
+    @staticmethod
+    def compat_config(config):
+        ret = config.copy()
+        url = 'gs://' + ret.pop(Config.SECTION_AWS_STORAGEPATH, '').lstrip('/')
+        ret[Config.SECTION_REMOTE_URL] = url
+        return ret
+
     @property
     def bucket(self):
         return urlparse(self.url).netloc
@@ -207,9 +214,9 @@ class RemoteGS(RemoteBase):
                 blob.upload_from_filename(from_info['path'])
             except Exception as exc:
                 msg = "Failed to upload '{}' to '{}/{}'"
-                Logger.error(msg.format(from_info['path'],
-                                        to_info['bucket'],
-                                        to_info['key']), exc)
+                Logger.warn(msg.format(from_info['path'],
+                                       to_info['bucket'],
+                                       to_info['key']), exc)
                 continue
 
             progress.finish_target(name)
@@ -256,9 +263,9 @@ class RemoteGS(RemoteBase):
                 blob.download_to_filename(tmp_file)
             except Exception as exc:
                 msg = "Failed to download '{}/{}' to '{}'"
-                Logger.error(msg.format(from_info['bucket'],
-                                        from_info['key'],
-                                        to_info['path']), exc)
+                Logger.warn(msg.format(from_info['bucket'],
+                                       from_info['key'],
+                                       to_info['path']), exc)
                 continue
 
             os.rename(tmp_file, to_info['path'])

dvc/remote/hdfs.py

@@ -34,10 +34,11 @@ class RemoteHDFS(RemoteBase):
         # See https://github.com/iterative/dvc/issues/1197.
         close_fds = (os.name != 'nt')
 
+        executable = os.getenv('SHELL') if os.name != 'nt' else None
         p = Popen(cmd,
                   shell=True,
                   close_fds=close_fds,
-                  executable=os.getenv('SHELL'),
+                  executable=executable,
                   env=fix_env(os.environ),
                   stdin=PIPE,
                   stdout=PIPE,

dvc/remote/local.py

@@ -1,4 +1,5 @@
 import os
+import stat
 import uuid
 import json
 import ntpath
@@ -36,6 +37,7 @@ class RemoteLOCAL(RemoteBase):
     def __init__(self, project, config):
         self.project = project
         self.state = self.project.state
+        self.protected = config.get(Config.SECTION_CACHE_PROTECTED, False)
         storagepath = config.get(Config.SECTION_AWS_STORAGEPATH, None)
         self.cache_dir = config.get(Config.SECTION_REMOTE_URL, storagepath)
 
@@ -50,6 +52,13 @@ class RemoteLOCAL(RemoteBase):
         if self.cache_dir is not None and not os.path.exists(self.cache_dir):
             os.mkdir(self.cache_dir)
 
+    @staticmethod
+    def compat_config(config):
+        ret = config.copy()
+        url = ret.pop(Config.SECTION_AWS_STORAGEPATH, '')
+        ret[Config.SECTION_REMOTE_URL] = url
+        return ret
+
     @property
     def url(self):
         return self.cache_dir
@@ -120,12 +129,25 @@ class RemoteLOCAL(RemoteBase):
         if not os.path.exists(dname):
             os.makedirs(dname)
 
+        # NOTE: just create an empty file for an empty cache
+        if os.path.getsize(cache) == 0:
+            open(path, 'w+').close()
+            msg = "Created empty file: {} -> {}"
+            self.project.logger.info(msg.format(os.path.relpath(cache),
+                                                os.path.relpath(path)))
+            return
+
         i = len(self.cache_types)
         while i > 0:
             try:
                 self.CACHE_TYPE_MAP[self.cache_types[0]](cache, path)
-                msg = "Created '{}': {} -> {}"
-                Logger.info(msg.format(self.cache_types[0],
+
+                if self.protected:
+                    os.chmod(path, stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH)
+
+                msg = "Created {}'{}': {} -> {}"
+                Logger.info(msg.format('protected ' if self.protected else '',
+                                       self.cache_types[0],
                                        os.path.relpath(cache),
                                        os.path.relpath(path)))
                 return
@@ -158,14 +180,15 @@ class RemoteLOCAL(RemoteBase):
     def collect_dir_cache(self, dname):
         dir_info = []
 
-        bar = False
         for root, dirs, files in os.walk(dname):
+            bar = False
+
             if len(files) > LARGE_DIR_SIZE:
                 msg = "Computing md5 for a large directory {}. " \
                       "This is only done once."
-                Logger.info(msg.format(os.path.relpath(dname)))
+                Logger.info(msg.format(os.path.relpath(root)))
                 bar = True
-                title = os.path.relpath(dname)
+                title = os.path.relpath(root)
                 processed = 0
                 total = len(files)
                 progress.update_target(title, 0, total)
@@ -182,8 +205,8 @@ class RemoteLOCAL(RemoteBase):
                 dir_info.append({self.PARAM_RELPATH: relpath,
                                  self.PARAM_MD5: md5})
 
-        if bar:
-            progress.finish_target(title)
+            if bar:
+                progress.finish_target(title)
 
         # NOTE: sorting the list by path to ensure reproducibility
         dir_info = sorted(dir_info, key=itemgetter(self.PARAM_RELPATH))
@@ -409,8 +432,8 @@ class RemoteLOCAL(RemoteBase):
                 copyfile(from_info['path'], to_info['path'], name=name)
             except Exception as exc:
                 msg = "Failed to upload '{}' tp '{}'"
-                Logger.error(msg.format(from_info['path'],
-                                        to_info['path']), exc)
+                Logger.warn(msg.format(from_info['path'],
+                                       to_info['path']), exc)
 
     def download(self,
                  from_infos,
@@ -441,36 +464,12 @@ class RemoteLOCAL(RemoteBase):
                          name=name)
             except Exception as exc:
                 msg = "Failed to download '{}' to '{}'"
-                Logger.error(msg.format(from_info['path'],
-                                        to_info['path']), exc)
+                Logger.warn(msg.format(from_info['path'],
+                                       to_info['path']), exc)
                 continue
 
             os.rename(tmp_file, to_info['path'])
 
-    def _collect(self, checksum_infos, push=False):
-        missing = []
-        collected = []
-        for info in checksum_infos:
-            md5 = info[self.PARAM_MD5]
-            cache = self.get(md5)
-
-            if not self.is_dir_cache(info[self.PARAM_MD5]):
-                continue
-
-            if not os.path.exists(cache):
-                missing.append(info)
-                continue
-
-            for i in self.load_dir_cache(md5):
-                if info.get('branch'):
-                    i['branch'] = info['branch']
-                i[self.PARAM_PATH] = os.path.join(info[self.PARAM_PATH],
-                                                  i[self.PARAM_RELPATH])
-                collected.append(i.copy())
-
-        collected.extend(checksum_infos)
-        return collected, missing
-
     def _group(self, checksum_infos, show_checksums=False):
         by_md5 = {}
 
@@ -496,7 +495,7 @@ class RemoteLOCAL(RemoteBase):
         return list(by_md5.keys()), list(by_md5.values())
 
     def gc(self, checksum_infos):
-        checksum_infos = self._collect(checksum_infos['local'])[0]
+        checksum_infos = checksum_infos['local']
         used_md5s = [info[self.PARAM_MD5] for info in checksum_infos]
 
         removed = False
@@ -515,9 +514,6 @@ class RemoteLOCAL(RemoteBase):
         progress.set_n_total(1)
         progress.update_target(title, 0, 100)
 
-        checksum_infos, missing = self._collect(checksum_infos)
-        checksum_infos += missing
-
         progress.update_target(title, 10, 100)
 
         md5s, names = self._group(checksum_infos,
@@ -541,11 +537,11 @@ class RemoteLOCAL(RemoteBase):
                                                                local_exists,
                                                                remote_exists)]
 
-    def _do_pull(self,
-                 checksum_infos,
-                 remote,
-                 jobs=None,
-                 show_checksums=False):
+    def pull(self,
+             checksum_infos,
+             remote,
+             jobs=None,
+             show_checksums=False):
         title = "Collecting information"
 
         progress.set_n_total(1)
@@ -608,23 +604,6 @@ class RemoteLOCAL(RemoteBase):
         for f in futures:
             f.result()
 
-    def pull(self, checksum_infos, remote, jobs=None, show_checksums=False):
-        Logger.info("Preparing to pull data from {}".format(remote.url))
-
-        # NOTE: try fetching missing dir info
-        checksum_infos, missing = self._collect(checksum_infos)
-        if len(missing) > 0:
-            self._do_pull(missing,
-                          remote,
-                          jobs,
-                          show_checksums=show_checksums)
-            checksum_infos += self._collect(missing)[0]
-
-        self._do_pull(checksum_infos,
-                      remote,
-                      jobs,
-                      show_checksums=show_checksums)
-
     def push(self, checksum_infos, remote, jobs=None, show_checksums=False):
         Logger.info("Preparing to push data to {}".format(remote.url))
         title = "Collecting information"
@@ -632,10 +611,6 @@ class RemoteLOCAL(RemoteBase):
         progress.set_n_total(1)
         progress.update_target(title, 0, 100)
 
-        checksum_infos = self._collect(checksum_infos)[0]
-
-        progress.update_target(title, 10, 100)
-
         # NOTE: verifying that our cache is not corrupted
         def func(info):
             return not self.changed_cache_file(info[self.PARAM_MD5])

dvc/remote/s3.py

@@ -64,6 +64,13 @@ class RemoteS3(RemoteBase):
         self.aws_access_key_id = creds.get('aws_access_key_id', None)
         self.aws_secret_access_key = creds.get('aws_secret_access_key', None)
 
+    @staticmethod
+    def compat_config(config):
+        ret = config.copy()
+        url = 's3://' + ret.pop(Config.SECTION_AWS_STORAGEPATH, '').lstrip('/')
+        ret[Config.SECTION_REMOTE_URL] = url
+        return ret
+
     @property
     def bucket(self):
         return urlparse(self.url).netloc
@@ -272,7 +279,7 @@ class RemoteS3(RemoteBase):
                                Callback=cb)
             except Exception as exc:
                 msg = "Failed to upload '{}'".format(from_info['path'])
-                Logger.error(msg, exc)
+                Logger.warn(msg, exc)
                 continue
 
             progress.finish_target(name)
@@ -306,16 +313,17 @@ class RemoteS3(RemoteBase):
             if not name:
                 name = os.path.basename(to_info['path'])
 
-            if no_progress_bar:
-                cb = None
-            else:
-                total = s3.head_object(Bucket=from_info['bucket'],
-                                       Key=from_info['key'])['ContentLength']
-                cb = Callback(name, total)
-
             self._makedirs(to_info['path'])
 
             try:
+                if no_progress_bar:
+                    cb = None
+                else:
+                    total = s3.head_object(
+                                    Bucket=from_info['bucket'],
+                                    Key=from_info['key'])['ContentLength']
+                    cb = Callback(name, total)
+
                 s3.download_file(from_info['bucket'],
                                  from_info['key'],
                                  tmp_file,
@@ -323,8 +331,8 @@ class RemoteS3(RemoteBase):
             except Exception as exc:
                 msg = "Failed to download '{}/{}'".format(from_info['bucket'],
                                                           from_info['key'])
-                Logger.error(msg, exc)
-                return
+                Logger.warn(msg, exc)
+                continue
 
             os.rename(tmp_file, to_info['path'])
 

dvc/remote/ssh.py

@@ -349,9 +349,9 @@ class RemoteSSH(RemoteBase):
                                     callback=create_cb(name))
             except Exception as exc:
                 msg = "Failed to download '{}/{}' to '{}'"
-                Logger.error(msg.format(from_info['host'],
-                                        from_info['path'],
-                                        to_info['path']), exc)
+                Logger.warn(msg.format(from_info['host'],
+                                       from_info['path'],
+                                       to_info['path']), exc)
                 continue
 
             os.rename(tmp_file, to_info['path'])
@@ -390,9 +390,9 @@ class RemoteSSH(RemoteBase):
                          callback=create_cb(name))
             except Exception as exc:
                 msg = "Failed to upload '{}' to '{}/{}'"
-                Logger.error(msg.format(from_info['path'],
-                                        to_info['host'],
-                                        to_info['path'], exc))
+                Logger.warn(msg.format(from_info['path'],
+                                       to_info['host'],
+                                       to_info['path'], exc))
                 continue
 
             progress.finish_target(name)

dvc/scm.py

@@ -18,6 +18,10 @@ class Base(object):
         self.project = project
         self.root_dir = root_dir
 
+    @property
+    def dir(self):
+        return None
+
     @staticmethod
     def is_repo(root_dir):
         return True
@@ -50,6 +54,9 @@ class Base(object):
     def branch(self, branch):
         pass
 
+    def tag(self, tag):
+        pass
+
     def brancher(self,
                  branches=None,
                  all_branches=False,
@@ -63,16 +70,19 @@ class Base(object):
         saved = self.active_branch()
         revs = []
 
-        if branches is not None:
-            revs.extend(branches)
-        elif all_branches:
-            revs.extend(self.list_branches())
-        elif tags is not None:
-            revs.extend(tags)
-        elif all_tags:
-            revs.extend(self.list_tags())
-        else:
+        if all_branches:
+            branches = self.list_branches()
+
+        if all_tags:
+            tags = self.list_tags()
+
+        if branches is None:
             revs.extend([saved])
+        else:
+            revs.extend(branches)
+
+        if tags is not None:
+            revs.extend(tags)
 
         for rev in revs:
             self.checkout(rev)
@@ -215,6 +225,9 @@ class Git(Base):
     def branch(self, branch):
         self.repo.git.branch(branch)
 
+    def tag(self, tag):
+        self.repo.git.tag(tag)
+
     def untracked_files(self):
         files = self.repo.untracked_files
         return [os.path.join(self.repo.working_dir, fname) for fname in files]
@@ -231,18 +244,22 @@ class Git(Base):
     def list_tags(self):
         return [t.name for t in self.repo.tags]
 
-    def install(self):
+    def _install_hook(self, name, cmd):
         hook = os.path.join(self.root_dir,
                             self.GIT_DIR,
                             'hooks',
-                            'post-checkout')
+                            name)
         if os.path.isfile(hook):
             msg = 'Git hook \'{}\' already exists.'
             raise SCMError(msg.format(os.path.relpath(hook)))
         with open(hook, 'w+') as fd:
-            fd.write('#!/bin/sh\nexec dvc checkout\n')
+            fd.write('#!/bin/sh\nexec dvc {}\n'.format(cmd))
         os.chmod(hook, 0o777)
 
+    def install(self):
+        self._install_hook('post-checkout', 'checkout')
+        self._install_hook('pre-commit', 'status')
+
 
 def SCM(root_dir, no_scm=False, project=None):
     if Git.is_repo(root_dir) or Git.is_submodule(root_dir):

dvc/stage.py

@@ -40,6 +40,12 @@ class StageFileBadNameError(DvcException):
         super(StageFileBadNameError, self).__init__(msg)
 
 
+class StageBadCwdError(DvcException):
+    def __init__(self, cwd):
+        msg = "Stage cwd '{}' is outside of the current dvc project"
+        super(StageBadCwdError, self).__init__(msg.format(cwd))
+
+
 class MissingDep(DvcException):
     def __init__(self, deps):
         assert len(deps) > 0
@@ -275,6 +281,13 @@ class Stage(object):
 
         return (fname, cwd)
 
+    @staticmethod
+    def _check_inside_project(project, cwd):
+        assert project is not None
+        proj_dir = os.path.realpath(project.root_dir)
+        if not os.path.realpath(cwd).startswith(proj_dir):
+            raise StageBadCwdError(cwd)
+
     @staticmethod
     def loads(project=None,
               cmd=None,
@@ -305,6 +318,8 @@ class Stage(object):
 
         fname, cwd = Stage._stage_fname_cwd(fname, cwd, stage.outs, add=add)
 
+        Stage._check_inside_project(project, cwd)
+
         cwd = os.path.abspath(cwd)
         path = os.path.join(cwd, fname)
 
@@ -425,11 +440,12 @@ class Stage(object):
 
             if not dry:
                 self._check_missing_deps()
+                executable = os.getenv('SHELL') if os.name != 'nt' else None
                 p = subprocess.Popen(self.cmd,
                                      cwd=self.cwd,
                                      shell=True,
                                      env=fix_env(os.environ),
-                                     executable=os.getenv('SHELL'))
+                                     executable=executable)
                 p.communicate()
                 if p.returncode != 0:
                     raise StageCmdFailedError(self)

dvc/system.py

@@ -9,9 +9,14 @@ class System(object):
     @staticmethod
     def hardlink(source, link_name):
         import ctypes
+        from dvc.exceptions import DvcException
+
         if System.is_unix():
-            os.link(source, link_name)
-            return
+            try:
+                os.link(source, link_name)
+                return
+            except Exception as exc:
+                raise DvcException('link', cause=exc)
 
         CreateHardLink = ctypes.windll.kernel32.CreateHardLinkW
         CreateHardLink.argtypes = [ctypes.c_wchar_p,
@@ -21,13 +26,19 @@ class System(object):
 
         res = CreateHardLink(link_name, source, None)
         if res == 0:
-            raise ctypes.WinError()
+            raise DvcException('CreateHardLinkW', cause=ctypes.WinError())
 
     @staticmethod
     def symlink(source, link_name):
         import ctypes
+        from dvc.exceptions import DvcException
+
         if System.is_unix():
-            return os.symlink(source, link_name)
+            try:
+                os.symlink(source, link_name)
+                return
+            except Exception as exc:
+                raise DvcException('symlink', cause=exc)
 
         flags = 0
         if source is not None and os.path.isdir(source):
@@ -38,7 +49,7 @@ class System(object):
         func.restype = ctypes.c_ubyte
 
         if func(link_name, source, flags) == 0:
-            raise ctypes.WinError()
+            raise DvcException('CreateSymbolicLinkW', cause=ctypes.WinError())
 
     @staticmethod
     def _reflink_darwin(src, dst):
@@ -178,7 +189,12 @@ class System(object):
     @staticmethod
     def inode(path):
         if System.is_unix():
+            import ctypes
             inode = os.lstat(path).st_ino
+            # NOTE: See https://bugs.python.org/issue29619 and
+            # https://stackoverflow.com/questions/34643289/
+            # pythons-os-stat-is-returning-wrong-inode-value
+            inode = ctypes.c_ulong(inode).value
         else:
             # getdirinfo from ntfsutils works on both files and dirs
             info = System.getdirinfo(path)

dvc/updater.py

@@ -1,6 +1,7 @@
 import os
 import time
 import requests
+import colorama
 
 from dvc import VERSION_BASE
 from dvc.logger import Logger
@@ -56,6 +57,19 @@ class Updater(object):  # pragma: no cover
            l_patch <= c_patch:
             return
 
-        msg = 'You are using dvc version {}, however version {} is ' \
-              'available. Consider upgrading.'
-        Logger.warn(msg.format(current, latest))
+        msg = (
+            'Update available {red}{current}{reset} -> {green}{latest}{reset}'
+            '\n'
+            'Run {yellow}pip{reset} install dvc {blue}--upgrade{reset}'
+            .format(
+                red=colorama.Fore.RED,
+                reset=colorama.Fore.RESET,
+                green=colorama.Fore.GREEN,
+                yellow=colorama.Fore.YELLOW,
+                blue=colorama.Fore.BLUE,
+                current=current,
+                latest=latest,
+            )
+        )
+
+        Logger.box(msg, border_color='yellow')

dvc/utils.py

@@ -3,6 +3,7 @@ Helpers for other modules.
 """
 import os
 import sys
+import stat
 import math
 import json
 import shutil
@@ -102,6 +103,7 @@ def copyfile(src, dest, no_progress_bar=False, name=None):
 
 
 def move(src, dst):
+    dst = os.path.abspath(dst)
     dname = os.path.dirname(dst)
     if not os.path.exists(dname):
         os.makedirs(dname)
@@ -119,10 +121,17 @@ def remove(path):
         return
 
     Logger.debug(u'Removing \'{}\''.format(os.path.relpath(path)))
+
+    def _chmod(func, p, excinfo):
+        perm = os.stat(p).st_mode
+        perm |= stat.S_IWRITE
+        os.chmod(p, perm)
+        func(p)
+
     if os.path.isfile(path):
-        os.unlink(path)
+        _chmod(os.unlink, path, None)
     else:
-        shutil.rmtree(path)
+        shutil.rmtree(path, onerror=_chmod)
 
 
 def to_chunks(l, jobs):

tests/test_checkout.py

@@ -246,3 +246,18 @@ class TestCheckoutNotCachedFile(TestDvc):
                              outs_no_cache=['out'])
 
         self.dvc.checkout()
+
+
+class TestCheckoutWithDeps(TestRepro):
+    def test(self):
+        os.unlink(self.FOO)
+        os.unlink(self.file1)
+
+        self.assertFalse(os.path.exists(self.FOO))
+        self.assertFalse(os.path.exists(self.file1))
+
+        ret = main(['checkout', self.file1_stage, '--with-deps'])
+        self.assertEqual(ret, 0)
+
+        self.assertTrue(os.path.exists(self.FOO))
+        self.assertTrue(os.path.exists(self.file1))

tests/test_config.py

@@ -28,6 +28,11 @@ class TestConfigCLI(TestDvc):
         ret = main(['root'])
         self.assertEqual(ret, 0)
 
+        # NOTE: check that `dvc root` is not blocked with dvc lock
+        with self.dvc.lock:
+            ret = main(['root'])
+        self.assertEqual(ret, 0)
+
     def _do_test(self, local=False):
         section = 'setsection'
         field = 'setfield'

tests/test_data_cloud.py

@@ -160,8 +160,8 @@ class TestDataCloud(TestDvc):
     def test(self):
         config = TEST_CONFIG
 
-        clist = [('s3://', RemoteS3),
-                 ('gs://', RemoteGS),
+        clist = [('s3://mybucket/', RemoteS3),
+                 ('gs://mybucket/', RemoteGS),
                  ('ssh://user@localhost:/', RemoteSSH),
                  (TestDvc.mkdtemp(), RemoteLOCAL)]
 
@@ -414,6 +414,18 @@ class TestDataCloudCLIBase(TestDvc):
 
         self.main(['status', '-c', '--show-checksums'] + args)
 
+        # NOTE: check if remote gc works correctly on directories
+        self.main(['gc', '-c', '-f'] + args)
+        shutil.rmtree(self.dvc.cache.local.cache_dir)
+        
+        sleep()
+        self.main(['pull'] + args)
+        self.assertTrue(os.path.exists(cache))
+        self.assertTrue(os.path.isfile(cache))
+        self.assertTrue(os.path.isfile(cache_dir))
+        self.assertTrue(os.path.isfile(self.FOO))
+        self.assertTrue(os.path.isdir(self.DATA_DIR))
+
     def _should_test(self):
         return True
 

tests/test_gc.py

@@ -40,3 +40,63 @@ class TestGC(TestDvc):
 
         for c in self.good_cache:
             self.assertTrue(os.path.exists(c))
+
+
+class TestGCBranchesTags(TestDvc):
+    def _check_cache(self, num):
+        total = 0
+        for root, dirs, files in os.walk(os.path.join('.dvc', 'cache')):
+            total += len(files)
+        self.assertEqual(total, num)
+
+    def test(self):
+        fname = 'file'
+
+        with open(fname, 'w+') as fobj:
+            fobj.write('v1.0')
+
+        stages = self.dvc.add(fname)
+        self.assertEqual(len(stages), 1)
+        self.dvc.scm.add(['.gitignore', stages[0].relpath])
+        self.dvc.scm.commit('v1.0')
+        self.dvc.scm.tag('v1.0')
+
+        self.dvc.scm.checkout('test', create_new=True)
+        self.dvc.remove(stages[0].relpath, outs_only=True)
+        with open(fname, 'w+') as fobj:
+            fobj.write('test')
+        stages = self.dvc.add(fname)
+        self.assertEqual(len(stages), 1)
+        self.dvc.scm.add(['.gitignore', stages[0].relpath])
+        self.dvc.scm.commit('test')
+
+        self.dvc.scm.checkout('master')
+        self.dvc.remove(stages[0].relpath, outs_only=True)
+        with open(fname, 'w+') as fobj:
+            fobj.write('trash')
+        stages = self.dvc.add(fname)
+        self.assertEqual(len(stages), 1)
+        self.dvc.scm.add(['.gitignore', stages[0].relpath])
+        self.dvc.scm.commit('trash')
+
+        self.dvc.remove(stages[0].relpath, outs_only=True)
+        with open(fname, 'w+') as fobj:
+            fobj.write('master')
+        stages = self.dvc.add(fname)
+        self.assertEqual(len(stages), 1)
+        self.dvc.scm.add(['.gitignore', stages[0].relpath])
+        self.dvc.scm.commit('master')
+
+        self._check_cache(4)
+
+        self.dvc.gc(all_tags=True, all_branches=True)
+
+        self._check_cache(3)
+
+        self.dvc.gc(all_tags=False, all_branches=True)
+
+        self._check_cache(2)
+
+        self.dvc.gc(all_tags=True, all_branches=False)
+
+        self._check_cache(1)

tests/test_install.py

@@ -13,7 +13,11 @@ class TestInstall(TestDvc):
         ret = main(['install'])
         self.assertNotEqual(ret, 0)
 
-        self.assertTrue(os.path.isfile('.git/hooks/post-checkout'))
+        def hook(name):
+            return os.path.join('.git', 'hooks', name)
+
+        self.assertTrue(os.path.isfile(hook('post-checkout')))
+        self.assertTrue(os.path.isfile(hook('pre-commit')))
 
         self.dvc.add(self.FOO)
         os.unlink(self.FOO)

tests/test_logger.py

@@ -1,13 +1,26 @@
+from mock import patch
+
+try:
+    from StringIO import StringIO
+except ImportError:
+    from io import StringIO
+
 import colorama
 import logging
 
 from dvc.command.base import CmdBase
 from dvc.config import Config
 from dvc.logger import Logger
+from dvc.main import main
 
 from tests.basic_env import TestDvc
 
 
+class StringContaining(str):
+    def __eq__(self, other):
+        return self in other
+
+
 class TestLogger(TestDvc):
     def test_config(self):
         Logger('debug')
@@ -34,6 +47,28 @@ class TestLogger(TestDvc):
             # This is not a tty, so it should not colorize anything
             self.assertEqual(msg, name)
 
+    @patch('sys.stderr', new_callable=StringIO)
+    @patch('sys.stdout', new_callable=StringIO)
+    def test_stderr(self, mock_stdout, mock_stderr):
+        error_message = 'error msg'
+        Logger.init()
+        Logger.error(error_message)
+        self.assertEqual('', mock_stdout.getvalue())
+        self.assertEqual('Error: {}\n\nHaving any troubles? '
+                                   'Hit us up at dvc.org/support, we '
+                                   'are always happy to help!\n'.format(error_message),
+                                   mock_stderr.getvalue())
+
+    @patch('sys.stderr', new_callable=StringIO)
+    @patch('sys.stdout', new_callable=StringIO)
+    def test_stdout(self, mock_stdout, mock_stderr):
+        non_error_message = 'non-error message'
+        Logger.init()
+        Logger.info(non_error_message)
+        self.assertEqual('', mock_stderr.getvalue())
+        self.assertEqual('{}\n'.format(non_error_message),
+                                   mock_stdout.getvalue())
+
 
 class TestLoggerCLI(TestDvc):
     def test(self):

tests/test_remove.py

@@ -2,9 +2,11 @@ import os
 
 from dvc.main import main
 from dvc.stage import Stage, StageFileDoesNotExistError
+from dvc.exceptions import DvcException
 from dvc.command.remove import CmdRemove
 
 from tests.basic_env import TestDvc
+from mock import patch
 
 
 class TestRemove(TestDvc):
@@ -57,3 +59,23 @@ class TestCmdRemove(TestDvc):
         ret = main(['remove',
                     'non-existing-dvc-file'])
         self.assertNotEqual(ret, 0)
+
+class TestRemovePurge(TestDvc):
+    def test(self):
+        dvcfile = self.dvc.add(self.FOO)[0].path
+        ret = main(['remove', '--purge', '--force', dvcfile])
+
+        self.assertEqual(ret, 0)
+        self.assertFalse(os.path.exists(self.FOO))
+        self.assertFalse(os.path.exists(dvcfile))
+
+    @patch('dvc.prompt.Prompt.prompt')
+    def test_force(self, mock_prompt):
+        mock_prompt.return_value = False
+
+        dvcfile = self.dvc.add(self.FOO)[0].path
+        ret = main(['remove', '--purge', dvcfile])
+
+        mock_prompt.assert_called()
+        self.assertEqual(ret, 1)
+        self.assertRaises(DvcException)

tests/test_repro.py

@@ -763,3 +763,9 @@ class TestReproShell(TestDvc):
 
         with open(fname, 'r') as fd:
             self.assertEqual(os.getenv('SHELL'), fd.read().strip())
+
+class TestReproNoSCM(TestRepro):
+    def test(self):
+        shutil.rmtree(self.dvc.scm.dir)
+        ret = main(['repro', self.file1_stage])
+        self.assertEqual(ret, 0)

tests/test_run.py

@@ -4,8 +4,9 @@ import filecmp
 from dvc.main import main
 from dvc.utils import file_md5
 from dvc.stage import Stage, StageFileBadNameError, MissingDep
+from dvc.stage import StageBadCwdError
 from dvc.command.run import CmdRun
-from dvc.exceptions import OutputDuplicationError
+from dvc.exceptions import OutputDuplicationError, CircularDependencyError
 
 from tests.basic_env import TestDvc
 
@@ -93,6 +94,36 @@ class TestRunNoExec(TestDvc):
         self.assertFalse(os.path.exists('out'))
 
 
+class TestRunCircularDependency(TestDvc):
+    def test(self):
+        with self.assertRaises(CircularDependencyError):
+            self.dvc.run(cmd='',
+                         deps=[self.FOO],
+                         outs=[self.FOO],
+                         fname='circular-dependency.dvc')
+
+    def test_outs_no_cache(self):
+        with self.assertRaises(CircularDependencyError):
+            self.dvc.run(cmd='',
+                         deps=[self.FOO],
+                         outs_no_cache=[self.FOO],
+                         fname='circular-dependency.dvc')
+
+    def test_non_normalized_paths(self):
+        with self.assertRaises(CircularDependencyError):
+            self.dvc.run(cmd='',
+                         deps=['./foo'],
+                         outs=['foo'],
+                         fname='circular-dependency.dvc')
+
+
+class TestRunBadCwd(TestDvc):
+    def test(self):
+        with self.assertRaises(StageBadCwdError):
+            self.dvc.run(cmd='',
+                         cwd=self.mkdtemp())
+
+
 class TestCmdRun(TestDvc):
     def test_run(self):
         ret = main(['run',

tests/test_unprotect.py

@@ -0,0 +1,31 @@
+import os
+import stat
+
+from dvc.main import main
+from dvc.project import Project
+
+from tests.basic_env import TestDvc
+
+
+class TestUnprotect(TestDvc):
+    def test(self):
+        ret = main(['config', 'cache.protected', 'true'])
+        self.assertEqual(ret, 0)
+
+        ret = main(['add', self.FOO])
+        self.assertEqual(ret, 0)
+
+        self.assertNotEqual(os.stat(self.FOO).st_mode & (stat.S_IREAD|
+                                                         stat.S_IRGRP|
+                                                         stat.S_IROTH), 0)
+        self.assertEqual(os.stat(self.FOO).st_mode & (stat.S_IWRITE|
+                                                      stat.S_IWGRP|
+                                                      stat.S_IWOTH), 0)
+
+        ret = main(['unprotect', self.FOO])
+        self.assertEqual(ret, 0)
+
+        self.assertNotEqual(os.stat(self.FOO).st_mode & (stat.S_IREAD|
+                                                         stat.S_IRGRP|
+                                                         stat.S_IROTH|
+                                                         stat.S_IWRITE), 0)