Merge pull request #1432 from efiop/master

Misc fixes

dvc/cli.py

@@ -19,7 +19,7 @@ from dvc.command.imp import CmdImport
 from dvc.command.config import CmdConfig
 from dvc.command.checkout import CmdCheckout
 from dvc.command.remote import CmdRemoteAdd, CmdRemoteRemove
-from dvc.command.remote import CmdRemoteModify, CmdRemoteList
+from dvc.command.remote import CmdRemoteModify, CmdRemoteList, CmdRemoteDefault
 from dvc.command.metrics import CmdMetricsShow, CmdMetricsAdd
 from dvc.command.metrics import CmdMetricsRemove, CmdMetricsModify
 from dvc.command.install import CmdInstall
@@ -524,6 +524,13 @@ def parse_args(argv=None):
                         action='store_true',
                         default=False,
                         help='Reproduce all pipelines in the project.')
+    repro_parser.add_argument(
+                        '--ignore-build-cache',
+                        action='store_true',
+                        default=False,
+                        help='Reproduce all descendants of a changed stage'
+                             'even if their direct dependencies didn\'t '
+                             'change.')
     repro_parser.set_defaults(func=CmdRepro)
 
     # Remove
@@ -726,6 +733,40 @@ def parse_args(argv=None):
                         help='Set as default remote.')
     remote_add_parser.set_defaults(func=CmdRemoteAdd)
 
+    REMOTE_DEFAULT_HELP = 'Set/unset default remote.'
+    remote_default_parser = remote_subparsers.add_parser(
+                        'default',
+                        parents=[parent_parser],
+                        description=REMOTE_DEFAULT_HELP,
+                        help=REMOTE_DEFAULT_HELP)
+    remote_default_parser.add_argument(
+                        'name',
+                        nargs='?',
+                        help='Name of the remote.')
+    remote_default_parser.add_argument(
+                        '-u',
+                        '--unset',
+                        action='store_true',
+                        default=False,
+                        help='Unset default remote.')
+    remote_default_parser.add_argument(
+                        '--global',
+                        dest='glob',
+                        action='store_true',
+                        default=False,
+                        help='Use global config.')
+    remote_default_parser.add_argument(
+                        '--system',
+                        action='store_true',
+                        default=False,
+                        help='Use system config.')
+    remote_default_parser.add_argument(
+                        '--local',
+                        action='store_true',
+                        default=False,
+                        help='Use local config.')
+    remote_default_parser.set_defaults(func=CmdRemoteDefault)
+
     REMOTE_REMOVE_HELP = 'Remove remote.'
     remote_remove_parser = remote_subparsers.add_parser(
                         'remove',

dvc/command/remote.py

@@ -62,6 +62,13 @@ class CmdRemoteModify(CmdConfig):
         return super(CmdRemoteModify, self).run()
 
 
+class CmdRemoteDefault(CmdConfig):
+    def run(self):
+        self.args.value = self.args.name
+        self.args.name = 'core.remote'
+        return super(CmdRemoteDefault, self).run()
+
+
 class CmdRemoteList(CmdConfig):
     def run(self):
         for section in self.configobj.keys():

dvc/command/repro.py

@@ -16,13 +16,14 @@ class CmdRepro(CmdBase):
         for target in self.args.targets:
             try:
                 stages = self.project.reproduce(
-                                       target,
-                                       recursive=recursive,
-                                       force=self.args.force,
-                                       dry=self.args.dry,
-                                       interactive=self.args.interactive,
-                                       pipeline=self.args.pipeline,
-                                       all_pipelines=self.args.all_pipelines)
+                               target,
+                               recursive=recursive,
+                               force=self.args.force,
+                               dry=self.args.dry,
+                               interactive=self.args.interactive,
+                               pipeline=self.args.pipeline,
+                               all_pipelines=self.args.all_pipelines,
+                               ignore_build_cache=self.args.ignore_build_cache)
 
                 if len(stages) == 0:
                     self.project.logger.info(CmdDataStatus.UP_TO_DATE_MSG)

dvc/config.py

@@ -150,6 +150,8 @@ class Config(object):
         SECTION_LOCAL_STORAGEPATH: str,
     }
 
+    SECTION_AZURE_CONNECTION_STRING = 'connection_string'
+
     SECTION_REMOTE_REGEX = r'^\s*remote\s*"(?P<name>.*)"\s*$'
     SECTION_REMOTE_FMT = 'remote "{}"'
     SECTION_REMOTE_URL = 'url'
@@ -177,6 +179,7 @@ class Config(object):
         Optional(SECTION_REMOTE_TIMEOUT): Use(int),
         Optional(SECTION_REMOTE_PASSWORD): str,
         Optional(SECTION_REMOTE_ASK_PASSWORD): And(str, is_bool, Use(to_bool)),
+        Optional(SECTION_AZURE_CONNECTION_STRING): str,
     }
 
     SECTION_STATE = 'state'

dvc/project.py

@@ -449,7 +449,8 @@ class Project(object):
                   dry=False,
                   interactive=False,
                   pipeline=False,
-                  all_pipelines=False):
+                  all_pipelines=False,
+                  ignore_build_cache=False):
         from dvc.stage import Stage
 
         if target is None and not all_pipelines:
@@ -485,7 +486,8 @@ class Project(object):
                                          recursive=recursive,
                                          force=force,
                                          dry=dry,
-                                         interactive=interactive)
+                                         interactive=interactive,
+                                         ignore_build_cache=ignore_build_cache)
                 ret.extend(stages)
 
         self._remind_to_git_add()
@@ -497,7 +499,8 @@ class Project(object):
                    recursive=True,
                    force=False,
                    dry=False,
-                   interactive=False):
+                   interactive=False,
+                   ignore_build_cache=False):
         import networkx as nx
         from dvc.stage import Stage
 
@@ -512,7 +515,8 @@ class Project(object):
                                          node,
                                          force,
                                          dry,
-                                         interactive)
+                                         interactive,
+                                         ignore_build_cache)
         else:
             ret = self._reproduce_stage(stages,
                                         node,
@@ -522,17 +526,34 @@ class Project(object):
 
         return ret
 
-    def _reproduce_stages(self, G, stages, node, force, dry, interactive):
+    def _reproduce_stages(self,
+                          G,
+                          stages,
+                          node,
+                          force,
+                          dry,
+                          interactive,
+                          ignore_build_cache):
         import networkx as nx
 
         result = []
         for n in nx.dfs_postorder_nodes(G, node):
             try:
-                result += self._reproduce_stage(stages,
-                                                n,
-                                                force,
-                                                dry,
-                                                interactive)
+                ret = self._reproduce_stage(stages,
+                                            n,
+                                            force,
+                                            dry,
+                                            interactive)
+
+                if len(ret) == 0 and ignore_build_cache:
+                    # NOTE: we are walking our pipeline from the top to the
+                    # bottom. If one stage is changed, it will be reproduced,
+                    # which tells us that we should force reproducing all of
+                    # the other stages down below, even if their direct
+                    # dependencies didn't change.
+                    force = True
+
+                result += ret
             except Exception as ex:
                 raise ReproductionError(stages[n].relpath, ex)
         return result

dvc/remote/azure.py

@@ -8,6 +8,11 @@ try:
 except ImportError:
     BlockBlobService = None
 
+try:
+    from urlparse import urlparse
+except ImportError:
+    from urllib.parse import urlparse
+
 from dvc.logger import logger
 from dvc.progress import progress
 from dvc.config import Config
@@ -24,9 +29,10 @@ class Callback(object):
 
 class RemoteAzure(RemoteBase):
     scheme = 'azure'
-    REGEX = (r'^azure://'
+    REGEX = (r'^azure://((?P<path>[^=;]*)|('
+             # backward compatibility
              r'(ContainerName=(?P<container_name>[^;]+);?)?'
-             r'(?P<connection_string>.+)?$')
+             r'(?P<connection_string>.+)?))$')
     REQUIRES = {'azure-storage-blob': BlockBlobService}
     PARAM_ETAG = 'etag'
     COPY_POLL_SECONDS = 5
@@ -36,17 +42,18 @@ class RemoteAzure(RemoteBase):
         self.project = project
 
         self.url = config.get(Config.SECTION_REMOTE_URL)
-        match = re.match(self.REGEX, self.url)
+        match = re.match(self.REGEX, self.url)  # backward compatibility
 
         self.bucket = (
-            match.group('container_name')
+            urlparse(self.url).netloc
+            or match.group('container_name')  # backward compatibility
             or os.getenv('AZURE_STORAGE_CONTAINER_NAME'))
 
-        # FIXME: currently Azure doesn't support prefixes
-        self.prefix = '/'
+        self.prefix = urlparse(self.url).path.lstrip('/') or '/'
 
         self.connection_string = (
-            match.group('connection_string')
+            config.get(Config.SECTION_AZURE_CONNECTION_STRING)
+            or match.group('connection_string')  # backward compatibility
             or os.getenv('AZURE_STORAGE_CONNECTION_STRING'))
 
         if not self.bucket:
@@ -138,7 +145,7 @@ class RemoteAzure(RemoteBase):
         return [{
             'scheme': self.scheme,
             'bucket': self.bucket,
-            'key': '{}/{}'.format(md5[0:2], md5[2:])
+            'key': '{}/{}/{}'.format(self.prefix, md5[0:2], md5[2:])
         } for md5 in md5s]
 
     def _all_keys(self, bucket, prefix):
@@ -235,8 +242,8 @@ class RemoteAzure(RemoteBase):
                     progress.finish_target(name)
 
     def _path_to_etag(self, path):
-        assert len(path.split('/')) == 2
-        return posixpath.dirname(path) + posixpath.basename(path)
+        relpath = posixpath.relpath(path, self.prefix)
+        return posixpath.dirname(relpath) + posixpath.basename(relpath)
 
     def _all(self):
         keys = self._all_keys(self.bucket, self.prefix)
@@ -253,7 +260,9 @@ class RemoteAzure(RemoteBase):
             if etag in used:
                 continue
             path_info = {'scheme': self.scheme,
-                         'key': posixpath.join(etag[0:2], etag[2:]),
+                         'key': posixpath.join(self.prefix,
+                                               etag[0:2],
+                                               etag[2:]),
                          'bucket': self.bucket}
             self.remove(path_info)
             removed = True

tests/test_data_cloud.py

@@ -141,10 +141,16 @@ def get_gcp_url():
     return 'gs://' + get_gcp_storagepath()
 
 
-def get_azure_url():
+def get_azure_url_compat():
     return 'azure://'
 
 
+def get_azure_url():
+    container_name = os.getenv("AZURE_STORAGE_CONTAINER_NAME")
+    assert container_name is not None
+    return 'azure://{}/{}'.format(container_name, str(uuid.uuid4()))
+
+
 def sleep():
     # Sleep some time to simulate realistic behavior.
     # Some filesystems have a bad date resolution for
@@ -312,17 +318,22 @@ class TestRemoteGS(TestDataCloudBase):
         return RemoteGS
 
 
-class TestRemoteAzure(TestDataCloudBase):
+class TestRemoteAzureCompat(TestDataCloudBase):
     def _should_test(self):
         return _should_test_azure()
 
     def _get_url(self):
-        return get_azure_url()
+        return get_azure_url_compat()
 
     def _get_cloud_class(self):
         return RemoteAzure
 
 
+class TestRemoteAzure(TestRemoteAzureCompat):
+    def _get_url(self):
+        return get_azure_url()
+
+
 class TestRemoteLOCAL(TestDataCloudBase):
     def _should_test(self):
         return True

tests/test_remote.py

@@ -96,3 +96,27 @@ class TestRemoteRemove(TestDvc):
 
         ret = main(['remote', 'remove', remote])
         self.assertEqual(ret, 0)
+
+
+class TestRemoteDefault(TestDvc):
+    def test(self):
+        remote = 'mys3'
+        ret = main(['remote', 'add', 'mys3', 's3://bucket/path'])
+        self.assertEqual(ret, 0)
+
+        ret = main(['remote', 'default', 'mys3'])
+        self.assertEqual(ret, 0)
+        config_file = os.path.join(self.dvc.dvc_dir, Config.CONFIG)
+        config = configobj.ConfigObj(config_file)
+        default = config[Config.SECTION_CORE][Config.SECTION_CORE_REMOTE]
+        self.assertEqual(default, remote)
+
+        ret = main(['remote', 'default', '--unset'])
+        self.assertEqual(ret, 0)
+        config = configobj.ConfigObj(config_file)
+        core = config.get(Config.SECTION_CORE)
+        if core is not None:
+            default = core.get(Config.SECTION_CORE_REMOTE)
+        else:
+            default = None
+        self.assertEqual(default, None)

tests/test_repro.py

@@ -256,6 +256,47 @@ class TestReproChangedDeepData(TestReproChangedData):
         self.assertEqual(len(stages), 3)
 
 
+class TestReproIgnoreBuildCache(TestDvc):
+    def test(self):
+        stages = self.dvc.add(self.FOO)
+        self.assertEqual(len(stages), 1)
+        foo_stage = stages[0]
+        self.assertTrue(foo_stage is not None)
+
+        code1 = 'code1.py'
+        shutil.copyfile(self.CODE, code1)
+        file1 = 'file1'
+        file1_stage = file1 + '.dvc'
+        stage = self.dvc.run(fname=file1_stage,
+                             outs=[file1],
+                             deps=[self.FOO, code1],
+                             cmd='python {} {} {}'.format(code1,
+                                                          self.FOO,
+                                                          file1))
+        self.assertTrue(stage is not None)
+
+        code2 = 'code2.py'
+        shutil.copyfile(self.CODE, code2)
+        file2 = 'file2'
+        file2_stage = file2 + '.dvc'
+        stage = self.dvc.run(fname=file2_stage,
+                             outs=[file2],
+                             deps=[file1, code2],
+                             cmd='python {} {} {}'.format(code2,
+                                                          file1,
+                                                          file2))
+        self.assertTrue(stage is not None)
+
+        with open(code1, 'a') as fobj:
+            fobj.write('\n\n')
+
+        stages = self.dvc.reproduce(file2_stage)
+        self.assertEqual(len(stages), 1)
+
+        stages = self.dvc.reproduce(file2_stage, ignore_build_cache=True)
+        self.assertEqual(len(stages), 2)
+
+
 class TestReproPipeline(TestReproChangedDeepData):
     def test(self):
         stages = self.dvc.reproduce(self.file1_stage,