diff --git a/dvc/cli.py b/dvc/cli.py index 10dcc09d7..d515bd208 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -19,7 +19,7 @@ from dvc.command.imp import CmdImport from dvc.command.config import CmdConfig from dvc.command.checkout import CmdCheckout from dvc.command.remote import CmdRemoteAdd, CmdRemoteRemove -from dvc.command.remote import CmdRemoteModify, CmdRemoteList +from dvc.command.remote import CmdRemoteModify, CmdRemoteList, CmdRemoteDefault from dvc.command.metrics import CmdMetricsShow, CmdMetricsAdd from dvc.command.metrics import CmdMetricsRemove, CmdMetricsModify from dvc.command.install import CmdInstall @@ -524,6 +524,13 @@ def parse_args(argv=None): action='store_true', default=False, help='Reproduce all pipelines in the project.') + repro_parser.add_argument( + '--ignore-build-cache', + action='store_true', + default=False, + help='Reproduce all descendants of a changed stage' + 'even if their direct dependencies didn\'t ' + 'change.') repro_parser.set_defaults(func=CmdRepro) # Remove @@ -726,6 +733,40 @@ def parse_args(argv=None): help='Set as default remote.') remote_add_parser.set_defaults(func=CmdRemoteAdd) + REMOTE_DEFAULT_HELP = 'Set/unset default remote.' + remote_default_parser = remote_subparsers.add_parser( + 'default', + parents=[parent_parser], + description=REMOTE_DEFAULT_HELP, + help=REMOTE_DEFAULT_HELP) + remote_default_parser.add_argument( + 'name', + nargs='?', + help='Name of the remote.') + remote_default_parser.add_argument( + '-u', + '--unset', + action='store_true', + default=False, + help='Unset default remote.') + remote_default_parser.add_argument( + '--global', + dest='glob', + action='store_true', + default=False, + help='Use global config.') + remote_default_parser.add_argument( + '--system', + action='store_true', + default=False, + help='Use system config.') + remote_default_parser.add_argument( + '--local', + action='store_true', + default=False, + help='Use local config.') + remote_default_parser.set_defaults(func=CmdRemoteDefault) + REMOTE_REMOVE_HELP = 'Remove remote.' remote_remove_parser = remote_subparsers.add_parser( 'remove', diff --git a/dvc/command/remote.py b/dvc/command/remote.py index 011823265..607ce2f71 100644 --- a/dvc/command/remote.py +++ b/dvc/command/remote.py @@ -62,6 +62,13 @@ class CmdRemoteModify(CmdConfig): return super(CmdRemoteModify, self).run() +class CmdRemoteDefault(CmdConfig): + def run(self): + self.args.value = self.args.name + self.args.name = 'core.remote' + return super(CmdRemoteDefault, self).run() + + class CmdRemoteList(CmdConfig): def run(self): for section in self.configobj.keys(): diff --git a/dvc/command/repro.py b/dvc/command/repro.py index 1d1f9c42d..726cbfeed 100644 --- a/dvc/command/repro.py +++ b/dvc/command/repro.py @@ -16,13 +16,14 @@ class CmdRepro(CmdBase): for target in self.args.targets: try: stages = self.project.reproduce( - target, - recursive=recursive, - force=self.args.force, - dry=self.args.dry, - interactive=self.args.interactive, - pipeline=self.args.pipeline, - all_pipelines=self.args.all_pipelines) + target, + recursive=recursive, + force=self.args.force, + dry=self.args.dry, + interactive=self.args.interactive, + pipeline=self.args.pipeline, + all_pipelines=self.args.all_pipelines, + ignore_build_cache=self.args.ignore_build_cache) if len(stages) == 0: self.project.logger.info(CmdDataStatus.UP_TO_DATE_MSG) diff --git a/dvc/config.py b/dvc/config.py index 70a3b88e5..aaa34a879 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -150,6 +150,8 @@ class Config(object): SECTION_LOCAL_STORAGEPATH: str, } + SECTION_AZURE_CONNECTION_STRING = 'connection_string' + SECTION_REMOTE_REGEX = r'^\s*remote\s*"(?P.*)"\s*$' SECTION_REMOTE_FMT = 'remote "{}"' SECTION_REMOTE_URL = 'url' @@ -177,6 +179,7 @@ class Config(object): Optional(SECTION_REMOTE_TIMEOUT): Use(int), Optional(SECTION_REMOTE_PASSWORD): str, Optional(SECTION_REMOTE_ASK_PASSWORD): And(str, is_bool, Use(to_bool)), + Optional(SECTION_AZURE_CONNECTION_STRING): str, } SECTION_STATE = 'state' diff --git a/dvc/project.py b/dvc/project.py index 628e66afb..dce1ade24 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -449,7 +449,8 @@ class Project(object): dry=False, interactive=False, pipeline=False, - all_pipelines=False): + all_pipelines=False, + ignore_build_cache=False): from dvc.stage import Stage if target is None and not all_pipelines: @@ -485,7 +486,8 @@ class Project(object): recursive=recursive, force=force, dry=dry, - interactive=interactive) + interactive=interactive, + ignore_build_cache=ignore_build_cache) ret.extend(stages) self._remind_to_git_add() @@ -497,7 +499,8 @@ class Project(object): recursive=True, force=False, dry=False, - interactive=False): + interactive=False, + ignore_build_cache=False): import networkx as nx from dvc.stage import Stage @@ -512,7 +515,8 @@ class Project(object): node, force, dry, - interactive) + interactive, + ignore_build_cache) else: ret = self._reproduce_stage(stages, node, @@ -522,17 +526,34 @@ class Project(object): return ret - def _reproduce_stages(self, G, stages, node, force, dry, interactive): + def _reproduce_stages(self, + G, + stages, + node, + force, + dry, + interactive, + ignore_build_cache): import networkx as nx result = [] for n in nx.dfs_postorder_nodes(G, node): try: - result += self._reproduce_stage(stages, - n, - force, - dry, - interactive) + ret = self._reproduce_stage(stages, + n, + force, + dry, + interactive) + + if len(ret) == 0 and ignore_build_cache: + # NOTE: we are walking our pipeline from the top to the + # bottom. If one stage is changed, it will be reproduced, + # which tells us that we should force reproducing all of + # the other stages down below, even if their direct + # dependencies didn't change. + force = True + + result += ret except Exception as ex: raise ReproductionError(stages[n].relpath, ex) return result diff --git a/dvc/remote/azure.py b/dvc/remote/azure.py index f971b4141..228c68551 100644 --- a/dvc/remote/azure.py +++ b/dvc/remote/azure.py @@ -8,6 +8,11 @@ try: except ImportError: BlockBlobService = None +try: + from urlparse import urlparse +except ImportError: + from urllib.parse import urlparse + from dvc.logger import logger from dvc.progress import progress from dvc.config import Config @@ -24,9 +29,10 @@ class Callback(object): class RemoteAzure(RemoteBase): scheme = 'azure' - REGEX = (r'^azure://' + REGEX = (r'^azure://((?P[^=;]*)|(' + # backward compatibility r'(ContainerName=(?P[^;]+);?)?' - r'(?P.+)?$') + r'(?P.+)?))$') REQUIRES = {'azure-storage-blob': BlockBlobService} PARAM_ETAG = 'etag' COPY_POLL_SECONDS = 5 @@ -36,17 +42,18 @@ class RemoteAzure(RemoteBase): self.project = project self.url = config.get(Config.SECTION_REMOTE_URL) - match = re.match(self.REGEX, self.url) + match = re.match(self.REGEX, self.url) # backward compatibility self.bucket = ( - match.group('container_name') + urlparse(self.url).netloc + or match.group('container_name') # backward compatibility or os.getenv('AZURE_STORAGE_CONTAINER_NAME')) - # FIXME: currently Azure doesn't support prefixes - self.prefix = '/' + self.prefix = urlparse(self.url).path.lstrip('/') or '/' self.connection_string = ( - match.group('connection_string') + config.get(Config.SECTION_AZURE_CONNECTION_STRING) + or match.group('connection_string') # backward compatibility or os.getenv('AZURE_STORAGE_CONNECTION_STRING')) if not self.bucket: @@ -138,7 +145,7 @@ class RemoteAzure(RemoteBase): return [{ 'scheme': self.scheme, 'bucket': self.bucket, - 'key': '{}/{}'.format(md5[0:2], md5[2:]) + 'key': '{}/{}/{}'.format(self.prefix, md5[0:2], md5[2:]) } for md5 in md5s] def _all_keys(self, bucket, prefix): @@ -235,8 +242,8 @@ class RemoteAzure(RemoteBase): progress.finish_target(name) def _path_to_etag(self, path): - assert len(path.split('/')) == 2 - return posixpath.dirname(path) + posixpath.basename(path) + relpath = posixpath.relpath(path, self.prefix) + return posixpath.dirname(relpath) + posixpath.basename(relpath) def _all(self): keys = self._all_keys(self.bucket, self.prefix) @@ -253,7 +260,9 @@ class RemoteAzure(RemoteBase): if etag in used: continue path_info = {'scheme': self.scheme, - 'key': posixpath.join(etag[0:2], etag[2:]), + 'key': posixpath.join(self.prefix, + etag[0:2], + etag[2:]), 'bucket': self.bucket} self.remove(path_info) removed = True diff --git a/tests/test_data_cloud.py b/tests/test_data_cloud.py index 35149a324..bff0abb1c 100644 --- a/tests/test_data_cloud.py +++ b/tests/test_data_cloud.py @@ -141,10 +141,16 @@ def get_gcp_url(): return 'gs://' + get_gcp_storagepath() -def get_azure_url(): +def get_azure_url_compat(): return 'azure://' +def get_azure_url(): + container_name = os.getenv("AZURE_STORAGE_CONTAINER_NAME") + assert container_name is not None + return 'azure://{}/{}'.format(container_name, str(uuid.uuid4())) + + def sleep(): # Sleep some time to simulate realistic behavior. # Some filesystems have a bad date resolution for @@ -312,17 +318,22 @@ class TestRemoteGS(TestDataCloudBase): return RemoteGS -class TestRemoteAzure(TestDataCloudBase): +class TestRemoteAzureCompat(TestDataCloudBase): def _should_test(self): return _should_test_azure() def _get_url(self): - return get_azure_url() + return get_azure_url_compat() def _get_cloud_class(self): return RemoteAzure +class TestRemoteAzure(TestRemoteAzureCompat): + def _get_url(self): + return get_azure_url() + + class TestRemoteLOCAL(TestDataCloudBase): def _should_test(self): return True diff --git a/tests/test_remote.py b/tests/test_remote.py index ad479fbe7..3762fa736 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -96,3 +96,27 @@ class TestRemoteRemove(TestDvc): ret = main(['remote', 'remove', remote]) self.assertEqual(ret, 0) + + +class TestRemoteDefault(TestDvc): + def test(self): + remote = 'mys3' + ret = main(['remote', 'add', 'mys3', 's3://bucket/path']) + self.assertEqual(ret, 0) + + ret = main(['remote', 'default', 'mys3']) + self.assertEqual(ret, 0) + config_file = os.path.join(self.dvc.dvc_dir, Config.CONFIG) + config = configobj.ConfigObj(config_file) + default = config[Config.SECTION_CORE][Config.SECTION_CORE_REMOTE] + self.assertEqual(default, remote) + + ret = main(['remote', 'default', '--unset']) + self.assertEqual(ret, 0) + config = configobj.ConfigObj(config_file) + core = config.get(Config.SECTION_CORE) + if core is not None: + default = core.get(Config.SECTION_CORE_REMOTE) + else: + default = None + self.assertEqual(default, None) diff --git a/tests/test_repro.py b/tests/test_repro.py index 29d227cdd..1d0c45edd 100644 --- a/tests/test_repro.py +++ b/tests/test_repro.py @@ -256,6 +256,47 @@ class TestReproChangedDeepData(TestReproChangedData): self.assertEqual(len(stages), 3) +class TestReproIgnoreBuildCache(TestDvc): + def test(self): + stages = self.dvc.add(self.FOO) + self.assertEqual(len(stages), 1) + foo_stage = stages[0] + self.assertTrue(foo_stage is not None) + + code1 = 'code1.py' + shutil.copyfile(self.CODE, code1) + file1 = 'file1' + file1_stage = file1 + '.dvc' + stage = self.dvc.run(fname=file1_stage, + outs=[file1], + deps=[self.FOO, code1], + cmd='python {} {} {}'.format(code1, + self.FOO, + file1)) + self.assertTrue(stage is not None) + + code2 = 'code2.py' + shutil.copyfile(self.CODE, code2) + file2 = 'file2' + file2_stage = file2 + '.dvc' + stage = self.dvc.run(fname=file2_stage, + outs=[file2], + deps=[file1, code2], + cmd='python {} {} {}'.format(code2, + file1, + file2)) + self.assertTrue(stage is not None) + + with open(code1, 'a') as fobj: + fobj.write('\n\n') + + stages = self.dvc.reproduce(file2_stage) + self.assertEqual(len(stages), 1) + + stages = self.dvc.reproduce(file2_stage, ignore_build_cache=True) + self.assertEqual(len(stages), 2) + + class TestReproPipeline(TestReproChangedDeepData): def test(self): stages = self.dvc.reproduce(self.file1_stage,