From bc2c84370d2fff7d709a1f8030d9d862de6369c2 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Mon, 13 Aug 2018 05:43:17 +0300 Subject: [PATCH 1/4] run: change prompt message Fixes #1005 Signed-off-by: Ruslan Kuprieiev --- dvc/stage.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dvc/stage.py b/dvc/stage.py index 73238086e..413d2e54e 100644 --- a/dvc/stage.py +++ b/dvc/stage.py @@ -253,9 +253,10 @@ class Stage(object): if os.path.exists(path): relpath = os.path.relpath(path) - msg = "{} already exists. Do you wish to overwrite it?" + msg = "'{}' already exists. " \ + "Do you wish to run the command and overwrite it?" if not overwrite and not prompt(msg.format(relpath), False): - raise DvcException("{} already exists".format(relpath)) + raise DvcException("'{}' already exists".format(relpath)) stage.cwd = cwd stage.path = path From db017543d5f8483274997a90484fc6629e90440b Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Mon, 13 Aug 2018 06:05:19 +0300 Subject: [PATCH 2/4] dvc: detect output duplication Fixes #1006 Signed-off-by: Ruslan Kuprieiev --- dvc/exceptions.py | 8 +++++--- dvc/project.py | 29 +++++++++++++++++++++++++++-- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/dvc/exceptions.py b/dvc/exceptions.py index f1d6c6ec1..e489a45a2 100644 --- a/dvc/exceptions.py +++ b/dvc/exceptions.py @@ -24,6 +24,8 @@ class OutputDuplicationError(DvcException): assert isinstance(output, str) assert isinstance(stages, list) assert all(isinstance(stage, str) for stage in stages) - msg = "File '{}' is specified as an output in more than one stage: {}" - super(OutputDuplicationError, self).__init__(msg.format(output, - str(stages))) + msg = "File '{}' is specified as an output in more than one stage:{}" + s = "" + for stage in stages: + s += "\n " + stage + super(OutputDuplicationError, self).__init__(msg.format(output, s)) diff --git a/dvc/project.py b/dvc/project.py index 590f9a227..e39bebe27 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -131,11 +131,23 @@ class Project(object): def to_dvc_path(self, path): return os.path.relpath(path, self.root_dir) + def _check_output_duplication(self, outs): + from dvc.exceptions import OutputDuplicationError + + for stage in self.stages(): + for o in stage.outs: + for out in outs: + if o.path == out.path and o.stage.path != out.stage.path: + stages = [o.stage.relpath, out.stage.relpath] + raise OutputDuplicationError(o.path, stages) + def add(self, fname): stage = Stage.loads(project=self, outs=[fname], add=True) + self._check_output_duplication(stage.outs) + stage.save() stage.dump() return stage @@ -220,6 +232,9 @@ class Project(object): metrics_no_cache=metrics_no_cache, deps=deps, overwrite=False) + + self._check_output_duplication(stage.outs) + if not no_exec: stage.run() stage.dump() @@ -231,6 +246,8 @@ class Project(object): deps=[url], outs=[out]) + self._check_output_duplication(stage.outs) + stage.run() stage.dump() return stage @@ -544,7 +561,7 @@ class Project(object): abs_path = os.path.abspath(path) matched = [out for out in outs if out.path == abs_path] - stages = [out.stage.path for out in matched] + stages = [out.stage.relpath for out in matched] if len(stages) > 1: raise OutputDuplicationError(path, stages) @@ -660,14 +677,22 @@ class Project(object): def graph(self): import networkx as nx + from dvc.exceptions import OutputDuplicationError G = nx.DiGraph() G_active = nx.DiGraph() stages = self.stages() outs = [] + outs_by_path = {} for stage in stages: - outs += stage.outs + for o in stage.outs: + existing = outs_by_path.get(o.path, None) + if existing is not None: + stages = [o.stage.relpath, existing.stage.relpath] + raise OutputDuplicationError(o.path, stages) + outs.append(o) + outs_by_path[o.path] = o # collect the whole DAG for stage in stages: From 2b19b6bbfaaab0620e28efb5ab71aaf30da2d6dd Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Mon, 13 Aug 2018 06:29:05 +0300 Subject: [PATCH 3/4] prompt: add y/n to msg Fixes #999 Signed-off-by: Ruslan Kuprieiev --- dvc/prompt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/prompt.py b/dvc/prompt.py index f8befe411..359114d2c 100644 --- a/dvc/prompt.py +++ b/dvc/prompt.py @@ -11,7 +11,7 @@ def prompt(msg, default=False): if not sys.stdout.isatty(): return default - answer = input(msg + u'\n').lower() + answer = input(msg + u' (y/n)\n').lower() while answer not in ['yes', 'no', 'y', 'n']: answer = input('Enter \'yes\' or \'no\'.\n').lower() From e84f231c34524b220014174a6c096d0d45d6f15f Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Mon, 13 Aug 2018 07:02:40 +0300 Subject: [PATCH 4/4] help: add cmd description Fixes #1007 Signed-off-by: Ruslan Kuprieiev --- dvc/cli.py | 129 +++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 95 insertions(+), 34 deletions(-) diff --git a/dvc/cli.py b/dvc/cli.py index aff2ec31c..93f0d3ac7 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -94,11 +94,13 @@ def parse_args(argv=None): _fix_subparsers(subparsers) # Init + INIT_HELP = 'Initialize dvc over a directory ' \ + '(should already be a git dir).' init_parser = subparsers.add_parser( 'init', parents=[parent_parser], - help='Initialize dvc over a directory ' - '(should already be a git dir).') + description=INIT_HELP, + help=INIT_HELP) init_parser.add_argument( '--no-scm', action='store_true', @@ -114,10 +116,12 @@ def parse_args(argv=None): init_parser.set_defaults(func=CmdInit) # Destroy + DESTROY_HELP = 'Destroy dvc.' destroy_parser = subparsers.add_parser( 'destroy', parents=[parent_parser], - help='Destroy dvc.') + description=DESTROY_HELP, + help=DESTROY_HELP) destroy_parser.add_argument( '-f', '--force', @@ -127,10 +131,12 @@ def parse_args(argv=None): destroy_parser.set_defaults(func=CmdDestroy) # Add + ADD_HELP = 'Add files/directories to dvc.' add_parser = subparsers.add_parser( 'add', parents=[parent_parser], - help='Add files/directories to dvc.') + description=ADD_HELP, + help=ADD_HELP) add_parser.add_argument( 'targets', nargs='+', @@ -138,10 +144,12 @@ def parse_args(argv=None): add_parser.set_defaults(func=CmdAdd) # Import + IMPORT_HELP = 'Import files from URL.' import_parser = subparsers.add_parser( 'import', parents=[parent_parser], - help='Import files from URL.') + description=IMPORT_HELP, + help=IMPORT_HELP) import_parser.add_argument( 'url', help='URL.') @@ -151,10 +159,12 @@ def parse_args(argv=None): import_parser.set_defaults(func=CmdImport) # Checkout + CHECKOUT_HELP = 'Checkout data files from cache.' checkout_parser = subparsers.add_parser( 'checkout', parents=[parent_parser], - help='Checkout data files from cache.') + description=CHECKOUT_HELP, + help=CHECKOUT_HELP) checkout_parser.add_argument( 'targets', nargs='*', @@ -162,11 +172,13 @@ def parse_args(argv=None): checkout_parser.set_defaults(func=CmdCheckout) # Run + RUN_HELP = 'Generate a stage file from a given ' \ + 'command and execute the command.' run_parser = subparsers.add_parser( 'run', parents=[parent_parser], - help='Generate a stage file from a given ' - 'command and execute the command.') + description=RUN_HELP, + help=RUN_HELP) run_parser.add_argument( '-d', '--deps', @@ -237,10 +249,12 @@ def parse_args(argv=None): help='DVC files.') # Pull + PULL_HELP = 'Pull data files from the cloud.' pull_parser = subparsers.add_parser( 'pull', parents=[parent_cache_parser], - help='Pull data files from the cloud.') + description=PULL_HELP, + help=PULL_HELP) pull_parser.add_argument( '-r', '--remote', @@ -254,10 +268,12 @@ def parse_args(argv=None): pull_parser.set_defaults(func=CmdDataPull) # Push + PUSH_HELP = 'Push data files to the cloud.' push_parser = subparsers.add_parser( 'push', parents=[parent_cache_parser], - help='Push data files to the cloud.') + description=PUSH_HELP, + help=PUSH_HELP) push_parser.add_argument( '-r', '--remote', @@ -271,10 +287,12 @@ def parse_args(argv=None): push_parser.set_defaults(func=CmdDataPush) # Fetch + FETCH_HELP = 'Fetch data files from the cloud.' fetch_parser = subparsers.add_parser( 'fetch', parents=[parent_cache_parser], - help='Fetch data files from the cloud.') + description=FETCH_HELP, + help=FETCH_HELP) fetch_parser.add_argument( '-r', '--remote', @@ -288,10 +306,12 @@ def parse_args(argv=None): fetch_parser.set_defaults(func=CmdDataFetch) # Status + STATUS_HELP = 'Show the project status.' status_parser = subparsers.add_parser( 'status', parents=[parent_cache_parser], - help='Show the project status.') + description=STATUS_HELP, + help=STATUS_HELP) status_parser.add_argument( '-c', '--cloud', @@ -306,11 +326,12 @@ def parse_args(argv=None): status_parser.set_defaults(func=CmdDataStatus) # Repro + REPRO_HELP = 'Reproduce DVC file. Default file name - \'Dvcfile\'.' repro_parser = subparsers.add_parser( 'repro', parents=[parent_parser], - help='Reproduce DVC file. Default file name ' - '- \'Dvcfile\'.') + description=REPRO_HELP, + help=REPRO_HELP) repro_parser.add_argument( 'targets', nargs='*', @@ -357,10 +378,12 @@ def parse_args(argv=None): repro_parser.set_defaults(func=CmdRepro) # Remove + REMOVE_HELP = 'Remove outputs of DVC file.' remove_parser = subparsers.add_parser( 'remove', parents=[parent_parser], - help='Remove outputs of DVC file.') + description=REMOVE_HELP, + help=REMOVE_HELP) remove_parser_group = remove_parser.add_mutually_exclusive_group() remove_parser_group.add_argument( '-o', @@ -381,10 +404,12 @@ def parse_args(argv=None): remove_parser.set_defaults(func=CmdRemove) # Move + MOVE_HELP = 'Move output of DVC file.' move_parser = subparsers.add_parser( 'move', parents=[parent_parser], - help='Move output of DVC file.') + description=MOVE_HELP, + help=MOVE_HELP) move_parser.add_argument( 'src', help='Source.') @@ -394,10 +419,12 @@ def parse_args(argv=None): move_parser.set_defaults(func=CmdMove) # Garbage collector + GC_HELP = 'Collect garbage.' gc_parser = subparsers.add_parser( 'gc', parents=[parent_parser], - help='Collect garbage.') + description=GC_HELP, + help=GC_HELP) gc_parser.add_argument( '-a', '--all-branches', @@ -417,10 +444,12 @@ def parse_args(argv=None): gc_parser.set_defaults(func=CmdGC) # Config + CONFIG_HELP = 'Get or set config options.' config_parser = subparsers.add_parser( 'config', parents=[parent_parser], - help='Get or set config options.') + description=CONFIG_HELP, + help=CONFIG_HELP) config_parser.add_argument( '-u', '--unset', @@ -443,10 +472,12 @@ def parse_args(argv=None): config_parser.set_defaults(func=CmdConfig) # Remote + REMOTE_HELP = 'Manage set of tracked repositories.' remote_parser = subparsers.add_parser( 'remote', parents=[parent_parser], - help='Manage set of tracked repositories.') + description=REMOTE_HELP, + help=REMOTE_HELP) remote_subparsers = remote_parser.add_subparsers( dest='cmd', @@ -455,10 +486,12 @@ def parse_args(argv=None): _fix_subparsers(remote_subparsers) + REMOTE_ADD_HELP = 'Add remote.' remote_add_parser = remote_subparsers.add_parser( 'add', parents=[parent_parser], - help='Add remote.') + description=REMOTE_ADD_HELP, + help=REMOTE_ADD_HELP) remote_add_parser.add_argument( 'name', help='Name.') @@ -478,10 +511,12 @@ def parse_args(argv=None): help='Set as default remote.') remote_add_parser.set_defaults(func=CmdRemoteAdd) + REMOTE_REMOVE_HELP = 'Remove remote.' remote_remove_parser = remote_subparsers.add_parser( 'remove', parents=[parent_parser], - help='Remove remote.') + description=REMOTE_REMOVE_HELP, + help=REMOTE_REMOVE_HELP) remote_remove_parser.add_argument( 'name', help='Name') @@ -492,10 +527,12 @@ def parse_args(argv=None): help='Use local config.') remote_remove_parser.set_defaults(func=CmdRemoteRemove) + REMOTE_MODIFY_HELP = 'Modify remote.' remote_modify_parser = remote_subparsers.add_parser( 'modify', parents=[parent_parser], - help='Modify remote.') + description=REMOTE_MODIFY_HELP, + help=REMOTE_MODIFY_HELP) remote_modify_parser.add_argument( 'name', help='Name.') @@ -519,10 +556,12 @@ def parse_args(argv=None): help='Use local config.') remote_modify_parser.set_defaults(func=CmdRemoteModify) + REMOTE_LIST_HELP = 'List remotes.' remote_list_parser = remote_subparsers.add_parser( 'list', parents=[parent_parser], - help='List remotes.') + description=REMOTE_LIST_HELP, + help=REMOTE_LIST_HELP) remote_list_parser.add_argument( '--local', action='store_true', @@ -531,10 +570,12 @@ def parse_args(argv=None): remote_list_parser.set_defaults(func=CmdRemoteList) # Metrics + METRICS_HELP = 'Get metrics from all branches.' metrics_parser = subparsers.add_parser( 'metrics', parents=[parent_parser], - help='Get metrics from all branches.') + description=METRICS_HELP, + help=METRICS_HELP) metrics_subparsers = metrics_parser.add_subparsers( dest='cmd', @@ -543,10 +584,12 @@ def parse_args(argv=None): _fix_subparsers(metrics_subparsers) + METRICS_SHOW_HELP = 'Show metrics.' metrics_show_parser = metrics_subparsers.add_parser( 'show', parents=[parent_parser], - help='Show metrics.') + description=METRICS_SHOW_HELP, + help=METRICS_SHOW_HELP) metrics_show_parser.add_argument( 'path', nargs='?', @@ -583,10 +626,12 @@ def parse_args(argv=None): help='Show metrics for all branches.') metrics_show_parser.set_defaults(func=CmdMetricsShow) + METRICS_ADD_HELP = 'Add metrics.' metrics_add_parser = metrics_subparsers.add_parser( 'add', parents=[parent_parser], - help='Add metrics.') + description=METRICS_ADD_HELP, + help=METRICS_ADD_HELP) metrics_add_parser.add_argument( '-t', '--type', @@ -600,10 +645,12 @@ def parse_args(argv=None): help='Path to metrics file.') metrics_add_parser.set_defaults(func=CmdMetricsAdd) + METRICS_MODIFY_HELP = 'Modify metrics.' metrics_modify_parser = metrics_subparsers.add_parser( 'modify', parents=[parent_parser], - help='Modify metrics.') + description=METRICS_MODIFY_HELP, + help=METRICS_MODIFY_HELP) metrics_modify_parser.add_argument( '-t', '--type', @@ -617,34 +664,42 @@ def parse_args(argv=None): help='Metrics file.') metrics_modify_parser.set_defaults(func=CmdMetricsModify) + METRICS_REMOVE_HELP = 'Remove metrics.' metrics_remove_parser = metrics_subparsers.add_parser( 'remove', parents=[parent_parser], - help='Remove metrics.') + description=METRICS_REMOVE_HELP, + help=METRICS_REMOVE_HELP) metrics_remove_parser.add_argument( 'path', help='Path to metrics file.') metrics_remove_parser.set_defaults(func=CmdMetricsRemove) # Install + INSTALL_HELP = 'Install dvc hooks into the repository.' install_parser = subparsers.add_parser( 'install', parents=[parent_parser], - help='Install dvc hooks into the repository.') + description=INSTALL_HELP, + help=INSTALL_HELP) install_parser.set_defaults(func=CmdInstall) # Root + ROOT_HELP = 'Relative path to project\'s directory.' root_parser = subparsers.add_parser( 'root', parents=[parent_parser], - help='Relative path to project\'s directory.') + description=ROOT_HELP, + help=ROOT_HELP) root_parser.set_defaults(func=CmdRoot) # Lock + LOCK_HELP = 'Lock DVC file.' lock_parser = subparsers.add_parser( 'lock', parents=[parent_parser], - help='Lock DVC file.') + description=LOCK_HELP, + help=LOCK_HELP) lock_parser.add_argument( 'targets', nargs='+', @@ -652,10 +707,12 @@ def parse_args(argv=None): lock_parser.set_defaults(func=CmdLock) # Unlock + UNLOCK_HELP = 'Unlock DVC file.' unlock_parser = subparsers.add_parser( 'unlock', parents=[parent_parser], - help='Unlock DVC file.') + description=UNLOCK_HELP, + help=UNLOCK_HELP) unlock_parser.add_argument( 'targets', nargs='+', @@ -663,10 +720,12 @@ def parse_args(argv=None): unlock_parser.set_defaults(func=CmdUnlock) # Pipeline + PIPELINE_HELP = 'Manage pipeline.' pipeline_parser = subparsers.add_parser( 'pipeline', parents=[parent_parser], - help='Manage pipeline.') + description=PIPELINE_HELP, + help=PIPELINE_HELP) pipeline_subparsers = pipeline_parser.add_subparsers( dest='cmd', @@ -675,10 +734,12 @@ def parse_args(argv=None): _fix_subparsers(pipeline_subparsers) + PIPELINE_SHOW_HELP = 'Show pipeline.' pipeline_show_parser = pipeline_subparsers.add_parser( 'show', parents=[parent_parser], - help='Show pipeline.') + description=PIPELINE_SHOW_HELP, + help=PIPELINE_SHOW_HELP) pipeline_show_group = pipeline_show_parser.add_mutually_exclusive_group() pipeline_show_group.add_argument( '-c',