From 50f736d9d9737d9f892976a0b1ca05a18302a997 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Sun, 26 Nov 2017 21:40:55 -0800 Subject: [PATCH 1/6] dvc run: regular file output. Close #314 --- dvc/cli.py | 9 ++++++++- dvc/command/add.py | 2 +- dvc/command/run.py | 48 +++++++++++++++++++++++++--------------------- 3 files changed, 35 insertions(+), 24 deletions(-) diff --git a/dvc/cli.py b/dvc/cli.py index 54ada8350..4d4cf04bd 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -96,11 +96,18 @@ def parse_args(argv=None): run_parser.add_argument('-d', '--deps', action='append', + default = [], help='Declare dependencies for reproducible cmd.') run_parser.add_argument('-o', '--out', action='append', - help='Declare output data items for reproducible cmd.') + default=[], + help='Declare output data file (sync to cloud) for reproducible cmd.') + run_parser.add_argument('-r', + '--reg', + action='append', + default=[], + help='Declare output regular file (sync to Git) for reproducible cmd.') run_parser.add_argument('-l', '--lock', action='store_true', diff --git a/dvc/command/add.py b/dvc/command/add.py index 945292b31..4f321cc41 100644 --- a/dvc/command/add.py +++ b/dvc/command/add.py @@ -40,7 +40,7 @@ class CmdAdd(CmdBase): for data_item in targets: Logger.debug('Creating state file for {}'.format(data_item.data.relative)) - command = CommandFile(None, [data_item.data.dvc], [], True, None) + command = CommandFile(None, [data_item.data.dvc], [], [], True, None) state_file = StateFile(data_item, self.settings, command.dict, diff --git a/dvc/command/run.py b/dvc/command/run.py index 7e9a750d2..c4e84db08 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -1,14 +1,9 @@ -import os -import sys import yaml from dvc.command.common.base import CmdBase from dvc.exceptions import DvcException -from dvc.git_wrapper import GitWrapper from dvc.logger import Logger -from dvc.repository_change import RepositoryChange from dvc.state_file import StateFile -from dvc.utils import cached_property from dvc.executor import Executor from dvc.state_file import StateFileBase @@ -24,12 +19,14 @@ class CommandFile(StateFileBase): PARAM_CMD = 'cmd' PARAM_OUT = 'out' + PARAM_REG = 'reg' PARAM_DEPS = 'deps' PARAM_LOCK = 'locked' - def __init__(self, cmd, out, deps, locked, fname): + def __init__(self, cmd, out, reg, deps, locked, fname): self.cmd = cmd self.out = out + self.reg = reg self.deps = deps self.locked = locked self.fname = fname @@ -39,6 +36,7 @@ class CommandFile(StateFileBase): data = { self.PARAM_CMD: self.cmd, self.PARAM_OUT: self.out, + self.PARAM_REG: self.reg, self.PARAM_DEPS: self.deps, self.PARAM_LOCK: self.locked } @@ -56,6 +54,7 @@ class CommandFile(StateFileBase): def loadd(data, fname=None): return CommandFile(data.get(CommandFile.PARAM_CMD, None), data.get(CommandFile.PARAM_OUT, None), + data.get(CommandFile.PARAM_REG, None), data.get(CommandFile.PARAM_DEPS, None), data.get(CommandFile.PARAM_LOCK, None), fname) @@ -75,7 +74,8 @@ class CmdRun(CmdBase): command = CommandFile.load(cmd) except Exception as exc: Logger.debug("Failed to load {}: {}".format(cmd, str(exc))) - command = CommandFile(cmd, self.parsed_args.out, self.parsed_args.deps, self.parsed_args.lock, None) + command = CommandFile(cmd, self.parsed_args.out, self.parsed_args.reg, + self.parsed_args.deps, self.parsed_args.lock, None) self.run_command(self.settings, command) return self.commit_if_needed('DVC run: {}'.format(command.cmd)) @@ -84,20 +84,24 @@ class CmdRun(CmdBase): def run_command(settings, command): Executor.exec_cmd_only_success(command.cmd, shell=True) - result = [] - items = settings.path_factory.to_data_items(command.out)[0] - for data_item in items: - Logger.debug('Move output file "{}" to cache dir "{}" and create a hardlink'.format( - data_item.data.relative, data_item.cache_dir_abs)) - data_item.move_data_to_cache() + map(lambda i: CmdRun._create_cache_and_state_files(i, command, settings), + settings.path_factory.to_data_items(command.out)[0]) + map(lambda i: CmdRun._create_state_file(i, command, settings), + settings.path_factory.to_data_items(command.reg)[0]) - Logger.debug('Create state file "{}"'.format(data_item.state.relative)) + @staticmethod + def _create_cache_and_state_files(data_item, command, settings): + Logger.debug('Move output file "{}" to cache dir "{}" and create a hardlink'.format( + data_item.data.relative, data_item.cache_dir_abs)) + data_item.move_data_to_cache() + return CmdRun._create_state_file(data_item, command, settings) - state_file = StateFile(data_item, - settings, - command.fname if command.fname else command.dict, - StateFile.parse_deps_state(settings, command.deps)) - state_file.save() - result.append(state_file) - - return result + @staticmethod + def _create_state_file(data_item, command, settings): + Logger.debug('Create state file "{}"'.format(data_item.state.relative)) + state_file = StateFile(data_item, + settings, + command.fname if command.fname else command.dict, + StateFile.parse_deps_state(settings, command.deps)) + state_file.save() + return state_file From 40660b5120817dc8d486bc37977d4866a7623356 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Sun, 26 Nov 2017 21:49:06 -0800 Subject: [PATCH 2/6] Refactoring: extract CommandFile class --- dvc/command/add.py | 2 +- dvc/command/common/command_file.py | 54 +++++++++++++++++++++++++++++ dvc/command/run.py | 55 +----------------------------- 3 files changed, 56 insertions(+), 55 deletions(-) create mode 100644 dvc/command/common/command_file.py diff --git a/dvc/command/add.py b/dvc/command/add.py index 4f321cc41..917e64499 100644 --- a/dvc/command/add.py +++ b/dvc/command/add.py @@ -3,7 +3,7 @@ import os from dvc.command.common.base import CmdBase from dvc.logger import Logger from dvc.state_file import StateFile -from dvc.command.run import CommandFile +from dvc.command.common.command_file import CommandFile class CmdAdd(CmdBase): diff --git a/dvc/command/common/command_file.py b/dvc/command/common/command_file.py new file mode 100644 index 000000000..62aee62e8 --- /dev/null +++ b/dvc/command/common/command_file.py @@ -0,0 +1,54 @@ +import yaml + +from dvc.state_file import StateFileBase + + +class CommandFile(StateFileBase): + MAGIC = 'DVC-Command-State' + VERSION = '0.1' + + PARAM_CMD = 'cmd' + PARAM_OUT = 'out' + PARAM_REG = 'reg' + PARAM_DEPS = 'deps' + PARAM_LOCK = 'locked' + + def __init__(self, cmd, out, reg, deps, locked, fname): + self.cmd = cmd + self.out = out + self.reg = reg + self.deps = deps + self.locked = locked + self.fname = fname + + @property + def dict(self): + data = { + self.PARAM_CMD: self.cmd, + self.PARAM_OUT: self.out, + self.PARAM_REG: self.reg, + self.PARAM_DEPS: self.deps, + self.PARAM_LOCK: self.locked + } + + return data + + def dumps(self): + return yaml.dump(self.dict) + + def dump(self, fname): + with open(fname, 'w+') as fd: + fd.write(self.dumps()) + + @staticmethod + def loadd(data, fname=None): + return CommandFile(data.get(CommandFile.PARAM_CMD, None), + data.get(CommandFile.PARAM_OUT, None), + data.get(CommandFile.PARAM_REG, None), + data.get(CommandFile.PARAM_DEPS, None), + data.get(CommandFile.PARAM_LOCK, None), + fname) + + @staticmethod + def load(fname): + return CommandFile._load(fname, CommandFile, fname) diff --git a/dvc/command/run.py b/dvc/command/run.py index c4e84db08..d027fafd0 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -1,11 +1,9 @@ -import yaml - from dvc.command.common.base import CmdBase +from dvc.command.common.command_file import CommandFile from dvc.exceptions import DvcException from dvc.logger import Logger from dvc.state_file import StateFile from dvc.executor import Executor -from dvc.state_file import StateFileBase class RunError(DvcException): @@ -13,57 +11,6 @@ class RunError(DvcException): DvcException.__init__(self, 'Run error: {}'.format(msg)) -class CommandFile(StateFileBase): - MAGIC = 'DVC-Command-State' - VERSION = '0.1' - - PARAM_CMD = 'cmd' - PARAM_OUT = 'out' - PARAM_REG = 'reg' - PARAM_DEPS = 'deps' - PARAM_LOCK = 'locked' - - def __init__(self, cmd, out, reg, deps, locked, fname): - self.cmd = cmd - self.out = out - self.reg = reg - self.deps = deps - self.locked = locked - self.fname = fname - - @property - def dict(self): - data = { - self.PARAM_CMD: self.cmd, - self.PARAM_OUT: self.out, - self.PARAM_REG: self.reg, - self.PARAM_DEPS: self.deps, - self.PARAM_LOCK: self.locked - } - - return data - - def dumps(self): - return yaml.dump(self.dict) - - def dump(self, fname): - with open(fname, 'w+') as fd: - fd.write(self.dumps()) - - @staticmethod - def loadd(data, fname=None): - return CommandFile(data.get(CommandFile.PARAM_CMD, None), - data.get(CommandFile.PARAM_OUT, None), - data.get(CommandFile.PARAM_REG, None), - data.get(CommandFile.PARAM_DEPS, None), - data.get(CommandFile.PARAM_LOCK, None), - fname) - - @staticmethod - def load(fname): - return CommandFile._load(fname, CommandFile, fname) - - class CmdRun(CmdBase): def __init__(self, settings): super(CmdRun, self).__init__(settings) From b022fa62fe501f8c10f23d277ca0b7f295ec7207 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Sun, 26 Nov 2017 22:47:47 -0800 Subject: [PATCH 3/6] Do not checkout files without cache_state --- dvc/path/factory.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/dvc/path/factory.py b/dvc/path/factory.py index 3502ecf41..48c0657a5 100644 --- a/dvc/path/factory.py +++ b/dvc/path/factory.py @@ -5,7 +5,7 @@ from dvc.path.data_item import DataItem, DataItemError, DataDirError from dvc.path.path import Path from dvc.path.stated_data_item import StatedDataItem from dvc.system import System -from dvc.logger import Logger + class PathFactory(object): def __init__(self, git, config): @@ -53,7 +53,7 @@ class PathFactory(object): return result, externally_created_files - def all_existing_data_items(self, subdir='.'): + def all_existing_data_items(self, subdir='.', cache_exists=True): states = [] for root, dirs, files in os.walk(os.path.join(ConfigI.STATE_DIR, subdir)): @@ -65,7 +65,11 @@ class PathFactory(object): states.append(path) - return self.data_items_from_states(states) + data_items = self.data_items_from_states(states) + if cache_exists: + data_items = filter(lambda i: os.path.exists(i.cache_state.relative), data_items) + + return data_items def data_items_from_states(self, states, existing=True): return map(lambda s: self.data_item_from_dvc_path(self.state_path_to_dvc_path(s), existing), From 86ad3cb3602869112c6308aa8efc84e89072f4d3 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Sun, 26 Nov 2017 22:50:09 -0800 Subject: [PATCH 4/6] It is okay if output file does not exist --- dvc/command/repro.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dvc/command/repro.py b/dvc/command/repro.py index 6997d02df..78c52ff4a 100644 --- a/dvc/command/repro.py +++ b/dvc/command/repro.py @@ -4,7 +4,6 @@ import copy from dvc.command.run import CmdRun, CommandFile from dvc.logger import Logger from dvc.exceptions import DvcException -from dvc.path.data_item import DataDirError from dvc.state_file import StateFile from dvc.system import System from dvc.data_cloud import file_md5 @@ -119,7 +118,7 @@ class ReproChange(object): os.remove(data_item.data.relative) except Exception as ex: msg = 'Data item {} cannot be removed before reproduction: {}' - Logger.error(msg.format(output_dvc, ex)) + Logger.debug(msg.format(output_dvc, ex)) def reproduce_run(self): Logger.info('Reproducing run command for data item {}. Args: {}'.format( From 4b3827255d34c871b77b9cd354fbb95759c836af Mon Sep 17 00:00:00 2001 From: Dmitry Date: Sun, 26 Nov 2017 22:57:39 -0800 Subject: [PATCH 5/6] Get rid of code duplication --- dvc/command/run.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/dvc/command/run.py b/dvc/command/run.py index d027fafd0..5afe93078 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -31,10 +31,13 @@ class CmdRun(CmdBase): def run_command(settings, command): Executor.exec_cmd_only_success(command.cmd, shell=True) - map(lambda i: CmdRun._create_cache_and_state_files(i, command, settings), - settings.path_factory.to_data_items(command.out)[0]) - map(lambda i: CmdRun._create_state_file(i, command, settings), - settings.path_factory.to_data_items(command.reg)[0]) + CmdRun.apply_to_files(command.out, command, CmdRun._create_cache_and_state_files, settings) + CmdRun.apply_to_files(command.reg, command, CmdRun._create_state_file, settings) + + @staticmethod + def apply_to_files(files, command, func, settings): + map(lambda i: func(i, command, settings), + settings.path_factory.to_data_items(files)[0]) @staticmethod def _create_cache_and_state_files(data_item, command, settings): From 2a980f6e57f3cc1bc0a4fb4c19170deb19fef2a4 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Mon, 27 Nov 2017 00:33:40 -0800 Subject: [PATCH 6/6] Fix func tests: get rid of python3 lazy evaluation --- dvc/command/run.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dvc/command/run.py b/dvc/command/run.py index 5afe93078..ddb00c352 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -36,8 +36,7 @@ class CmdRun(CmdBase): @staticmethod def apply_to_files(files, command, func, settings): - map(lambda i: func(i, command, settings), - settings.path_factory.to_data_items(files)[0]) + [func(i, command, settings) for i in settings.path_factory.to_data_items(files)[0]] @staticmethod def _create_cache_and_state_files(data_item, command, settings):