diff --git a/dvc/cli.py b/dvc/cli.py index 54ada8350..4d4cf04bd 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -96,11 +96,18 @@ def parse_args(argv=None): run_parser.add_argument('-d', '--deps', action='append', + default = [], help='Declare dependencies for reproducible cmd.') run_parser.add_argument('-o', '--out', action='append', - help='Declare output data items for reproducible cmd.') + default=[], + help='Declare output data file (sync to cloud) for reproducible cmd.') + run_parser.add_argument('-r', + '--reg', + action='append', + default=[], + help='Declare output regular file (sync to Git) for reproducible cmd.') run_parser.add_argument('-l', '--lock', action='store_true', diff --git a/dvc/command/add.py b/dvc/command/add.py index 945292b31..917e64499 100644 --- a/dvc/command/add.py +++ b/dvc/command/add.py @@ -3,7 +3,7 @@ import os from dvc.command.common.base import CmdBase from dvc.logger import Logger from dvc.state_file import StateFile -from dvc.command.run import CommandFile +from dvc.command.common.command_file import CommandFile class CmdAdd(CmdBase): @@ -40,7 +40,7 @@ class CmdAdd(CmdBase): for data_item in targets: Logger.debug('Creating state file for {}'.format(data_item.data.relative)) - command = CommandFile(None, [data_item.data.dvc], [], True, None) + command = CommandFile(None, [data_item.data.dvc], [], [], True, None) state_file = StateFile(data_item, self.settings, command.dict, diff --git a/dvc/command/common/command_file.py b/dvc/command/common/command_file.py new file mode 100644 index 000000000..62aee62e8 --- /dev/null +++ b/dvc/command/common/command_file.py @@ -0,0 +1,54 @@ +import yaml + +from dvc.state_file import StateFileBase + + +class CommandFile(StateFileBase): + MAGIC = 'DVC-Command-State' + VERSION = '0.1' + + PARAM_CMD = 'cmd' + PARAM_OUT = 'out' + PARAM_REG = 'reg' + PARAM_DEPS = 'deps' + PARAM_LOCK = 'locked' + + def __init__(self, cmd, out, reg, deps, locked, fname): + self.cmd = cmd + self.out = out + self.reg = reg + self.deps = deps + self.locked = locked + self.fname = fname + + @property + def dict(self): + data = { + self.PARAM_CMD: self.cmd, + self.PARAM_OUT: self.out, + self.PARAM_REG: self.reg, + self.PARAM_DEPS: self.deps, + self.PARAM_LOCK: self.locked + } + + return data + + def dumps(self): + return yaml.dump(self.dict) + + def dump(self, fname): + with open(fname, 'w+') as fd: + fd.write(self.dumps()) + + @staticmethod + def loadd(data, fname=None): + return CommandFile(data.get(CommandFile.PARAM_CMD, None), + data.get(CommandFile.PARAM_OUT, None), + data.get(CommandFile.PARAM_REG, None), + data.get(CommandFile.PARAM_DEPS, None), + data.get(CommandFile.PARAM_LOCK, None), + fname) + + @staticmethod + def load(fname): + return CommandFile._load(fname, CommandFile, fname) diff --git a/dvc/command/repro.py b/dvc/command/repro.py index 6997d02df..78c52ff4a 100644 --- a/dvc/command/repro.py +++ b/dvc/command/repro.py @@ -4,7 +4,6 @@ import copy from dvc.command.run import CmdRun, CommandFile from dvc.logger import Logger from dvc.exceptions import DvcException -from dvc.path.data_item import DataDirError from dvc.state_file import StateFile from dvc.system import System from dvc.data_cloud import file_md5 @@ -119,7 +118,7 @@ class ReproChange(object): os.remove(data_item.data.relative) except Exception as ex: msg = 'Data item {} cannot be removed before reproduction: {}' - Logger.error(msg.format(output_dvc, ex)) + Logger.debug(msg.format(output_dvc, ex)) def reproduce_run(self): Logger.info('Reproducing run command for data item {}. Args: {}'.format( diff --git a/dvc/command/run.py b/dvc/command/run.py index 7e9a750d2..ddb00c352 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -1,16 +1,9 @@ -import os -import sys -import yaml - from dvc.command.common.base import CmdBase +from dvc.command.common.command_file import CommandFile from dvc.exceptions import DvcException -from dvc.git_wrapper import GitWrapper from dvc.logger import Logger -from dvc.repository_change import RepositoryChange from dvc.state_file import StateFile -from dvc.utils import cached_property from dvc.executor import Executor -from dvc.state_file import StateFileBase class RunError(DvcException): @@ -18,53 +11,6 @@ class RunError(DvcException): DvcException.__init__(self, 'Run error: {}'.format(msg)) -class CommandFile(StateFileBase): - MAGIC = 'DVC-Command-State' - VERSION = '0.1' - - PARAM_CMD = 'cmd' - PARAM_OUT = 'out' - PARAM_DEPS = 'deps' - PARAM_LOCK = 'locked' - - def __init__(self, cmd, out, deps, locked, fname): - self.cmd = cmd - self.out = out - self.deps = deps - self.locked = locked - self.fname = fname - - @property - def dict(self): - data = { - self.PARAM_CMD: self.cmd, - self.PARAM_OUT: self.out, - self.PARAM_DEPS: self.deps, - self.PARAM_LOCK: self.locked - } - - return data - - def dumps(self): - return yaml.dump(self.dict) - - def dump(self, fname): - with open(fname, 'w+') as fd: - fd.write(self.dumps()) - - @staticmethod - def loadd(data, fname=None): - return CommandFile(data.get(CommandFile.PARAM_CMD, None), - data.get(CommandFile.PARAM_OUT, None), - data.get(CommandFile.PARAM_DEPS, None), - data.get(CommandFile.PARAM_LOCK, None), - fname) - - @staticmethod - def load(fname): - return CommandFile._load(fname, CommandFile, fname) - - class CmdRun(CmdBase): def __init__(self, settings): super(CmdRun, self).__init__(settings) @@ -75,7 +21,8 @@ class CmdRun(CmdBase): command = CommandFile.load(cmd) except Exception as exc: Logger.debug("Failed to load {}: {}".format(cmd, str(exc))) - command = CommandFile(cmd, self.parsed_args.out, self.parsed_args.deps, self.parsed_args.lock, None) + command = CommandFile(cmd, self.parsed_args.out, self.parsed_args.reg, + self.parsed_args.deps, self.parsed_args.lock, None) self.run_command(self.settings, command) return self.commit_if_needed('DVC run: {}'.format(command.cmd)) @@ -84,20 +31,26 @@ class CmdRun(CmdBase): def run_command(settings, command): Executor.exec_cmd_only_success(command.cmd, shell=True) - result = [] - items = settings.path_factory.to_data_items(command.out)[0] - for data_item in items: - Logger.debug('Move output file "{}" to cache dir "{}" and create a hardlink'.format( - data_item.data.relative, data_item.cache_dir_abs)) - data_item.move_data_to_cache() + CmdRun.apply_to_files(command.out, command, CmdRun._create_cache_and_state_files, settings) + CmdRun.apply_to_files(command.reg, command, CmdRun._create_state_file, settings) - Logger.debug('Create state file "{}"'.format(data_item.state.relative)) + @staticmethod + def apply_to_files(files, command, func, settings): + [func(i, command, settings) for i in settings.path_factory.to_data_items(files)[0]] - state_file = StateFile(data_item, - settings, - command.fname if command.fname else command.dict, - StateFile.parse_deps_state(settings, command.deps)) - state_file.save() - result.append(state_file) + @staticmethod + def _create_cache_and_state_files(data_item, command, settings): + Logger.debug('Move output file "{}" to cache dir "{}" and create a hardlink'.format( + data_item.data.relative, data_item.cache_dir_abs)) + data_item.move_data_to_cache() + return CmdRun._create_state_file(data_item, command, settings) - return result + @staticmethod + def _create_state_file(data_item, command, settings): + Logger.debug('Create state file "{}"'.format(data_item.state.relative)) + state_file = StateFile(data_item, + settings, + command.fname if command.fname else command.dict, + StateFile.parse_deps_state(settings, command.deps)) + state_file.save() + return state_file diff --git a/dvc/path/factory.py b/dvc/path/factory.py index 3502ecf41..48c0657a5 100644 --- a/dvc/path/factory.py +++ b/dvc/path/factory.py @@ -5,7 +5,7 @@ from dvc.path.data_item import DataItem, DataItemError, DataDirError from dvc.path.path import Path from dvc.path.stated_data_item import StatedDataItem from dvc.system import System -from dvc.logger import Logger + class PathFactory(object): def __init__(self, git, config): @@ -53,7 +53,7 @@ class PathFactory(object): return result, externally_created_files - def all_existing_data_items(self, subdir='.'): + def all_existing_data_items(self, subdir='.', cache_exists=True): states = [] for root, dirs, files in os.walk(os.path.join(ConfigI.STATE_DIR, subdir)): @@ -65,7 +65,11 @@ class PathFactory(object): states.append(path) - return self.data_items_from_states(states) + data_items = self.data_items_from_states(states) + if cache_exists: + data_items = filter(lambda i: os.path.exists(i.cache_state.relative), data_items) + + return data_items def data_items_from_states(self, states, existing=True): return map(lambda s: self.data_item_from_dvc_path(self.state_path_to_dvc_path(s), existing),