Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-10 07:18:27 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #317 from dataversioncontrol/new_design_fixes

Browse Source 
New design fixes
This commit is contained in:
Dmitry Petrov
2017-11-27 01:52:42 -08:00
committed by GitHub
parent e5fe6b3732 2a980f6e57
commit add5552c02
6 changed files with 95 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/cli.py
+8 -1
View File
@@ -96,11 +96,18 @@ def parse_args(argv=None):
run_parser.add_argument('-d',
'--deps',
action='append',
default = [],
help='Declare dependencies for reproducible cmd.')
run_parser.add_argument('-o',
'--out',
action='append',
help='Declare output data items for reproducible cmd.')
default=[],
help='Declare output data file (sync to cloud) for reproducible cmd.')
run_parser.add_argument('-r',
'--reg',
action='append',
default=[],
help='Declare output regular file (sync to Git) for reproducible cmd.')
run_parser.add_argument('-l',
'--lock',
action='store_true',
dvc/command/add.py
+2 -2
View File
@@ -3,7 +3,7 @@ import os
from dvc.command.common.base import CmdBase
from dvc.logger import Logger
from dvc.state_file import StateFile
from dvc.command.run import CommandFile
from dvc.command.common.command_file import CommandFile
class CmdAdd(CmdBase):
@@ -40,7 +40,7 @@ class CmdAdd(CmdBase):
for data_item in targets:
Logger.debug('Creating state file for {}'.format(data_item.data.relative))
command = CommandFile(None, [data_item.data.dvc], [], True, None)
command = CommandFile(None, [data_item.data.dvc], [], [], True, None)
state_file = StateFile(data_item,
self.settings,
command.dict,
dvc/command/common/command_file.py
+54
View File
@@ -0,0 +1,54 @@
import yaml
from dvc.state_file import StateFileBase
class CommandFile(StateFileBase):
MAGIC = 'DVC-Command-State'
VERSION = '0.1'
PARAM_CMD = 'cmd'
PARAM_OUT = 'out'
PARAM_REG = 'reg'
PARAM_DEPS = 'deps'
PARAM_LOCK = 'locked'
def __init__(self, cmd, out, reg, deps, locked, fname):
self.cmd = cmd
self.out = out
self.reg = reg
self.deps = deps
self.locked = locked
self.fname = fname
@property
def dict(self):
data = {
self.PARAM_CMD: self.cmd,
self.PARAM_OUT: self.out,
self.PARAM_REG: self.reg,
self.PARAM_DEPS: self.deps,
self.PARAM_LOCK: self.locked
}
return data
def dumps(self):
return yaml.dump(self.dict)
def dump(self, fname):
with open(fname, 'w+') as fd:
fd.write(self.dumps())
@staticmethod
def loadd(data, fname=None):
return CommandFile(data.get(CommandFile.PARAM_CMD, None),
data.get(CommandFile.PARAM_OUT, None),
data.get(CommandFile.PARAM_REG, None),
data.get(CommandFile.PARAM_DEPS, None),
data.get(CommandFile.PARAM_LOCK, None),
fname)
@staticmethod
def load(fname):
return CommandFile._load(fname, CommandFile, fname)
dvc/command/repro.py
+1 -2
View File
@@ -4,7 +4,6 @@ import copy
from dvc.command.run import CmdRun, CommandFile
from dvc.logger import Logger
from dvc.exceptions import DvcException
from dvc.path.data_item import DataDirError
from dvc.state_file import StateFile
from dvc.system import System
from dvc.data_cloud import file_md5
@@ -119,7 +118,7 @@ class ReproChange(object):
os.remove(data_item.data.relative)
except Exception as ex:
msg = 'Data item {} cannot be removed before reproduction: {}'
Logger.error(msg.format(output_dvc, ex))
Logger.debug(msg.format(output_dvc, ex))
def reproduce_run(self):
Logger.info('Reproducing run command for data item {}. Args: {}'.format(
dvc/command/run.py
+23 -70
View File
@@ -1,16 +1,9 @@
import os
import sys
import yaml
from dvc.command.common.base import CmdBase
from dvc.command.common.command_file import CommandFile
from dvc.exceptions import DvcException
from dvc.git_wrapper import GitWrapper
from dvc.logger import Logger
from dvc.repository_change import RepositoryChange
from dvc.state_file import StateFile
from dvc.utils import cached_property
from dvc.executor import Executor
from dvc.state_file import StateFileBase
class RunError(DvcException):
@@ -18,53 +11,6 @@ class RunError(DvcException):
DvcException.__init__(self, 'Run error: {}'.format(msg))
class CommandFile(StateFileBase):
MAGIC = 'DVC-Command-State'
VERSION = '0.1'
PARAM_CMD = 'cmd'
PARAM_OUT = 'out'
PARAM_DEPS = 'deps'
PARAM_LOCK = 'locked'
def __init__(self, cmd, out, deps, locked, fname):
self.cmd = cmd
self.out = out
self.deps = deps
self.locked = locked
self.fname = fname
@property
def dict(self):
data = {
self.PARAM_CMD: self.cmd,
self.PARAM_OUT: self.out,
self.PARAM_DEPS: self.deps,
self.PARAM_LOCK: self.locked
}
return data
def dumps(self):
return yaml.dump(self.dict)
def dump(self, fname):
with open(fname, 'w+') as fd:
fd.write(self.dumps())
@staticmethod
def loadd(data, fname=None):
return CommandFile(data.get(CommandFile.PARAM_CMD, None),
data.get(CommandFile.PARAM_OUT, None),
data.get(CommandFile.PARAM_DEPS, None),
data.get(CommandFile.PARAM_LOCK, None),
fname)
@staticmethod
def load(fname):
return CommandFile._load(fname, CommandFile, fname)
class CmdRun(CmdBase):
def __init__(self, settings):
super(CmdRun, self).__init__(settings)
@@ -75,7 +21,8 @@ class CmdRun(CmdBase):
command = CommandFile.load(cmd)
except Exception as exc:
Logger.debug("Failed to load {}: {}".format(cmd, str(exc)))
command = CommandFile(cmd, self.parsed_args.out, self.parsed_args.deps, self.parsed_args.lock, None)
command = CommandFile(cmd, self.parsed_args.out, self.parsed_args.reg,
self.parsed_args.deps, self.parsed_args.lock, None)
self.run_command(self.settings, command)
return self.commit_if_needed('DVC run: {}'.format(command.cmd))
@@ -84,20 +31,26 @@ class CmdRun(CmdBase):
def run_command(settings, command):
Executor.exec_cmd_only_success(command.cmd, shell=True)
result = []
items = settings.path_factory.to_data_items(command.out)[0]
for data_item in items:
Logger.debug('Move output file "{}" to cache dir "{}" and create a hardlink'.format(
data_item.data.relative, data_item.cache_dir_abs))
data_item.move_data_to_cache()
CmdRun.apply_to_files(command.out, command, CmdRun._create_cache_and_state_files, settings)
CmdRun.apply_to_files(command.reg, command, CmdRun._create_state_file, settings)
Logger.debug('Create state file "{}"'.format(data_item.state.relative))
@staticmethod
def apply_to_files(files, command, func, settings):
[func(i, command, settings) for i in settings.path_factory.to_data_items(files)[0]]
state_file = StateFile(data_item,
settings,
command.fname if command.fname else command.dict,
StateFile.parse_deps_state(settings, command.deps))
state_file.save()
result.append(state_file)
@staticmethod
def _create_cache_and_state_files(data_item, command, settings):
Logger.debug('Move output file "{}" to cache dir "{}" and create a hardlink'.format(
data_item.data.relative, data_item.cache_dir_abs))
data_item.move_data_to_cache()
return CmdRun._create_state_file(data_item, command, settings)
return result
@staticmethod
def _create_state_file(data_item, command, settings):
Logger.debug('Create state file "{}"'.format(data_item.state.relative))
state_file = StateFile(data_item,
settings,
command.fname if command.fname else command.dict,
StateFile.parse_deps_state(settings, command.deps))
state_file.save()
return state_file
dvc/path/factory.py
+7 -3
View File
@@ -5,7 +5,7 @@ from dvc.path.data_item import DataItem, DataItemError, DataDirError
from dvc.path.path import Path
from dvc.path.stated_data_item import StatedDataItem
from dvc.system import System
from dvc.logger import Logger
class PathFactory(object):
def __init__(self, git, config):
@@ -53,7 +53,7 @@ class PathFactory(object):
return result, externally_created_files
def all_existing_data_items(self, subdir='.'):
def all_existing_data_items(self, subdir='.', cache_exists=True):
states = []
for root, dirs, files in os.walk(os.path.join(ConfigI.STATE_DIR, subdir)):
@@ -65,7 +65,11 @@ class PathFactory(object):
states.append(path)
return self.data_items_from_states(states)
data_items = self.data_items_from_states(states)
if cache_exists:
data_items = filter(lambda i: os.path.exists(i.cache_state.relative), data_items)
return data_items
def data_items_from_states(self, states, existing=True):
return map(lambda s: self.data_item_from_dvc_path(self.state_path_to_dvc_path(s), existing),