Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-02 07:18:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #52 from dataversioncontrol/sync_small_fixes

Browse Source 
Sync small fixes
This commit is contained in:
Dmitry Petrov
2017-04-28 11:08:24 -07:00
committed by GitHub
parent 6b3559a628 dac5d36f9f
commit dc0d4b6054
9 changed files with 76 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/command/data_sync.py
+14 -9
View File
@@ -7,6 +7,7 @@ from boto.s3.connection import S3Connection
from google.cloud import storage as gc
from dvc.command.base import CmdBase
from dvc.command.init import CmdInit
from dvc.logger import Logger
from dvc.exceptions import DvcException
from dvc.runtime import Runtime
@@ -44,7 +45,10 @@ class CmdDataSync(CmdBase):
super(CmdDataSync, self).__init__(settings)
def define_args(self, parser):
self.add_string_arg(parser, 'target', 'Target to sync - file or directory')
parser.add_argument('targets',
metavar='',
help='File or directory to sync',
nargs='*')
def run(self):
if self.is_locker:
@@ -55,14 +59,15 @@ class CmdDataSync(CmdBase):
return 1
try:
if System.islink(self.parsed_args.target):
data_item = self.settings.path_factory.data_item(self.parsed_args.target)
return self.sync_symlink(data_item)
if os.path.isdir(self.parsed_args.target):
return self.sync_dir(self.parsed_args.target)
raise DataSyncError('File "{}" does not exit'.format(self.parsed_args.target))
for target in self.parsed_args.targets:
data_item = self.settings.path_factory.data_item(target)
if System.islink(target):
self.sync_symlink(data_item)
elif os.path.isdir(target):
self.sync_dir(target)
else:
# if data_item.data.dvc != empty_data_item.data.dvc:
raise DataSyncError('File "{}" does not exit'.format(target))
finally:
if self.is_locker:
lock.release()
dvc/command/import_bulk.py
+1 -1
View File
@@ -21,7 +21,7 @@ class CmdImportBulk(CmdBase):
self.add_string_arg(parser, 'output', 'Output file')
parser.add_argument('-i', '--is-reproducible', action='store_false', default=False,
parser.add_argument('-r', '--is-reproducible', action='store_false', default=True,
help='Is data file reproducible')
pass
dvc/command/import_file.py
+2 -2
View File
@@ -28,7 +28,7 @@ class CmdImportFile(CmdBase):
self.add_string_arg(parser, 'input', 'Input file')
self.add_string_arg(parser, 'output', 'Output file')
parser.add_argument('-i', '--is-reproducible', action='store_false', default=False,
parser.add_argument('-r', '--is-reproducible', action='store_false', default=True,
help='Is data file reproducible')
pass
@@ -55,7 +55,7 @@ class CmdImportFile(CmdBase):
self.import_file(input, output, is_reproducible)
message = 'DVC import file: {} {}'.format(' '.join(input), output)
message = 'DVC import file: {} {}'.format(input, output)
return self.commit_if_needed(message)
def import_file(self, input, output, is_reproducible=True):
dvc/command/init.py
+39 -3
View File
@@ -6,6 +6,8 @@ from dvc.logger import Logger
from dvc.config import Config
from dvc.exceptions import DvcException
from dvc.runtime import Runtime
from dvc.state_file import StateFile
from dvc.system import System
class InitError(DvcException):
@@ -51,6 +53,9 @@ StoragePath =
ProjectName =
'''
EMPTY_FILE_NAME = 'empty'
EMPTY_FILE_CHECKSUM = '0000000'
def __init__(self, settings):
super(CmdInit, self).__init__(settings)
@@ -78,10 +83,20 @@ ProjectName =
if not self.skip_git_actions and not self.git.is_ready_to_go():
return 1
if os.path.realpath(os.path.curdir) != self.settings.git.git_dir_abs:
Logger.error('DVC error: initialization could be done only from git root directory {}'.format(
self.settings.git.git_dir_abs
))
return 1
data_dir_path = self.get_not_existing_dir(self.parsed_args.data_dir)
cache_dir_path = self.get_not_existing_dir(self.parsed_args.cache_dir)
state_dir_path = self.get_not_existing_dir(self.parsed_args.state_dir)
self.settings.config.set(self.parsed_args.data_dir,
self.parsed_args.cache_dir,
self.parsed_args.state_dir)
conf_file_name = self.get_not_existing_conf_file_name()
data_dir_path.mkdir()
@@ -92,20 +107,41 @@ ProjectName =
cache_dir_path.name,
state_dir_path.name))
self.create_empty_file()
conf_file = open(conf_file_name, 'wt')
conf_file.write(self.CONFIG_TEMPLATE.format(data_dir_path.name,
cache_dir_path.name,
state_dir_path.name))
conf_file.close()
self.modify_gitignore(cache_dir_path.name)
message = 'DVC init. data dir {}, cache dir {}, state dir {}'.format(
data_dir_path.name,
cache_dir_path.name,
state_dir_path.name
)
return self.commit_if_needed(message)
if self.commit_if_needed(message) == 1:
return 1
self.modify_gitignore(cache_dir_path.name)
return self.commit_if_needed('DVC init. Commit .gitignore file')
def create_empty_file(self):
empty_data_path = os.path.join(self.parsed_args.data_dir, self.EMPTY_FILE_NAME)
cache_file_suffix = self.EMPTY_FILE_NAME + '_' + self.EMPTY_FILE_CHECKSUM
empty_cache_path = os.path.join(self.parsed_args.cache_dir, cache_file_suffix)
empty_state_path = os.path.join(self.parsed_args.state_dir, self.EMPTY_FILE_NAME + '.state')
open(empty_cache_path, 'w').close()
System.symlink(os.path.join('..', empty_cache_path), empty_data_path)
StateFile(StateFile.COMMAND_EMPTY_FILE,
empty_state_path,
self.settings,
input_files=[],
output_files=[],
is_reproducible=False).save()
pass
def modify_gitignore(self, cache_dir_name):
gitignore_file = os.path.join(self.git.git_dir, '.gitignore')
dvc/command/repro.py
+12 -6
View File
@@ -9,6 +9,7 @@ from dvc.exceptions import DvcException
from dvc.path.data_item import NotInDataDirError
from dvc.runtime import Runtime
from dvc.state_file import StateFile
from dvc.system import System
class ReproError(DvcException):
@@ -54,7 +55,6 @@ class CmdRepro(CmdRun):
if not gotten:
self.warning_dvc_is_busy()
return 1
try:
return self.repro_target(self.parsed_args.target, self.parsed_args.force)
finally:
@@ -122,8 +122,8 @@ class ReproChange(object):
self.git = cmd_obj.git
self._cmd_obj = cmd_obj
if not os.path.exists(data_item.data.relative):
raise ReproError('data item {} does not exist'.format(data_item.data.relative))
if not System.islink(data_item.data.relative):
raise ReproError('data item {} is no symlink'.format(data_item.data.relative))
try:
self._state = StateFile.load(data_item.state.relative, self.git)
@@ -133,8 +133,6 @@ class ReproChange(object):
cmd_obj._code = self.state.code_dependencies
# self._target_commit = target_commit
if not self.state.argv:
raise ReproError('Error: parameter {} is not defined in state file "{}"'.
format(StateFile.PARAM_ARGV, data_item.state.relative))
@@ -143,6 +141,9 @@ class ReproChange(object):
format(self.state.file))
pass
def is_cache_exists(self):
return os.path.exists(self._data_item.cache.relative)
@property
def cmd_obj(self):
return self._cmd_obj
@@ -186,7 +187,7 @@ class ReproChange(object):
if cmd.import_and_commit_if_needed(input, output, is_reproducible=True, check_if_ready=False) != 0:
raise ReproError('Import command reproduction failed')
return True
else:
elif self.state.is_run:
cmd = CmdRun(settings)
cmd.set_git_action(True)
cmd.set_locker(False)
@@ -204,6 +205,10 @@ class ReproChange(object):
check_if_ready=False) != 0:
raise ReproError('Run command reproduction failed')
return True
else:
# Ignore EMPTY_FILE command
pass
pass
def reproduce(self, changed_files, globally_changed_files, force=False):
input_files_dependencies = self.dependencies
@@ -239,6 +244,7 @@ class ReproChange(object):
)
is_change_needed = were_sources_changed or were_input_files_changed or force
# if not self.is_cache_exists() or (is_change_needed and data_item_dvc not in changed_files):
if is_change_needed and data_item_dvc not in changed_files:
self.reproduce_data_item()
changed_files.add(data_item_dvc)
dvc/config.py
+3 -1
View File
@@ -12,10 +12,12 @@ class ConfigError(DvcException):
class ConfigI(object):
def __init__(self, data_dir=None, cache_dir=None, state_dir=None):
self.set(data_dir, cache_dir, state_dir)
def set(self, data_dir, cache_dir, state_dir):
self._data_dir = data_dir
self._cache_dir = cache_dir
self._state_dir = state_dir
pass
@property
def data_dir(self):
dvc/path/factory.py
-3
View File
@@ -26,9 +26,6 @@ class PathFactory(object):
return self.existing_data_item(path.relative)
def existing_data_item(self, file):
if not os.path.exists(file):
raise DataItemError(u'Data file "%s" is not exist' % file)
if not System.islink(file):
raise DataItemError(u'Data file "%s" must be a symbolic link' % file)
resolved_symlink = System.realpath(file)
dvc/state_file.py
+2 -12
View File
@@ -22,7 +22,8 @@ class StateFile(object):
COMMAND_RUN = 'run'
COMMAND_IMPORT_FILE = 'import-file'
ACCEPTED_COMMANDS = {COMMAND_IMPORT_FILE, COMMAND_RUN}
COMMAND_EMPTY_FILE = 'empty'
ACCEPTED_COMMANDS = {COMMAND_IMPORT_FILE, COMMAND_RUN, COMMAND_EMPTY_FILE}
PARAM_COMMAND = 'Command'
PARAM_TYPE = 'Type'
@@ -138,17 +139,6 @@ class StateFile(object):
json.dump(res, fd, indent=2)
pass
# def process_args(self, argv):
# if len(argv) >= 2 and argv[0].endswith(self.DVC_PYTHON_FILE_NAME):
# if argv[1] in self.ACCEPTED_COMMANDS:
# return argv[1], self._argv_paths_normalization(argv[2:])
# else:
# msg = 'File generation error: command "{}" is not allowed. Argv={}'
# raise StateFileError(msg.format(argv[1], argv))
# else:
# msg = 'File generation error: dvc python command "{}" format error. Argv={}'
# raise StateFileError(msg.format(self.DVC_PYTHON_FILE_NAME, argv))
def _argv_paths_normalization(self, argv):
result = []
dvc2.py
+3 -4
View File
@@ -42,11 +42,10 @@ def print_usage():
print('\n'.join(usage))
if __name__ == '__main__':
cmds = ['init', 'run', 'sync', 'repro', 'data', 'data-sync', 'remove', 'import', 'import-file', 'cloud', \
cmds = ['init', 'run', 'sync', 'repro', 'data', 'remove', 'import', 'import-file', 'cloud', \
'cloud', 'cloud-run', 'cloud-instance-create', 'cloud-instance-remove', 'cloud-instance-describe', \
'test', 'test-aws', 'test-gcloud', 'test-cloud']
cmds_expand = {'data': ['sync'],
'cloud': ['run', 'instance-create', 'instance-remove', 'instance-describe'],
cmds_expand = {'cloud': ['run', 'instance-create', 'instance-remove', 'instance-describe'],
'test': ['aws', 'cloud', 'gcloud'],
}
@@ -73,7 +72,7 @@ if __name__ == '__main__':
Runtime.run(CmdRun, args_start_loc=2)
elif cmd == 'repro':
Runtime.run(CmdRepro, args_start_loc=2)
elif cmd == 'data-sync' or (cmd == 'data' and subcmd == 'sync'):
elif cmd == 'sync':
Runtime.run(CmdDataSync, args_start_loc=argv_offset)
elif cmd == 'import':
Runtime.run(CmdImportBulk, args_start_loc=argv_offset)