diff --git a/dvc/command/data_sync.py b/dvc/command/data_sync.py index c461feb87..66ec4b7ac 100644 --- a/dvc/command/data_sync.py +++ b/dvc/command/data_sync.py @@ -7,6 +7,7 @@ from boto.s3.connection import S3Connection from google.cloud import storage as gc from dvc.command.base import CmdBase +from dvc.command.init import CmdInit from dvc.logger import Logger from dvc.exceptions import DvcException from dvc.runtime import Runtime @@ -44,7 +45,10 @@ class CmdDataSync(CmdBase): super(CmdDataSync, self).__init__(settings) def define_args(self, parser): - self.add_string_arg(parser, 'target', 'Target to sync - file or directory') + parser.add_argument('targets', + metavar='', + help='File or directory to sync', + nargs='*') def run(self): if self.is_locker: @@ -55,14 +59,15 @@ class CmdDataSync(CmdBase): return 1 try: - if System.islink(self.parsed_args.target): - data_item = self.settings.path_factory.data_item(self.parsed_args.target) - return self.sync_symlink(data_item) - - if os.path.isdir(self.parsed_args.target): - return self.sync_dir(self.parsed_args.target) - - raise DataSyncError('File "{}" does not exit'.format(self.parsed_args.target)) + for target in self.parsed_args.targets: + data_item = self.settings.path_factory.data_item(target) + if System.islink(target): + self.sync_symlink(data_item) + elif os.path.isdir(target): + self.sync_dir(target) + else: + # if data_item.data.dvc != empty_data_item.data.dvc: + raise DataSyncError('File "{}" does not exit'.format(target)) finally: if self.is_locker: lock.release() diff --git a/dvc/command/import_bulk.py b/dvc/command/import_bulk.py index 3c620e7a6..c989b0f76 100644 --- a/dvc/command/import_bulk.py +++ b/dvc/command/import_bulk.py @@ -21,7 +21,7 @@ class CmdImportBulk(CmdBase): self.add_string_arg(parser, 'output', 'Output file') - parser.add_argument('-i', '--is-reproducible', action='store_false', default=False, + parser.add_argument('-r', '--is-reproducible', action='store_false', default=True, help='Is data file reproducible') pass diff --git a/dvc/command/import_file.py b/dvc/command/import_file.py index 7cc2e447d..67718b8cf 100644 --- a/dvc/command/import_file.py +++ b/dvc/command/import_file.py @@ -28,7 +28,7 @@ class CmdImportFile(CmdBase): self.add_string_arg(parser, 'input', 'Input file') self.add_string_arg(parser, 'output', 'Output file') - parser.add_argument('-i', '--is-reproducible', action='store_false', default=False, + parser.add_argument('-r', '--is-reproducible', action='store_false', default=True, help='Is data file reproducible') pass @@ -55,7 +55,7 @@ class CmdImportFile(CmdBase): self.import_file(input, output, is_reproducible) - message = 'DVC import file: {} {}'.format(' '.join(input), output) + message = 'DVC import file: {} {}'.format(input, output) return self.commit_if_needed(message) def import_file(self, input, output, is_reproducible=True): diff --git a/dvc/command/init.py b/dvc/command/init.py index 0e7f33a93..87570cf8b 100644 --- a/dvc/command/init.py +++ b/dvc/command/init.py @@ -6,6 +6,8 @@ from dvc.logger import Logger from dvc.config import Config from dvc.exceptions import DvcException from dvc.runtime import Runtime +from dvc.state_file import StateFile +from dvc.system import System class InitError(DvcException): @@ -51,6 +53,9 @@ StoragePath = ProjectName = ''' + EMPTY_FILE_NAME = 'empty' + EMPTY_FILE_CHECKSUM = '0000000' + def __init__(self, settings): super(CmdInit, self).__init__(settings) @@ -78,10 +83,20 @@ ProjectName = if not self.skip_git_actions and not self.git.is_ready_to_go(): return 1 + if os.path.realpath(os.path.curdir) != self.settings.git.git_dir_abs: + Logger.error('DVC error: initialization could be done only from git root directory {}'.format( + self.settings.git.git_dir_abs + )) + return 1 + data_dir_path = self.get_not_existing_dir(self.parsed_args.data_dir) cache_dir_path = self.get_not_existing_dir(self.parsed_args.cache_dir) state_dir_path = self.get_not_existing_dir(self.parsed_args.state_dir) + self.settings.config.set(self.parsed_args.data_dir, + self.parsed_args.cache_dir, + self.parsed_args.state_dir) + conf_file_name = self.get_not_existing_conf_file_name() data_dir_path.mkdir() @@ -92,20 +107,41 @@ ProjectName = cache_dir_path.name, state_dir_path.name)) + self.create_empty_file() + conf_file = open(conf_file_name, 'wt') conf_file.write(self.CONFIG_TEMPLATE.format(data_dir_path.name, cache_dir_path.name, state_dir_path.name)) conf_file.close() - self.modify_gitignore(cache_dir_path.name) - message = 'DVC init. data dir {}, cache dir {}, state dir {}'.format( data_dir_path.name, cache_dir_path.name, state_dir_path.name ) - return self.commit_if_needed(message) + if self.commit_if_needed(message) == 1: + return 1 + + self.modify_gitignore(cache_dir_path.name) + return self.commit_if_needed('DVC init. Commit .gitignore file') + + def create_empty_file(self): + empty_data_path = os.path.join(self.parsed_args.data_dir, self.EMPTY_FILE_NAME) + cache_file_suffix = self.EMPTY_FILE_NAME + '_' + self.EMPTY_FILE_CHECKSUM + empty_cache_path = os.path.join(self.parsed_args.cache_dir, cache_file_suffix) + empty_state_path = os.path.join(self.parsed_args.state_dir, self.EMPTY_FILE_NAME + '.state') + + open(empty_cache_path, 'w').close() + System.symlink(os.path.join('..', empty_cache_path), empty_data_path) + + StateFile(StateFile.COMMAND_EMPTY_FILE, + empty_state_path, + self.settings, + input_files=[], + output_files=[], + is_reproducible=False).save() + pass def modify_gitignore(self, cache_dir_name): gitignore_file = os.path.join(self.git.git_dir, '.gitignore') diff --git a/dvc/command/repro.py b/dvc/command/repro.py index 266f8edf3..4e1a58e87 100644 --- a/dvc/command/repro.py +++ b/dvc/command/repro.py @@ -9,6 +9,7 @@ from dvc.exceptions import DvcException from dvc.path.data_item import NotInDataDirError from dvc.runtime import Runtime from dvc.state_file import StateFile +from dvc.system import System class ReproError(DvcException): @@ -54,7 +55,6 @@ class CmdRepro(CmdRun): if not gotten: self.warning_dvc_is_busy() return 1 - try: return self.repro_target(self.parsed_args.target, self.parsed_args.force) finally: @@ -122,8 +122,8 @@ class ReproChange(object): self.git = cmd_obj.git self._cmd_obj = cmd_obj - if not os.path.exists(data_item.data.relative): - raise ReproError('data item {} does not exist'.format(data_item.data.relative)) + if not System.islink(data_item.data.relative): + raise ReproError('data item {} is no symlink'.format(data_item.data.relative)) try: self._state = StateFile.load(data_item.state.relative, self.git) @@ -133,8 +133,6 @@ class ReproChange(object): cmd_obj._code = self.state.code_dependencies - # self._target_commit = target_commit - if not self.state.argv: raise ReproError('Error: parameter {} is not defined in state file "{}"'. format(StateFile.PARAM_ARGV, data_item.state.relative)) @@ -143,6 +141,9 @@ class ReproChange(object): format(self.state.file)) pass + def is_cache_exists(self): + return os.path.exists(self._data_item.cache.relative) + @property def cmd_obj(self): return self._cmd_obj @@ -186,7 +187,7 @@ class ReproChange(object): if cmd.import_and_commit_if_needed(input, output, is_reproducible=True, check_if_ready=False) != 0: raise ReproError('Import command reproduction failed') return True - else: + elif self.state.is_run: cmd = CmdRun(settings) cmd.set_git_action(True) cmd.set_locker(False) @@ -204,6 +205,10 @@ class ReproChange(object): check_if_ready=False) != 0: raise ReproError('Run command reproduction failed') return True + else: + # Ignore EMPTY_FILE command + pass + pass def reproduce(self, changed_files, globally_changed_files, force=False): input_files_dependencies = self.dependencies @@ -239,6 +244,7 @@ class ReproChange(object): ) is_change_needed = were_sources_changed or were_input_files_changed or force + # if not self.is_cache_exists() or (is_change_needed and data_item_dvc not in changed_files): if is_change_needed and data_item_dvc not in changed_files: self.reproduce_data_item() changed_files.add(data_item_dvc) diff --git a/dvc/config.py b/dvc/config.py index 8dd6e1125..c2b1611d6 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -12,10 +12,12 @@ class ConfigError(DvcException): class ConfigI(object): def __init__(self, data_dir=None, cache_dir=None, state_dir=None): + self.set(data_dir, cache_dir, state_dir) + + def set(self, data_dir, cache_dir, state_dir): self._data_dir = data_dir self._cache_dir = cache_dir self._state_dir = state_dir - pass @property def data_dir(self): diff --git a/dvc/path/factory.py b/dvc/path/factory.py index f34008cf6..2570a1c94 100644 --- a/dvc/path/factory.py +++ b/dvc/path/factory.py @@ -26,9 +26,6 @@ class PathFactory(object): return self.existing_data_item(path.relative) def existing_data_item(self, file): - if not os.path.exists(file): - raise DataItemError(u'Data file "%s" is not exist' % file) - if not System.islink(file): raise DataItemError(u'Data file "%s" must be a symbolic link' % file) resolved_symlink = System.realpath(file) diff --git a/dvc/state_file.py b/dvc/state_file.py index 88fbd5717..3cef69266 100644 --- a/dvc/state_file.py +++ b/dvc/state_file.py @@ -22,7 +22,8 @@ class StateFile(object): COMMAND_RUN = 'run' COMMAND_IMPORT_FILE = 'import-file' - ACCEPTED_COMMANDS = {COMMAND_IMPORT_FILE, COMMAND_RUN} + COMMAND_EMPTY_FILE = 'empty' + ACCEPTED_COMMANDS = {COMMAND_IMPORT_FILE, COMMAND_RUN, COMMAND_EMPTY_FILE} PARAM_COMMAND = 'Command' PARAM_TYPE = 'Type' @@ -138,17 +139,6 @@ class StateFile(object): json.dump(res, fd, indent=2) pass - # def process_args(self, argv): - # if len(argv) >= 2 and argv[0].endswith(self.DVC_PYTHON_FILE_NAME): - # if argv[1] in self.ACCEPTED_COMMANDS: - # return argv[1], self._argv_paths_normalization(argv[2:]) - # else: - # msg = 'File generation error: command "{}" is not allowed. Argv={}' - # raise StateFileError(msg.format(argv[1], argv)) - # else: - # msg = 'File generation error: dvc python command "{}" format error. Argv={}' - # raise StateFileError(msg.format(self.DVC_PYTHON_FILE_NAME, argv)) - def _argv_paths_normalization(self, argv): result = [] diff --git a/dvc2.py b/dvc2.py index bfa1d297a..7a8119dce 100755 --- a/dvc2.py +++ b/dvc2.py @@ -42,11 +42,10 @@ def print_usage(): print('\n'.join(usage)) if __name__ == '__main__': - cmds = ['init', 'run', 'sync', 'repro', 'data', 'data-sync', 'remove', 'import', 'import-file', 'cloud', \ + cmds = ['init', 'run', 'sync', 'repro', 'data', 'remove', 'import', 'import-file', 'cloud', \ 'cloud', 'cloud-run', 'cloud-instance-create', 'cloud-instance-remove', 'cloud-instance-describe', \ 'test', 'test-aws', 'test-gcloud', 'test-cloud'] - cmds_expand = {'data': ['sync'], - 'cloud': ['run', 'instance-create', 'instance-remove', 'instance-describe'], + cmds_expand = {'cloud': ['run', 'instance-create', 'instance-remove', 'instance-describe'], 'test': ['aws', 'cloud', 'gcloud'], } @@ -73,7 +72,7 @@ if __name__ == '__main__': Runtime.run(CmdRun, args_start_loc=2) elif cmd == 'repro': Runtime.run(CmdRepro, args_start_loc=2) - elif cmd == 'data-sync' or (cmd == 'data' and subcmd == 'sync'): + elif cmd == 'sync': Runtime.run(CmdDataSync, args_start_loc=argv_offset) elif cmd == 'import': Runtime.run(CmdImportBulk, args_start_loc=argv_offset)