diff --git a/dvc/command/data_sync.py b/dvc/command/data_sync.py index 38da1fab0..66ec4b7ac 100644 --- a/dvc/command/data_sync.py +++ b/dvc/command/data_sync.py @@ -7,6 +7,7 @@ from boto.s3.connection import S3Connection from google.cloud import storage as gc from dvc.command.base import CmdBase +from dvc.command.init import CmdInit from dvc.logger import Logger from dvc.exceptions import DvcException from dvc.runtime import Runtime @@ -59,14 +60,14 @@ class CmdDataSync(CmdBase): try: for target in self.parsed_args.targets: + data_item = self.settings.path_factory.data_item(target) if System.islink(target): - data_item = self.settings.path_factory.data_item(target) - return self.sync_symlink(data_item) - - if os.path.isdir(target): - return self.sync_dir(target) - - raise DataSyncError('File "{}" does not exit'.format(self.parsed_args.target)) + self.sync_symlink(data_item) + elif os.path.isdir(target): + self.sync_dir(target) + else: + # if data_item.data.dvc != empty_data_item.data.dvc: + raise DataSyncError('File "{}" does not exit'.format(target)) finally: if self.is_locker: lock.release() diff --git a/dvc/command/import_bulk.py b/dvc/command/import_bulk.py index 3c620e7a6..c989b0f76 100644 --- a/dvc/command/import_bulk.py +++ b/dvc/command/import_bulk.py @@ -21,7 +21,7 @@ class CmdImportBulk(CmdBase): self.add_string_arg(parser, 'output', 'Output file') - parser.add_argument('-i', '--is-reproducible', action='store_false', default=False, + parser.add_argument('-r', '--is-reproducible', action='store_false', default=True, help='Is data file reproducible') pass diff --git a/dvc/command/import_file.py b/dvc/command/import_file.py index 27ce6d19a..67718b8cf 100644 --- a/dvc/command/import_file.py +++ b/dvc/command/import_file.py @@ -28,7 +28,7 @@ class CmdImportFile(CmdBase): self.add_string_arg(parser, 'input', 'Input file') self.add_string_arg(parser, 'output', 'Output file') - parser.add_argument('-i', '--is-reproducible', action='store_false', default=False, + parser.add_argument('-r', '--is-reproducible', action='store_false', default=True, help='Is data file reproducible') pass diff --git a/dvc/command/init.py b/dvc/command/init.py index 8d3ef4647..87570cf8b 100644 --- a/dvc/command/init.py +++ b/dvc/command/init.py @@ -6,6 +6,8 @@ from dvc.logger import Logger from dvc.config import Config from dvc.exceptions import DvcException from dvc.runtime import Runtime +from dvc.state_file import StateFile +from dvc.system import System class InitError(DvcException): @@ -52,6 +54,7 @@ ProjectName = ''' EMPTY_FILE_NAME = 'empty' + EMPTY_FILE_CHECKSUM = '0000000' def __init__(self, settings): super(CmdInit, self).__init__(settings) @@ -80,10 +83,20 @@ ProjectName = if not self.skip_git_actions and not self.git.is_ready_to_go(): return 1 + if os.path.realpath(os.path.curdir) != self.settings.git.git_dir_abs: + Logger.error('DVC error: initialization could be done only from git root directory {}'.format( + self.settings.git.git_dir_abs + )) + return 1 + data_dir_path = self.get_not_existing_dir(self.parsed_args.data_dir) cache_dir_path = self.get_not_existing_dir(self.parsed_args.cache_dir) state_dir_path = self.get_not_existing_dir(self.parsed_args.state_dir) + self.settings.config.set(self.parsed_args.data_dir, + self.parsed_args.cache_dir, + self.parsed_args.state_dir) + conf_file_name = self.get_not_existing_conf_file_name() data_dir_path.mkdir() @@ -94,18 +107,7 @@ ProjectName = cache_dir_path.name, state_dir_path.name)) - data_empty_file = os.path.join(self.parsed_args.data_dir, self.EMPTY_FILE_NAME) - cache_empty_file = os.path.join(self.parsed_args.cache_dir, self.EMPTY_FILE_NAME) - state_empty_file = os.path.join(self.parsed_args.state_dir, self.EMPTY_FILE_NAME) - - open(data_empty_file, 'w').close() - open(cache_empty_file, 'w').close() - open(state_empty_file, 'w').close() - - Logger.info('Empty files {}, {} and {} were created'.format( - data_empty_file, - cache_empty_file, - state_empty_file)) + self.create_empty_file() conf_file = open(conf_file_name, 'wt') conf_file.write(self.CONFIG_TEMPLATE.format(data_dir_path.name, @@ -124,6 +126,23 @@ ProjectName = self.modify_gitignore(cache_dir_path.name) return self.commit_if_needed('DVC init. Commit .gitignore file') + def create_empty_file(self): + empty_data_path = os.path.join(self.parsed_args.data_dir, self.EMPTY_FILE_NAME) + cache_file_suffix = self.EMPTY_FILE_NAME + '_' + self.EMPTY_FILE_CHECKSUM + empty_cache_path = os.path.join(self.parsed_args.cache_dir, cache_file_suffix) + empty_state_path = os.path.join(self.parsed_args.state_dir, self.EMPTY_FILE_NAME + '.state') + + open(empty_cache_path, 'w').close() + System.symlink(os.path.join('..', empty_cache_path), empty_data_path) + + StateFile(StateFile.COMMAND_EMPTY_FILE, + empty_state_path, + self.settings, + input_files=[], + output_files=[], + is_reproducible=False).save() + pass + def modify_gitignore(self, cache_dir_name): gitignore_file = os.path.join(self.git.git_dir, '.gitignore') if not os.path.exists(gitignore_file): diff --git a/dvc/command/repro.py b/dvc/command/repro.py index 266f8edf3..4e1a58e87 100644 --- a/dvc/command/repro.py +++ b/dvc/command/repro.py @@ -9,6 +9,7 @@ from dvc.exceptions import DvcException from dvc.path.data_item import NotInDataDirError from dvc.runtime import Runtime from dvc.state_file import StateFile +from dvc.system import System class ReproError(DvcException): @@ -54,7 +55,6 @@ class CmdRepro(CmdRun): if not gotten: self.warning_dvc_is_busy() return 1 - try: return self.repro_target(self.parsed_args.target, self.parsed_args.force) finally: @@ -122,8 +122,8 @@ class ReproChange(object): self.git = cmd_obj.git self._cmd_obj = cmd_obj - if not os.path.exists(data_item.data.relative): - raise ReproError('data item {} does not exist'.format(data_item.data.relative)) + if not System.islink(data_item.data.relative): + raise ReproError('data item {} is no symlink'.format(data_item.data.relative)) try: self._state = StateFile.load(data_item.state.relative, self.git) @@ -133,8 +133,6 @@ class ReproChange(object): cmd_obj._code = self.state.code_dependencies - # self._target_commit = target_commit - if not self.state.argv: raise ReproError('Error: parameter {} is not defined in state file "{}"'. format(StateFile.PARAM_ARGV, data_item.state.relative)) @@ -143,6 +141,9 @@ class ReproChange(object): format(self.state.file)) pass + def is_cache_exists(self): + return os.path.exists(self._data_item.cache.relative) + @property def cmd_obj(self): return self._cmd_obj @@ -186,7 +187,7 @@ class ReproChange(object): if cmd.import_and_commit_if_needed(input, output, is_reproducible=True, check_if_ready=False) != 0: raise ReproError('Import command reproduction failed') return True - else: + elif self.state.is_run: cmd = CmdRun(settings) cmd.set_git_action(True) cmd.set_locker(False) @@ -204,6 +205,10 @@ class ReproChange(object): check_if_ready=False) != 0: raise ReproError('Run command reproduction failed') return True + else: + # Ignore EMPTY_FILE command + pass + pass def reproduce(self, changed_files, globally_changed_files, force=False): input_files_dependencies = self.dependencies @@ -239,6 +244,7 @@ class ReproChange(object): ) is_change_needed = were_sources_changed or were_input_files_changed or force + # if not self.is_cache_exists() or (is_change_needed and data_item_dvc not in changed_files): if is_change_needed and data_item_dvc not in changed_files: self.reproduce_data_item() changed_files.add(data_item_dvc) diff --git a/dvc/config.py b/dvc/config.py index 8dd6e1125..c2b1611d6 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -12,10 +12,12 @@ class ConfigError(DvcException): class ConfigI(object): def __init__(self, data_dir=None, cache_dir=None, state_dir=None): + self.set(data_dir, cache_dir, state_dir) + + def set(self, data_dir, cache_dir, state_dir): self._data_dir = data_dir self._cache_dir = cache_dir self._state_dir = state_dir - pass @property def data_dir(self): diff --git a/dvc/path/factory.py b/dvc/path/factory.py index f34008cf6..2570a1c94 100644 --- a/dvc/path/factory.py +++ b/dvc/path/factory.py @@ -26,9 +26,6 @@ class PathFactory(object): return self.existing_data_item(path.relative) def existing_data_item(self, file): - if not os.path.exists(file): - raise DataItemError(u'Data file "%s" is not exist' % file) - if not System.islink(file): raise DataItemError(u'Data file "%s" must be a symbolic link' % file) resolved_symlink = System.realpath(file) diff --git a/dvc/state_file.py b/dvc/state_file.py index 88fbd5717..3cef69266 100644 --- a/dvc/state_file.py +++ b/dvc/state_file.py @@ -22,7 +22,8 @@ class StateFile(object): COMMAND_RUN = 'run' COMMAND_IMPORT_FILE = 'import-file' - ACCEPTED_COMMANDS = {COMMAND_IMPORT_FILE, COMMAND_RUN} + COMMAND_EMPTY_FILE = 'empty' + ACCEPTED_COMMANDS = {COMMAND_IMPORT_FILE, COMMAND_RUN, COMMAND_EMPTY_FILE} PARAM_COMMAND = 'Command' PARAM_TYPE = 'Type' @@ -138,17 +139,6 @@ class StateFile(object): json.dump(res, fd, indent=2) pass - # def process_args(self, argv): - # if len(argv) >= 2 and argv[0].endswith(self.DVC_PYTHON_FILE_NAME): - # if argv[1] in self.ACCEPTED_COMMANDS: - # return argv[1], self._argv_paths_normalization(argv[2:]) - # else: - # msg = 'File generation error: command "{}" is not allowed. Argv={}' - # raise StateFileError(msg.format(argv[1], argv)) - # else: - # msg = 'File generation error: dvc python command "{}" format error. Argv={}' - # raise StateFileError(msg.format(self.DVC_PYTHON_FILE_NAME, argv)) - def _argv_paths_normalization(self, argv): result = [] diff --git a/dvc2.py b/dvc2.py index bfa1d297a..7a8119dce 100755 --- a/dvc2.py +++ b/dvc2.py @@ -42,11 +42,10 @@ def print_usage(): print('\n'.join(usage)) if __name__ == '__main__': - cmds = ['init', 'run', 'sync', 'repro', 'data', 'data-sync', 'remove', 'import', 'import-file', 'cloud', \ + cmds = ['init', 'run', 'sync', 'repro', 'data', 'remove', 'import', 'import-file', 'cloud', \ 'cloud', 'cloud-run', 'cloud-instance-create', 'cloud-instance-remove', 'cloud-instance-describe', \ 'test', 'test-aws', 'test-gcloud', 'test-cloud'] - cmds_expand = {'data': ['sync'], - 'cloud': ['run', 'instance-create', 'instance-remove', 'instance-describe'], + cmds_expand = {'cloud': ['run', 'instance-create', 'instance-remove', 'instance-describe'], 'test': ['aws', 'cloud', 'gcloud'], } @@ -73,7 +72,7 @@ if __name__ == '__main__': Runtime.run(CmdRun, args_start_loc=2) elif cmd == 'repro': Runtime.run(CmdRepro, args_start_loc=2) - elif cmd == 'data-sync' or (cmd == 'data' and subcmd == 'sync'): + elif cmd == 'sync': Runtime.run(CmdDataSync, args_start_loc=argv_offset) elif cmd == 'import': Runtime.run(CmdImportBulk, args_start_loc=argv_offset)