diff --git a/neatlynx/cmd_data_import.py b/neatlynx/cmd_data_import.py index 6d59a3483..3d7ee30fc 100644 --- a/neatlynx/cmd_data_import.py +++ b/neatlynx/cmd_data_import.py @@ -32,6 +32,9 @@ class CmdDataImport(CmdBase): nargs='*') self.add_string_arg(parser, 'output', 'Output file') + + parser.add_argument('-i', '--is-reproducible', action='store_false', default=False, + help='Is data file reproducible') pass def run(self): @@ -47,7 +50,7 @@ class CmdDataImport(CmdBase): output = self.args.output for file in self.args.input: - self.import_file(file, output) + self.import_file(file, output, self.args.is_reproducible) if self.skip_git_actions: self.not_committed_changes_warning() @@ -59,7 +62,7 @@ class CmdDataImport(CmdBase): lock.release() pass - def import_file(self, input, output): + def import_file(self, input, output, is_reproducible): if not CmdDataImport.is_url(input): if not os.path.exists(input): raise DataImportError('Input file "{}" does not exist'.format(input)) @@ -77,7 +80,10 @@ class CmdDataImport(CmdBase): raise DataImportError('Output file directory "{}" does not exists'.format( os.path.dirname(dobj.data_file_relative))) - os.makedirs(os.path.dirname(dobj.cache_file_relative), exist_ok=True) + cache_dir = os.path.dirname(dobj.cache_file_relative) + if not os.path.exists(cache_dir): + os.makedirs(cache_dir) + if CmdDataImport.is_url(input): Logger.debug('Downloading file {} ...'.format(input)) self.download_file(input, dobj.cache_file_relative) @@ -92,7 +98,12 @@ class CmdDataImport(CmdBase): Logger.debug('Symlink from data file "{}" to the cache file "{}" was created'. format(dobj.data_file_relative, dobj.cache_file_relative)) - state_file = StateFile(dobj.state_file_relative, self.git) + state_file = StateFile(dobj.state_file_relative, + self.git, + [input], + [output], + [], + is_reproducible) state_file.save() Logger.debug('State file "{}" was created'.format(dobj.state_file_relative)) pass diff --git a/neatlynx/cmd_init.py b/neatlynx/cmd_init.py index d41c118fe..5e62530bc 100644 --- a/neatlynx/cmd_init.py +++ b/neatlynx/cmd_init.py @@ -49,6 +49,8 @@ SecurityGroup = neatlynx-group''' pass def define_args(self, parser): + self.set_skip_git_actions(parser) + self.add_string_arg(parser, '--data-dir', 'NeatLynx data directory', 'data') self.add_string_arg(parser, '--cache-dir', 'NeatLynx cache directory', 'cache') self.add_string_arg(parser, '--state-dir', 'NeatLynx state directory', 'state') @@ -60,18 +62,18 @@ SecurityGroup = neatlynx-group''' raise InitError('Directory "{}" already exist'.format(path.name)) return path - def get_not_existing_conf_file(self): - path = Path(os.path.join(self.git.git_dir, Config.CONFIG)) - if path.exists(): - raise InitError('Configuration file "{}" already exist'.format(path.name)) - return path + def get_not_existing_conf_file_name(self): + file_name = os.path.join(self.git.git_dir, Config.CONFIG) + if os.path.exists(file_name): + raise InitError('Configuration file "{}" already exist'.format(file_name)) + return file_name def run(self): data_dir_path = self.get_not_existing_dir(self.args.data_dir) cache_dir_path = self.get_not_existing_dir(self.args.cache_dir) state_dir_path = self.get_not_existing_dir(self.args.state_dir) - conf = self.get_not_existing_conf_file() + conf_file_name = self.get_not_existing_conf_file_name() data_dir_path.mkdir() cache_dir_path.mkdir() @@ -81,11 +83,24 @@ SecurityGroup = neatlynx-group''' cache_dir_path.name, state_dir_path.name)) - conf.write_text(self.CONFIG_TEMPLATE.format(data_dir_path.name, + conf_file = open(conf_file_name, 'wt') + conf_file.write(self.CONFIG_TEMPLATE.format(data_dir_path.name, cache_dir_path.name, state_dir_path.name)) + conf_file.close() self.modify_gitignore(cache_dir_path.name) + + if self.skip_git_actions: + self.not_committed_changes_warning() + return 0 + + message = 'NLX init. data dir {}, cache dir {}, state dir {}'.format( + data_dir_path.name, + cache_dir_path.name, + state_dir_path.name + ) + self.git.commit_all_changes_and_log_status(message) pass def modify_gitignore(self, cache_dir_name): diff --git a/neatlynx/cmd_repro.py b/neatlynx/cmd_repro.py index 95f3a353b..b53497959 100644 --- a/neatlynx/cmd_repro.py +++ b/neatlynx/cmd_repro.py @@ -7,7 +7,7 @@ from neatlynx.cmd_run import CmdRun from neatlynx.git_wrapper import GitWrapper from neatlynx.logger import Logger from neatlynx.exceptions import NeatLynxException -from neatlynx.data_file_obj import DataFileObj +from neatlynx.data_file_obj import DataFileObj, NotInDataDirError from neatlynx.state_file import StateFile @@ -19,8 +19,14 @@ class ReproError(NeatLynxException): class CmdRepro(CmdRun): def __init__(self): CmdBase.__init__(self) + + self._code =[] pass + @property + def code(self): + return self._code + def define_args(self, parser): self.set_skip_git_actions(parser) @@ -56,64 +62,109 @@ class CmdRepro(CmdRun): return 1 error = False + changed = False for dobj in dobjs: - self.repro(dobj) - - # if returncode != 0: - # error = True - # Logger.error('Error: cannot reproduce file "{}"\n{}'.format( - # dobj.data_file_relative, out)) - # sys.stderr.write('{}\n'.format(err)) + try: + repro_change = ReproChange(dobj, self.git, self) + if repro_change.reproduce(): + changed = True + Logger.info(u'Data file "{}" was reproduced.'.format( + dobj.data_file_relative + )) + else: + Logger.info(u'Reproduction is not required for data file "{}".'.format( + dobj.data_file_relative + )) + except ReproError as err: + Logger.error('Error in reproducing data file {}: {}'.format( + dobj.data_file_relative, str(err) + )) + error = True + break if error and not self.skip_git_actions: Logger.error('Errors occurred. One or more repro command was not successful.') self.not_committed_changes_warning() return 1 - if self.skip_git_actions: - self.not_committed_changes_warning() - return 0 + if changed: + if self.skip_git_actions: + self.not_committed_changes_warning() + return 1 - message = 'NLX repro: {}'.format(' '.join(self.args.target)) - self.git.commit_all_changes_and_log_status(message) + message = 'NLX repro: {}'.format(' '.join(self.args.target)) + self.git.commit_all_changes_and_log_status(message) finally: lock.release() - - # dobj = DataFileObj(self.args.target, self.git, self.config) - # os.remove(self.args.target) - # - # state_file = StateFile(dobj.state_file_relative, self.git) - # returncode, out, err = state_file.repro() - # - # print(out) - # sys.stderr.write(err) - # - # return returncode return 0 - def repro(self, dobj): - state = StateFile.load(dobj.state_file_relative, self.git) - argv = state.norm_argv +class ReproChange(object): + def __init__(self, dobj, git, cmd_obj): + self._dobj = dobj + self._state = StateFile.load(dobj.state_file_relative, git) + self.git = git + self._cmd_obj = cmd_obj + + cmd_obj._code = self._state.code_sources # HACK!!! + + argv = self._state.norm_argv if not argv: raise ReproError('Error: parameter {} is nor defined in state file "{}"'. format(StateFile.PARAM_NORM_ARGV, dobj.state_file_relative)) if len(argv) < 2: raise ReproError('Error: reproducible command in state file "{}" is too short'. - format(self.file)) + format(self._state.file)) - if argv[0][-3:] != '.py': - raise ReproError('Error: reproducible command format error in state file "{}"'. - format(self.file)) - argv.pop(0) + # if argv[0][-3:] != '.py': + # raise ReproError('Error: reproducible command format error in state file "{}"'. + # format(self._state.file)) - Logger.debug('Removing data file "{}"'.format(dobj.data_file_relative)) - os.remove(dobj.data_file_relative) + self._repro_argv = argv + pass - Logger.debug("Repro cmd:\n\t{}".format(' '.join(argv))) - return self.run_command(argv) + def were_direct_dependencies_changed(self): + return True + def reproduce_data_file(self): + Logger.debug('Reproducing data file "{}". Removing the file...'.format( + self._dobj.data_file_relative)) + os.remove(self._dobj.data_file_relative) + + Logger.debug('Reproducing data file "{}". Re-runs command: {}'.format( + self._dobj.data_file_relative, ' '.join(self._repro_argv))) + return self._cmd_obj.run_command(self._repro_argv) + + def reproduce(self, force=False): + were_input_files_changed = False + + if not self._state.is_reproducible: + Logger.debug('Data file "{}" is not reproducible'.format(self._dobj.data_file_relative)) + return False + + for input_file in self._state.input_files: + try: + dobj = DataFileObj(input_file, self.git, self._cmd_obj.config) + except NotInDataDirError: + raise ReproError(u'The dependency files "{}" is not a data file'.format(input_file)) + except Exception as ex: + raise ReproError(u'The dependency files "{}" can not be reproduced: {}'.format( + input_file, ex)) + + change = ReproChange(dobj, self.git, self._cmd_obj) + if change.reproduce(force): + were_input_files_changed = True + + was_source_code_changed = self.git.were_files_changed(self._dobj.data_file_relative, + self._state.code_sources) + + if not force and not was_source_code_changed and not were_input_files_changed: + Logger.debug('Data file "{}" is up to date'.format( + self._dobj.data_file_relative)) + return False + + return self.reproduce_data_file() if __name__ == '__main__': import sys diff --git a/neatlynx/cmd_run.py b/neatlynx/cmd_run.py index e6f9d3ad9..f4d812f48 100644 --- a/neatlynx/cmd_run.py +++ b/neatlynx/cmd_run.py @@ -31,8 +31,14 @@ class CmdRun(CmdBase): help='Declare input file for reproducible command') parser.add_argument('--output-file', '-o', action='append', help='Declare output file for reproducible command') + parser.add_argument('--code', '-c', action='append', + help='Code file or code directory which produces the output') pass + @property + def code(self): + return self.args.code + @property def declaration_input_files(self): if self.args.input_file: @@ -73,16 +79,32 @@ class CmdRun(CmdBase): def run_command(self, argv, stdout=None, stderr=None): repo_change = RepositoryChange(argv, stdout, stderr, self.git, self.config) + # print('===== new_files={}'.format(repo_change.new_files)) + # print('===== modified_files={}'.format(repo_change.modified_files)) + # print('===== removed_files={}'.format(repo_change.removed_files)) + # print('===== externally_created_files={}'.format(repo_change.externally_created_files)) + # # raise Exception() + + print('========= {} {}'.format(not self.skip_git_actions, not self.validate_file_states(repo_change))) + if not self.skip_git_actions and not self.validate_file_states(repo_change): self.remove_new_files(repo_change) return False - output_files = self.git.abs_paths_to_nlx(repo_change.new_files + self.declaration_output_files) - input_files_from_args = list(set(self.get_data_files_from_args(argv)) - set(repo_change.new_files)) + changed_files_nlx = self.git.abs_paths_to_nlx(repo_change.changed_files) + output_files = changed_files_nlx + self.git.abs_paths_to_nlx(self.declaration_output_files) + args_files_nlx = self.git.abs_paths_to_nlx(self.get_data_files_from_args(argv)) + + input_files_from_args = list(set(args_files_nlx) - set(changed_files_nlx)) input_files = self.git.abs_paths_to_nlx(input_files_from_args + self.declaration_input_files) - for dobj in repo_change.dobj_for_new_files: - os.makedirs(os.path.dirname(dobj.cache_file_relative), exist_ok=True) + print('================== changed files = {}'.format(len(repo_change.dobj_for_changed_files))) + + for dobj in repo_change.dobj_for_changed_files: + print('================== move ... {}'.format(dobj.data_file_relative)) + dirname = os.path.dirname(dobj.cache_file_relative) + if not os.path.isdir(dirname): + os.makedirs(dirname) Logger.debug('Move output file "{}" to cache dir "{}" and create a symlink'.format( dobj.data_file_relative, dobj.cache_file_relative)) @@ -90,8 +112,14 @@ class CmdRun(CmdBase): dobj.create_symlink() + nlx_code_sources = map(lambda x: self.git.abs_paths_to_nlx([x])[0], self.code) + Logger.debug('Create state file "{}"'.format(dobj.state_file_relative)) - state_file = StateFile(dobj.state_file_relative, self.git, input_files, output_files) + state_file = StateFile(dobj.state_file_relative, self.git, + input_files, + output_files, + nlx_code_sources, + argv=argv) state_file.save() pass @@ -111,13 +139,14 @@ class CmdRun(CmdBase): Logger.error('Error: file "{}" was removed'.format(file)) error = True - for file in GitWrapper.abs_paths_to_relative(files_states.modified_files): - Logger.error('Error: file "{}" was modified'.format(file)) - error = True + # for file in GitWrapper.abs_paths_to_relative(files_states.modified_files): + # Logger.error('Error: file "{}" was modified'.format(file)) + # error = True - for file in GitWrapper.abs_paths_to_relative(files_states.unusual_state_files): - Logger.error('Error: file "{}" is in not acceptable state'.format(file)) - error = True + # # This code doesn't cover repro case + # for file in GitWrapper.abs_paths_to_relative(files_states.unusual_state_files): + # Logger.error('Error: file "{}" is in not acceptable state'.format(file)) + # error = True for file in GitWrapper.abs_paths_to_relative(files_states.externally_created_files): Logger.error('Error: file "{}" was created outside of the data directory'.format(file)) @@ -129,9 +158,9 @@ class CmdRun(CmdBase): format(self.config.data_dir)) return False - if not files_states.new_files: - Logger.error('Errors occurred. No files were changed in run command.') - return False + # if not files_states.new_files: + # Logger.error('Errors occurred. No files were changed in run command.') + # return False return True diff --git a/neatlynx/git_wrapper.py b/neatlynx/git_wrapper.py index d7d5fce9a..a1f7c013b 100644 --- a/neatlynx/git_wrapper.py +++ b/neatlynx/git_wrapper.py @@ -98,7 +98,7 @@ class GitWrapper(GitWrapperI): @property def curr_dir_nlx(self): - return os.path.relpath(self.curr_dir_abs, self.git_dir_abs) + return os.path.relpath(os.path.abspath(os.curdir), self.git_dir_abs) @property def git_dir(self): @@ -180,3 +180,39 @@ class GitWrapper(GitWrapperI): result.append(os.path.relpath(os.path.abspath(file), self.git_dir_abs)) return result + + def nlx_paths_to_abs(self, files): + results = [] + + for file in files: + results.append(os.path.join(self.git_dir_abs, file)) + + return results + + def were_files_changed(self, file, code_sources): + commit = self.exec_cmd_only_success(['git', 'log', '-1', '--pretty=format:"%h"', file]) + commit = commit.strip('"') + + changed_files = self.exec_cmd_only_success(['git', 'diff', '--name-only', 'HEAD', commit]) + changed_files = changed_files.strip('"') + + code_sources_abs = self.nlx_paths_to_abs(code_sources) + code_files = [] + code_dirs = [] + for code in code_sources_abs: + if os.path.isdir(code): + code_dirs.append(code) + else: + code_files.append(code) + + code_files_set = set(code_files) + for changed_file in changed_files.split('\n'): + changed_file = os.path.realpath(changed_file) + if changed_file in code_files_set: + return True + + for dir in code_dirs: + if changed_file.startswith(dir): + return True + + return False diff --git a/neatlynx/logger.py b/neatlynx/logger.py index 282e2866c..f8b76e8d9 100644 --- a/neatlynx/logger.py +++ b/neatlynx/logger.py @@ -18,3 +18,7 @@ class Logger(object): @staticmethod def debug(msg): print(u'Debug. {}'.format(msg)) + + @staticmethod + def info(msg): + print(u'Info. {}'.format(msg)) \ No newline at end of file diff --git a/neatlynx/repository_change.py b/neatlynx/repository_change.py index dbd56d6cd..0f3a2780b 100644 --- a/neatlynx/repository_change.py +++ b/neatlynx/repository_change.py @@ -14,6 +14,7 @@ class FileModificationState(object): STATUS_UNTRACKED = '?' STATUS_DELETE = 'D' STATUS_MODIFIED = 'M' + STATUS_TYPE_CHANGED = 'T' def __init__(self, state, file): self.state = state @@ -28,7 +29,8 @@ class FileModificationState(object): @property def is_modified(self): - return self._check_status(self.STATUS_MODIFIED) + return self._check_status(self.STATUS_MODIFIED) \ + or self._check_status(self.STATUS_TYPE_CHANGED) @property def is_new(self): @@ -49,8 +51,8 @@ class RepositoryChange(object): GitWrapper.exec_cmd_only_success(args, stdout, stderr) self._file_states = self._get_file_states() - self._new_dobj, self._externally_created_files = DataFileObj.files_to_dobjs( - self.new_files, self.git, self.config) + self._changed_dobj, self._externally_created_files = DataFileObj.files_to_dobjs( + self.changed_files, self.git, self.config) @staticmethod def exec_cmd(args, stdout, stderr, git): @@ -72,6 +74,10 @@ class RepositoryChange(object): def new_files(self): return self.get_filenames(filter(lambda x: x.is_new, self._file_states)) + @property + def changed_files(self): + return self.new_files + self.modified_files + @property def unusual_state_files(self): return self.get_filenames(filter(lambda x: not x.is_unusual, self._file_states)) @@ -87,7 +93,7 @@ class RepositoryChange(object): else: files = [] self.get_all_files_from_dir(file_path, files) - state = self.STATUS_UNKNOWN + self.STATUS_UNKNOWN + state = FileModificationState.STATUS_UNTRACKED + FileModificationState.STATUS_UNTRACKED for f in files: result.append(FileModificationState(state, f)) @@ -107,8 +113,8 @@ class RepositoryChange(object): pass @property - def dobj_for_new_files(self): - return self._new_dobj + def dobj_for_changed_files(self): + return self._changed_dobj @property def externally_created_files(self): diff --git a/neatlynx/state_file.py b/neatlynx/state_file.py index 12ff32935..028ab25ad 100644 --- a/neatlynx/state_file.py +++ b/neatlynx/state_file.py @@ -25,15 +25,23 @@ class StateFile(object): PARAM_CREATED_AT = 'CreatedAt' PARAM_INPUT_FILES = 'InputFiles' PARAM_OUTPUT_FILES = 'OutputFiles' + PARAM_CODE_SOURCES = 'CodeSources' + PARAM_NOT_REPRODUCIBLE = 'NotReproducible' def __init__(self, file, git, input_files, output_files, - argv=sys.argv, norm_argv=None, + code_sources=[], + is_reproducible=True, + argv=sys.argv, + norm_argv=None, created_at=time.strftime('%Y-%m-%d %H:%M:%S %z'), cwd=None): self.file = file self.git = git self.input_files = input_files self.output_files = output_files + self.is_reproducible = is_reproducible + self.code_sources = code_sources + self.argv = argv if norm_argv: self.norm_argv = norm_argv @@ -55,28 +63,34 @@ class StateFile(object): return StateFile(filename, git, - data.get('InputFiles'), - data.get('OutputFiles'), - data.get('Argv'), - data.get('NormArgv'), - data.get('CreatedAt'), - data.get('Cwd')) + data.get(StateFile.PARAM_INPUT_FILES, []), + data.get(StateFile.PARAM_OUTPUT_FILES, []), + data.get(StateFile.PARAM_CODE_SOURCES, []), + not data.get(StateFile.PARAM_NOT_REPRODUCIBLE, False), + data.get(StateFile.PARAM_ARGV), + data.get(StateFile.PARAM_NORM_ARGV), + data.get(StateFile.PARAM_CREATED_AT), + data.get(StateFile.PARAM_CWD)) def save(self): res = { - self.PARAM_TYPE: self.MAGIC, - self.PARAM_VERSION: self.VERSION, - self.PARAM_ARGV: self.argv, - self.PARAM_NORM_ARGV: self.norm_argv, - self.PARAM_CWD: self.cwd, - self.PARAM_CREATED_AT: self.created_at, - self.PARAM_INPUT_FILES: self.input_files, - self.PARAM_OUTPUT_FILES:self.output_files + self.PARAM_TYPE: self.MAGIC, + self.PARAM_VERSION: self.VERSION, + self.PARAM_ARGV: self.argv, + self.PARAM_NORM_ARGV: self.norm_argv, + self.PARAM_CWD: self.cwd, + self.PARAM_CREATED_AT: self.created_at, + self.PARAM_INPUT_FILES: self.input_files, + self.PARAM_OUTPUT_FILES: self.output_files, + self.PARAM_CODE_SOURCES: self.code_sources } + if not self.is_reproducible: + res[self.PARAM_NOT_REPRODUCIBLE] = True + file_dir = os.path.dirname(self.file) - if file_dir != '': - os.makedirs(file_dir, exist_ok=True) + if file_dir != '' and not os.path.isdir(file_dir): + os.makedirs(file_dir) with open(self.file, 'w') as fd: json.dump(res, fd, indent=2) @@ -85,14 +99,14 @@ class StateFile(object): def normalized_args(self): result = [] - if len(sys.argv) > 0: - cmd = sys.argv[0] + if len(self.argv) > 0: + cmd = self.argv [0] pos = cmd.rfind(os.sep) if pos >= 0: cmd = cmd[pos+1:] result.append(cmd) - for arg in sys.argv[1:]: + for arg in self.argv [1:]: if os.path.isfile(arg): path = os.path.abspath(arg) nlx_path = os.path.relpath(path, self.git.git_dir_abs) @@ -108,23 +122,3 @@ class StateFile(object): raise StateFileError('the file cannot be created outside of a git repository') return os.path.relpath(pwd, self.git.git_dir_abs) - - # def repro(self): - # with open(self.file, 'r') as fd: - # res = json.load(fd) - # - # args = res['NormArgs'] - # if not args: - # raise StateFileError('Error: parameter NormalizedArgs is nor defined in state file "{}"'. - # format(self.file)) - # if len(args) < 2: - # raise StateFileError('Error: reproducible command in state file "{}" is too short'. - # format(self.file)) - # - # if args[0][-3:] != '.py': - # raise StateFileError('Error: reproducible command format error in state file "{}"'. - # format(self.file)) - # args.pop(0) - # - # Logger.info("Repro cmd:\n\t{}".format(' '.join(args))) - # return GitWrapper.exec_cmd(args, cwd=self.git.git_dir_abs)