diff --git a/neatlynx/cmd_data_import.py b/neatlynx/cmd_data_import.py index 9cdc3e8cb..6d59a3483 100644 --- a/neatlynx/cmd_data_import.py +++ b/neatlynx/cmd_data_import.py @@ -38,7 +38,7 @@ class CmdDataImport(CmdBase): lock = fasteners.InterProcessLock(self.git.lock_file) gotten = lock.acquire(timeout=5) if not gotten: - Logger.info('Cannot perform the command since NLX is busy and locked. Please retry the command later.') + Logger.printing('Cannot perform the command since NLX is busy and locked. Please retry the command later.') return 1 try: @@ -79,22 +79,22 @@ class CmdDataImport(CmdBase): os.makedirs(os.path.dirname(dobj.cache_file_relative), exist_ok=True) if CmdDataImport.is_url(input): - Logger.verbose('Downloading file {} ...'.format(input)) + Logger.debug('Downloading file {} ...'.format(input)) self.download_file(input, dobj.cache_file_relative) - Logger.verbose('Input file "{}" was downloaded to cache "{}"'.format( + Logger.debug('Input file "{}" was downloaded to cache "{}"'.format( input, dobj.cache_file_relative)) else: copyfile(input, dobj.cache_file_relative) - Logger.verbose('Input file "{}" was copied to cache "{}"'.format( + Logger.debug('Input file "{}" was copied to cache "{}"'.format( input, dobj.cache_file_relative)) dobj.create_symlink() - Logger.verbose('Symlink from data file "{}" to the cache file "{}" was created'. - format(dobj.data_file_relative, dobj.cache_file_relative)) + Logger.debug('Symlink from data file "{}" to the cache file "{}" was created'. + format(dobj.data_file_relative, dobj.cache_file_relative)) state_file = StateFile(dobj.state_file_relative, self.git) state_file.save() - Logger.verbose('State file "{}" was created'.format(dobj.state_file_relative)) + Logger.debug('State file "{}" was created'.format(dobj.state_file_relative)) pass URL_REGEX = re.compile( @@ -124,7 +124,7 @@ class CmdDataImport(CmdBase): last_reported += chunk_size if last_reported >= report_bucket: last_reported = 0 - Logger.verbose('Downloaded {}'.format(sizeof_fmt(downloaded))) + Logger.debug('Downloaded {}'.format(sizeof_fmt(downloaded))) f.write(chunk) return diff --git a/neatlynx/cmd_data_remove.py b/neatlynx/cmd_data_remove.py index 15859841b..b0eee1af9 100644 --- a/neatlynx/cmd_data_remove.py +++ b/neatlynx/cmd_data_remove.py @@ -30,7 +30,7 @@ class CmdDataRemove(CmdBase): lock = fasteners.InterProcessLock(self.git.lock_file) gotten = lock.acquire(timeout=5) if not gotten: - Logger.info('Cannot perform the command since NLX is busy and locked. Please retry the command later.') + Logger.printing('Cannot perform the command since NLX is busy and locked. Please retry the command later.') return 1 try: @@ -92,7 +92,7 @@ class CmdDataRemove(CmdBase): Logger.warn('S3 remove warning: file "{}" does not exist in S3'.format(aws_file_name)) else: key.delete() - Logger.info('File "{}" was removed from S3'.format(aws_file_name)) + Logger.printing('File "{}" was removed from S3'.format(aws_file_name)) def remove_dir(self, data_dir): for f in os.listdir(data_dir): diff --git a/neatlynx/cmd_data_sync.py b/neatlynx/cmd_data_sync.py index 828126aec..06611a068 100644 --- a/neatlynx/cmd_data_sync.py +++ b/neatlynx/cmd_data_sync.py @@ -23,7 +23,7 @@ def sizeof_fmt(num, suffix='B'): def percent_cb(complete, total): - Logger.verbose('{} transferred out of {}'.format(sizeof_fmt(complete), sizeof_fmt(total))) + Logger.debug('{} transferred out of {}'.format(sizeof_fmt(complete), sizeof_fmt(total))) def file_md5(fname): @@ -44,7 +44,7 @@ class CmdDataSync(CmdBase): self._bucket = conn.lookup(bucket_name) if not self._bucket: self._bucket = conn.create_bucket(bucket_name) - Logger.info('S3 bucket "{}" was created'.format(bucket_name)) + Logger.printing('S3 bucket "{}" was created'.format(bucket_name)) pass def define_args(self, parser): @@ -84,30 +84,30 @@ class CmdDataSync(CmdBase): if not key: raise DataSyncError('File "{}" does not exist in the cloud'.format(dobj.cache_file_aws_key)) - Logger.info('Downloading cache file from S3 "{}/{}"'.format(self._bucket.name, - dobj.cache_file_aws_key)) + Logger.printing('Downloading cache file from S3 "{}/{}"'.format(self._bucket.name, + dobj.cache_file_aws_key)) key.get_contents_to_filename(dobj.cache_file_relative, cb=percent_cb) - Logger.info('Downloading completed') + Logger.printing('Downloading completed') pass def sync_to_cloud(self, dobj): key = self._bucket.get_key(dobj.cache_file_aws_key) if key: - Logger.verbose('File already uploaded to the cloud. Checksum validation...') + Logger.debug('File already uploaded to the cloud. Checksum validation...') md5_cloud = key.etag[1:-1] md5_local = file_md5(dobj.cache_file_relative) if md5_cloud == md5_local: - Logger.verbose('File checksum matches. No uploading is needed.') + Logger.debug('File checksum matches. No uploading is needed.') return - Logger.info('Checksum miss-match. Re-uploading is required.') + Logger.printing('Checksum miss-match. Re-uploading is required.') - Logger.info('Uploading cache file "{}" to S3 "{}"'.format(dobj.cache_file_relative, - dobj.cache_file_aws_key)) + Logger.printing('Uploading cache file "{}" to S3 "{}"'.format(dobj.cache_file_relative, + dobj.cache_file_aws_key)) key = self._bucket.new_key(dobj.cache_file_aws_key) key.set_contents_from_filename(dobj.cache_file_relative, cb=percent_cb) - Logger.info('Uploading completed') + Logger.printing('Uploading completed') pass diff --git a/neatlynx/cmd_init.py b/neatlynx/cmd_init.py index 86a2c17f3..d41c118fe 100644 --- a/neatlynx/cmd_init.py +++ b/neatlynx/cmd_init.py @@ -76,7 +76,7 @@ SecurityGroup = neatlynx-group''' data_dir_path.mkdir() cache_dir_path.mkdir() state_dir_path.mkdir() - Logger.info('Directories {}, {} and {} were created'.format( + Logger.printing('Directories {}, {} and {} were created'.format( data_dir_path.name, cache_dir_path.name, state_dir_path.name)) @@ -92,12 +92,12 @@ SecurityGroup = neatlynx-group''' gitignore_file = os.path.join(self.git.git_dir, '.gitignore') if not os.path.exists(gitignore_file): open(gitignore_file, 'a').close() - Logger.info('File .gitignore was created') + Logger.printing('File .gitignore was created') with open(gitignore_file, 'a') as fd: fd.write('\n{}'.format(cache_dir_name)) fd.write('\n{}'.format(os.path.basename(self.git.lock_file))) - Logger.info('Directory {} was added to .gitignore file'.format(cache_dir_name)) + Logger.printing('Directory {} was added to .gitignore file'.format(cache_dir_name)) if __name__ == '__main__': diff --git a/neatlynx/cmd_repro.py b/neatlynx/cmd_repro.py index 52bce8063..95f3a353b 100644 --- a/neatlynx/cmd_repro.py +++ b/neatlynx/cmd_repro.py @@ -1,6 +1,10 @@ import os +import fasteners + from neatlynx.cmd_base import CmdBase +from neatlynx.cmd_run import CmdRun +from neatlynx.git_wrapper import GitWrapper from neatlynx.logger import Logger from neatlynx.exceptions import NeatLynxException from neatlynx.data_file_obj import DataFileObj @@ -12,29 +16,103 @@ class ReproError(NeatLynxException): NeatLynxException.__init__(self, 'Run error: {}'.format(msg)) -class CmdRepro(CmdBase): +class CmdRepro(CmdRun): def __init__(self): CmdBase.__init__(self) pass def define_args(self, parser): - self.add_string_arg(parser, 'target', 'Reproduce data file') + self.set_skip_git_actions(parser) + + # self.add_string_arg(parser, 'target', 'Reproduce data file') + parser.add_argument('target', metavar='', help='Data file to reproduce', nargs='*') pass + # Overridden methods: + + @property + def declaration_input_files(self): + return [] + + @property + def declaration_output_files(self): + return [] + def run(self): - if not self.git.is_ready_to_go(): + lock = fasteners.InterProcessLock(self.git.lock_file) + gotten = lock.acquire(timeout=5) + if not gotten: + Logger.printing('Cannot perform the command since NLX is busy and locked. Please retry the command later.') return 1 - dobj = DataFileObj(self.args.target, self.git, self.config) - os.remove(self.args.target) + try: + if not self.skip_git_actions and not self.git.is_ready_to_go(): + return 1 - state_file = StateFile(dobj.state_file_relative, self.git) - returncode, out, err = state_file.repro() + dobjs, externally_created_files = DataFileObj.files_to_dobjs(self.args.target, self.git, self.config) + if externally_created_files: + Logger.error('Files from outside of the data directory "{}" could not be reproduced: {}'. + format(self.config.data_dir, ' '.join(externally_created_files))) + return 1 - print(out) - sys.stderr.write(err) + error = False + for dobj in dobjs: + self.repro(dobj) - return returncode + # if returncode != 0: + # error = True + # Logger.error('Error: cannot reproduce file "{}"\n{}'.format( + # dobj.data_file_relative, out)) + # sys.stderr.write('{}\n'.format(err)) + + if error and not self.skip_git_actions: + Logger.error('Errors occurred. One or more repro command was not successful.') + self.not_committed_changes_warning() + return 1 + + if self.skip_git_actions: + self.not_committed_changes_warning() + return 0 + + message = 'NLX repro: {}'.format(' '.join(self.args.target)) + self.git.commit_all_changes_and_log_status(message) + finally: + lock.release() + + # dobj = DataFileObj(self.args.target, self.git, self.config) + # os.remove(self.args.target) + # + # state_file = StateFile(dobj.state_file_relative, self.git) + # returncode, out, err = state_file.repro() + # + # print(out) + # sys.stderr.write(err) + # + # return returncode + return 0 + + def repro(self, dobj): + state = StateFile.load(dobj.state_file_relative, self.git) + + argv = state.norm_argv + + if not argv: + raise ReproError('Error: parameter {} is nor defined in state file "{}"'. + format(StateFile.PARAM_NORM_ARGV, dobj.state_file_relative)) + if len(argv) < 2: + raise ReproError('Error: reproducible command in state file "{}" is too short'. + format(self.file)) + + if argv[0][-3:] != '.py': + raise ReproError('Error: reproducible command format error in state file "{}"'. + format(self.file)) + argv.pop(0) + + Logger.debug('Removing data file "{}"'.format(dobj.data_file_relative)) + os.remove(dobj.data_file_relative) + + Logger.debug("Repro cmd:\n\t{}".format(' '.join(argv))) + return self.run_command(argv) if __name__ == '__main__': diff --git a/neatlynx/cmd_run.py b/neatlynx/cmd_run.py index 2301402a5..e6f9d3ad9 100644 --- a/neatlynx/cmd_run.py +++ b/neatlynx/cmd_run.py @@ -1,13 +1,13 @@ import os -import sys import shutil import fasteners +from neatlynx.data_file_obj import DataFileObj, NotInDataDirError from neatlynx.git_wrapper import GitWrapper from neatlynx.cmd_base import CmdBase from neatlynx.logger import Logger from neatlynx.exceptions import NeatLynxException -from neatlynx.data_file_obj import DataFileObj, NotInDataDirError +from neatlynx.repository_change import RepositoryChange from neatlynx.state_file import StateFile @@ -27,59 +27,38 @@ class CmdRun(CmdBase): parser.add_argument('--random', help='not reproducible, output is random', action='store_true') parser.add_argument('--stdout', help='output std output to a file') parser.add_argument('--stderr', help='output std error to a file') + parser.add_argument('--input-file', '-i', action='append', + help='Declare input file for reproducible command') + parser.add_argument('--output-file', '-o', action='append', + help='Declare output file for reproducible command') pass + @property + def declaration_input_files(self): + if self.args.input_file: + return self.args.input_file + return [] + + @property + def declaration_output_files(self): + if self.args.output_file: + return self.args.output_file + return [] + def run(self): lock = fasteners.InterProcessLock(self.git.lock_file) gotten = lock.acquire(timeout=5) if not gotten: - Logger.info('Cannot perform the command since NLX is busy and locked. Please retry the command later.') + Logger.printing('Cannot perform the command since NLX is busy and locked. Please retry the command later.') return 1 try: if not self.skip_git_actions and not self.git.is_ready_to_go(): return 1 - old_files = set(self.get_changed_files()) - - GitWrapper.exec_cmd(self._args_unkn, self.args.stdout, self.args.stderr) - - new_files = set(self.get_changed_files()) - - removed_files = old_files - new_files - if removed_files != set(): - Logger.error('Error: existing files were removed in run command'.format(' '.join(removed_files))) - - created_files = new_files - old_files - - if created_files == set(): - Logger.error('Error: no files were changes in run command') + if not self.run_command(self._args_unkn, self.args.stdout, self.args.stderr): return 1 - dobjs, errors = self.files_to_dobjs(created_files) - - if errors: - for file in GitWrapper.abs_paths_to_relative(errors): - Logger.error('Error: file "{}" was created outside of the data directory'.format(file)) - - for file in created_files: - rel_path = GitWrapper.abs_paths_to_relative([file])[0] - Logger.error('Removing created file: {}'.format(rel_path)) - os.remove(file) - - Logger.error('Errors occurred. Please fix the errors and re-run the command.') - return 1 - - for dobj in dobjs: - os.makedirs(os.path.dirname(dobj.cache_file_relative), exist_ok=True) - shutil.move(dobj.data_file_relative, dobj.cache_file_relative) - - dobj.create_symlink() - - state_file = StateFile(dobj.state_file_relative, self.git) - state_file.save() - pass - if self.skip_git_actions: self.not_committed_changes_warning() return 0 @@ -91,49 +70,88 @@ class CmdRun(CmdBase): return 0 - def files_to_dobjs(self, files): - result = [] - errors = [] + def run_command(self, argv, stdout=None, stderr=None): + repo_change = RepositoryChange(argv, stdout, stderr, self.git, self.config) - for file in files: + if not self.skip_git_actions and not self.validate_file_states(repo_change): + self.remove_new_files(repo_change) + return False + + output_files = self.git.abs_paths_to_nlx(repo_change.new_files + self.declaration_output_files) + input_files_from_args = list(set(self.get_data_files_from_args(argv)) - set(repo_change.new_files)) + input_files = self.git.abs_paths_to_nlx(input_files_from_args + self.declaration_input_files) + + for dobj in repo_change.dobj_for_new_files: + os.makedirs(os.path.dirname(dobj.cache_file_relative), exist_ok=True) + + Logger.debug('Move output file "{}" to cache dir "{}" and create a symlink'.format( + dobj.data_file_relative, dobj.cache_file_relative)) + shutil.move(dobj.data_file_relative, dobj.cache_file_relative) + + dobj.create_symlink() + + Logger.debug('Create state file "{}"'.format(dobj.state_file_relative)) + state_file = StateFile(dobj.state_file_relative, self.git, input_files, output_files) + state_file.save() + pass + + return True + + @staticmethod + def remove_new_files(repo_change): + for file in repo_change.new_files: + rel_path = GitWrapper.abs_paths_to_relative([file])[0] + Logger.error('Removing created file: {}'.format(rel_path)) + os.remove(file) + pass + + def validate_file_states(self, files_states): + error = False + for file in GitWrapper.abs_paths_to_relative(files_states.removed_files): + Logger.error('Error: file "{}" was removed'.format(file)) + error = True + + for file in GitWrapper.abs_paths_to_relative(files_states.modified_files): + Logger.error('Error: file "{}" was modified'.format(file)) + error = True + + for file in GitWrapper.abs_paths_to_relative(files_states.unusual_state_files): + Logger.error('Error: file "{}" is in not acceptable state'.format(file)) + error = True + + for file in GitWrapper.abs_paths_to_relative(files_states.externally_created_files): + Logger.error('Error: file "{}" was created outside of the data directory'.format(file)) + error = True + + if error: + Logger.error('Errors occurred. ' + \ + 'Reproducible commands allow only file creation only in data directory "{}".'. + format(self.config.data_dir)) + return False + + if not files_states.new_files: + Logger.error('Errors occurred. No files were changed in run command.') + return False + + return True + + def get_data_files_from_args(self, argv): + result = [] + + for arg in argv: try: - result.append(DataFileObj(file, self.git, self.config)) + if os.path.isfile(arg): + DataFileObj(arg, self.git, self.config) + result.append(arg) except NotInDataDirError: - errors.append(file) - - return result, errors - - def get_changed_files(self): - statuses = GitWrapper.status_files() - - result = [] - for status, file in statuses: - file_path = os.path.join(self.git.git_dir_abs, file) - if os.path.isfile(file_path): - result.append(file_path) - else: - files = [] - self.get_all_files_from_dir(file_path, files) - for f in files: - result.append(f) + pass return result - def get_all_files_from_dir(self, dir, result): - if not os.path.isdir(dir): - raise RunError('Changed path {} is not directory'.format(dir)) - - files = os.listdir(dir) - for f in files: - path = os.path.join(dir, f) - if os.path.isfile(path): - result.append(path) - else: - self.get_all_files_from_dir(path, result) - pass if __name__ == '__main__': import sys + try: sys.exit(CmdRun().run()) except NeatLynxException as e: diff --git a/neatlynx/data_file_obj.py b/neatlynx/data_file_obj.py index cff0617ca..69f87f2d0 100644 --- a/neatlynx/data_file_obj.py +++ b/neatlynx/data_file_obj.py @@ -106,6 +106,19 @@ class DataFileObj(object): def state_file_relative(self): return os.path.relpath(self.state_file_abs, self._curr_dir_abs) + @staticmethod + def files_to_dobjs(files, git, config): + result = [] + externally_created_files = [] + + for file in files: + try: + result.append(DataFileObj(file, git, config)) + except NotInDataDirError: + externally_created_files.append(file) + + return result, externally_created_files + class DataFileObjExisting(DataFileObj): def __init__(self, data_file, git, config): diff --git a/neatlynx/git_wrapper.py b/neatlynx/git_wrapper.py index d23f26e47..ad81509d1 100644 --- a/neatlynx/git_wrapper.py +++ b/neatlynx/git_wrapper.py @@ -70,7 +70,7 @@ class GitWrapper(GitWrapperI): @staticmethod def exec_cmd_only_success(cmd, stdout_file=None, stderr_file=None, cwd=None): - code, out, err = GitWrapper.exec_cmd(cmd, stdout_file=None, stderr_file=None, cwd=None) + code, out, err = GitWrapper.exec_cmd(cmd, stdout_file=stdout_file, stderr_file=stderr_file, cwd=cwd) if code != 0: if err: sys.stderr.write(err + '\n') @@ -88,6 +88,10 @@ class GitWrapper(GitWrapperI): return True + @property + def curr_dir_nlx(self): + return os.path.relpath(os.path.abspath(os.curdir), self.git_dir_abs) + @property def git_dir(self): if self._git_dir: @@ -143,9 +147,9 @@ class GitWrapper(GitWrapperI): def commit_all_changes_and_log_status(self, message): statuses = self.commit_all_changes(message) - Logger.info('A new commit {} was made in the current branch. Added files:'.format(self.curr_commit)) + Logger.printing('A new commit {} was made in the current branch. Added files:'.format(self.curr_commit)) for status, file in statuses: - Logger.info('\t{} {}'.format(status, file)) + Logger.printing('\t{} {}'.format(status, file)) pass @property @@ -158,6 +162,13 @@ class GitWrapper(GitWrapperI): result = [] for file in files: - result.append(os.path.relpath(file, cur_dir)) + result.append(os.path.relpath(os.path.realpath(file), cur_dir)) - return result \ No newline at end of file + return result + + def abs_paths_to_nlx(self, files): + result = [] + for file in files: + result.append(os.path.relpath(os.path.abspath(file), self.git_dir_abs)) + + return result diff --git a/neatlynx/logger.py b/neatlynx/logger.py index 4a2065335..282e2866c 100644 --- a/neatlynx/logger.py +++ b/neatlynx/logger.py @@ -3,17 +3,18 @@ import sys class Logger(object): @staticmethod - def info(msg): - print('{}'.format(msg)) + def printing(msg): + print(u'{}'.format(msg)) @staticmethod def warn(msg): - print('{}'.format(msg)) + print(u'Warning. {}'.format(msg)) @staticmethod def error(msg): - sys.stderr.write('{}\n'.format(msg)) + #sys.stderr.write('Error. {}\n'.format(msg)) + print(u'Error. {}'.format(msg)) @staticmethod - def verbose(msg): - print('{}'.format(msg)) \ No newline at end of file + def debug(msg): + print(u'Debug. {}'.format(msg)) diff --git a/neatlynx/repository_change.py b/neatlynx/repository_change.py new file mode 100644 index 000000000..dbd56d6cd --- /dev/null +++ b/neatlynx/repository_change.py @@ -0,0 +1,115 @@ +import os + +from neatlynx.data_file_obj import DataFileObj, NotInDataDirError +from neatlynx.exceptions import NeatLynxException +from neatlynx.git_wrapper import GitWrapper + + +class RepositoryChangeError(NeatLynxException): + def __init__(self, msg): + NeatLynxException.__init__(self, 'Repository change error: {}'.format(msg)) + + +class FileModificationState(object): + STATUS_UNTRACKED = '?' + STATUS_DELETE = 'D' + STATUS_MODIFIED = 'M' + + def __init__(self, state, file): + self.state = state + self.file = file + + def _check_status(self, status): + return self.state.find(status) >= 0 + + @property + def is_removed(self): + return self._check_status(self.STATUS_DELETE) + + @property + def is_modified(self): + return self._check_status(self.STATUS_MODIFIED) + + @property + def is_new(self): + return self._check_status(self.STATUS_UNTRACKED) + + @property + def is_unusual(self): + return self.is_new or self.is_modified or self.is_removed + + +class RepositoryChange(object): + """Pre-condition: git repository has no changes""" + + def __init__(self, args, stdout, stderr, git, config): + self.git = git + self.config = config + + GitWrapper.exec_cmd_only_success(args, stdout, stderr) + self._file_states = self._get_file_states() + + self._new_dobj, self._externally_created_files = DataFileObj.files_to_dobjs( + self.new_files, self.git, self.config) + + @staticmethod + def exec_cmd(args, stdout, stderr, git): + return RepositoryChange(args, stdout, stderr, git) + + @staticmethod + def get_filenames(mod_state_list): + return list(map(lambda x: x.file, mod_state_list)) + + @property + def removed_files(self): + return self.get_filenames(filter(lambda x: x.is_removed, self._file_states)) + + @property + def modified_files(self): + return self.get_filenames(filter(lambda x: x.is_modified, self._file_states)) + + @property + def new_files(self): + return self.get_filenames(filter(lambda x: x.is_new, self._file_states)) + + @property + def unusual_state_files(self): + return self.get_filenames(filter(lambda x: not x.is_unusual, self._file_states)) + + def _get_file_states(self): + statuses = GitWrapper.status_files() + + result = [] + for status, file in statuses: + file_path = os.path.join(self.git.git_dir_abs, file) + if os.path.isfile(file_path): + result.append(FileModificationState(status, file_path)) + else: + files = [] + self.get_all_files_from_dir(file_path, files) + state = self.STATUS_UNKNOWN + self.STATUS_UNKNOWN + for f in files: + result.append(FileModificationState(state, f)) + + return result + + def get_all_files_from_dir(self, dir, result): + if not os.path.isdir(dir): + raise RepositoryChangeError('Changed path {} is not directory'.format(dir)) + + files = os.listdir(dir) + for f in files: + path = os.path.join(dir, f) + if os.path.isfile(path): + result.append(path) + else: + self.get_all_files_from_dir(path, result) + pass + + @property + def dobj_for_new_files(self): + return self._new_dobj + + @property + def externally_created_files(self): + return self._externally_created_files diff --git a/neatlynx/state_file.py b/neatlynx/state_file.py index 146fc7c9a..12ff32935 100644 --- a/neatlynx/state_file.py +++ b/neatlynx/state_file.py @@ -5,6 +5,7 @@ import time from neatlynx.exceptions import NeatLynxException from neatlynx.git_wrapper import GitWrapper +from neatlynx.logger import Logger class StateFileError(NeatLynxException): @@ -16,17 +17,61 @@ class StateFile(object): MAGIC = 'NLX-State' VERSION = '0.1' - def __init__(self, file, git): + PARAM_TYPE = 'Type' + PARAM_VERSION = 'Version' + PARAM_ARGV = 'Argv' + PARAM_NORM_ARGV = 'NormArgv' + PARAM_CWD = 'Cwd' + PARAM_CREATED_AT = 'CreatedAt' + PARAM_INPUT_FILES = 'InputFiles' + PARAM_OUTPUT_FILES = 'OutputFiles' + + def __init__(self, file, git, input_files, output_files, + argv=sys.argv, norm_argv=None, + created_at=time.strftime('%Y-%m-%d %H:%M:%S %z'), + cwd=None): self.file = file self.git = git + self.input_files = input_files + self.output_files = output_files + self.argv = argv + if norm_argv: + self.norm_argv = norm_argv + else: + self.norm_argv = self.normalized_args() + + self.created_at = created_at + + if cwd: + self.cwd = cwd + else: + self.cwd = self.get_nlx_path() + pass + + @staticmethod + def load(filename, git): + with open(filename, 'r') as fd: + data = json.load(fd) + + return StateFile(filename, + git, + data.get('InputFiles'), + data.get('OutputFiles'), + data.get('Argv'), + data.get('NormArgv'), + data.get('CreatedAt'), + data.get('Cwd')) def save(self): res = { - 'Type': self.MAGIC, - 'Version': self.VERSION, - 'Argv': sys.argv, - 'NLX_cwd': self.get_nlx_path(), - 'CreatedAt': time.strftime('%Y-%m-%d %H:%M:%S %z') + self.PARAM_TYPE: self.MAGIC, + self.PARAM_VERSION: self.VERSION, + self.PARAM_ARGV: self.argv, + self.PARAM_NORM_ARGV: self.norm_argv, + self.PARAM_CWD: self.cwd, + self.PARAM_CREATED_AT: self.created_at, + self.PARAM_INPUT_FILES: self.input_files, + self.PARAM_OUTPUT_FILES:self.output_files } file_dir = os.path.dirname(self.file) @@ -37,6 +82,26 @@ class StateFile(object): json.dump(res, fd, indent=2) pass + def normalized_args(self): + result = [] + + if len(sys.argv) > 0: + cmd = sys.argv[0] + pos = cmd.rfind(os.sep) + if pos >= 0: + cmd = cmd[pos+1:] + result.append(cmd) + + for arg in sys.argv[1:]: + if os.path.isfile(arg): + path = os.path.abspath(arg) + nlx_path = os.path.relpath(path, self.git.git_dir_abs) + result.append(nlx_path) + else: + result.append(arg) + + return result + def get_nlx_path(self): pwd = os.path.realpath(os.curdir) if not pwd.startswith(self.git.git_dir_abs): @@ -44,13 +109,22 @@ class StateFile(object): return os.path.relpath(pwd, self.git.git_dir_abs) - def repro(self): - with open(self.file, 'r') as fd: - res = json.load(fd) - - argv = res['Argv'] - argv.insert(0, 'python') - argv.insert(2, '--ignore-git-status') - - cwd = os.path.join(self.git.git_dir_abs, res['NLX_cwd']) - return GitWrapper.exec_cmd(argv, cwd=cwd) + # def repro(self): + # with open(self.file, 'r') as fd: + # res = json.load(fd) + # + # args = res['NormArgs'] + # if not args: + # raise StateFileError('Error: parameter NormalizedArgs is nor defined in state file "{}"'. + # format(self.file)) + # if len(args) < 2: + # raise StateFileError('Error: reproducible command in state file "{}" is too short'. + # format(self.file)) + # + # if args[0][-3:] != '.py': + # raise StateFileError('Error: reproducible command format error in state file "{}"'. + # format(self.file)) + # args.pop(0) + # + # Logger.info("Repro cmd:\n\t{}".format(' '.join(args))) + # return GitWrapper.exec_cmd(args, cwd=self.git.git_dir_abs)