Finalizing repro with HACK

neatlynx/cmd_data_import.py

@@ -32,6 +32,9 @@ class CmdDataImport(CmdBase):
                             nargs='*')
 
         self.add_string_arg(parser, 'output', 'Output file')
+
+        parser.add_argument('-i', '--is-reproducible', action='store_false', default=False,
+                            help='Is data file reproducible')
         pass
 
     def run(self):
@@ -47,7 +50,7 @@ class CmdDataImport(CmdBase):
 
             output = self.args.output
             for file in self.args.input:
-                self.import_file(file, output)
+                self.import_file(file, output, self.args.is_reproducible)
 
             if self.skip_git_actions:
                 self.not_committed_changes_warning()
@@ -59,7 +62,7 @@ class CmdDataImport(CmdBase):
             lock.release()
         pass
 
-    def import_file(self, input, output):
+    def import_file(self, input, output, is_reproducible):
         if not CmdDataImport.is_url(input):
             if not os.path.exists(input):
                 raise DataImportError('Input file "{}" does not exist'.format(input))
@@ -77,7 +80,10 @@ class CmdDataImport(CmdBase):
             raise DataImportError('Output file directory "{}" does not exists'.format(
                 os.path.dirname(dobj.data_file_relative)))
 
-        os.makedirs(os.path.dirname(dobj.cache_file_relative), exist_ok=True)
+        cache_dir = os.path.dirname(dobj.cache_file_relative)
+        if not os.path.exists(cache_dir):
+            os.makedirs(cache_dir)
+
         if CmdDataImport.is_url(input):
             Logger.debug('Downloading file {} ...'.format(input))
             self.download_file(input, dobj.cache_file_relative)
@@ -92,7 +98,12 @@ class CmdDataImport(CmdBase):
         Logger.debug('Symlink from data file "{}" to the cache file "{}" was created'.
                      format(dobj.data_file_relative, dobj.cache_file_relative))
 
-        state_file = StateFile(dobj.state_file_relative, self.git)
+        state_file = StateFile(dobj.state_file_relative,
+                               self.git,
+                               [input],
+                               [output],
+                               [],
+                               is_reproducible)
         state_file.save()
         Logger.debug('State file "{}" was created'.format(dobj.state_file_relative))
         pass

neatlynx/cmd_init.py

@@ -49,6 +49,8 @@ SecurityGroup = neatlynx-group'''
         pass
 
     def define_args(self, parser):
+        self.set_skip_git_actions(parser)
+
         self.add_string_arg(parser, '--data-dir',  'NeatLynx data directory', 'data')
         self.add_string_arg(parser, '--cache-dir', 'NeatLynx cache directory', 'cache')
         self.add_string_arg(parser, '--state-dir', 'NeatLynx state directory', 'state')
@@ -60,18 +62,18 @@ SecurityGroup = neatlynx-group'''
             raise InitError('Directory "{}" already exist'.format(path.name))
         return path
 
-    def get_not_existing_conf_file(self):
-        path = Path(os.path.join(self.git.git_dir, Config.CONFIG))
-        if path.exists():
-            raise InitError('Configuration file "{}" already exist'.format(path.name))
-        return path
+    def get_not_existing_conf_file_name(self):
+        file_name = os.path.join(self.git.git_dir, Config.CONFIG)
+        if os.path.exists(file_name):
+            raise InitError('Configuration file "{}" already exist'.format(file_name))
+        return file_name
 
     def run(self):
         data_dir_path = self.get_not_existing_dir(self.args.data_dir)
         cache_dir_path = self.get_not_existing_dir(self.args.cache_dir)
         state_dir_path = self.get_not_existing_dir(self.args.state_dir)
 
-        conf = self.get_not_existing_conf_file()
+        conf_file_name = self.get_not_existing_conf_file_name()
 
         data_dir_path.mkdir()
         cache_dir_path.mkdir()
@@ -81,11 +83,24 @@ SecurityGroup = neatlynx-group'''
             cache_dir_path.name,
             state_dir_path.name))
 
-        conf.write_text(self.CONFIG_TEMPLATE.format(data_dir_path.name,
+        conf_file = open(conf_file_name, 'wt')
+        conf_file.write(self.CONFIG_TEMPLATE.format(data_dir_path.name,
                                                     cache_dir_path.name,
                                                     state_dir_path.name))
+        conf_file.close()
 
         self.modify_gitignore(cache_dir_path.name)
+
+        if self.skip_git_actions:
+            self.not_committed_changes_warning()
+            return 0
+
+        message = 'NLX init. data dir {}, cache dir {}, state dir {}'.format(
+                        data_dir_path.name,
+                        cache_dir_path.name,
+                        state_dir_path.name
+        )
+        self.git.commit_all_changes_and_log_status(message)
         pass
 
     def modify_gitignore(self, cache_dir_name):

neatlynx/cmd_repro.py

@@ -7,7 +7,7 @@ from neatlynx.cmd_run import CmdRun
 from neatlynx.git_wrapper import GitWrapper
 from neatlynx.logger import Logger
 from neatlynx.exceptions import NeatLynxException
-from neatlynx.data_file_obj import DataFileObj
+from neatlynx.data_file_obj import DataFileObj, NotInDataDirError
 from neatlynx.state_file import StateFile
 
 
@@ -19,8 +19,14 @@ class ReproError(NeatLynxException):
 class CmdRepro(CmdRun):
     def __init__(self):
         CmdBase.__init__(self)
+
+        self._code =[]
         pass
 
+    @property
+    def code(self):
+        return self._code
+
     def define_args(self, parser):
         self.set_skip_git_actions(parser)
 
@@ -56,64 +62,109 @@ class CmdRepro(CmdRun):
                 return 1
 
             error = False
+            changed = False
             for dobj in dobjs:
-                self.repro(dobj)
-
-                # if returncode != 0:
-                #     error = True
-                #     Logger.error('Error: cannot reproduce file "{}"\n{}'.format(
-                #         dobj.data_file_relative, out))
-                #     sys.stderr.write('{}\n'.format(err))
+                try:
+                    repro_change = ReproChange(dobj, self.git, self)
+                    if repro_change.reproduce():
+                        changed = True
+                        Logger.info(u'Data file "{}" was reproduced.'.format(
+                            dobj.data_file_relative
+                        ))
+                    else:
+                        Logger.info(u'Reproduction is not required for data file "{}".'.format(
+                            dobj.data_file_relative
+                        ))
+                except ReproError as err:
+                    Logger.error('Error in reproducing data file {}: {}'.format(
+                        dobj.data_file_relative, str(err)
+                    ))
+                    error = True
+                    break
 
             if error and not self.skip_git_actions:
                 Logger.error('Errors occurred. One or more repro command was not successful.')
                 self.not_committed_changes_warning()
                 return 1
 
-            if self.skip_git_actions:
-                self.not_committed_changes_warning()
-                return 0
+            if changed:
+                if self.skip_git_actions:
+                    self.not_committed_changes_warning()
+                    return 1
 
-            message = 'NLX repro: {}'.format(' '.join(self.args.target))
-            self.git.commit_all_changes_and_log_status(message)
+                message = 'NLX repro: {}'.format(' '.join(self.args.target))
+                self.git.commit_all_changes_and_log_status(message)
         finally:
             lock.release()
-
-        # dobj = DataFileObj(self.args.target, self.git, self.config)
-        # os.remove(self.args.target)
-        #
-        # state_file = StateFile(dobj.state_file_relative, self.git)
-        # returncode, out, err = state_file.repro()
-        #
-        # print(out)
-        # sys.stderr.write(err)
-        #
-        # return returncode
         return 0
 
-    def repro(self, dobj):
-        state = StateFile.load(dobj.state_file_relative, self.git)
 
-        argv = state.norm_argv
+class ReproChange(object):
+    def __init__(self, dobj, git, cmd_obj):
+        self._dobj = dobj
+        self._state = StateFile.load(dobj.state_file_relative, git)
+        self.git = git
+        self._cmd_obj = cmd_obj
+
+        cmd_obj._code = self._state.code_sources # HACK!!!
+
+        argv = self._state.norm_argv
 
         if not argv:
             raise ReproError('Error: parameter {} is nor defined in state file "{}"'.
                              format(StateFile.PARAM_NORM_ARGV, dobj.state_file_relative))
         if len(argv) < 2:
             raise ReproError('Error: reproducible command in state file "{}" is too short'.
-                             format(self.file))
+                             format(self._state.file))
 
-        if argv[0][-3:] != '.py':
-            raise ReproError('Error: reproducible command format error in state file "{}"'.
-                             format(self.file))
-        argv.pop(0)
+        # if argv[0][-3:] != '.py':
+        #     raise ReproError('Error: reproducible command format error in state file "{}"'.
+        #                      format(self._state.file))
 
-        Logger.debug('Removing data file "{}"'.format(dobj.data_file_relative))
-        os.remove(dobj.data_file_relative)
+        self._repro_argv = argv
+        pass
 
-        Logger.debug("Repro cmd:\n\t{}".format(' '.join(argv)))
-        return self.run_command(argv)
+    def were_direct_dependencies_changed(self):
+        return True
 
+    def reproduce_data_file(self):
+        Logger.debug('Reproducing data file "{}". Removing the file...'.format(
+            self._dobj.data_file_relative))
+        os.remove(self._dobj.data_file_relative)
+
+        Logger.debug('Reproducing data file "{}". Re-runs command: {}'.format(
+            self._dobj.data_file_relative, ' '.join(self._repro_argv)))
+        return self._cmd_obj.run_command(self._repro_argv)
+
+    def reproduce(self, force=False):
+        were_input_files_changed = False
+
+        if not self._state.is_reproducible:
+            Logger.debug('Data file "{}" is not reproducible'.format(self._dobj.data_file_relative))
+            return False
+
+        for input_file in self._state.input_files:
+            try:
+                dobj = DataFileObj(input_file, self.git, self._cmd_obj.config)
+            except NotInDataDirError:
+                raise ReproError(u'The dependency files "{}" is not a data file'.format(input_file))
+            except Exception as ex:
+                raise ReproError(u'The dependency files "{}" can not be reproduced: {}'.format(
+                                 input_file, ex))
+
+            change = ReproChange(dobj, self.git, self._cmd_obj)
+            if change.reproduce(force):
+                were_input_files_changed = True
+
+        was_source_code_changed = self.git.were_files_changed(self._dobj.data_file_relative,
+                                                              self._state.code_sources)
+
+        if not force and not was_source_code_changed and not were_input_files_changed:
+            Logger.debug('Data file "{}" is up to date'.format(
+                self._dobj.data_file_relative))
+            return False
+
+        return self.reproduce_data_file()
 
 if __name__ == '__main__':
     import sys

neatlynx/cmd_run.py

@@ -31,8 +31,14 @@ class CmdRun(CmdBase):
                             help='Declare input file for reproducible command')
         parser.add_argument('--output-file', '-o', action='append',
                             help='Declare output file for reproducible command')
+        parser.add_argument('--code', '-c', action='append',
+                            help='Code file or code directory which produces the output')
         pass
 
+    @property
+    def code(self):
+        return self.args.code
+
     @property
     def declaration_input_files(self):
         if self.args.input_file:
@@ -73,16 +79,32 @@ class CmdRun(CmdBase):
     def run_command(self, argv, stdout=None, stderr=None):
         repo_change = RepositoryChange(argv, stdout, stderr, self.git, self.config)
 
+        # print('===== new_files={}'.format(repo_change.new_files))
+        # print('===== modified_files={}'.format(repo_change.modified_files))
+        # print('===== removed_files={}'.format(repo_change.removed_files))
+        # print('===== externally_created_files={}'.format(repo_change.externally_created_files))
+        # # raise Exception()
+
+        print('========= {} {}'.format(not self.skip_git_actions, not self.validate_file_states(repo_change)))
+
         if not self.skip_git_actions and not self.validate_file_states(repo_change):
             self.remove_new_files(repo_change)
             return False
 
-        output_files = self.git.abs_paths_to_nlx(repo_change.new_files + self.declaration_output_files)
-        input_files_from_args = list(set(self.get_data_files_from_args(argv)) - set(repo_change.new_files))
+        changed_files_nlx = self.git.abs_paths_to_nlx(repo_change.changed_files)
+        output_files = changed_files_nlx + self.git.abs_paths_to_nlx(self.declaration_output_files)
+        args_files_nlx = self.git.abs_paths_to_nlx(self.get_data_files_from_args(argv))
+
+        input_files_from_args = list(set(args_files_nlx) - set(changed_files_nlx))
         input_files = self.git.abs_paths_to_nlx(input_files_from_args + self.declaration_input_files)
 
-        for dobj in repo_change.dobj_for_new_files:
-            os.makedirs(os.path.dirname(dobj.cache_file_relative), exist_ok=True)
+        print('================== changed files = {}'.format(len(repo_change.dobj_for_changed_files)))
+
+        for dobj in repo_change.dobj_for_changed_files:
+            print('================== move ... {}'.format(dobj.data_file_relative))
+            dirname = os.path.dirname(dobj.cache_file_relative)
+            if not os.path.isdir(dirname):
+                os.makedirs(dirname)
 
             Logger.debug('Move output file "{}" to cache dir "{}" and create a symlink'.format(
                 dobj.data_file_relative, dobj.cache_file_relative))
@@ -90,8 +112,14 @@ class CmdRun(CmdBase):
 
             dobj.create_symlink()
 
+            nlx_code_sources = map(lambda x: self.git.abs_paths_to_nlx([x])[0], self.code)
+
             Logger.debug('Create state file "{}"'.format(dobj.state_file_relative))
-            state_file = StateFile(dobj.state_file_relative, self.git, input_files, output_files)
+            state_file = StateFile(dobj.state_file_relative, self.git,
+                                   input_files,
+                                   output_files,
+                                   nlx_code_sources,
+                                   argv=argv)
             state_file.save()
             pass
 
@@ -111,13 +139,14 @@ class CmdRun(CmdBase):
             Logger.error('Error: file "{}" was removed'.format(file))
             error = True
 
-        for file in GitWrapper.abs_paths_to_relative(files_states.modified_files):
-            Logger.error('Error: file "{}" was modified'.format(file))
-            error = True
+        # for file in GitWrapper.abs_paths_to_relative(files_states.modified_files):
+        #     Logger.error('Error: file "{}" was modified'.format(file))
+        #     error = True
 
-        for file in GitWrapper.abs_paths_to_relative(files_states.unusual_state_files):
-            Logger.error('Error: file "{}" is in not acceptable state'.format(file))
-            error = True
+        # # This code doesn't cover repro case
+        # for file in GitWrapper.abs_paths_to_relative(files_states.unusual_state_files):
+        #     Logger.error('Error: file "{}" is in not acceptable state'.format(file))
+        #     error = True
 
         for file in GitWrapper.abs_paths_to_relative(files_states.externally_created_files):
             Logger.error('Error: file "{}" was created outside of the data directory'.format(file))
@@ -129,9 +158,9 @@ class CmdRun(CmdBase):
                          format(self.config.data_dir))
             return False
 
-        if not files_states.new_files:
-            Logger.error('Errors occurred. No files were changed in run command.')
-            return False
+        # if not files_states.new_files:
+        #     Logger.error('Errors occurred. No files were changed in run command.')
+        #     return False
 
         return True
 

neatlynx/git_wrapper.py

@@ -98,7 +98,7 @@ class GitWrapper(GitWrapperI):
 
     @property
     def curr_dir_nlx(self):
-        return os.path.relpath(self.curr_dir_abs, self.git_dir_abs)
+        return os.path.relpath(os.path.abspath(os.curdir), self.git_dir_abs)
 
     @property
     def git_dir(self):
@@ -180,3 +180,39 @@ class GitWrapper(GitWrapperI):
             result.append(os.path.relpath(os.path.abspath(file), self.git_dir_abs))
 
         return result
+
+    def nlx_paths_to_abs(self, files):
+        results = []
+
+        for file in files:
+            results.append(os.path.join(self.git_dir_abs, file))
+
+        return results
+
+    def were_files_changed(self, file, code_sources):
+        commit = self.exec_cmd_only_success(['git', 'log', '-1', '--pretty=format:"%h"', file])
+        commit = commit.strip('"')
+
+        changed_files = self.exec_cmd_only_success(['git', 'diff', '--name-only', 'HEAD', commit])
+        changed_files = changed_files.strip('"')
+
+        code_sources_abs = self.nlx_paths_to_abs(code_sources)
+        code_files = []
+        code_dirs = []
+        for code in code_sources_abs:
+            if os.path.isdir(code):
+                code_dirs.append(code)
+            else:
+                code_files.append(code)
+
+        code_files_set = set(code_files)
+        for changed_file in changed_files.split('\n'):
+            changed_file = os.path.realpath(changed_file)
+            if changed_file in code_files_set:
+                return True
+
+            for dir in code_dirs:
+                if changed_file.startswith(dir):
+                    return True
+
+        return False

neatlynx/logger.py

@@ -18,3 +18,7 @@ class Logger(object):
     @staticmethod
     def debug(msg):
         print(u'Debug. {}'.format(msg))
+
+    @staticmethod
+    def info(msg):
+        print(u'Info. {}'.format(msg))

neatlynx/repository_change.py

@@ -14,6 +14,7 @@ class FileModificationState(object):
     STATUS_UNTRACKED = '?'
     STATUS_DELETE = 'D'
     STATUS_MODIFIED = 'M'
+    STATUS_TYPE_CHANGED = 'T'
 
     def __init__(self, state, file):
         self.state = state
@@ -28,7 +29,8 @@ class FileModificationState(object):
 
     @property
     def is_modified(self):
-        return self._check_status(self.STATUS_MODIFIED)
+        return self._check_status(self.STATUS_MODIFIED) \
+               or self._check_status(self.STATUS_TYPE_CHANGED)
 
     @property
     def is_new(self):
@@ -49,8 +51,8 @@ class RepositoryChange(object):
         GitWrapper.exec_cmd_only_success(args, stdout, stderr)
         self._file_states = self._get_file_states()
 
-        self._new_dobj, self._externally_created_files = DataFileObj.files_to_dobjs(
-            self.new_files, self.git, self.config)
+        self._changed_dobj, self._externally_created_files = DataFileObj.files_to_dobjs(
+            self.changed_files, self.git, self.config)
 
     @staticmethod
     def exec_cmd(args, stdout, stderr, git):
@@ -72,6 +74,10 @@ class RepositoryChange(object):
     def new_files(self):
         return self.get_filenames(filter(lambda x: x.is_new, self._file_states))
 
+    @property
+    def changed_files(self):
+        return self.new_files + self.modified_files
+
     @property
     def unusual_state_files(self):
         return self.get_filenames(filter(lambda x: not x.is_unusual, self._file_states))
@@ -87,7 +93,7 @@ class RepositoryChange(object):
             else:
                 files = []
                 self.get_all_files_from_dir(file_path, files)
-                state = self.STATUS_UNKNOWN + self.STATUS_UNKNOWN
+                state = FileModificationState.STATUS_UNTRACKED + FileModificationState.STATUS_UNTRACKED
                 for f in files:
                     result.append(FileModificationState(state, f))
 
@@ -107,8 +113,8 @@ class RepositoryChange(object):
         pass
 
     @property
-    def dobj_for_new_files(self):
-        return self._new_dobj
+    def dobj_for_changed_files(self):
+        return self._changed_dobj
 
     @property
     def externally_created_files(self):

neatlynx/state_file.py

@@ -25,15 +25,23 @@ class StateFile(object):
     PARAM_CREATED_AT = 'CreatedAt'
     PARAM_INPUT_FILES = 'InputFiles'
     PARAM_OUTPUT_FILES = 'OutputFiles'
+    PARAM_CODE_SOURCES = 'CodeSources'
+    PARAM_NOT_REPRODUCIBLE = 'NotReproducible'
 
     def __init__(self, file, git, input_files, output_files,
-                 argv=sys.argv, norm_argv=None,
+                 code_sources=[],
+                 is_reproducible=True,
+                 argv=sys.argv,
+                 norm_argv=None,
                  created_at=time.strftime('%Y-%m-%d %H:%M:%S %z'),
                  cwd=None):
         self.file = file
         self.git = git
         self.input_files = input_files
         self.output_files = output_files
+        self.is_reproducible = is_reproducible
+        self.code_sources = code_sources
+
         self.argv = argv
         if norm_argv:
             self.norm_argv = norm_argv
@@ -55,28 +63,34 @@ class StateFile(object):
 
         return StateFile(filename,
                          git,
-                         data.get('InputFiles'),
-                         data.get('OutputFiles'),
-                         data.get('Argv'),
-                         data.get('NormArgv'),
-                         data.get('CreatedAt'),
-                         data.get('Cwd'))
+                         data.get(StateFile.PARAM_INPUT_FILES, []),
+                         data.get(StateFile.PARAM_OUTPUT_FILES, []),
+                         data.get(StateFile.PARAM_CODE_SOURCES, []),
+                         not data.get(StateFile.PARAM_NOT_REPRODUCIBLE, False),
+                         data.get(StateFile.PARAM_ARGV),
+                         data.get(StateFile.PARAM_NORM_ARGV),
+                         data.get(StateFile.PARAM_CREATED_AT),
+                         data.get(StateFile.PARAM_CWD))
 
     def save(self):
         res = {
-            self.PARAM_TYPE:        self.MAGIC,
-            self.PARAM_VERSION:     self.VERSION,
-            self.PARAM_ARGV:        self.argv,
-            self.PARAM_NORM_ARGV:   self.norm_argv,
-            self.PARAM_CWD:         self.cwd,
-            self.PARAM_CREATED_AT:  self.created_at,
-            self.PARAM_INPUT_FILES: self.input_files,
-            self.PARAM_OUTPUT_FILES:self.output_files
+            self.PARAM_TYPE:            self.MAGIC,
+            self.PARAM_VERSION:         self.VERSION,
+            self.PARAM_ARGV:            self.argv,
+            self.PARAM_NORM_ARGV:       self.norm_argv,
+            self.PARAM_CWD:             self.cwd,
+            self.PARAM_CREATED_AT:      self.created_at,
+            self.PARAM_INPUT_FILES:     self.input_files,
+            self.PARAM_OUTPUT_FILES:    self.output_files,
+            self.PARAM_CODE_SOURCES:    self.code_sources
         }
 
+        if not self.is_reproducible:
+            res[self.PARAM_NOT_REPRODUCIBLE] = True
+
         file_dir = os.path.dirname(self.file)
-        if file_dir != '':
-            os.makedirs(file_dir, exist_ok=True)
+        if file_dir != '' and not os.path.isdir(file_dir):
+            os.makedirs(file_dir)
 
         with open(self.file, 'w') as fd:
             json.dump(res, fd, indent=2)
@@ -85,14 +99,14 @@ class StateFile(object):
     def normalized_args(self):
         result = []
 
-        if len(sys.argv) > 0:
-            cmd = sys.argv[0]
+        if len(self.argv) > 0:
+            cmd = self.argv [0]
             pos = cmd.rfind(os.sep)
             if pos >= 0:
                 cmd = cmd[pos+1:]
             result.append(cmd)
 
-            for arg in sys.argv[1:]:
+            for arg in self.argv [1:]:
                 if os.path.isfile(arg):
                     path = os.path.abspath(arg)
                     nlx_path = os.path.relpath(path, self.git.git_dir_abs)
@@ -108,23 +122,3 @@ class StateFile(object):
             raise StateFileError('the file cannot be created outside of a git repository')
 
         return os.path.relpath(pwd, self.git.git_dir_abs)
-
-    # def repro(self):
-    #     with open(self.file, 'r') as fd:
-    #         res = json.load(fd)
-    #
-    #     args = res['NormArgs']
-    #     if not args:
-    #         raise StateFileError('Error: parameter NormalizedArgs is nor defined in state file "{}"'.
-    #                              format(self.file))
-    #     if len(args) < 2:
-    #         raise StateFileError('Error: reproducible command in state file "{}" is too short'.
-    #                              format(self.file))
-    #
-    #     if args[0][-3:] != '.py':
-    #         raise StateFileError('Error: reproducible command format error in state file "{}"'.
-    #                              format(self.file))
-    #     args.pop(0)
-    #
-    #     Logger.info("Repro cmd:\n\t{}".format(' '.join(args)))
-    #     return GitWrapper.exec_cmd(args, cwd=self.git.git_dir_abs)