Repro cmd and run cmd change

neatlynx/cmd_data_import.py

@@ -38,7 +38,7 @@ class CmdDataImport(CmdBase):
         lock = fasteners.InterProcessLock(self.git.lock_file)
         gotten = lock.acquire(timeout=5)
         if not gotten:
-            Logger.info('Cannot perform the command since NLX is busy and locked. Please retry the command later.')
+            Logger.printing('Cannot perform the command since NLX is busy and locked. Please retry the command later.')
             return 1
 
         try:
@@ -79,22 +79,22 @@ class CmdDataImport(CmdBase):
 
         os.makedirs(os.path.dirname(dobj.cache_file_relative), exist_ok=True)
         if CmdDataImport.is_url(input):
-            Logger.verbose('Downloading file {} ...'.format(input))
+            Logger.debug('Downloading file {} ...'.format(input))
             self.download_file(input, dobj.cache_file_relative)
-            Logger.verbose('Input file "{}" was downloaded to cache "{}"'.format(
+            Logger.debug('Input file "{}" was downloaded to cache "{}"'.format(
                 input, dobj.cache_file_relative))
         else:
             copyfile(input, dobj.cache_file_relative)
-            Logger.verbose('Input file "{}" was copied to cache "{}"'.format(
+            Logger.debug('Input file "{}" was copied to cache "{}"'.format(
                 input, dobj.cache_file_relative))
 
         dobj.create_symlink()
-        Logger.verbose('Symlink from data file "{}" to the cache file "{}" was created'.
-                       format(dobj.data_file_relative, dobj.cache_file_relative))
+        Logger.debug('Symlink from data file "{}" to the cache file "{}" was created'.
+                     format(dobj.data_file_relative, dobj.cache_file_relative))
 
         state_file = StateFile(dobj.state_file_relative, self.git)
         state_file.save()
-        Logger.verbose('State file "{}" was created'.format(dobj.state_file_relative))
+        Logger.debug('State file "{}" was created'.format(dobj.state_file_relative))
         pass
 
     URL_REGEX = re.compile(
@@ -124,7 +124,7 @@ class CmdDataImport(CmdBase):
                     last_reported += chunk_size
                     if last_reported >= report_bucket:
                         last_reported = 0
-                        Logger.verbose('Downloaded {}'.format(sizeof_fmt(downloaded)))
+                        Logger.debug('Downloaded {}'.format(sizeof_fmt(downloaded)))
                     f.write(chunk)
         return
 

neatlynx/cmd_data_remove.py

@@ -30,7 +30,7 @@ class CmdDataRemove(CmdBase):
         lock = fasteners.InterProcessLock(self.git.lock_file)
         gotten = lock.acquire(timeout=5)
         if not gotten:
-            Logger.info('Cannot perform the command since NLX is busy and locked. Please retry the command later.')
+            Logger.printing('Cannot perform the command since NLX is busy and locked. Please retry the command later.')
             return 1
 
         try:
@@ -92,7 +92,7 @@ class CmdDataRemove(CmdBase):
                 Logger.warn('S3 remove warning: file "{}" does not exist in S3'.format(aws_file_name))
             else:
                 key.delete()
-                Logger.info('File "{}" was removed from S3'.format(aws_file_name))
+                Logger.printing('File "{}" was removed from S3'.format(aws_file_name))
 
     def remove_dir(self, data_dir):
         for f in os.listdir(data_dir):

neatlynx/cmd_data_sync.py

@@ -23,7 +23,7 @@ def sizeof_fmt(num, suffix='B'):
 
 
 def percent_cb(complete, total):
-    Logger.verbose('{} transferred out of {}'.format(sizeof_fmt(complete), sizeof_fmt(total)))
+    Logger.debug('{} transferred out of {}'.format(sizeof_fmt(complete), sizeof_fmt(total)))
 
 
 def file_md5(fname):
@@ -44,7 +44,7 @@ class CmdDataSync(CmdBase):
         self._bucket = conn.lookup(bucket_name)
         if not self._bucket:
             self._bucket = conn.create_bucket(bucket_name)
-            Logger.info('S3 bucket "{}" was created'.format(bucket_name))
+            Logger.printing('S3 bucket "{}" was created'.format(bucket_name))
         pass
 
     def define_args(self, parser):
@@ -84,30 +84,30 @@ class CmdDataSync(CmdBase):
         if not key:
             raise DataSyncError('File "{}" does not exist in the cloud'.format(dobj.cache_file_aws_key))
 
-        Logger.info('Downloading cache file from S3 "{}/{}"'.format(self._bucket.name,
-                                                                    dobj.cache_file_aws_key))
+        Logger.printing('Downloading cache file from S3 "{}/{}"'.format(self._bucket.name,
+                                                                        dobj.cache_file_aws_key))
         key.get_contents_to_filename(dobj.cache_file_relative, cb=percent_cb)
-        Logger.info('Downloading completed')
+        Logger.printing('Downloading completed')
         pass
 
     def sync_to_cloud(self, dobj):
         key = self._bucket.get_key(dobj.cache_file_aws_key)
         if key:
-            Logger.verbose('File already uploaded to the cloud. Checksum validation...')
+            Logger.debug('File already uploaded to the cloud. Checksum validation...')
 
             md5_cloud = key.etag[1:-1]
             md5_local = file_md5(dobj.cache_file_relative)
             if md5_cloud == md5_local:
-                Logger.verbose('File checksum matches. No uploading is needed.')
+                Logger.debug('File checksum matches. No uploading is needed.')
                 return
 
-            Logger.info('Checksum miss-match. Re-uploading is required.')
+            Logger.printing('Checksum miss-match. Re-uploading is required.')
 
-        Logger.info('Uploading cache file "{}" to S3 "{}"'.format(dobj.cache_file_relative,
-                                                                  dobj.cache_file_aws_key))
+        Logger.printing('Uploading cache file "{}" to S3 "{}"'.format(dobj.cache_file_relative,
+                                                                      dobj.cache_file_aws_key))
         key = self._bucket.new_key(dobj.cache_file_aws_key)
         key.set_contents_from_filename(dobj.cache_file_relative, cb=percent_cb)
-        Logger.info('Uploading completed')
+        Logger.printing('Uploading completed')
         pass
 
 

neatlynx/cmd_init.py

@@ -76,7 +76,7 @@ SecurityGroup = neatlynx-group'''
         data_dir_path.mkdir()
         cache_dir_path.mkdir()
         state_dir_path.mkdir()
-        Logger.info('Directories {}, {} and {} were created'.format(
+        Logger.printing('Directories {}, {} and {} were created'.format(
             data_dir_path.name,
             cache_dir_path.name,
             state_dir_path.name))
@@ -92,12 +92,12 @@ SecurityGroup = neatlynx-group'''
         gitignore_file = os.path.join(self.git.git_dir, '.gitignore')
         if not os.path.exists(gitignore_file):
             open(gitignore_file, 'a').close()
-            Logger.info('File .gitignore was created')
+            Logger.printing('File .gitignore was created')
         with open(gitignore_file, 'a') as fd:
             fd.write('\n{}'.format(cache_dir_name))
             fd.write('\n{}'.format(os.path.basename(self.git.lock_file)))
 
-        Logger.info('Directory {} was added to .gitignore file'.format(cache_dir_name))
+        Logger.printing('Directory {} was added to .gitignore file'.format(cache_dir_name))
 
 
 if __name__ == '__main__':

neatlynx/cmd_repro.py

@@ -1,6 +1,10 @@
 import os
 
+import fasteners
+
 from neatlynx.cmd_base import CmdBase
+from neatlynx.cmd_run import CmdRun
+from neatlynx.git_wrapper import GitWrapper
 from neatlynx.logger import Logger
 from neatlynx.exceptions import NeatLynxException
 from neatlynx.data_file_obj import DataFileObj
@@ -12,29 +16,103 @@ class ReproError(NeatLynxException):
         NeatLynxException.__init__(self, 'Run error: {}'.format(msg))
 
 
-class CmdRepro(CmdBase):
+class CmdRepro(CmdRun):
     def __init__(self):
         CmdBase.__init__(self)
         pass
 
     def define_args(self, parser):
-        self.add_string_arg(parser, 'target', 'Reproduce data file')
+        self.set_skip_git_actions(parser)
+
+        # self.add_string_arg(parser, 'target', 'Reproduce data file')
+        parser.add_argument('target', metavar='', help='Data file to reproduce', nargs='*')
         pass
 
+    # Overridden methods:
+
+    @property
+    def declaration_input_files(self):
+        return []
+
+    @property
+    def declaration_output_files(self):
+        return []
+
     def run(self):
-        if not self.git.is_ready_to_go():
+        lock = fasteners.InterProcessLock(self.git.lock_file)
+        gotten = lock.acquire(timeout=5)
+        if not gotten:
+            Logger.printing('Cannot perform the command since NLX is busy and locked. Please retry the command later.')
             return 1
 
-        dobj = DataFileObj(self.args.target, self.git, self.config)
-        os.remove(self.args.target)
+        try:
+            if not self.skip_git_actions and not self.git.is_ready_to_go():
+                return 1
 
-        state_file = StateFile(dobj.state_file_relative, self.git)
-        returncode, out, err = state_file.repro()
+            dobjs, externally_created_files = DataFileObj.files_to_dobjs(self.args.target, self.git, self.config)
+            if externally_created_files:
+                Logger.error('Files from outside of the data directory "{}" could not be reproduced: {}'.
+                             format(self.config.data_dir, ' '.join(externally_created_files)))
+                return 1
 
-        print(out)
-        sys.stderr.write(err)
+            error = False
+            for dobj in dobjs:
+                self.repro(dobj)
 
-        return returncode
+                # if returncode != 0:
+                #     error = True
+                #     Logger.error('Error: cannot reproduce file "{}"\n{}'.format(
+                #         dobj.data_file_relative, out))
+                #     sys.stderr.write('{}\n'.format(err))
+
+            if error and not self.skip_git_actions:
+                Logger.error('Errors occurred. One or more repro command was not successful.')
+                self.not_committed_changes_warning()
+                return 1
+
+            if self.skip_git_actions:
+                self.not_committed_changes_warning()
+                return 0
+
+            message = 'NLX repro: {}'.format(' '.join(self.args.target))
+            self.git.commit_all_changes_and_log_status(message)
+        finally:
+            lock.release()
+
+        # dobj = DataFileObj(self.args.target, self.git, self.config)
+        # os.remove(self.args.target)
+        #
+        # state_file = StateFile(dobj.state_file_relative, self.git)
+        # returncode, out, err = state_file.repro()
+        #
+        # print(out)
+        # sys.stderr.write(err)
+        #
+        # return returncode
+        return 0
+
+    def repro(self, dobj):
+        state = StateFile.load(dobj.state_file_relative, self.git)
+
+        argv = state.norm_argv
+
+        if not argv:
+            raise ReproError('Error: parameter {} is nor defined in state file "{}"'.
+                             format(StateFile.PARAM_NORM_ARGV, dobj.state_file_relative))
+        if len(argv) < 2:
+            raise ReproError('Error: reproducible command in state file "{}" is too short'.
+                             format(self.file))
+
+        if argv[0][-3:] != '.py':
+            raise ReproError('Error: reproducible command format error in state file "{}"'.
+                             format(self.file))
+        argv.pop(0)
+
+        Logger.debug('Removing data file "{}"'.format(dobj.data_file_relative))
+        os.remove(dobj.data_file_relative)
+
+        Logger.debug("Repro cmd:\n\t{}".format(' '.join(argv)))
+        return self.run_command(argv)
 
 
 if __name__ == '__main__':

neatlynx/cmd_run.py

@@ -1,13 +1,13 @@
 import os
-import sys
 import shutil
 import fasteners
 
+from neatlynx.data_file_obj import DataFileObj, NotInDataDirError
 from neatlynx.git_wrapper import GitWrapper
 from neatlynx.cmd_base import CmdBase
 from neatlynx.logger import Logger
 from neatlynx.exceptions import NeatLynxException
-from neatlynx.data_file_obj import DataFileObj, NotInDataDirError
+from neatlynx.repository_change import RepositoryChange
 from neatlynx.state_file import StateFile
 
 
@@ -27,59 +27,38 @@ class CmdRun(CmdBase):
         parser.add_argument('--random', help='not reproducible, output is random', action='store_true')
         parser.add_argument('--stdout', help='output std output to a file')
         parser.add_argument('--stderr', help='output std error to a file')
+        parser.add_argument('--input-file', '-i', action='append',
+                            help='Declare input file for reproducible command')
+        parser.add_argument('--output-file', '-o', action='append',
+                            help='Declare output file for reproducible command')
         pass
 
+    @property
+    def declaration_input_files(self):
+        if self.args.input_file:
+            return self.args.input_file
+        return []
+
+    @property
+    def declaration_output_files(self):
+        if self.args.output_file:
+            return self.args.output_file
+        return []
+
     def run(self):
         lock = fasteners.InterProcessLock(self.git.lock_file)
         gotten = lock.acquire(timeout=5)
         if not gotten:
-            Logger.info('Cannot perform the command since NLX is busy and locked. Please retry the command later.')
+            Logger.printing('Cannot perform the command since NLX is busy and locked. Please retry the command later.')
             return 1
 
         try:
             if not self.skip_git_actions and not self.git.is_ready_to_go():
                 return 1
 
-            old_files = set(self.get_changed_files())
-
-            GitWrapper.exec_cmd(self._args_unkn, self.args.stdout, self.args.stderr)
-
-            new_files = set(self.get_changed_files())
-
-            removed_files = old_files - new_files
-            if removed_files != set():
-                Logger.error('Error: existing files were removed in run command'.format(' '.join(removed_files)))
-
-            created_files = new_files - old_files
-
-            if created_files == set():
-                Logger.error('Error: no files were changes in run command')
+            if not self.run_command(self._args_unkn, self.args.stdout, self.args.stderr):
                 return 1
 
-            dobjs, errors = self.files_to_dobjs(created_files)
-
-            if errors:
-                for file in GitWrapper.abs_paths_to_relative(errors):
-                    Logger.error('Error: file "{}" was created outside of the data directory'.format(file))
-
-                for file in created_files:
-                    rel_path = GitWrapper.abs_paths_to_relative([file])[0]
-                    Logger.error('Removing created file: {}'.format(rel_path))
-                    os.remove(file)
-
-                Logger.error('Errors occurred. Please fix the errors and re-run the command.')
-                return 1
-
-            for dobj in dobjs:
-                os.makedirs(os.path.dirname(dobj.cache_file_relative), exist_ok=True)
-                shutil.move(dobj.data_file_relative, dobj.cache_file_relative)
-
-                dobj.create_symlink()
-
-                state_file = StateFile(dobj.state_file_relative, self.git)
-                state_file.save()
-                pass
-
             if self.skip_git_actions:
                 self.not_committed_changes_warning()
                 return 0
@@ -91,49 +70,88 @@ class CmdRun(CmdBase):
 
         return 0
 
-    def files_to_dobjs(self, files):
-        result = []
-        errors = []
+    def run_command(self, argv, stdout=None, stderr=None):
+        repo_change = RepositoryChange(argv, stdout, stderr, self.git, self.config)
 
-        for file in files:
+        if not self.skip_git_actions and not self.validate_file_states(repo_change):
+            self.remove_new_files(repo_change)
+            return False
+
+        output_files = self.git.abs_paths_to_nlx(repo_change.new_files + self.declaration_output_files)
+        input_files_from_args = list(set(self.get_data_files_from_args(argv)) - set(repo_change.new_files))
+        input_files = self.git.abs_paths_to_nlx(input_files_from_args + self.declaration_input_files)
+
+        for dobj in repo_change.dobj_for_new_files:
+            os.makedirs(os.path.dirname(dobj.cache_file_relative), exist_ok=True)
+
+            Logger.debug('Move output file "{}" to cache dir "{}" and create a symlink'.format(
+                dobj.data_file_relative, dobj.cache_file_relative))
+            shutil.move(dobj.data_file_relative, dobj.cache_file_relative)
+
+            dobj.create_symlink()
+
+            Logger.debug('Create state file "{}"'.format(dobj.state_file_relative))
+            state_file = StateFile(dobj.state_file_relative, self.git, input_files, output_files)
+            state_file.save()
+            pass
+
+        return True
+
+    @staticmethod
+    def remove_new_files(repo_change):
+        for file in repo_change.new_files:
+            rel_path = GitWrapper.abs_paths_to_relative([file])[0]
+            Logger.error('Removing created file: {}'.format(rel_path))
+            os.remove(file)
+        pass
+
+    def validate_file_states(self, files_states):
+        error = False
+        for file in GitWrapper.abs_paths_to_relative(files_states.removed_files):
+            Logger.error('Error: file "{}" was removed'.format(file))
+            error = True
+
+        for file in GitWrapper.abs_paths_to_relative(files_states.modified_files):
+            Logger.error('Error: file "{}" was modified'.format(file))
+            error = True
+
+        for file in GitWrapper.abs_paths_to_relative(files_states.unusual_state_files):
+            Logger.error('Error: file "{}" is in not acceptable state'.format(file))
+            error = True
+
+        for file in GitWrapper.abs_paths_to_relative(files_states.externally_created_files):
+            Logger.error('Error: file "{}" was created outside of the data directory'.format(file))
+            error = True
+
+        if error:
+            Logger.error('Errors occurred. ' + \
+                         'Reproducible commands allow only file creation only in data directory "{}".'.
+                         format(self.config.data_dir))
+            return False
+
+        if not files_states.new_files:
+            Logger.error('Errors occurred. No files were changed in run command.')
+            return False
+
+        return True
+
+    def get_data_files_from_args(self, argv):
+        result = []
+
+        for arg in argv:
             try:
-                result.append(DataFileObj(file, self.git, self.config))
+                if os.path.isfile(arg):
+                    DataFileObj(arg, self.git, self.config)
+                    result.append(arg)
             except NotInDataDirError:
-                errors.append(file)
-
-        return result, errors
-
-    def get_changed_files(self):
-        statuses = GitWrapper.status_files()
-
-        result = []
-        for status, file in statuses:
-            file_path = os.path.join(self.git.git_dir_abs, file)
-            if os.path.isfile(file_path):
-                result.append(file_path)
-            else:
-                files = []
-                self.get_all_files_from_dir(file_path, files)
-                for f in files:
-                    result.append(f)
+                pass
 
         return result
 
-    def get_all_files_from_dir(self, dir, result):
-        if not os.path.isdir(dir):
-            raise RunError('Changed path {} is not directory'.format(dir))
-
-        files = os.listdir(dir)
-        for f in files:
-            path = os.path.join(dir, f)
-            if os.path.isfile(path):
-                result.append(path)
-            else:
-                self.get_all_files_from_dir(path, result)
-        pass
 
 if __name__ == '__main__':
     import sys
+
     try:
         sys.exit(CmdRun().run())
     except NeatLynxException as e:

neatlynx/data_file_obj.py

@@ -106,6 +106,19 @@ class DataFileObj(object):
     def state_file_relative(self):
         return os.path.relpath(self.state_file_abs, self._curr_dir_abs)
 
+    @staticmethod
+    def files_to_dobjs(files, git, config):
+        result = []
+        externally_created_files = []
+
+        for file in files:
+            try:
+                result.append(DataFileObj(file, git, config))
+            except NotInDataDirError:
+                externally_created_files.append(file)
+
+        return result, externally_created_files
+
 
 class DataFileObjExisting(DataFileObj):
     def __init__(self, data_file, git, config):

neatlynx/git_wrapper.py

@@ -70,7 +70,7 @@ class GitWrapper(GitWrapperI):
 
     @staticmethod
     def exec_cmd_only_success(cmd, stdout_file=None, stderr_file=None, cwd=None):
-        code, out, err = GitWrapper.exec_cmd(cmd, stdout_file=None, stderr_file=None, cwd=None)
+        code, out, err = GitWrapper.exec_cmd(cmd, stdout_file=stdout_file, stderr_file=stderr_file, cwd=cwd)
         if code != 0:
             if err:
                 sys.stderr.write(err + '\n')
@@ -88,6 +88,10 @@ class GitWrapper(GitWrapperI):
 
         return True
 
+    @property
+    def curr_dir_nlx(self):
+        return os.path.relpath(os.path.abspath(os.curdir), self.git_dir_abs)
+
     @property
     def git_dir(self):
         if self._git_dir:
@@ -143,9 +147,9 @@ class GitWrapper(GitWrapperI):
 
     def commit_all_changes_and_log_status(self, message):
         statuses = self.commit_all_changes(message)
-        Logger.info('A new commit {} was made in the current branch. Added files:'.format(self.curr_commit))
+        Logger.printing('A new commit {} was made in the current branch. Added files:'.format(self.curr_commit))
         for status, file in statuses:
-            Logger.info('\t{} {}'.format(status, file))
+            Logger.printing('\t{} {}'.format(status, file))
         pass
 
     @property
@@ -158,6 +162,13 @@ class GitWrapper(GitWrapperI):
 
         result = []
         for file in files:
-            result.append(os.path.relpath(file, cur_dir))
+            result.append(os.path.relpath(os.path.realpath(file), cur_dir))
 
-        return result
+        return result
+
+    def abs_paths_to_nlx(self, files):
+        result = []
+        for file in files:
+            result.append(os.path.relpath(os.path.abspath(file), self.git_dir_abs))
+
+        return result

neatlynx/logger.py

@@ -3,17 +3,18 @@ import sys
 
 class Logger(object):
     @staticmethod
-    def info(msg):
-        print('{}'.format(msg))
+    def printing(msg):
+        print(u'{}'.format(msg))
 
     @staticmethod
     def warn(msg):
-        print('{}'.format(msg))
+        print(u'Warning. {}'.format(msg))
 
     @staticmethod
     def error(msg):
-        sys.stderr.write('{}\n'.format(msg))
+        #sys.stderr.write('Error. {}\n'.format(msg))
+        print(u'Error. {}'.format(msg))
 
     @staticmethod
-    def verbose(msg):
-        print('{}'.format(msg))
+    def debug(msg):
+        print(u'Debug. {}'.format(msg))

neatlynx/repository_change.py

@@ -0,0 +1,115 @@
+import os
+
+from neatlynx.data_file_obj import DataFileObj, NotInDataDirError
+from neatlynx.exceptions import NeatLynxException
+from neatlynx.git_wrapper import GitWrapper
+
+
+class RepositoryChangeError(NeatLynxException):
+    def __init__(self, msg):
+        NeatLynxException.__init__(self, 'Repository change error: {}'.format(msg))
+
+
+class FileModificationState(object):
+    STATUS_UNTRACKED = '?'
+    STATUS_DELETE = 'D'
+    STATUS_MODIFIED = 'M'
+
+    def __init__(self, state, file):
+        self.state = state
+        self.file = file
+
+    def _check_status(self, status):
+        return self.state.find(status) >= 0
+
+    @property
+    def is_removed(self):
+        return self._check_status(self.STATUS_DELETE)
+
+    @property
+    def is_modified(self):
+        return self._check_status(self.STATUS_MODIFIED)
+
+    @property
+    def is_new(self):
+        return self._check_status(self.STATUS_UNTRACKED)
+
+    @property
+    def is_unusual(self):
+        return self.is_new or self.is_modified or self.is_removed
+
+
+class RepositoryChange(object):
+    """Pre-condition: git repository has no changes"""
+
+    def __init__(self, args, stdout, stderr, git, config):
+        self.git = git
+        self.config = config
+
+        GitWrapper.exec_cmd_only_success(args, stdout, stderr)
+        self._file_states = self._get_file_states()
+
+        self._new_dobj, self._externally_created_files = DataFileObj.files_to_dobjs(
+            self.new_files, self.git, self.config)
+
+    @staticmethod
+    def exec_cmd(args, stdout, stderr, git):
+        return RepositoryChange(args, stdout, stderr, git)
+
+    @staticmethod
+    def get_filenames(mod_state_list):
+        return list(map(lambda x: x.file, mod_state_list))
+
+    @property
+    def removed_files(self):
+        return self.get_filenames(filter(lambda x: x.is_removed, self._file_states))
+
+    @property
+    def modified_files(self):
+        return self.get_filenames(filter(lambda x: x.is_modified, self._file_states))
+
+    @property
+    def new_files(self):
+        return self.get_filenames(filter(lambda x: x.is_new, self._file_states))
+
+    @property
+    def unusual_state_files(self):
+        return self.get_filenames(filter(lambda x: not x.is_unusual, self._file_states))
+
+    def _get_file_states(self):
+        statuses = GitWrapper.status_files()
+
+        result = []
+        for status, file in statuses:
+            file_path = os.path.join(self.git.git_dir_abs, file)
+            if os.path.isfile(file_path):
+                result.append(FileModificationState(status, file_path))
+            else:
+                files = []
+                self.get_all_files_from_dir(file_path, files)
+                state = self.STATUS_UNKNOWN + self.STATUS_UNKNOWN
+                for f in files:
+                    result.append(FileModificationState(state, f))
+
+        return result
+
+    def get_all_files_from_dir(self, dir, result):
+        if not os.path.isdir(dir):
+            raise RepositoryChangeError('Changed path {} is not directory'.format(dir))
+
+        files = os.listdir(dir)
+        for f in files:
+            path = os.path.join(dir, f)
+            if os.path.isfile(path):
+                result.append(path)
+            else:
+                self.get_all_files_from_dir(path, result)
+        pass
+
+    @property
+    def dobj_for_new_files(self):
+        return self._new_dobj
+
+    @property
+    def externally_created_files(self):
+        return self._externally_created_files

neatlynx/state_file.py

@@ -5,6 +5,7 @@ import time
 
 from neatlynx.exceptions import NeatLynxException
 from neatlynx.git_wrapper import GitWrapper
+from neatlynx.logger import Logger
 
 
 class StateFileError(NeatLynxException):
@@ -16,17 +17,61 @@ class StateFile(object):
     MAGIC = 'NLX-State'
     VERSION = '0.1'
 
-    def __init__(self, file, git):
+    PARAM_TYPE = 'Type'
+    PARAM_VERSION = 'Version'
+    PARAM_ARGV = 'Argv'
+    PARAM_NORM_ARGV = 'NormArgv'
+    PARAM_CWD = 'Cwd'
+    PARAM_CREATED_AT = 'CreatedAt'
+    PARAM_INPUT_FILES = 'InputFiles'
+    PARAM_OUTPUT_FILES = 'OutputFiles'
+
+    def __init__(self, file, git, input_files, output_files,
+                 argv=sys.argv, norm_argv=None,
+                 created_at=time.strftime('%Y-%m-%d %H:%M:%S %z'),
+                 cwd=None):
         self.file = file
         self.git = git
+        self.input_files = input_files
+        self.output_files = output_files
+        self.argv = argv
+        if norm_argv:
+            self.norm_argv = norm_argv
+        else:
+            self.norm_argv = self.normalized_args()
+
+        self.created_at = created_at
+
+        if cwd:
+            self.cwd = cwd
+        else:
+            self.cwd = self.get_nlx_path()
+        pass
+
+    @staticmethod
+    def load(filename, git):
+        with open(filename, 'r') as fd:
+            data = json.load(fd)
+
+        return StateFile(filename,
+                         git,
+                         data.get('InputFiles'),
+                         data.get('OutputFiles'),
+                         data.get('Argv'),
+                         data.get('NormArgv'),
+                         data.get('CreatedAt'),
+                         data.get('Cwd'))
 
     def save(self):
         res = {
-            'Type': self.MAGIC,
-            'Version': self.VERSION,
-            'Argv': sys.argv,
-            'NLX_cwd': self.get_nlx_path(),
-            'CreatedAt': time.strftime('%Y-%m-%d %H:%M:%S %z')
+            self.PARAM_TYPE:        self.MAGIC,
+            self.PARAM_VERSION:     self.VERSION,
+            self.PARAM_ARGV:        self.argv,
+            self.PARAM_NORM_ARGV:   self.norm_argv,
+            self.PARAM_CWD:         self.cwd,
+            self.PARAM_CREATED_AT:  self.created_at,
+            self.PARAM_INPUT_FILES: self.input_files,
+            self.PARAM_OUTPUT_FILES:self.output_files
         }
 
         file_dir = os.path.dirname(self.file)
@@ -37,6 +82,26 @@ class StateFile(object):
             json.dump(res, fd, indent=2)
         pass
 
+    def normalized_args(self):
+        result = []
+
+        if len(sys.argv) > 0:
+            cmd = sys.argv[0]
+            pos = cmd.rfind(os.sep)
+            if pos >= 0:
+                cmd = cmd[pos+1:]
+            result.append(cmd)
+
+            for arg in sys.argv[1:]:
+                if os.path.isfile(arg):
+                    path = os.path.abspath(arg)
+                    nlx_path = os.path.relpath(path, self.git.git_dir_abs)
+                    result.append(nlx_path)
+                else:
+                    result.append(arg)
+
+        return result
+
     def get_nlx_path(self):
         pwd = os.path.realpath(os.curdir)
         if not pwd.startswith(self.git.git_dir_abs):
@@ -44,13 +109,22 @@ class StateFile(object):
 
         return os.path.relpath(pwd, self.git.git_dir_abs)
 
-    def repro(self):
-        with open(self.file, 'r') as fd:
-            res = json.load(fd)
-
-        argv = res['Argv']
-        argv.insert(0, 'python')
-        argv.insert(2, '--ignore-git-status')
-
-        cwd = os.path.join(self.git.git_dir_abs, res['NLX_cwd'])
-        return GitWrapper.exec_cmd(argv, cwd=cwd)
+    # def repro(self):
+    #     with open(self.file, 'r') as fd:
+    #         res = json.load(fd)
+    #
+    #     args = res['NormArgs']
+    #     if not args:
+    #         raise StateFileError('Error: parameter NormalizedArgs is nor defined in state file "{}"'.
+    #                              format(self.file))
+    #     if len(args) < 2:
+    #         raise StateFileError('Error: reproducible command in state file "{}" is too short'.
+    #                              format(self.file))
+    #
+    #     if args[0][-3:] != '.py':
+    #         raise StateFileError('Error: reproducible command format error in state file "{}"'.
+    #                              format(self.file))
+    #     args.pop(0)
+    #
+    #     Logger.info("Repro cmd:\n\t{}".format(' '.join(args)))
+    #     return GitWrapper.exec_cmd(args, cwd=self.git.git_dir_abs)