DataImport: commit after finish and bugfixes

neatlynx/cmd_data_import.py

@@ -5,6 +5,7 @@ import re
 import requests
 
 from neatlynx.cmd_base import CmdBase
+from neatlynx.cmd_data_sync import sizeof_fmt
 from neatlynx.logger import Logger
 from neatlynx.data_file_obj import DataFileObj
 from neatlynx.exceptions import NeatLynxException
@@ -59,14 +60,16 @@ class CmdDataImport(CmdBase):
             Logger.verbose('Input file "{}" was copied to cache "{}"'.format(
                 self.args.input, dobj.cache_file_relative))
 
-        cache_relative_to_data = os.path.relpath(dobj.cache_file_relative, os.path.dirname(dobj.data_file_relative))
-        os.symlink(cache_relative_to_data, dobj.data_file_relative)
+        dobj.create_symlink()
         Logger.verbose('Symlink from data file "{}" to the cache file "{}" was created'.
-                       format(dobj.data_file_relative, cache_relative_to_data))
+                       format(dobj.data_file_relative, dobj.cache_file_relative))
 
         state_file = StateFile(dobj.state_file_relative, self.git)
         state_file.save()
         Logger.verbose('State file "{}" was created'.format(dobj.state_file_relative))
+
+        message = 'NLX data import: {} {}'.format(self.args.input, self.args.output)
+        self.git.commit_all_changes_and_log_status(message)
         pass
 
     URL_REGEX = re.compile(
@@ -84,9 +87,19 @@ class CmdDataImport(CmdBase):
     @staticmethod
     def download_file(from_url, to_file):
         r = requests.get(from_url, stream=True)
+
+        chunk_size = 1024 * 100
+        downloaded = 0
+        last_reported = 0
+        report_bucket = 100*1024*1024
         with open(to_file, 'wb') as f:
-            for chunk in r.iter_content(chunk_size=1024*100):
+            for chunk in r.iter_content(chunk_size=chunk_size):
                 if chunk:  # filter out keep-alive new chunks
+                    downloaded += chunk_size
+                    last_reported += chunk_size
+                    if last_reported >= report_bucket:
+                        last_reported = 0
+                        Logger.verbose('Downloaded {}'.format(sizeof_fmt(downloaded)))
                     f.write(chunk)
         return
 

neatlynx/cmd_init.py

@@ -76,12 +76,28 @@ SecurityGroup = neatlynx-group'''
         data_dir_path.mkdir()
         cache_dir_path.mkdir()
         state_dir_path.mkdir()
+        Logger.info('Directories {}, {} and {} were created'.format(
+            data_dir_path.name,
+            cache_dir_path.name,
+            state_dir_path.name))
 
         conf.write_text(self.CONFIG_TEMPLATE.format(data_dir_path.name,
                                                     cache_dir_path.name,
                                                     state_dir_path.name))
+
+        self.modify_gitignore(cache_dir_path.name)
         pass
 
+    def modify_gitignore(self, cache_dir_name):
+        gitignore_file = os.path.join(self.git.git_dir, '.gitignore')
+        if not os.path.exists(gitignore_file):
+            open(gitignore_file, 'a').close()
+            Logger.info('File .gitignore was created')
+        with open(gitignore_file, 'a') as fd:
+            fd.write('\ncache')
+        Logger.info('Directory {} was added to .gitignore file'.format(cache_dir_name))
+
+
 if __name__ == '__main__':
     try:
         CmdInit().run()

neatlynx/cmd_run.py

@@ -1,4 +1,5 @@
 import os
+import sys
 import shutil
 
 from neatlynx.git_wrapper import GitWrapper
@@ -32,32 +33,59 @@ class CmdRun(CmdBase):
 
         GitWrapper.exec_cmd(self._args_unkn, self.args.stdout, self.args.stderr)
 
+        dobjs = self.get_new_file_objects()
+
+        for dobj in dobjs:
+            os.makedirs(os.path.dirname(dobj.cache_file_relative), exist_ok=True)
+            shutil.move(dobj.data_file_relative, dobj.cache_file_relative)
+
+            dobj.create_symlink()
+
+            state_file = StateFile(dobj.state_file_relative, self.git)
+            state_file.save()
+            pass
+
+        message = 'NLX run: {}'.format(' '.join(sys.argv))
+        self.git.commit_all_changes_and_log_status(message)
+        return 0
+
+    def get_new_file_objects(self):
         statuses = GitWrapper.status_files()
+
         error = False
         dobjs = []
         for status, file in statuses:
             try:
                 file_path = os.path.join(self.git.git_dir_abs, file)
-                dobjs.append(DataFileObj(file_path, self.git, self.config))
+                if os.path.isfile(file_path):
+                    dobjs.append(DataFileObj(file_path, self.git, self.config))
+                else:
+                    files = []
+                    self.get_all_files_from_dir(file_path, files)
+                    for f in files:
+                        dobjs.append(DataFileObj(f, self.git, self.config))
             except NotInDataDirError:
                 Logger.error('Error: file "{}" was created outside of the data directory'.format(file_path))
                 error = True
 
         if error:
             Logger.error('Please fix the errors and re-run the command')
-            return 1
+            return None
 
-        for dobj in dobjs:
-            os.makedirs(os.path.dirname(dobj.cache_file_relative), exist_ok=True)
-            shutil.move(dobj.data_file_relative, dobj.cache_file_relative)
-            os.symlink(dobj.cache_file_relative, dobj.data_file_relative)
+        return dobjs
 
-            state_file = StateFile(dobj.state_file_relative, self.git)
-            state_file.save()
-            pass
-
-        return 0
+    def get_all_files_from_dir(self, dir, result):
+        if not os.path.isdir(dir):
+            raise RunError('Changed path {} is not directory'.format(dir))
 
+        files = os.listdir(dir)
+        for f in files:
+            path = os.path.join(dir, f)
+            if os.path.isfile(path):
+                result.append(path)
+            else:
+                self.get_all_files_from_dir(path, result)
+        pass
 
 if __name__ == '__main__':
     import sys

neatlynx/data_file_obj.py

@@ -33,6 +33,11 @@ class DataFileObj(object):
             raise NotInDataDirError(self._data_file, self._config.data_dir)
         pass
 
+    def create_symlink(self):
+        data_dir = os.path.dirname(self.data_file_relative)
+        cache_relative_to_data_dir = os.path.relpath(self.cache_file_relative, data_dir)
+        os.symlink(cache_relative_to_data_dir, self.data_file_relative)
+
     @property
     def data_dir_abs(self):
         return os.path.join(self._git.git_dir_abs, self._config.data_dir)

neatlynx/git_wrapper.py

@@ -60,6 +60,13 @@ class GitWrapper(GitWrapperI):
 
         return p.returncode, out, err
 
+    @staticmethod
+    def exec_cmd_only_success(cmd, stdout_file=None, stderr_file=None, cwd=None):
+        code, out, err = GitWrapper.exec_cmd(cmd, stdout_file=None, stderr_file=None, cwd=None)
+        if code != 0:
+            raise GitCmdError('Git command error ({}): {}'.format(' '.join(cmd), err))
+        return out
+
     def is_ready_to_go(self):
         statuses = self.status_files()
         if len(statuses) > 0:
@@ -95,6 +102,10 @@ class GitWrapper(GitWrapperI):
         if code != 0:
             raise GitCmdError('Git command error - {}'.format(err))
 
+        return GitWrapper.parse_porcelain_files(out)
+
+    @staticmethod
+    def parse_porcelain_files(out):
         result = []
         if len(out) > 0:
             lines = out.split('\n')
@@ -102,14 +113,26 @@ class GitWrapper(GitWrapperI):
                 status = line[:2]
                 file = line[3:]
                 result.append((status, file))
-
         return result
 
     @property
     def curr_commit(self):
         if self._commit is None:
-            code, out, err = GitWrapper.exec_cmd(['git', 'rev-parse', 'HEAD'])
+            code, out, err = GitWrapper.exec_cmd(['git', 'rev-parse', '--short', 'HEAD'])
             if code != 0:
                 raise GitCmdError('Git command error - {}'.format(err))
             self._commit = out
         return self._commit
+
+    def commit_all_changes(self, message):
+        GitWrapper.exec_cmd_only_success(['git', 'add', '--all'])
+        out_status = GitWrapper.exec_cmd_only_success(['git', 'status', '--porcelain'])
+        GitWrapper.exec_cmd_only_success(['git', 'commit', '-m', message])
+        return GitWrapper.parse_porcelain_files(out_status)
+
+    def commit_all_changes_and_log_status(self, message):
+        statuses = self.commit_all_changes(message)
+        Logger.info('A new commit {} was made in the current branch. Added files:'.format(self.curr_commit))
+        for status, file in statuses:
+            Logger.info('\t{} {}'.format(status, file))
+        pass