From d3cb95d94f32715fcc253e75182cb6eeb6ba52db Mon Sep 17 00:00:00 2001 From: Dmitry Petrov Date: Sun, 12 Mar 2017 01:02:36 -0800 Subject: [PATCH] DataImport: commit after finish and bugfixes --- neatlynx/cmd_data_import.py | 21 +++++++++++++--- neatlynx/cmd_init.py | 16 ++++++++++++ neatlynx/cmd_run.py | 50 +++++++++++++++++++++++++++++-------- neatlynx/data_file_obj.py | 5 ++++ neatlynx/git_wrapper.py | 27 ++++++++++++++++++-- 5 files changed, 102 insertions(+), 17 deletions(-) diff --git a/neatlynx/cmd_data_import.py b/neatlynx/cmd_data_import.py index dc88a58d2..39a49edd4 100644 --- a/neatlynx/cmd_data_import.py +++ b/neatlynx/cmd_data_import.py @@ -5,6 +5,7 @@ import re import requests from neatlynx.cmd_base import CmdBase +from neatlynx.cmd_data_sync import sizeof_fmt from neatlynx.logger import Logger from neatlynx.data_file_obj import DataFileObj from neatlynx.exceptions import NeatLynxException @@ -59,14 +60,16 @@ class CmdDataImport(CmdBase): Logger.verbose('Input file "{}" was copied to cache "{}"'.format( self.args.input, dobj.cache_file_relative)) - cache_relative_to_data = os.path.relpath(dobj.cache_file_relative, os.path.dirname(dobj.data_file_relative)) - os.symlink(cache_relative_to_data, dobj.data_file_relative) + dobj.create_symlink() Logger.verbose('Symlink from data file "{}" to the cache file "{}" was created'. - format(dobj.data_file_relative, cache_relative_to_data)) + format(dobj.data_file_relative, dobj.cache_file_relative)) state_file = StateFile(dobj.state_file_relative, self.git) state_file.save() Logger.verbose('State file "{}" was created'.format(dobj.state_file_relative)) + + message = 'NLX data import: {} {}'.format(self.args.input, self.args.output) + self.git.commit_all_changes_and_log_status(message) pass URL_REGEX = re.compile( @@ -84,9 +87,19 @@ class CmdDataImport(CmdBase): @staticmethod def download_file(from_url, to_file): r = requests.get(from_url, stream=True) + + chunk_size = 1024 * 100 + downloaded = 0 + last_reported = 0 + report_bucket = 100*1024*1024 with open(to_file, 'wb') as f: - for chunk in r.iter_content(chunk_size=1024*100): + for chunk in r.iter_content(chunk_size=chunk_size): if chunk: # filter out keep-alive new chunks + downloaded += chunk_size + last_reported += chunk_size + if last_reported >= report_bucket: + last_reported = 0 + Logger.verbose('Downloaded {}'.format(sizeof_fmt(downloaded))) f.write(chunk) return diff --git a/neatlynx/cmd_init.py b/neatlynx/cmd_init.py index 0f3a969b9..104b6758e 100644 --- a/neatlynx/cmd_init.py +++ b/neatlynx/cmd_init.py @@ -76,12 +76,28 @@ SecurityGroup = neatlynx-group''' data_dir_path.mkdir() cache_dir_path.mkdir() state_dir_path.mkdir() + Logger.info('Directories {}, {} and {} were created'.format( + data_dir_path.name, + cache_dir_path.name, + state_dir_path.name)) conf.write_text(self.CONFIG_TEMPLATE.format(data_dir_path.name, cache_dir_path.name, state_dir_path.name)) + + self.modify_gitignore(cache_dir_path.name) pass + def modify_gitignore(self, cache_dir_name): + gitignore_file = os.path.join(self.git.git_dir, '.gitignore') + if not os.path.exists(gitignore_file): + open(gitignore_file, 'a').close() + Logger.info('File .gitignore was created') + with open(gitignore_file, 'a') as fd: + fd.write('\ncache') + Logger.info('Directory {} was added to .gitignore file'.format(cache_dir_name)) + + if __name__ == '__main__': try: CmdInit().run() diff --git a/neatlynx/cmd_run.py b/neatlynx/cmd_run.py index 04acb6fe6..4d2d9cef8 100644 --- a/neatlynx/cmd_run.py +++ b/neatlynx/cmd_run.py @@ -1,4 +1,5 @@ import os +import sys import shutil from neatlynx.git_wrapper import GitWrapper @@ -32,32 +33,59 @@ class CmdRun(CmdBase): GitWrapper.exec_cmd(self._args_unkn, self.args.stdout, self.args.stderr) + dobjs = self.get_new_file_objects() + + for dobj in dobjs: + os.makedirs(os.path.dirname(dobj.cache_file_relative), exist_ok=True) + shutil.move(dobj.data_file_relative, dobj.cache_file_relative) + + dobj.create_symlink() + + state_file = StateFile(dobj.state_file_relative, self.git) + state_file.save() + pass + + message = 'NLX run: {}'.format(' '.join(sys.argv)) + self.git.commit_all_changes_and_log_status(message) + return 0 + + def get_new_file_objects(self): statuses = GitWrapper.status_files() + error = False dobjs = [] for status, file in statuses: try: file_path = os.path.join(self.git.git_dir_abs, file) - dobjs.append(DataFileObj(file_path, self.git, self.config)) + if os.path.isfile(file_path): + dobjs.append(DataFileObj(file_path, self.git, self.config)) + else: + files = [] + self.get_all_files_from_dir(file_path, files) + for f in files: + dobjs.append(DataFileObj(f, self.git, self.config)) except NotInDataDirError: Logger.error('Error: file "{}" was created outside of the data directory'.format(file_path)) error = True if error: Logger.error('Please fix the errors and re-run the command') - return 1 + return None - for dobj in dobjs: - os.makedirs(os.path.dirname(dobj.cache_file_relative), exist_ok=True) - shutil.move(dobj.data_file_relative, dobj.cache_file_relative) - os.symlink(dobj.cache_file_relative, dobj.data_file_relative) + return dobjs - state_file = StateFile(dobj.state_file_relative, self.git) - state_file.save() - pass - - return 0 + def get_all_files_from_dir(self, dir, result): + if not os.path.isdir(dir): + raise RunError('Changed path {} is not directory'.format(dir)) + files = os.listdir(dir) + for f in files: + path = os.path.join(dir, f) + if os.path.isfile(path): + result.append(path) + else: + self.get_all_files_from_dir(path, result) + pass if __name__ == '__main__': import sys diff --git a/neatlynx/data_file_obj.py b/neatlynx/data_file_obj.py index 929fcb260..cff0617ca 100644 --- a/neatlynx/data_file_obj.py +++ b/neatlynx/data_file_obj.py @@ -33,6 +33,11 @@ class DataFileObj(object): raise NotInDataDirError(self._data_file, self._config.data_dir) pass + def create_symlink(self): + data_dir = os.path.dirname(self.data_file_relative) + cache_relative_to_data_dir = os.path.relpath(self.cache_file_relative, data_dir) + os.symlink(cache_relative_to_data_dir, self.data_file_relative) + @property def data_dir_abs(self): return os.path.join(self._git.git_dir_abs, self._config.data_dir) diff --git a/neatlynx/git_wrapper.py b/neatlynx/git_wrapper.py index 89df5557e..4c66cbd8c 100644 --- a/neatlynx/git_wrapper.py +++ b/neatlynx/git_wrapper.py @@ -60,6 +60,13 @@ class GitWrapper(GitWrapperI): return p.returncode, out, err + @staticmethod + def exec_cmd_only_success(cmd, stdout_file=None, stderr_file=None, cwd=None): + code, out, err = GitWrapper.exec_cmd(cmd, stdout_file=None, stderr_file=None, cwd=None) + if code != 0: + raise GitCmdError('Git command error ({}): {}'.format(' '.join(cmd), err)) + return out + def is_ready_to_go(self): statuses = self.status_files() if len(statuses) > 0: @@ -95,6 +102,10 @@ class GitWrapper(GitWrapperI): if code != 0: raise GitCmdError('Git command error - {}'.format(err)) + return GitWrapper.parse_porcelain_files(out) + + @staticmethod + def parse_porcelain_files(out): result = [] if len(out) > 0: lines = out.split('\n') @@ -102,14 +113,26 @@ class GitWrapper(GitWrapperI): status = line[:2] file = line[3:] result.append((status, file)) - return result @property def curr_commit(self): if self._commit is None: - code, out, err = GitWrapper.exec_cmd(['git', 'rev-parse', 'HEAD']) + code, out, err = GitWrapper.exec_cmd(['git', 'rev-parse', '--short', 'HEAD']) if code != 0: raise GitCmdError('Git command error - {}'.format(err)) self._commit = out return self._commit + + def commit_all_changes(self, message): + GitWrapper.exec_cmd_only_success(['git', 'add', '--all']) + out_status = GitWrapper.exec_cmd_only_success(['git', 'status', '--porcelain']) + GitWrapper.exec_cmd_only_success(['git', 'commit', '-m', message]) + return GitWrapper.parse_porcelain_files(out_status) + + def commit_all_changes_and_log_status(self, message): + statuses = self.commit_all_changes(message) + Logger.info('A new commit {} was made in the current branch. Added files:'.format(self.curr_commit)) + for status, file in statuses: + Logger.info('\t{} {}'.format(status, file)) + pass \ No newline at end of file