From 63c8aa605094e800362341818bac9063f9da45db Mon Sep 17 00:00:00 2001 From: Dmitry Petrov Date: Sun, 12 Mar 2017 19:09:38 -0700 Subject: [PATCH] Multiple inputs for import cmd and locks --- neatlynx/cmd_base.py | 2 +- neatlynx/cmd_data_import.py | 57 ++++++++++++++++++++++++------------- neatlynx/cmd_data_remove.py | 34 ++++++++++++++-------- neatlynx/cmd_init.py | 4 ++- neatlynx/cmd_run.py | 35 +++++++++++++++-------- neatlynx/config.py | 5 ++++ neatlynx/git_wrapper.py | 7 ++++- 7 files changed, 98 insertions(+), 46 deletions(-) diff --git a/neatlynx/cmd_base.py b/neatlynx/cmd_base.py index e213183c0..5836c507f 100644 --- a/neatlynx/cmd_base.py +++ b/neatlynx/cmd_base.py @@ -15,7 +15,7 @@ class CmdBase(object): self._config = None if parse_config: - self._config = Config(os.path.join(self.git.git_dir, self.CONFIG)) + self._config = Config(os.path.realpath(os.path.join(self.git.git_dir, self.CONFIG))) parser = argparse.ArgumentParser() self.define_args(parser) diff --git a/neatlynx/cmd_data_import.py b/neatlynx/cmd_data_import.py index 39a49edd4..01623b17e 100644 --- a/neatlynx/cmd_data_import.py +++ b/neatlynx/cmd_data_import.py @@ -2,6 +2,7 @@ import sys import os from shutil import copyfile import re +import fasteners import requests from neatlynx.cmd_base import CmdBase @@ -23,23 +24,44 @@ class CmdDataImport(CmdBase): pass def define_args(self, parser): - self.add_string_arg(parser, 'input', 'Input file') + parser.add_argument('input', + metavar='', + help='Input file', + nargs='*') + self.add_string_arg(parser, 'output', 'Output file') pass def run(self): - if not self.git.is_ready_to_go(): + lock = fasteners.InterProcessLock(self.git.lock_file) + gotten = lock.acquire(timeout=5) + if not gotten: + Logger.info('Cannot perform the command since NLX is busy and locked. Please retry the command later.') return 1 - if not CmdDataImport.is_url(self.args.input): - if not os.path.exists(self.args.input): - raise DataImportError('Input file "{}" does not exist'.format(self.args.input)) - if not os.path.isfile(self.args.input): - raise DataImportError('Input file "{}" has to be a regular file'.format(self.args.input)) + try: + if not self.git.is_ready_to_go(): + return 1 - output = self.args.output - if os.path.isdir(self.args.output): - output = os.path.join(output, os.path.basename(self.args.input)) + output = self.args.output + for file in self.args.input: + self.import_file(file, output) + + message = 'NLX data import: {} {}'.format(' '.join(self.args.input), self.args.output) + self.git.commit_all_changes_and_log_status(message) + finally: + lock.release() + pass + + def import_file(self, input, output): + if not CmdDataImport.is_url(input): + if not os.path.exists(input): + raise DataImportError('Input file "{}" does not exist'.format(input)) + if not os.path.isfile(input): + raise DataImportError('Input file "{}" has to be a regular file'.format(input)) + + if os.path.isdir(output): + output = os.path.join(output, os.path.basename(input)) dobj = DataFileObj(output, self.git, self.config) @@ -50,15 +72,15 @@ class CmdDataImport(CmdBase): os.path.dirname(dobj.data_file_relative))) os.makedirs(os.path.dirname(dobj.cache_file_relative), exist_ok=True) - if CmdDataImport.is_url(self.args.input): - Logger.verbose('Downloading file {} ...'.format(self.args.input)) - self.download_file(self.args.input, dobj.cache_file_relative) + if CmdDataImport.is_url(input): + Logger.verbose('Downloading file {} ...'.format(input)) + self.download_file(input, dobj.cache_file_relative) Logger.verbose('Input file "{}" was downloaded to cache "{}"'.format( - self.args.input, dobj.cache_file_relative)) + input, dobj.cache_file_relative)) else: - copyfile(self.args.input, dobj.cache_file_relative) + copyfile(input, dobj.cache_file_relative) Logger.verbose('Input file "{}" was copied to cache "{}"'.format( - self.args.input, dobj.cache_file_relative)) + input, dobj.cache_file_relative)) dobj.create_symlink() Logger.verbose('Symlink from data file "{}" to the cache file "{}" was created'. @@ -67,9 +89,6 @@ class CmdDataImport(CmdBase): state_file = StateFile(dobj.state_file_relative, self.git) state_file.save() Logger.verbose('State file "{}" was created'.format(dobj.state_file_relative)) - - message = 'NLX data import: {} {}'.format(self.args.input, self.args.output) - self.git.commit_all_changes_and_log_status(message) pass URL_REGEX = re.compile( diff --git a/neatlynx/cmd_data_remove.py b/neatlynx/cmd_data_remove.py index 73569023a..fb61ad47b 100644 --- a/neatlynx/cmd_data_remove.py +++ b/neatlynx/cmd_data_remove.py @@ -1,5 +1,6 @@ import os from boto.s3.connection import S3Connection +import fasteners from neatlynx.cmd_base import CmdBase from neatlynx.logger import Logger @@ -24,23 +25,32 @@ class CmdDataRemove(CmdBase): pass def run(self): - target = self.args.target + lock = fasteners.InterProcessLock(self.git.lock_file) + gotten = lock.acquire(timeout=5) + if not gotten: + Logger.info('Cannot perform the command since NLX is busy and locked. Please retry the command later.') + return 1 - if os.path.isdir(target): - if not self.args.recursive: - raise DataRemoveError('Directory cannot be removed. Use --recurcive flag.') + try: + target = self.args.target - if os.path.realpath(target) == \ - os.path.realpath(os.path.join(self.git.git_dir_abs, self.config.data_dir)): - raise DataRemoveError('data directory cannot be removed') + if os.path.isdir(target): + if not self.args.recursive: + raise DataRemoveError('Directory cannot be removed. Use --recurcive flag.') - return self.remove_dir(target) + if os.path.realpath(target) == \ + os.path.realpath(os.path.join(self.git.git_dir_abs, self.config.data_dir)): + raise DataRemoveError('data directory cannot be removed') - dobj = DataFileObjExisting(target, self.git, self.config) - if os.path.islink(dobj.data_file_relative): - return self.remove_symlink(dobj.data_file_relative) + return self.remove_dir(target) - raise DataRemoveError('Cannot remove a regular file "{}"'.format(target)) + dobj = DataFileObjExisting(target, self.git, self.config) + if os.path.islink(dobj.data_file_relative): + return self.remove_symlink(dobj.data_file_relative) + + raise DataRemoveError('Cannot remove a regular file "{}"'.format(target)) + finally: + lock.release() def remove_symlink(self, file): dobj = DataFileObjExisting(file, self.git, self.config) diff --git a/neatlynx/cmd_init.py b/neatlynx/cmd_init.py index 104b6758e..86a2c17f3 100644 --- a/neatlynx/cmd_init.py +++ b/neatlynx/cmd_init.py @@ -94,7 +94,9 @@ SecurityGroup = neatlynx-group''' open(gitignore_file, 'a').close() Logger.info('File .gitignore was created') with open(gitignore_file, 'a') as fd: - fd.write('\ncache') + fd.write('\n{}'.format(cache_dir_name)) + fd.write('\n{}'.format(os.path.basename(self.git.lock_file))) + Logger.info('Directory {} was added to .gitignore file'.format(cache_dir_name)) diff --git a/neatlynx/cmd_run.py b/neatlynx/cmd_run.py index 4d2d9cef8..384c0f020 100644 --- a/neatlynx/cmd_run.py +++ b/neatlynx/cmd_run.py @@ -1,6 +1,7 @@ import os import sys import shutil +import fasteners from neatlynx.git_wrapper import GitWrapper from neatlynx.cmd_base import CmdBase @@ -28,25 +29,35 @@ class CmdRun(CmdBase): pass def run(self): - if not self.args.ignore_git_status and not self.git.is_ready_to_go(): + lock = fasteners.InterProcessLock(self.lock.lock_file) + gotten = lock.acquire(timeout=5) + if not gotten: + Logger.info('Cannot perform the command since NLX is busy and locked. Please retry the command later.') return 1 - GitWrapper.exec_cmd(self._args_unkn, self.args.stdout, self.args.stderr) + try: + if not self.args.ignore_git_status and not self.git.is_ready_to_go(): + return 1 - dobjs = self.get_new_file_objects() + GitWrapper.exec_cmd(self._args_unkn, self.args.stdout, self.args.stderr) - for dobj in dobjs: - os.makedirs(os.path.dirname(dobj.cache_file_relative), exist_ok=True) - shutil.move(dobj.data_file_relative, dobj.cache_file_relative) + dobjs = self.get_new_file_objects() - dobj.create_symlink() + for dobj in dobjs: + os.makedirs(os.path.dirname(dobj.cache_file_relative), exist_ok=True) + shutil.move(dobj.data_file_relative, dobj.cache_file_relative) - state_file = StateFile(dobj.state_file_relative, self.git) - state_file.save() - pass + dobj.create_symlink() + + state_file = StateFile(dobj.state_file_relative, self.git) + state_file.save() + pass + + message = 'NLX run: {}'.format(' '.join(sys.argv)) + self.git.commit_all_changes_and_log_status(message) + finally: + lock.release() - message = 'NLX run: {}'.format(' '.join(sys.argv)) - self.git.commit_all_changes_and_log_status(message) return 0 def get_new_file_objects(self): diff --git a/neatlynx/config.py b/neatlynx/config.py index ac42f9d2f..d34313414 100644 --- a/neatlynx/config.py +++ b/neatlynx/config.py @@ -33,6 +33,7 @@ class Config(ConfigI): CONFIG = 'neatlynx.conf' def __init__(self, conf_file): + self._conf_file = conf_file self._config = configparser.ConfigParser() if not os.path.isfile(conf_file): @@ -49,6 +50,10 @@ class Config(ConfigI): # def get_data_file_obj(self): # return DataFileObj(self.config) + @property + def file(self): + return self._conf_file + @property def data_dir(self): return self._config['Global']['DataDir'] diff --git a/neatlynx/git_wrapper.py b/neatlynx/git_wrapper.py index 4c66cbd8c..cfc377d09 100644 --- a/neatlynx/git_wrapper.py +++ b/neatlynx/git_wrapper.py @@ -3,6 +3,7 @@ import subprocess from neatlynx.exceptions import NeatLynxException from neatlynx.logger import Logger +from neatlynx.config import Config class GitCmdError(NeatLynxException): @@ -135,4 +136,8 @@ class GitWrapper(GitWrapperI): Logger.info('A new commit {} was made in the current branch. Added files:'.format(self.curr_commit)) for status, file in statuses: Logger.info('\t{} {}'.format(status, file)) - pass \ No newline at end of file + pass + + @property + def lock_file(self): + return os.path.join(self.git_dir_abs, '.' + Config.CONFIG + '.lock')