Multiple inputs for import cmd and locks

neatlynx/cmd_base.py

@@ -15,7 +15,7 @@ class CmdBase(object):
 
         self._config = None
         if parse_config:
-            self._config = Config(os.path.join(self.git.git_dir, self.CONFIG))
+            self._config = Config(os.path.realpath(os.path.join(self.git.git_dir, self.CONFIG)))
 
         parser = argparse.ArgumentParser()
         self.define_args(parser)

neatlynx/cmd_data_import.py

@@ -2,6 +2,7 @@ import sys
 import os
 from shutil import copyfile
 import re
+import fasteners
 import requests
 
 from neatlynx.cmd_base import CmdBase
@@ -23,23 +24,44 @@ class CmdDataImport(CmdBase):
         pass
 
     def define_args(self, parser):
-        self.add_string_arg(parser, 'input', 'Input file')
+        parser.add_argument('input',
+                            metavar='',
+                            help='Input file',
+                            nargs='*')
+
         self.add_string_arg(parser, 'output', 'Output file')
         pass
 
     def run(self):
-        if not self.git.is_ready_to_go():
+        lock = fasteners.InterProcessLock(self.git.lock_file)
+        gotten = lock.acquire(timeout=5)
+        if not gotten:
+            Logger.info('Cannot perform the command since NLX is busy and locked. Please retry the command later.')
             return 1
 
-        if not CmdDataImport.is_url(self.args.input):
-            if not os.path.exists(self.args.input):
-                raise DataImportError('Input file "{}" does not exist'.format(self.args.input))
-            if not os.path.isfile(self.args.input):
-                raise DataImportError('Input file "{}" has to be a regular file'.format(self.args.input))
+        try:
+            if not self.git.is_ready_to_go():
+                return 1
 
-        output = self.args.output
-        if os.path.isdir(self.args.output):
-            output = os.path.join(output, os.path.basename(self.args.input))
+            output = self.args.output
+            for file in self.args.input:
+                self.import_file(file, output)
+
+            message = 'NLX data import: {} {}'.format(' '.join(self.args.input), self.args.output)
+            self.git.commit_all_changes_and_log_status(message)
+        finally:
+            lock.release()
+        pass
+
+    def import_file(self, input, output):
+        if not CmdDataImport.is_url(input):
+            if not os.path.exists(input):
+                raise DataImportError('Input file "{}" does not exist'.format(input))
+            if not os.path.isfile(input):
+                raise DataImportError('Input file "{}" has to be a regular file'.format(input))
+
+        if os.path.isdir(output):
+            output = os.path.join(output, os.path.basename(input))
 
         dobj = DataFileObj(output, self.git, self.config)
 
@@ -50,15 +72,15 @@ class CmdDataImport(CmdBase):
                 os.path.dirname(dobj.data_file_relative)))
 
         os.makedirs(os.path.dirname(dobj.cache_file_relative), exist_ok=True)
-        if CmdDataImport.is_url(self.args.input):
-            Logger.verbose('Downloading file {} ...'.format(self.args.input))
-            self.download_file(self.args.input, dobj.cache_file_relative)
+        if CmdDataImport.is_url(input):
+            Logger.verbose('Downloading file {} ...'.format(input))
+            self.download_file(input, dobj.cache_file_relative)
             Logger.verbose('Input file "{}" was downloaded to cache "{}"'.format(
-                self.args.input, dobj.cache_file_relative))
+                input, dobj.cache_file_relative))
         else:
-            copyfile(self.args.input, dobj.cache_file_relative)
+            copyfile(input, dobj.cache_file_relative)
             Logger.verbose('Input file "{}" was copied to cache "{}"'.format(
-                self.args.input, dobj.cache_file_relative))
+                input, dobj.cache_file_relative))
 
         dobj.create_symlink()
         Logger.verbose('Symlink from data file "{}" to the cache file "{}" was created'.
@@ -67,9 +89,6 @@ class CmdDataImport(CmdBase):
         state_file = StateFile(dobj.state_file_relative, self.git)
         state_file.save()
         Logger.verbose('State file "{}" was created'.format(dobj.state_file_relative))
-
-        message = 'NLX data import: {} {}'.format(self.args.input, self.args.output)
-        self.git.commit_all_changes_and_log_status(message)
         pass
 
     URL_REGEX = re.compile(

neatlynx/cmd_data_remove.py

@@ -1,5 +1,6 @@
 import os
 from boto.s3.connection import S3Connection
+import fasteners
 
 from neatlynx.cmd_base import CmdBase
 from neatlynx.logger import Logger
@@ -24,23 +25,32 @@ class CmdDataRemove(CmdBase):
         pass
 
     def run(self):
-        target = self.args.target
+        lock = fasteners.InterProcessLock(self.git.lock_file)
+        gotten = lock.acquire(timeout=5)
+        if not gotten:
+            Logger.info('Cannot perform the command since NLX is busy and locked. Please retry the command later.')
+            return 1
 
-        if os.path.isdir(target):
-            if not self.args.recursive:
-                raise DataRemoveError('Directory cannot be removed. Use --recurcive flag.')
+        try:
+            target = self.args.target
 
-            if os.path.realpath(target) == \
-                    os.path.realpath(os.path.join(self.git.git_dir_abs, self.config.data_dir)):
-                raise DataRemoveError('data directory cannot be removed')
+            if os.path.isdir(target):
+                if not self.args.recursive:
+                    raise DataRemoveError('Directory cannot be removed. Use --recurcive flag.')
 
-            return self.remove_dir(target)
+                if os.path.realpath(target) == \
+                        os.path.realpath(os.path.join(self.git.git_dir_abs, self.config.data_dir)):
+                    raise DataRemoveError('data directory cannot be removed')
 
-        dobj = DataFileObjExisting(target, self.git, self.config)
-        if os.path.islink(dobj.data_file_relative):
-            return self.remove_symlink(dobj.data_file_relative)
+                return self.remove_dir(target)
 
-        raise DataRemoveError('Cannot remove a regular file "{}"'.format(target))
+            dobj = DataFileObjExisting(target, self.git, self.config)
+            if os.path.islink(dobj.data_file_relative):
+                return self.remove_symlink(dobj.data_file_relative)
+
+            raise DataRemoveError('Cannot remove a regular file "{}"'.format(target))
+        finally:
+            lock.release()
 
     def remove_symlink(self, file):
         dobj = DataFileObjExisting(file, self.git, self.config)

neatlynx/cmd_init.py

@@ -94,7 +94,9 @@ SecurityGroup = neatlynx-group'''
             open(gitignore_file, 'a').close()
             Logger.info('File .gitignore was created')
         with open(gitignore_file, 'a') as fd:
-            fd.write('\ncache')
+            fd.write('\n{}'.format(cache_dir_name))
+            fd.write('\n{}'.format(os.path.basename(self.git.lock_file)))
+
         Logger.info('Directory {} was added to .gitignore file'.format(cache_dir_name))
 
 

neatlynx/cmd_run.py

@@ -1,6 +1,7 @@
 import os
 import sys
 import shutil
+import fasteners
 
 from neatlynx.git_wrapper import GitWrapper
 from neatlynx.cmd_base import CmdBase
@@ -28,25 +29,35 @@ class CmdRun(CmdBase):
         pass
 
     def run(self):
-        if not self.args.ignore_git_status and not self.git.is_ready_to_go():
+        lock = fasteners.InterProcessLock(self.lock.lock_file)
+        gotten = lock.acquire(timeout=5)
+        if not gotten:
+            Logger.info('Cannot perform the command since NLX is busy and locked. Please retry the command later.')
             return 1
 
-        GitWrapper.exec_cmd(self._args_unkn, self.args.stdout, self.args.stderr)
+        try:
+            if not self.args.ignore_git_status and not self.git.is_ready_to_go():
+                return 1
 
-        dobjs = self.get_new_file_objects()
+            GitWrapper.exec_cmd(self._args_unkn, self.args.stdout, self.args.stderr)
 
-        for dobj in dobjs:
-            os.makedirs(os.path.dirname(dobj.cache_file_relative), exist_ok=True)
-            shutil.move(dobj.data_file_relative, dobj.cache_file_relative)
+            dobjs = self.get_new_file_objects()
 
-            dobj.create_symlink()
+            for dobj in dobjs:
+                os.makedirs(os.path.dirname(dobj.cache_file_relative), exist_ok=True)
+                shutil.move(dobj.data_file_relative, dobj.cache_file_relative)
 
-            state_file = StateFile(dobj.state_file_relative, self.git)
-            state_file.save()
-            pass
+                dobj.create_symlink()
+
+                state_file = StateFile(dobj.state_file_relative, self.git)
+                state_file.save()
+                pass
+
+            message = 'NLX run: {}'.format(' '.join(sys.argv))
+            self.git.commit_all_changes_and_log_status(message)
+        finally:
+            lock.release()
 
-        message = 'NLX run: {}'.format(' '.join(sys.argv))
-        self.git.commit_all_changes_and_log_status(message)
         return 0
 
     def get_new_file_objects(self):

neatlynx/config.py

@@ -33,6 +33,7 @@ class Config(ConfigI):
     CONFIG = 'neatlynx.conf'
 
     def __init__(self, conf_file):
+        self._conf_file = conf_file
         self._config = configparser.ConfigParser()
 
         if not os.path.isfile(conf_file):
@@ -49,6 +50,10 @@ class Config(ConfigI):
     # def get_data_file_obj(self):
     #     return DataFileObj(self.config)
 
+    @property
+    def file(self):
+        return self._conf_file
+
     @property
     def data_dir(self):
         return self._config['Global']['DataDir']

neatlynx/git_wrapper.py

@@ -3,6 +3,7 @@ import subprocess
 
 from neatlynx.exceptions import NeatLynxException
 from neatlynx.logger import Logger
+from neatlynx.config import Config
 
 
 class GitCmdError(NeatLynxException):
@@ -135,4 +136,8 @@ class GitWrapper(GitWrapperI):
         Logger.info('A new commit {} was made in the current branch. Added files:'.format(self.curr_commit))
         for status, file in statuses:
             Logger.info('\t{} {}'.format(status, file))
-        pass
+        pass
+
+    @property
+    def lock_file(self):
+        return os.path.join(self.git_dir_abs, '.' + Config.CONFIG + '.lock')