Data import cmd and data obj class abstraction

bin/nlx-data-import

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+PYTHONPATH=$NEATLYNX_HOME python $NEATLYNX_HOME/neatlynx/cmd_data_import.py $@

bin/nlx-init

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-python $NEATLYNX_HOME/lib/cmd_init.py $@
+PYTHONPATH=$NEATLYNX_HOME python $NEATLYNX_HOME/neatlynx/cmd_init.py $@

neatlynx/base_cmd.py

@@ -1,10 +1,9 @@
-import configparser
 import argparse
 import os
-import git
-import subprocess
 
-from neatlynx.exceptions import GitCmdError, ConfigError
+from neatlynx.git_wrapper import GitWrapper
+from neatlynx.config import Config, ConfigError
+
 
 class Logger(object):
     @staticmethod
@@ -21,16 +20,13 @@ class Logger(object):
 
 
 class BaseCmd(object):
-    CONFIG = 'neatlynx.conf'
-
     def __init__(self, parse_config=True):
-        self._git_dir = None
+        self._git = GitWrapper()
         self._args = None
         self._lnx_home = None
 
         if parse_config:
-            self._config = configparser.ConfigParser()
-            self._config.read(os.path.join(self.git_dir, self.CONFIG))
+            self._config = Config(os.path.join(self.git.git_dir, self.CONFIG))
 
         parser = argparse.ArgumentParser()
         self.define_args(parser)
@@ -53,31 +49,13 @@ class BaseCmd(object):
         return self._args
 
     @property
-    def git_dir(self):
-        if self._git_dir:
-            return self._git_dir
-
-        try:
-            p = subprocess.Popen(['git', 'rev-parse', '--show-toplevel'],
-                                 stdout=subprocess.PIPE,
-                                 stderr=subprocess.PIPE)
-            out, err = map(lambda s: s.decode().strip('\n\r'), p.communicate())
-
-            if p.returncode != 0:
-                raise GitCmdError('Git command error - {}'.format(err))
-
-            self._git_dir = out
-            return self._git_dir
-        except GitCmdError as e:
-            raise
-        except Exception as e:
-            raise GitCmdError('Unable to run git command: {}'.format(e))
-        pass
+    def git(self):
+        return self._git
 
     def define_args(self):
         pass
 
-    def add_string_arg(self, parser, name, default, message,
+    def add_string_arg(self, parser, name, message, default = None,
                        conf_section=None, conf_name=None):
         if conf_section and conf_name:
             section = self._config[conf_section]
@@ -93,4 +71,4 @@ class BaseCmd(object):
                             help=message)
 
     def run(self):
-        pass
+        pass

neatlynx/cmd_data_import.py

@@ -0,0 +1,81 @@
+import sys
+import os
+from pathlib import Path
+from shutil import copyfile
+
+from neatlynx.base_cmd import BaseCmd, Logger
+from neatlynx.data_file_obj import DataFileObj
+from neatlynx.exceptions import NeatLynxException
+from neatlynx.git_wrapper import GitWrapper
+
+
+class DataImportError(NeatLynxException):
+    def __init__(self, msg):
+        NeatLynxException.__init__(self, 'Import error: {}'.format(msg))
+
+
+class DataImport(BaseCmd):
+    def __init__(self):
+        BaseCmd.__init__(self)
+        pass
+
+    def define_args(self, parser):
+        self.add_string_arg(parser, 'input', 'Input file')
+        self.add_string_arg(parser, 'output', 'Output file')
+        pass
+
+    def run(self):
+        input = Path(self.args.input)
+        if not input.exists():
+            raise DataImportError('Input file "{}" does not exist'.format(input))
+        if not input.is_file():
+            raise DataImportError('Input file "{}" has to be a regular file'.format(input))
+
+        dataFileObj = DataFileObj(self.args.output, self._config, GitWrapper.curr_commit())
+
+        #output = Path(self.args.output)
+        if dataFileObj.data_file_relative.exists():
+            raise DataImportError('Output file "{}" already exists'.format(dataFileObj.data_file_relative))
+        if os.path.isdir(dataFileObj.data_dir_relative):
+            raise DataImportError('Output file directory "{}" does not exists'.format(dataFileObj.data_dir_relative))
+
+        data_dir_path = Path(self.config.DataDir)
+        if output.parent < data_dir_path:
+            raise DataImportError('Output file has to be in data dir - {}'.format(data_dir_path))
+
+        # data_dir_path_str = str(data_dir_path)
+        # output_dir_str = str(output.parent)
+        # relative_dir = output_dir_str[len(data_dir_path_str):].strip(os.path.sep)
+        #
+        # cache_file_dir = os.path.join(self.config.CachDir, relative_dir)
+        # cache_file_dir_path = Path(cache_file_dir)
+        #
+        # state_file_dir = os.path.join(self.config.StateDir, relative_dir)
+        # state_file_dir_path = Path(state_file_dir)
+        #
+        # commit = GitWrapper.curr_commit()
+        # cache_file_name = output.name + '_' + commit
+        # cache_file = cache_file_dir_path / cache_file_name
+        # state_file = state_file_dir_path / output.name
+
+        # Perform actions
+        cache_file.parent.mkdir(parents=True, exist_ok=True)
+        state_file.parent.mkdir(parents=True, exist_ok=True)
+
+        copyfile(self.args.input, str(cache_file))
+        Logger.verbose('Input file "{}" was copied to cache "{}"'.format(self.args.input, cache_file))
+
+        output.symlink_to(cache_file)
+        Logger.verbose('Symlink from data file "{}" the cache file "{}" was created'.
+                       format(output, cache_file))
+
+        StateFile.create(state_file, input, output.absolute(), cache_file.absolute())
+        pass
+
+
+if __name__ == '__main__':
+    try:
+        sys.exit(DataImport().run())
+    except NeatLynxException as e:
+        Logger.error(e)
+        sys.exit(1)

neatlynx/cmd_init.py

@@ -3,6 +3,7 @@ import sys
 from pathlib import Path
 
 from neatlynx.base_cmd import BaseCmd, Logger
+from neatlynx.config import Config
 from neatlynx.exceptions import NeatLynxException
 
 
@@ -46,19 +47,19 @@ SecurityGroup = neatlynx-group'''
         pass
 
     def define_args(self, parser):
-        self.add_string_arg(parser, '--data-dir',  'data',  'NeatLynx data directory')
-        self.add_string_arg(parser, '--cache-dir', 'cache', 'NeatLynx cache directory')
-        self.add_string_arg(parser, '--state-dir', 'state', 'NeatLynx state directory')
+        self.add_string_arg(parser, '--data-dir',  'NeatLynx data directory', 'data')
+        self.add_string_arg(parser, '--cache-dir', 'NeatLynx cache directory', 'cache')
+        self.add_string_arg(parser, '--state-dir', 'NeatLynx state directory', 'state')
         pass
 
     def get_not_existing_dir(self, dir):
-        path = Path(os.path.join(self.git_dir, dir))
+        path = Path(os.path.join(self.git.git_dir, dir))
         if path.exists():
             raise InitError('Directory "{}" already exist'.format(path.name))
         return path
 
     def get_not_existing_conf_file(self):
-        path = Path(os.path.join(self.git_dir, self.CONFIG))
+        path = Path(os.path.join(self.git.git_dir, Config.CONFIG))
         if path.exists():
             raise InitError('Configuration file "{}" already exist'.format(path.name))
         return path

neatlynx/config.py

@@ -0,0 +1,59 @@
+import os
+import configparser
+
+from neatlynx.exceptions import NeatLynxException
+
+
+class ConfigError(NeatLynxException):
+    def __init__(self, msg):
+        NeatLynxException.__init__(self, 'Config file error: {}'.format(msg))
+
+
+class ConfigI(object):
+    def __init__(self, data_dir, cache_dir, state_dir):
+        self._data_dir = data_dir
+        self._cache_dir = cache_dir
+        self._state_dir = state_dir
+        pass
+
+    @property
+    def data_dir(self):
+        return self._data_dir
+
+    @property
+    def cache_dir(self):
+        return self._cache_dir
+
+    @property
+    def state_dir(self):
+        return self._state_dir
+
+
+class Config(ConfigI):
+    CONFIG = 'neatlynx.conf'
+
+    def __init__(self, git_dir, file=None, data_dir=None, cache_dir=None, state_dir=None):
+        self._config = configparser.ConfigParser()
+        self._config.read(os.path.join(git_dir, self.CONFIG))
+
+        self._global = self._config['Global']
+
+        self._data_dir = self._global['DataDir']
+        self._cache_dir = self._global['CacheDir']
+        self._state_dir = self._global['StateDir']
+        pass
+
+    # def get_data_file_obj(self):
+    #     return DataFileObj(self.config)
+
+    @property
+    def data_dir(self):
+        return self._data_dir
+
+    @property
+    def cache_dir(self):
+        return self._cache_dir
+
+    @property
+    def state_dir(self):
+        return self._state_dir

neatlynx/data_file_obj.py

@@ -0,0 +1,104 @@
+import os
+
+from neatlynx.exceptions import NeatLynxException
+
+
+class DataFilePathError(NeatLynxException):
+    def __init__(self, msg):
+        NeatLynxException.__init__(self, 'Data file error: {}'.format(msg))
+
+
+class NotInDataDirError(NeatLynxException):
+    def __init__(self):
+        NeatLynxException.__init__(self, 'Data file location error: the file has to be in the data directory')
+
+
+class DataFileObj(object):
+    STATE_FILE_SUFFIX = '.state'
+    CACHE_FILE_SEP = '_'
+
+    def __init__(self, data_file, git, config):
+        self._git = git
+        self._config = config
+
+        data_file = os.path.realpath(data_file)
+
+        self._data_file_name = os.path.basename(data_file)
+        data_file_dir = os.path.realpath(os.path.dirname(data_file))
+
+        data_dir = os.path.join(self._git.git_dir_abs, config.data_dir)
+        self._rel_path = os.path.relpath(data_file_dir, data_dir)
+
+        if not data_file_dir.startswith(data_dir):
+            raise NotInDataDirError()
+
+        if self._rel_path == '.':
+            self._rel_path = ''
+        pass
+
+    # Data file properties
+    @property
+    def data_file_name(self):
+        return self._data_file_name
+
+    @property
+    def data_file_nlx(self):
+        return os.path.join(self._config.data_dir, self._rel_path, self.data_file_name)
+    
+    @property
+    def data_file_abs(self):
+        return os.path.join(self._git.git_dir_abs, self.data_file_nlx)
+
+    @property
+    def data_file_relative(self):
+        return os.path.relpath(self.data_file_abs, self._git.git_dir_abs())
+
+    @property
+    def data_dir_relative(self):
+        return os.path.basename(self.data_file_relative)
+
+    # Cache file properties
+    @property
+    def cache_file_name(self):
+        if not self._git.curr_commit:
+            return None
+        return self.data_file_name + self.CACHE_FILE_SEP + self._git.curr_commit
+
+    @property
+    def cache_file_nlx(self):
+        if not self._git.curr_commit:
+            return None
+        return os.path.join(self._config.cache_dir, self._rel_path, self.cache_file_name)
+
+    @property
+    def cache_file_abs(self):
+        return os.path.join(self._git.git_dir_abs, self.cache_file_nlx)
+
+    @property
+    def cache_file_relative(self):
+        return os.path.relpath(self.cache_file_abs, self._git.git_dir_abs)
+
+    @property
+    def cache_dir_relative(self):
+        return os.path.basename(self.cache_file_relative)
+
+    # State file properties
+    @property
+    def state_file_name(self):
+        return self.data_file_name + self.STATE_FILE_SUFFIX
+
+    @property
+    def state_file_nlx(self):
+        return os.path.join(self._config.state_dir, self._rel_path, self.state_file_name)
+
+    @property
+    def state_file_abs(self):
+        return os.path.join(self._git.git_dir_abs, self.state_file_nlx)
+
+    @property
+    def state_file_relative(self):
+        return os.path.relpath(self.state_file_abs, self._git.git_dir_abs)
+
+    @property
+    def state_dir_relative(self):
+        return os.path.basename(self.state_file_relative)

neatlynx/exceptions.py

@@ -1,13 +1,3 @@
 class NeatLynxException(Exception):
     def __init__(self, msg):
         Exception.__init__(self, msg)
-
-
-class GitCmdError(NeatLynxException):
-    def __init__(self, msg):
-        NeatLynxException.__init__(self, msg)
-
-
-class ConfigError(NeatLynxException):
-    def __init__(self, msg):
-        NeatLynxException.__init__(self, 'Config file error: {}'.format(msg))

neatlynx/git_wrapper.py

@@ -0,0 +1,84 @@
+import os
+import subprocess
+
+from neatlynx.exceptions import NeatLynxException
+
+
+class GitCmdError(NeatLynxException):
+    def __init__(self, msg):
+        NeatLynxException.__init__(self, msg)
+
+
+class GitWrapperI(object):
+    def __init__(self, git_dir=None, commit=None):
+        self._git_dir = git_dir
+        self._commit = commit
+
+    @property
+    def git_dir(self):
+        return self._git_dir
+
+    @property
+    def git_dir_abs(self):
+        return os.path.realpath(self.git_dir)
+
+    @property
+    def curr_commit(self):
+        return self._commit
+
+
+class GitWrapper(GitWrapperI):
+    def __init__(self):
+        GitWrapperI.__init__(self)
+
+    @staticmethod
+    def _exec_cmd(cmd):
+        p = subprocess.Popen(cmd,
+                             stdout=subprocess.PIPE,
+                             stderr=subprocess.PIPE)
+        out, err = map(lambda s: s.decode().strip('\n\r'), p.communicate())
+
+        return p.returncode, out, err
+
+    @property
+    def git_dir(self):
+        if self._git_dir:
+            return self._git_dir
+
+        try:
+            code, out, err = GitWrapper._exec_cmd(['git', 'rev-parse', '--show-toplevel'])
+
+            if code != 0:
+                raise GitCmdError('Git command error - {}'.format(err))
+
+            self._git_dir = out
+            return self._git_dir
+        except GitCmdError as e:
+            raise
+        except Exception as e:
+            raise GitCmdError('Unable to run git command: {}'.format(e))
+        pass
+
+    @staticmethod
+    def status_files():
+        code, out, err = GitWrapper._exec_cmd(['git', 'status' '--porcelain'])
+        if code != 0:
+            raise GitCmdError('Git command error - {}'.format(err))
+
+        result = []
+        if len(err) > 0:
+            lines = out.split('\n')
+            for line in lines:
+                status, file = line.s.strip().split(' ', 1)
+                result.append((status, file ))
+
+        return result
+
+    @property
+    def curr_commit(self):
+        if self._commit is None:
+            code, out, err = GitWrapper._exec_cmd(['git', 'rev-parse' 'HEAD'])
+            if code != 0:
+                raise GitCmdError('Git command error - {}'.format(err))
+            self._commit = out
+        return self._commit

tests/test_data_file_obj.py

@@ -0,0 +1,132 @@
+from unittest import TestCase
+import os
+import shutil
+
+from neatlynx.config import ConfigI
+from neatlynx.data_file_obj import DataFileObj, NotInDataDirError, DataFilePathError
+from neatlynx.git_wrapper import GitWrapperI
+
+
+class BasicDataDirTest(TestCase):
+    def setUp(self, test_dir, curr_dir=None):
+        self._test_dir = os.path.realpath(test_dir)
+        self._test_git_dir = os.path.join(self._test_dir, 'proj')
+        self._old_curr_dir_abs = os.path.realpath(os.curdir)
+
+        shutil.rmtree(self._test_dir, ignore_errors=True)
+
+        if curr_dir:
+            self._curr_dir = os.path.realpath(curr_dir)
+            os.makedirs(self._curr_dir, exist_ok=True)
+        else:
+            self._curr_dir = self._test_git_dir
+
+        os.makedirs(self._test_git_dir, exist_ok=True)
+        os.chdir(self._curr_dir)
+
+    def tearDown(self):
+        os.chdir(self._old_curr_dir_abs)
+        shutil.rmtree(self._test_dir, ignore_errors=True)
+
+
+class TestDataFileObjBasic(BasicDataDirTest):
+    def setUp(self):
+        BasicDataDirTest.setUp(self, test_dir=os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'))
+
+        git = GitWrapperI(git_dir=self._test_git_dir, commit='ad45ba8')
+        config = ConfigI('data', 'cache', 'state')
+
+        self._dobj = DataFileObj(os.path.join('data', 'file.txt'), git, config)
+        pass
+
+    def test_data_file(self):
+        self.assertEqual(self._dobj.data_file_abs, os.path.join(self._test_git_dir, 'data', 'file.txt'))
+
+    def test_cache_file(self):
+        self.assertEqual(self._dobj.cache_file_abs, os.path.join(self._test_git_dir, 'cache', 'file.txt_ad45ba8'))
+
+    def test_state_file(self):
+        self.assertEqual(self._dobj.state_file_abs, os.path.join(self._test_git_dir, 'state', 'file.txt.state'))
+
+
+class TestDataFileObjInDataDir(BasicDataDirTest):
+    def setUp(self):
+        BasicDataDirTest.setUp(self, os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'))
+
+        git = GitWrapperI(git_dir=self._test_git_dir, commit='eeeff8f')
+        config = ConfigI('da', 'ca', 'st')
+
+        self._dobj = DataFileObj(os.path.join('da', 'dir1', 'd2', 'file.txt'), git, config)
+        pass
+
+    def test_data_file(self):
+        target = os.path.join(self._test_git_dir, 'da', 'dir1', 'd2', 'file.txt')
+        self.assertEqual(self._dobj.data_file_abs, target)
+
+    def test_cache_file(self):
+        target = os.path.join(self._test_git_dir, 'ca', 'dir1', 'd2', 'file.txt_eeeff8f')
+        self.assertEqual(self._dobj.cache_file_abs, target)
+
+    def test_state_file(self):
+        target = os.path.join(self._test_git_dir, 'st', 'dir1', 'd2', 'file.txt.state')
+        self.assertEqual(self._dobj.state_file_abs, target)
+
+
+class TestDataFileObjLongPath(BasicDataDirTest):
+    def setUp(self):
+        curr_dir = os.path.join(os.path.sep, 'tmp', 'ntx_unit_test', 'proj', 'mydir', 'dd3')
+
+        BasicDataDirTest.setUp(self,
+                               os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'),
+                               curr_dir)
+
+        self._git = GitWrapperI(git_dir=self._test_git_dir, commit='123ed8')
+        self._config = ConfigI('data', 'cache', 'state')
+
+        self._dobj = DataFileObj(os.path.join('..', '..', 'data', 'file1.txt'),
+                                 self._git, self._config)
+        pass
+
+    def test_data_file(self):
+        target = os.path.join(self._test_git_dir, 'data', 'file1.txt')
+        self.assertEqual(self._dobj.data_file_abs, target)
+
+    def test_cache_file(self):
+        target = os.path.join(self._test_git_dir, 'cache', 'file1.txt_123ed8')
+        self.assertEqual(self._dobj.cache_file_abs, target)
+
+    def test_state_file(self):
+        target = os.path.join(self._test_git_dir, 'state', 'file1.txt.state')
+        self.assertEqual(self._dobj.state_file_abs, target)
+
+    def test_file_name_only(self):
+        with self.assertRaises(NotInDataDirError):
+            DataFileObj('file.txt', self._git, self._config)
+        pass
+
+    def test_relative_git_dir(self):
+        with self.assertRaises(NotInDataDirError):
+            DataFileObj('data/file.txt', self._git, self._config)
+        pass
+
+    def test_relative_path_error(self):
+        with self.assertRaises(NotInDataDirError):
+            DataFileObj('../data/file.txt', self._git, self._config)
+        pass
+
+
+class RunOutsideGitRepoTest(BasicDataDirTest):
+    def setUp(self):
+        curr_dir = os.path.join(os.path.sep, 'tmp')
+
+        BasicDataDirTest.setUp(self,
+                               os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'),
+                               curr_dir)
+
+    def test_outside_git_dir(self):
+        git = GitWrapperI(git_dir=self._test_git_dir, commit='123ed8')
+        config = ConfigI('data', 'cache', 'state')
+
+        with self.assertRaises(NotInDataDirError):
+            DataFileObj('file.txt', git, config)
+        pass

tests/test_git_wrapper.py

@@ -0,0 +1,6 @@
+from unittest import TestCase
+
+
+class TestGitWrapper(TestCase):
+    def test1(self):
+        pass