diff --git a/bin/nlx-data-import b/bin/nlx-data-import index e69de29bb..e5706e8fe 100755 --- a/bin/nlx-data-import +++ b/bin/nlx-data-import @@ -0,0 +1,3 @@ +#!/bin/bash + +PYTHONPATH=$NEATLYNX_HOME python $NEATLYNX_HOME/neatlynx/cmd_data_import.py $@ diff --git a/bin/nlx-init b/bin/nlx-init index 6f74d8ba9..523e28730 100755 --- a/bin/nlx-init +++ b/bin/nlx-init @@ -1,3 +1,3 @@ #!/bin/bash -python $NEATLYNX_HOME/lib/cmd_init.py $@ +PYTHONPATH=$NEATLYNX_HOME python $NEATLYNX_HOME/neatlynx/cmd_init.py $@ diff --git a/neatlynx/__init__.py b/neatlynx/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/neatlynx/base_cmd.py b/neatlynx/base_cmd.py index 190aec7e2..c1cce3bf4 100644 --- a/neatlynx/base_cmd.py +++ b/neatlynx/base_cmd.py @@ -1,10 +1,9 @@ -import configparser import argparse import os -import git -import subprocess -from neatlynx.exceptions import GitCmdError, ConfigError +from neatlynx.git_wrapper import GitWrapper +from neatlynx.config import Config, ConfigError + class Logger(object): @staticmethod @@ -21,16 +20,13 @@ class Logger(object): class BaseCmd(object): - CONFIG = 'neatlynx.conf' - def __init__(self, parse_config=True): - self._git_dir = None + self._git = GitWrapper() self._args = None self._lnx_home = None if parse_config: - self._config = configparser.ConfigParser() - self._config.read(os.path.join(self.git_dir, self.CONFIG)) + self._config = Config(os.path.join(self.git.git_dir, self.CONFIG)) parser = argparse.ArgumentParser() self.define_args(parser) @@ -53,31 +49,13 @@ class BaseCmd(object): return self._args @property - def git_dir(self): - if self._git_dir: - return self._git_dir - - try: - p = subprocess.Popen(['git', 'rev-parse', '--show-toplevel'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - out, err = map(lambda s: s.decode().strip('\n\r'), p.communicate()) - - if p.returncode != 0: - raise GitCmdError('Git command error - {}'.format(err)) - - self._git_dir = out - return self._git_dir - except GitCmdError as e: - raise - except Exception as e: - raise GitCmdError('Unable to run git command: {}'.format(e)) - pass + def git(self): + return self._git def define_args(self): pass - def add_string_arg(self, parser, name, default, message, + def add_string_arg(self, parser, name, message, default = None, conf_section=None, conf_name=None): if conf_section and conf_name: section = self._config[conf_section] @@ -93,4 +71,4 @@ class BaseCmd(object): help=message) def run(self): - pass \ No newline at end of file + pass diff --git a/neatlynx/cmd_data_import.py b/neatlynx/cmd_data_import.py new file mode 100644 index 000000000..3c7727720 --- /dev/null +++ b/neatlynx/cmd_data_import.py @@ -0,0 +1,81 @@ +import sys +import os +from pathlib import Path +from shutil import copyfile + +from neatlynx.base_cmd import BaseCmd, Logger +from neatlynx.data_file_obj import DataFileObj +from neatlynx.exceptions import NeatLynxException +from neatlynx.git_wrapper import GitWrapper + + +class DataImportError(NeatLynxException): + def __init__(self, msg): + NeatLynxException.__init__(self, 'Import error: {}'.format(msg)) + + +class DataImport(BaseCmd): + def __init__(self): + BaseCmd.__init__(self) + pass + + def define_args(self, parser): + self.add_string_arg(parser, 'input', 'Input file') + self.add_string_arg(parser, 'output', 'Output file') + pass + + def run(self): + input = Path(self.args.input) + if not input.exists(): + raise DataImportError('Input file "{}" does not exist'.format(input)) + if not input.is_file(): + raise DataImportError('Input file "{}" has to be a regular file'.format(input)) + + dataFileObj = DataFileObj(self.args.output, self._config, GitWrapper.curr_commit()) + + #output = Path(self.args.output) + if dataFileObj.data_file_relative.exists(): + raise DataImportError('Output file "{}" already exists'.format(dataFileObj.data_file_relative)) + if os.path.isdir(dataFileObj.data_dir_relative): + raise DataImportError('Output file directory "{}" does not exists'.format(dataFileObj.data_dir_relative)) + + data_dir_path = Path(self.config.DataDir) + if output.parent < data_dir_path: + raise DataImportError('Output file has to be in data dir - {}'.format(data_dir_path)) + + # data_dir_path_str = str(data_dir_path) + # output_dir_str = str(output.parent) + # relative_dir = output_dir_str[len(data_dir_path_str):].strip(os.path.sep) + # + # cache_file_dir = os.path.join(self.config.CachDir, relative_dir) + # cache_file_dir_path = Path(cache_file_dir) + # + # state_file_dir = os.path.join(self.config.StateDir, relative_dir) + # state_file_dir_path = Path(state_file_dir) + # + # commit = GitWrapper.curr_commit() + # cache_file_name = output.name + '_' + commit + # cache_file = cache_file_dir_path / cache_file_name + # state_file = state_file_dir_path / output.name + + # Perform actions + cache_file.parent.mkdir(parents=True, exist_ok=True) + state_file.parent.mkdir(parents=True, exist_ok=True) + + copyfile(self.args.input, str(cache_file)) + Logger.verbose('Input file "{}" was copied to cache "{}"'.format(self.args.input, cache_file)) + + output.symlink_to(cache_file) + Logger.verbose('Symlink from data file "{}" the cache file "{}" was created'. + format(output, cache_file)) + + StateFile.create(state_file, input, output.absolute(), cache_file.absolute()) + pass + + +if __name__ == '__main__': + try: + sys.exit(DataImport().run()) + except NeatLynxException as e: + Logger.error(e) + sys.exit(1) diff --git a/neatlynx/cmd_init.py b/neatlynx/cmd_init.py index c067dff4a..07256e3ca 100644 --- a/neatlynx/cmd_init.py +++ b/neatlynx/cmd_init.py @@ -3,6 +3,7 @@ import sys from pathlib import Path from neatlynx.base_cmd import BaseCmd, Logger +from neatlynx.config import Config from neatlynx.exceptions import NeatLynxException @@ -46,19 +47,19 @@ SecurityGroup = neatlynx-group''' pass def define_args(self, parser): - self.add_string_arg(parser, '--data-dir', 'data', 'NeatLynx data directory') - self.add_string_arg(parser, '--cache-dir', 'cache', 'NeatLynx cache directory') - self.add_string_arg(parser, '--state-dir', 'state', 'NeatLynx state directory') + self.add_string_arg(parser, '--data-dir', 'NeatLynx data directory', 'data') + self.add_string_arg(parser, '--cache-dir', 'NeatLynx cache directory', 'cache') + self.add_string_arg(parser, '--state-dir', 'NeatLynx state directory', 'state') pass def get_not_existing_dir(self, dir): - path = Path(os.path.join(self.git_dir, dir)) + path = Path(os.path.join(self.git.git_dir, dir)) if path.exists(): raise InitError('Directory "{}" already exist'.format(path.name)) return path def get_not_existing_conf_file(self): - path = Path(os.path.join(self.git_dir, self.CONFIG)) + path = Path(os.path.join(self.git.git_dir, Config.CONFIG)) if path.exists(): raise InitError('Configuration file "{}" already exist'.format(path.name)) return path diff --git a/neatlynx/config.py b/neatlynx/config.py new file mode 100644 index 000000000..ca426cd15 --- /dev/null +++ b/neatlynx/config.py @@ -0,0 +1,59 @@ +import os +import configparser + +from neatlynx.exceptions import NeatLynxException + + +class ConfigError(NeatLynxException): + def __init__(self, msg): + NeatLynxException.__init__(self, 'Config file error: {}'.format(msg)) + + +class ConfigI(object): + def __init__(self, data_dir, cache_dir, state_dir): + self._data_dir = data_dir + self._cache_dir = cache_dir + self._state_dir = state_dir + pass + + @property + def data_dir(self): + return self._data_dir + + @property + def cache_dir(self): + return self._cache_dir + + @property + def state_dir(self): + return self._state_dir + + +class Config(ConfigI): + CONFIG = 'neatlynx.conf' + + def __init__(self, git_dir, file=None, data_dir=None, cache_dir=None, state_dir=None): + self._config = configparser.ConfigParser() + self._config.read(os.path.join(git_dir, self.CONFIG)) + + self._global = self._config['Global'] + + self._data_dir = self._global['DataDir'] + self._cache_dir = self._global['CacheDir'] + self._state_dir = self._global['StateDir'] + pass + + # def get_data_file_obj(self): + # return DataFileObj(self.config) + + @property + def data_dir(self): + return self._data_dir + + @property + def cache_dir(self): + return self._cache_dir + + @property + def state_dir(self): + return self._state_dir \ No newline at end of file diff --git a/neatlynx/data_file_obj.py b/neatlynx/data_file_obj.py new file mode 100644 index 000000000..36bb45e25 --- /dev/null +++ b/neatlynx/data_file_obj.py @@ -0,0 +1,104 @@ +import os + +from neatlynx.exceptions import NeatLynxException + + +class DataFilePathError(NeatLynxException): + def __init__(self, msg): + NeatLynxException.__init__(self, 'Data file error: {}'.format(msg)) + + +class NotInDataDirError(NeatLynxException): + def __init__(self): + NeatLynxException.__init__(self, 'Data file location error: the file has to be in the data directory') + + +class DataFileObj(object): + STATE_FILE_SUFFIX = '.state' + CACHE_FILE_SEP = '_' + + def __init__(self, data_file, git, config): + self._git = git + self._config = config + + data_file = os.path.realpath(data_file) + + self._data_file_name = os.path.basename(data_file) + data_file_dir = os.path.realpath(os.path.dirname(data_file)) + + data_dir = os.path.join(self._git.git_dir_abs, config.data_dir) + self._rel_path = os.path.relpath(data_file_dir, data_dir) + + if not data_file_dir.startswith(data_dir): + raise NotInDataDirError() + + if self._rel_path == '.': + self._rel_path = '' + pass + + # Data file properties + @property + def data_file_name(self): + return self._data_file_name + + @property + def data_file_nlx(self): + return os.path.join(self._config.data_dir, self._rel_path, self.data_file_name) + + @property + def data_file_abs(self): + return os.path.join(self._git.git_dir_abs, self.data_file_nlx) + + @property + def data_file_relative(self): + return os.path.relpath(self.data_file_abs, self._git.git_dir_abs()) + + @property + def data_dir_relative(self): + return os.path.basename(self.data_file_relative) + + # Cache file properties + @property + def cache_file_name(self): + if not self._git.curr_commit: + return None + return self.data_file_name + self.CACHE_FILE_SEP + self._git.curr_commit + + @property + def cache_file_nlx(self): + if not self._git.curr_commit: + return None + return os.path.join(self._config.cache_dir, self._rel_path, self.cache_file_name) + + @property + def cache_file_abs(self): + return os.path.join(self._git.git_dir_abs, self.cache_file_nlx) + + @property + def cache_file_relative(self): + return os.path.relpath(self.cache_file_abs, self._git.git_dir_abs) + + @property + def cache_dir_relative(self): + return os.path.basename(self.cache_file_relative) + + # State file properties + @property + def state_file_name(self): + return self.data_file_name + self.STATE_FILE_SUFFIX + + @property + def state_file_nlx(self): + return os.path.join(self._config.state_dir, self._rel_path, self.state_file_name) + + @property + def state_file_abs(self): + return os.path.join(self._git.git_dir_abs, self.state_file_nlx) + + @property + def state_file_relative(self): + return os.path.relpath(self.state_file_abs, self._git.git_dir_abs) + + @property + def state_dir_relative(self): + return os.path.basename(self.state_file_relative) diff --git a/neatlynx/exceptions.py b/neatlynx/exceptions.py index d30f29b29..b9ab1a9ed 100644 --- a/neatlynx/exceptions.py +++ b/neatlynx/exceptions.py @@ -1,13 +1,3 @@ class NeatLynxException(Exception): def __init__(self, msg): Exception.__init__(self, msg) - - -class GitCmdError(NeatLynxException): - def __init__(self, msg): - NeatLynxException.__init__(self, msg) - - -class ConfigError(NeatLynxException): - def __init__(self, msg): - NeatLynxException.__init__(self, 'Config file error: {}'.format(msg)) \ No newline at end of file diff --git a/neatlynx/git_wrapper.py b/neatlynx/git_wrapper.py new file mode 100644 index 000000000..277c39064 --- /dev/null +++ b/neatlynx/git_wrapper.py @@ -0,0 +1,84 @@ +import os +import subprocess + +from neatlynx.exceptions import NeatLynxException + + +class GitCmdError(NeatLynxException): + def __init__(self, msg): + NeatLynxException.__init__(self, msg) + + +class GitWrapperI(object): + def __init__(self, git_dir=None, commit=None): + self._git_dir = git_dir + self._commit = commit + + @property + def git_dir(self): + return self._git_dir + + @property + def git_dir_abs(self): + return os.path.realpath(self.git_dir) + + @property + def curr_commit(self): + return self._commit + + +class GitWrapper(GitWrapperI): + def __init__(self): + GitWrapperI.__init__(self) + + @staticmethod + def _exec_cmd(cmd): + p = subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, err = map(lambda s: s.decode().strip('\n\r'), p.communicate()) + + return p.returncode, out, err + + @property + def git_dir(self): + if self._git_dir: + return self._git_dir + + try: + code, out, err = GitWrapper._exec_cmd(['git', 'rev-parse', '--show-toplevel']) + + if code != 0: + raise GitCmdError('Git command error - {}'.format(err)) + + self._git_dir = out + return self._git_dir + except GitCmdError as e: + raise + except Exception as e: + raise GitCmdError('Unable to run git command: {}'.format(e)) + pass + + @staticmethod + def status_files(): + code, out, err = GitWrapper._exec_cmd(['git', 'status' '--porcelain']) + if code != 0: + raise GitCmdError('Git command error - {}'.format(err)) + + result = [] + if len(err) > 0: + lines = out.split('\n') + for line in lines: + status, file = line.s.strip().split(' ', 1) + result.append((status, file )) + + return result + + @property + def curr_commit(self): + if self._commit is None: + code, out, err = GitWrapper._exec_cmd(['git', 'rev-parse' 'HEAD']) + if code != 0: + raise GitCmdError('Git command error - {}'.format(err)) + self._commit = out + return self._commit diff --git a/tests/test_data_file_obj.py b/tests/test_data_file_obj.py new file mode 100644 index 000000000..dc235b167 --- /dev/null +++ b/tests/test_data_file_obj.py @@ -0,0 +1,132 @@ +from unittest import TestCase +import os +import shutil + +from neatlynx.config import ConfigI +from neatlynx.data_file_obj import DataFileObj, NotInDataDirError, DataFilePathError +from neatlynx.git_wrapper import GitWrapperI + + +class BasicDataDirTest(TestCase): + def setUp(self, test_dir, curr_dir=None): + self._test_dir = os.path.realpath(test_dir) + self._test_git_dir = os.path.join(self._test_dir, 'proj') + self._old_curr_dir_abs = os.path.realpath(os.curdir) + + shutil.rmtree(self._test_dir, ignore_errors=True) + + if curr_dir: + self._curr_dir = os.path.realpath(curr_dir) + os.makedirs(self._curr_dir, exist_ok=True) + else: + self._curr_dir = self._test_git_dir + + os.makedirs(self._test_git_dir, exist_ok=True) + os.chdir(self._curr_dir) + + def tearDown(self): + os.chdir(self._old_curr_dir_abs) + shutil.rmtree(self._test_dir, ignore_errors=True) + + +class TestDataFileObjBasic(BasicDataDirTest): + def setUp(self): + BasicDataDirTest.setUp(self, test_dir=os.path.join(os.path.sep, 'tmp', 'ntx_unit_test')) + + git = GitWrapperI(git_dir=self._test_git_dir, commit='ad45ba8') + config = ConfigI('data', 'cache', 'state') + + self._dobj = DataFileObj(os.path.join('data', 'file.txt'), git, config) + pass + + def test_data_file(self): + self.assertEqual(self._dobj.data_file_abs, os.path.join(self._test_git_dir, 'data', 'file.txt')) + + def test_cache_file(self): + self.assertEqual(self._dobj.cache_file_abs, os.path.join(self._test_git_dir, 'cache', 'file.txt_ad45ba8')) + + def test_state_file(self): + self.assertEqual(self._dobj.state_file_abs, os.path.join(self._test_git_dir, 'state', 'file.txt.state')) + + +class TestDataFileObjInDataDir(BasicDataDirTest): + def setUp(self): + BasicDataDirTest.setUp(self, os.path.join(os.path.sep, 'tmp', 'ntx_unit_test')) + + git = GitWrapperI(git_dir=self._test_git_dir, commit='eeeff8f') + config = ConfigI('da', 'ca', 'st') + + self._dobj = DataFileObj(os.path.join('da', 'dir1', 'd2', 'file.txt'), git, config) + pass + + def test_data_file(self): + target = os.path.join(self._test_git_dir, 'da', 'dir1', 'd2', 'file.txt') + self.assertEqual(self._dobj.data_file_abs, target) + + def test_cache_file(self): + target = os.path.join(self._test_git_dir, 'ca', 'dir1', 'd2', 'file.txt_eeeff8f') + self.assertEqual(self._dobj.cache_file_abs, target) + + def test_state_file(self): + target = os.path.join(self._test_git_dir, 'st', 'dir1', 'd2', 'file.txt.state') + self.assertEqual(self._dobj.state_file_abs, target) + + +class TestDataFileObjLongPath(BasicDataDirTest): + def setUp(self): + curr_dir = os.path.join(os.path.sep, 'tmp', 'ntx_unit_test', 'proj', 'mydir', 'dd3') + + BasicDataDirTest.setUp(self, + os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'), + curr_dir) + + self._git = GitWrapperI(git_dir=self._test_git_dir, commit='123ed8') + self._config = ConfigI('data', 'cache', 'state') + + self._dobj = DataFileObj(os.path.join('..', '..', 'data', 'file1.txt'), + self._git, self._config) + pass + + def test_data_file(self): + target = os.path.join(self._test_git_dir, 'data', 'file1.txt') + self.assertEqual(self._dobj.data_file_abs, target) + + def test_cache_file(self): + target = os.path.join(self._test_git_dir, 'cache', 'file1.txt_123ed8') + self.assertEqual(self._dobj.cache_file_abs, target) + + def test_state_file(self): + target = os.path.join(self._test_git_dir, 'state', 'file1.txt.state') + self.assertEqual(self._dobj.state_file_abs, target) + + def test_file_name_only(self): + with self.assertRaises(NotInDataDirError): + DataFileObj('file.txt', self._git, self._config) + pass + + def test_relative_git_dir(self): + with self.assertRaises(NotInDataDirError): + DataFileObj('data/file.txt', self._git, self._config) + pass + + def test_relative_path_error(self): + with self.assertRaises(NotInDataDirError): + DataFileObj('../data/file.txt', self._git, self._config) + pass + + +class RunOutsideGitRepoTest(BasicDataDirTest): + def setUp(self): + curr_dir = os.path.join(os.path.sep, 'tmp') + + BasicDataDirTest.setUp(self, + os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'), + curr_dir) + + def test_outside_git_dir(self): + git = GitWrapperI(git_dir=self._test_git_dir, commit='123ed8') + config = ConfigI('data', 'cache', 'state') + + with self.assertRaises(NotInDataDirError): + DataFileObj('file.txt', git, config) + pass diff --git a/tests/test_git_wrapper.py b/tests/test_git_wrapper.py new file mode 100644 index 000000000..b109150d4 --- /dev/null +++ b/tests/test_git_wrapper.py @@ -0,0 +1,6 @@ +from unittest import TestCase + + +class TestGitWrapper(TestCase): + def test1(self): + pass