diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..82ffd30f8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +neatlynx.conf +neatlynx/__pycache__ +.idea +cache diff --git a/neatlynx/cmd_data_import.py b/neatlynx/cmd_data_import.py index 655711b2b..7057bf0bc 100644 --- a/neatlynx/cmd_data_import.py +++ b/neatlynx/cmd_data_import.py @@ -1,8 +1,10 @@ import sys import os -from pathlib import Path from shutil import copyfile +import re +import requests + from neatlynx.cmd_base import CmdBase, Logger from neatlynx.data_file_obj import DataFileObj from neatlynx.exceptions import NeatLynxException @@ -24,11 +26,12 @@ class CmdDataImport(CmdBase): pass def run(self): - input = Path(self.args.input) - if not input.exists(): - raise DataImportError('Input file "{}" does not exist'.format(input)) - if not input.is_file(): - raise DataImportError('Input file "{}" has to be a regular file'.format(input)) + + if not CmdDataImport.is_url(self.args.input): + if not os.path.exists(self.args.input): + raise DataImportError('Input file "{}" does not exist'.format(self.args.input)) + if not os.path.isfile(self.args.input): + raise DataImportError('Input file "{}" has to be a regular file'.format(self.args.input)) output = self.args.output if os.path.isdir(self.args.output): @@ -43,9 +46,15 @@ class CmdDataImport(CmdBase): os.path.dirname(dobj.data_file_relative))) os.makedirs(os.path.dirname(dobj.cache_file_relative), exist_ok=True) - copyfile(self.args.input, dobj.cache_file_relative) - Logger.verbose('Input file "{}" was copied to cache "{}"'.format( - self.args.input, dobj.cache_file_relative)) + if CmdDataImport.is_url(self.args.input): + Logger.verbose('Downloading file {} ...'.format(self.args.input)) + self.download_file(self.args.input, dobj.cache_file_relative) + Logger.verbose('Input file "{}" was downloaded to cache "{}"'.format( + self.args.input, dobj.cache_file_relative)) + else: + copyfile(self.args.input, dobj.cache_file_relative) + Logger.verbose('Input file "{}" was copied to cache "{}"'.format( + self.args.input, dobj.cache_file_relative)) cache_relative_to_data = os.path.relpath(dobj.cache_file_relative, os.path.dirname(dobj.data_file_relative)) os.symlink(cache_relative_to_data, dobj.data_file_relative) @@ -59,6 +68,27 @@ class CmdDataImport(CmdBase): Logger.verbose('State file "{}" was created'.format(dobj.state_file_relative)) pass + URL_REGEX = re.compile( + r'^(?:http|ftp)s?://' # http:// or https:// + r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain... + r'localhost|' # localhost... + r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip + r'(?::\d+)?' # optional port + r'(?:/?|[/?]\S+)$', re.IGNORECASE) + + @staticmethod + def is_url(url): + return CmdDataImport.URL_REGEX.match(url) is not None + + @staticmethod + def download_file(from_url, to_file): + r = requests.get(from_url, stream=True) + with open(to_file, 'wb') as f: + for chunk in r.iter_content(chunk_size=1024*100): + if chunk: # filter out keep-alive new chunks + f.write(chunk) + return + if __name__ == '__main__': try: diff --git a/neatlynx/cmd_data_remove.py b/neatlynx/cmd_data_remove.py index af5ab7ab3..4fac5fc3f 100644 --- a/neatlynx/cmd_data_remove.py +++ b/neatlynx/cmd_data_remove.py @@ -1,8 +1,9 @@ import os -from boto.s3.connection import S3Connection, Key +from boto.s3.connection import S3Connection from neatlynx.cmd_base import CmdBase, Logger from neatlynx.exceptions import NeatLynxException +from neatlynx.data_file_obj import DataFileObjExisting class DataRemoveError(NeatLynxException): @@ -32,46 +33,42 @@ class CmdDataRemove(CmdBase): def run(self): target = self.args.target - if os.path.islink(target): - return self.remove_symlink(target) - if os.path.isdir(target): if not self.args.recursive: raise DataRemoveError('Directory cannot be removed. Use --recurcive flag.') - if os.path.realpath(target) == os.path.realpath(self.config.data_dir): + if os.path.realpath(target) == \ + os.path.realpath(os.path.join(self.git.git_dir_abs, self.config.data_dir)): raise DataRemoveError('data directory cannot be removed') + return self.remove_dir(target) - raise DataRemoveError('File "{}" does not exit'.format(target)) + dobj = DataFileObjExisting(target, self.git, self.config) + if os.path.islink(dobj.data_file_relative): + return self.remove_symlink(dobj.data_file_relative) + + raise DataRemoveError('Cannot remove a regular file "{}"'.format(target)) def remove_symlink(self, file): - if not file.startswith(self.config.data_dir): - raise DataRemoveError('File "{}" supposes to be in data dir'.format(file)) + dobj = DataFileObjExisting(file, self.git, self.config) - cache_file_rel_data = os.path.join(os.path.dirname(file), os.readlink(file)) - cache_file = os.path.relpath(os.path.realpath(cache_file_rel_data), os.path.realpath(os.curdir)) + if os.path.isfile(dobj.cache_file_relative): + os.remove(dobj.cache_file_relative) + dobj.remove_cache_dir_if_empty() - rel_data_file = os.path.relpath(file, self.config.data_dir) - state_file = os.path.join(self.config.state_dir, rel_data_file) + if os.path.isfile(dobj.state_file_relative): + os.remove(dobj.state_file_relative) + dobj.remove_state_dir_if_empty() - if os.path.isfile(cache_file): - os.remove(cache_file) - os.remove(file) - - if not os.path.isfile(state_file): - Logger.warn('Warning: state file "{}" does not exist'.format(state_file)) + if not self.args.keep_in_cloud: + key = self._bucket.get_key(dobj.cache_file_aws_key) + if not key: + Logger.warn('S3 remove warning: file "{}" does not exist in S3'.format(dobj.cache_file_aws_key)) else: - os.remove(state_file) + key.delete() + Logger.info('File "{}" was removed from S3'.format(dobj.cache_file_aws_key)) - if not self.args.keep_in_cloud: - s3_name = self.get_cache_file_s3_name(cache_file) - key = self._bucket.get_key(s3_name) - if not key: - Logger.warn('S3 remove warning: file "{}" does not exist in S3'.format(s3_name)) - else: - key.delete() - Logger.info('File "{}" was removed from S3'.format(s3_name)) + os.remove(file) pass def remove_dir(self, data_dir): @@ -85,10 +82,6 @@ class CmdDataRemove(CmdBase): raise DataRemoveError('Unsupported file type "{}"'.format(fname)) os.rmdir(data_dir) - - rel_data_dir = os.path.relpath(data_dir, self.config.data_dir) - cache_dir = os.path.join(self.config.cache_dir, rel_data_dir) - os.rmdir(cache_dir) pass diff --git a/neatlynx/cmd_data_sync.py b/neatlynx/cmd_data_sync.py index 97ccf6a15..802f6024e 100644 --- a/neatlynx/cmd_data_sync.py +++ b/neatlynx/cmd_data_sync.py @@ -4,6 +4,7 @@ import os from boto.s3.connection import S3Connection from neatlynx.cmd_base import CmdBase, Logger +from neatlynx.data_file_obj import DataFileObjExisting from neatlynx.exceptions import NeatLynxException @@ -50,17 +51,12 @@ class CmdDataSync(CmdBase): pass def run(self): - target = self.args.target - rel_data_path = os.path.join(os.path.realpath(self.git.git_dir), self.config.data_dir) - if not os.path.abspath(target).startswith(os.path.realpath(rel_data_path)): - raise DataSyncError('File supposes to be in data dir - "{}"'. - format(self.config.data_dir)) + if os.path.islink(self.args.target): + dobj = DataFileObjExisting(self.args.target, self.git, self.config) + return self.sync_symlink(dobj) - if os.path.islink(target): - return self.sync_symlink(target) - - if os.path.isdir(target): - return self.sync_dir(target) + if os.path.isdir(self.args.target): + return self.sync_dir(self.args.target) raise DataSyncError('File "{}" does not exit'.format(target)) @@ -70,51 +66,46 @@ class CmdDataSync(CmdBase): if os.path.isdir(fname): self.sync_dir(fname) elif os.path.islink(fname): - self.sync_symlink(fname) + self.sync_symlink(DataFileObjExisting(fname, self.git, self.config)) else: raise DataSyncError('Unsupported file type "{}"'.format(fname)) pass - def sync_symlink(self, file): - cache_file_rel_data = os.path.join(os.path.dirname(file), os.readlink(file)) - cache_file = os.path.relpath(os.path.realpath(cache_file_rel_data), os.path.realpath(os.curdir)) - - if os.path.isfile(cache_file): - self.sync_to_cloud(cache_file) + def sync_symlink(self, dobj): + if os.path.isfile(dobj.cache_file_relative): + self.sync_to_cloud(dobj) else: - self.sync_from_cloud(cache_file) - pass + self.sync_from_cloud(dobj) pass - def sync_from_cloud(self, cache_file): - s3_file = self.get_cache_file_s3_name(cache_file) - key = self._bucket.get_key(s3_file) + def sync_from_cloud(self, dobj): + key = self._bucket.get_key(dobj.cache_file_aws_key) if not key: - raise DataSyncError('File "{}" is not exist in the cloud'.format(cache_file)) + raise DataSyncError('File "{}" does not exist in the cloud'.format(dobj.cache_file_aws_key)) - Logger.info('Downloading cache file "{}" from S3 {}/{}'.format(cache_file, self._bucket.name, s3_file)) - key.get_contents_to_filename(cache_file, cb=percent_cb) + Logger.info('Downloading cache file from S3 "{}/{}"'.format(self._bucket.name, + dobj.cache_file_aws_key)) + key.get_contents_to_filename(dobj.cache_file_relative, cb=percent_cb) Logger.info('Downloading completed') pass - def sync_to_cloud(self, cache_file): - target_file = self.get_cache_file_s3_name(cache_file) - - key = self._bucket.get_key(target_file) + def sync_to_cloud(self, dobj): + key = self._bucket.get_key(dobj.cache_file_aws_key) if key: - Logger.verbose('File already uploaded to the cloud. Checking checksum...') + Logger.verbose('File already uploaded to the cloud. Checksum validation...') md5_cloud = key.etag[1:-1] - md5_local = file_md5(cache_file) + md5_local = file_md5(dobj.cache_file_relative) if md5_cloud == md5_local: Logger.verbose('File checksum matches. No uploading is needed.') return Logger.info('Checksum miss-match. Re-uploading is required.') - Logger.info('Uploading cache file "{}" to S3 {}/{}'.format(cache_file, self._bucket.name, target_file)) - key = self._bucket.new_key(target_file) - key.set_contents_from_filename(cache_file, cb=percent_cb) + Logger.info('Uploading cache file "{}" to S3 "{}"'.format(dobj.cache_file_relative, + dobj.cache_file_aws_key)) + key = self._bucket.new_key(dobj.cache_file_aws_key) + key.set_contents_from_filename(dobj.cache_file_relative, cb=percent_cb) Logger.info('Uploading completed') pass diff --git a/neatlynx/data_file_obj.py b/neatlynx/data_file_obj.py index 3d8829541..cf9ad8dcf 100644 --- a/neatlynx/data_file_obj.py +++ b/neatlynx/data_file_obj.py @@ -9,10 +9,10 @@ class DataFilePathError(NeatLynxException): class NotInDataDirError(NeatLynxException): - def __init__(self, data_dir): + def __init__(self, file, data_dir): NeatLynxException.__init__(self, - 'Data file location error: the file has to be in the data directory "{}"'. - format(data_dir)) + 'Data file location error: the file "{}" has to be in the data directory "{}"'. + format(file, data_dir)) class DataFileObj(object): @@ -20,41 +20,44 @@ class DataFileObj(object): CACHE_FILE_SEP = '_' def __init__(self, data_file, git, config): + ''' + nlx file name - system file name without data/cache/state prefix + ''' self._git = git self._config = config self._curr_dir_abs = os.path.realpath(os.curdir) - data_file = os.path.realpath(data_file) + self._data_file = data_file - self._data_file_name = os.path.basename(data_file) - data_file_dir = os.path.realpath(os.path.dirname(data_file)) - - data_dir = os.path.join(self._git.git_dir_abs, config.data_dir) - self._rel_path = os.path.relpath(data_file_dir, data_dir) - - if not data_file_dir.startswith(data_dir): - raise NotInDataDirError(config.data_dir) - - if self._rel_path == '.': - self._rel_path = '' + if not self.data_file_abs.startswith(self.data_dir_abs): + raise NotInDataDirError(self._data_file, self._config.data_dir) pass + @property + def data_dir_abs(self): + return os.path.join(self._git.git_dir_abs, self._config.data_dir) + # Data file properties @property def data_file_name(self): - return self._data_file_name + return os.path.basename(self._data_file) @property def data_file_nlx(self): - return os.path.join(self._config.data_dir, self._rel_path, self.data_file_name) + return os.path.relpath(self.data_file_abs, self.data_dir_abs) + # @property + # def data_file_abs(self): + # return os.path.realpath(self.data_file_relative) + @property def data_file_abs(self): - return os.path.join(self._git.git_dir_abs, self.data_file_nlx) + '''Do not fully resolve file name since it is a link''' + return os.path.abspath(self.data_file_relative) @property def data_file_relative(self): - return os.path.relpath(self.data_file_abs, self._curr_dir_abs) + return self._data_file # Cache file properties @property @@ -67,11 +70,15 @@ class DataFileObj(object): def cache_file_nlx(self): if not self._git.curr_commit: return None - return os.path.join(self._config.cache_dir, self._rel_path, self.cache_file_name) + return self.data_file_nlx + self.CACHE_FILE_SEP + self._git.curr_commit + + @property + def cache_file_aws_key(self): + return '{}/{}'.format(self._config.aws_storage_prefix, self.cache_file_nlx).strip('/') @property def cache_file_abs(self): - return os.path.join(self._git.git_dir_abs, self.cache_file_nlx) + return os.path.join(self._git.git_dir_abs, self._config.cache_dir, self.cache_file_nlx) @property def cache_file_relative(self): @@ -84,12 +91,41 @@ class DataFileObj(object): @property def state_file_nlx(self): - return os.path.join(self._config.state_dir, self._rel_path, self.state_file_name) + return self.data_file_nlx + self.STATE_FILE_SUFFIX @property def state_file_abs(self): - return os.path.join(self._git.git_dir_abs, self.state_file_nlx) + return os.path.join(self._git.git_dir_abs, self._config.state_dir, self.state_file_nlx) @property def state_file_relative(self): return os.path.relpath(self.state_file_abs, self._curr_dir_abs) + + +class DataFileObjExisting(DataFileObj): + def __init__(self, data_file, git, config): + DataFileObj.__init__(self, data_file, git, config) + + if not os.path.islink(data_file): + raise DataFilePathError('Data file must be a symbolic link') + pass + + @property + def cache_file_name(self): + return os.path.basename(self.cache_file_abs) + + @property + def cache_file_abs(self): + return os.path.realpath(self.data_file_relative) + + @staticmethod + def remove_dir_if_empty(dir): + cache_file_dir = os.path.dirname(dir) + if not os.listdir(cache_file_dir): + os.rmdir(cache_file_dir) + + def remove_state_dir_if_empty(self): + self.remove_dir_if_empty(self.state_file_relative) + + def remove_cache_dir_if_empty(self): + self.remove_dir_if_empty(self.cache_file_relative) diff --git a/tests/test_data_file_obj.py b/tests/test_data_file_obj.py index dc235b167..95dc79cbb 100644 --- a/tests/test_data_file_obj.py +++ b/tests/test_data_file_obj.py @@ -49,6 +49,21 @@ class TestDataFileObjBasic(BasicDataDirTest): self.assertEqual(self._dobj.state_file_abs, os.path.join(self._test_git_dir, 'state', 'file.txt.state')) +class TestNlxNames(BasicDataDirTest): + def setUp(self): + BasicDataDirTest.setUp(self, test_dir=os.path.join(os.path.sep, 'tmp', 'ntx_unit_test')) + + git = GitWrapperI(git_dir=self._test_git_dir, commit='ad45ba8') + config = ConfigI('data', 'cache', 'state') + + self._dobj = DataFileObj(os.path.join('data', 'file.txt'), git, config) + pass + + def test_data_nlx(self): + self.assertEqual(self._dobj.data_file_nlx, 'file.txt') + pass + + class TestDataFileObjInDataDir(BasicDataDirTest): def setUp(self): BasicDataDirTest.setUp(self, os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'))