diff --git a/bin/nlx-data-remove b/bin/nlx-data-remove index e69de29bb..3a758a151 100755 --- a/bin/nlx-data-remove +++ b/bin/nlx-data-remove @@ -0,0 +1,3 @@ +#!/bin/bash + +PYTHONPATH=$NEATLYNX_HOME python $NEATLYNX_HOME/neatlynx/cmd_data_remove.py $@ diff --git a/neatlynx/cmd_base.py b/neatlynx/cmd_base.py index f56ad65c9..10ec0ff2b 100644 --- a/neatlynx/cmd_base.py +++ b/neatlynx/cmd_base.py @@ -82,4 +82,10 @@ class CmdBase(object): help=message) def run(self): - pass \ No newline at end of file + pass + + def get_cache_file_s3_name(self, cache_file): + cache_prefix_file_name = os.path.relpath(os.path.realpath(cache_file), os.path.realpath(self.git.git_dir)) + file_name = os.path.relpath(cache_prefix_file_name, self.config.cache_dir) + nlx_file_path_trim = file_name.replace(os.sep, '/').strip('/') + return self.config.aws_storage_prefix + '/' + nlx_file_path_trim diff --git a/neatlynx/cmd_data_remove.py b/neatlynx/cmd_data_remove.py new file mode 100644 index 000000000..af5ab7ab3 --- /dev/null +++ b/neatlynx/cmd_data_remove.py @@ -0,0 +1,101 @@ +import os +from boto.s3.connection import S3Connection, Key + +from neatlynx.cmd_base import CmdBase, Logger +from neatlynx.exceptions import NeatLynxException + + +class DataRemoveError(NeatLynxException): + def __init__(self, msg): + NeatLynxException.__init__(self, 'Data remove error: {}'.format(msg)) + + +class CmdDataRemove(CmdBase): + def __init__(self): + CmdBase.__init__(self) + + conn = S3Connection(self.config.aws_access_key_id, self.config.aws_secret_access_key) + + bucket_name = self.config.aws_storage_bucket + self._bucket = conn.lookup(bucket_name) + if not self._bucket: + self._bucket = conn.create_bucket(bucket_name) + Logger.info('S3 bucket "{}" was created'.format(bucket_name)) + pass + + def define_args(self, parser): + self.add_string_arg(parser, 'target', 'Target to remove - file or directory') + parser.add_argument('-r', '--recursive', action='store_true', help='Remove directory recursively') + parser.add_argument('-k', '--keep-in-cloud', action='store_true', help='Keep file in cloud') + pass + + def run(self): + target = self.args.target + + if os.path.islink(target): + return self.remove_symlink(target) + + if os.path.isdir(target): + if not self.args.recursive: + raise DataRemoveError('Directory cannot be removed. Use --recurcive flag.') + + if os.path.realpath(target) == os.path.realpath(self.config.data_dir): + raise DataRemoveError('data directory cannot be removed') + return self.remove_dir(target) + + raise DataRemoveError('File "{}" does not exit'.format(target)) + + def remove_symlink(self, file): + if not file.startswith(self.config.data_dir): + raise DataRemoveError('File "{}" supposes to be in data dir'.format(file)) + + cache_file_rel_data = os.path.join(os.path.dirname(file), os.readlink(file)) + cache_file = os.path.relpath(os.path.realpath(cache_file_rel_data), os.path.realpath(os.curdir)) + + rel_data_file = os.path.relpath(file, self.config.data_dir) + state_file = os.path.join(self.config.state_dir, rel_data_file) + + if os.path.isfile(cache_file): + os.remove(cache_file) + os.remove(file) + + if not os.path.isfile(state_file): + Logger.warn('Warning: state file "{}" does not exist'.format(state_file)) + else: + os.remove(state_file) + + if not self.args.keep_in_cloud: + s3_name = self.get_cache_file_s3_name(cache_file) + key = self._bucket.get_key(s3_name) + if not key: + Logger.warn('S3 remove warning: file "{}" does not exist in S3'.format(s3_name)) + else: + key.delete() + Logger.info('File "{}" was removed from S3'.format(s3_name)) + pass + + def remove_dir(self, data_dir): + for f in os.listdir(data_dir): + fname = os.path.join(data_dir, f) + if os.path.isdir(fname): + self.remove_dir(fname) + elif os.path.islink(fname): + self.remove_symlink(fname) + else: + raise DataRemoveError('Unsupported file type "{}"'.format(fname)) + + os.rmdir(data_dir) + + rel_data_dir = os.path.relpath(data_dir, self.config.data_dir) + cache_dir = os.path.join(self.config.cache_dir, rel_data_dir) + os.rmdir(cache_dir) + pass + + +if __name__ == '__main__': + import sys + try: + sys.exit(CmdDataRemove().run()) + except NeatLynxException as e: + Logger.error(e) + sys.exit(1) diff --git a/neatlynx/cmd_data_sync.py b/neatlynx/cmd_data_sync.py index 43a879f22..97ccf6a15 100644 --- a/neatlynx/cmd_data_sync.py +++ b/neatlynx/cmd_data_sync.py @@ -1,5 +1,6 @@ -import os import hashlib +import os + from boto.s3.connection import S3Connection from neatlynx.cmd_base import CmdBase, Logger @@ -12,7 +13,7 @@ class DataSyncError(NeatLynxException): def sizeof_fmt(num, suffix='B'): - for unit in ['','K','M','G','T','P','E','Z']: + for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']: if abs(num) < 1024.0: return "%3.1f%s%s" % (num, unit, suffix) num /= 1024.0 @@ -26,7 +27,7 @@ def percent_cb(complete, total): def file_md5(fname): hash_md5 = hashlib.md5() with open(fname, "rb") as f: - for chunk in iter(lambda: f.read(1024*100), b""): + for chunk in iter(lambda: f.read(1024*1000), b""): hash_md5.update(chunk) return hash_md5.hexdigest() @@ -50,6 +51,11 @@ class CmdDataSync(CmdBase): def run(self): target = self.args.target + rel_data_path = os.path.join(os.path.realpath(self.git.git_dir), self.config.data_dir) + if not os.path.abspath(target).startswith(os.path.realpath(rel_data_path)): + raise DataSyncError('File supposes to be in data dir - "{}"'. + format(self.config.data_dir)) + if os.path.islink(target): return self.sync_symlink(target) @@ -71,7 +77,7 @@ class CmdDataSync(CmdBase): def sync_symlink(self, file): cache_file_rel_data = os.path.join(os.path.dirname(file), os.readlink(file)) - cache_file = os.path.relpath(os.path.relpath(cache_file_rel_data), os.path.realpath(os.curdir)) + cache_file = os.path.relpath(os.path.realpath(cache_file_rel_data), os.path.realpath(os.curdir)) if os.path.isfile(cache_file): self.sync_to_cloud(cache_file) @@ -81,7 +87,7 @@ class CmdDataSync(CmdBase): pass def sync_from_cloud(self, cache_file): - s3_file = self._get_target_s3_name(cache_file) + s3_file = self.get_cache_file_s3_name(cache_file) key = self._bucket.get_key(s3_file) if not key: raise DataSyncError('File "{}" is not exist in the cloud'.format(cache_file)) @@ -91,15 +97,8 @@ class CmdDataSync(CmdBase): Logger.info('Downloading completed') pass - def _get_target_s3_name(self, cache_file): - cache_file_rel = os.path.relpath(cache_file, self.config.cache_dir) - cache_file_rel = cache_file_rel.replace(os.sep, '/').strip('/') - - target_file = self.config.aws_storage_prefix + '/' + cache_file_rel - return target_file - def sync_to_cloud(self, cache_file): - target_file = self._get_target_s3_name(cache_file) + target_file = self.get_cache_file_s3_name(cache_file) key = self._bucket.get_key(target_file) if key: