Download files and add .gitignore

This commit is contained in:
Dmitry Petrov
2017-03-09 23:47:43 -08:00
parent 6f08f44b8c
commit 4f035bc35c
6 changed files with 166 additions and 97 deletions
+4
View File
@@ -0,0 +1,4 @@
neatlynx.conf
neatlynx/__pycache__
.idea
cache
+39 -9
View File
@@ -1,8 +1,10 @@
import sys
import os
from pathlib import Path
from shutil import copyfile
import re
import requests
from neatlynx.cmd_base import CmdBase, Logger
from neatlynx.data_file_obj import DataFileObj
from neatlynx.exceptions import NeatLynxException
@@ -24,11 +26,12 @@ class CmdDataImport(CmdBase):
pass
def run(self):
input = Path(self.args.input)
if not input.exists():
raise DataImportError('Input file "{}" does not exist'.format(input))
if not input.is_file():
raise DataImportError('Input file "{}" has to be a regular file'.format(input))
if not CmdDataImport.is_url(self.args.input):
if not os.path.exists(self.args.input):
raise DataImportError('Input file "{}" does not exist'.format(self.args.input))
if not os.path.isfile(self.args.input):
raise DataImportError('Input file "{}" has to be a regular file'.format(self.args.input))
output = self.args.output
if os.path.isdir(self.args.output):
@@ -43,9 +46,15 @@ class CmdDataImport(CmdBase):
os.path.dirname(dobj.data_file_relative)))
os.makedirs(os.path.dirname(dobj.cache_file_relative), exist_ok=True)
copyfile(self.args.input, dobj.cache_file_relative)
Logger.verbose('Input file "{}" was copied to cache "{}"'.format(
self.args.input, dobj.cache_file_relative))
if CmdDataImport.is_url(self.args.input):
Logger.verbose('Downloading file {} ...'.format(self.args.input))
self.download_file(self.args.input, dobj.cache_file_relative)
Logger.verbose('Input file "{}" was downloaded to cache "{}"'.format(
self.args.input, dobj.cache_file_relative))
else:
copyfile(self.args.input, dobj.cache_file_relative)
Logger.verbose('Input file "{}" was copied to cache "{}"'.format(
self.args.input, dobj.cache_file_relative))
cache_relative_to_data = os.path.relpath(dobj.cache_file_relative, os.path.dirname(dobj.data_file_relative))
os.symlink(cache_relative_to_data, dobj.data_file_relative)
@@ -59,6 +68,27 @@ class CmdDataImport(CmdBase):
Logger.verbose('State file "{}" was created'.format(dobj.state_file_relative))
pass
URL_REGEX = re.compile(
r'^(?:http|ftp)s?://' # http:// or https://
r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain...
r'localhost|' # localhost...
r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip
r'(?::\d+)?' # optional port
r'(?:/?|[/?]\S+)$', re.IGNORECASE)
@staticmethod
def is_url(url):
return CmdDataImport.URL_REGEX.match(url) is not None
@staticmethod
def download_file(from_url, to_file):
r = requests.get(from_url, stream=True)
with open(to_file, 'wb') as f:
for chunk in r.iter_content(chunk_size=1024*100):
if chunk: # filter out keep-alive new chunks
f.write(chunk)
return
if __name__ == '__main__':
try:
+24 -31
View File
@@ -1,8 +1,9 @@
import os
from boto.s3.connection import S3Connection, Key
from boto.s3.connection import S3Connection
from neatlynx.cmd_base import CmdBase, Logger
from neatlynx.exceptions import NeatLynxException
from neatlynx.data_file_obj import DataFileObjExisting
class DataRemoveError(NeatLynxException):
@@ -32,46 +33,42 @@ class CmdDataRemove(CmdBase):
def run(self):
target = self.args.target
if os.path.islink(target):
return self.remove_symlink(target)
if os.path.isdir(target):
if not self.args.recursive:
raise DataRemoveError('Directory cannot be removed. Use --recurcive flag.')
if os.path.realpath(target) == os.path.realpath(self.config.data_dir):
if os.path.realpath(target) == \
os.path.realpath(os.path.join(self.git.git_dir_abs, self.config.data_dir)):
raise DataRemoveError('data directory cannot be removed')
return self.remove_dir(target)
raise DataRemoveError('File "{}" does not exit'.format(target))
dobj = DataFileObjExisting(target, self.git, self.config)
if os.path.islink(dobj.data_file_relative):
return self.remove_symlink(dobj.data_file_relative)
raise DataRemoveError('Cannot remove a regular file "{}"'.format(target))
def remove_symlink(self, file):
if not file.startswith(self.config.data_dir):
raise DataRemoveError('File "{}" supposes to be in data dir'.format(file))
dobj = DataFileObjExisting(file, self.git, self.config)
cache_file_rel_data = os.path.join(os.path.dirname(file), os.readlink(file))
cache_file = os.path.relpath(os.path.realpath(cache_file_rel_data), os.path.realpath(os.curdir))
if os.path.isfile(dobj.cache_file_relative):
os.remove(dobj.cache_file_relative)
dobj.remove_cache_dir_if_empty()
rel_data_file = os.path.relpath(file, self.config.data_dir)
state_file = os.path.join(self.config.state_dir, rel_data_file)
if os.path.isfile(dobj.state_file_relative):
os.remove(dobj.state_file_relative)
dobj.remove_state_dir_if_empty()
if os.path.isfile(cache_file):
os.remove(cache_file)
os.remove(file)
if not os.path.isfile(state_file):
Logger.warn('Warning: state file "{}" does not exist'.format(state_file))
if not self.args.keep_in_cloud:
key = self._bucket.get_key(dobj.cache_file_aws_key)
if not key:
Logger.warn('S3 remove warning: file "{}" does not exist in S3'.format(dobj.cache_file_aws_key))
else:
os.remove(state_file)
key.delete()
Logger.info('File "{}" was removed from S3'.format(dobj.cache_file_aws_key))
if not self.args.keep_in_cloud:
s3_name = self.get_cache_file_s3_name(cache_file)
key = self._bucket.get_key(s3_name)
if not key:
Logger.warn('S3 remove warning: file "{}" does not exist in S3'.format(s3_name))
else:
key.delete()
Logger.info('File "{}" was removed from S3'.format(s3_name))
os.remove(file)
pass
def remove_dir(self, data_dir):
@@ -85,10 +82,6 @@ class CmdDataRemove(CmdBase):
raise DataRemoveError('Unsupported file type "{}"'.format(fname))
os.rmdir(data_dir)
rel_data_dir = os.path.relpath(data_dir, self.config.data_dir)
cache_dir = os.path.join(self.config.cache_dir, rel_data_dir)
os.rmdir(cache_dir)
pass
+25 -34
View File
@@ -4,6 +4,7 @@ import os
from boto.s3.connection import S3Connection
from neatlynx.cmd_base import CmdBase, Logger
from neatlynx.data_file_obj import DataFileObjExisting
from neatlynx.exceptions import NeatLynxException
@@ -50,17 +51,12 @@ class CmdDataSync(CmdBase):
pass
def run(self):
target = self.args.target
rel_data_path = os.path.join(os.path.realpath(self.git.git_dir), self.config.data_dir)
if not os.path.abspath(target).startswith(os.path.realpath(rel_data_path)):
raise DataSyncError('File supposes to be in data dir - "{}"'.
format(self.config.data_dir))
if os.path.islink(self.args.target):
dobj = DataFileObjExisting(self.args.target, self.git, self.config)
return self.sync_symlink(dobj)
if os.path.islink(target):
return self.sync_symlink(target)
if os.path.isdir(target):
return self.sync_dir(target)
if os.path.isdir(self.args.target):
return self.sync_dir(self.args.target)
raise DataSyncError('File "{}" does not exit'.format(target))
@@ -70,51 +66,46 @@ class CmdDataSync(CmdBase):
if os.path.isdir(fname):
self.sync_dir(fname)
elif os.path.islink(fname):
self.sync_symlink(fname)
self.sync_symlink(DataFileObjExisting(fname, self.git, self.config))
else:
raise DataSyncError('Unsupported file type "{}"'.format(fname))
pass
def sync_symlink(self, file):
cache_file_rel_data = os.path.join(os.path.dirname(file), os.readlink(file))
cache_file = os.path.relpath(os.path.realpath(cache_file_rel_data), os.path.realpath(os.curdir))
if os.path.isfile(cache_file):
self.sync_to_cloud(cache_file)
def sync_symlink(self, dobj):
if os.path.isfile(dobj.cache_file_relative):
self.sync_to_cloud(dobj)
else:
self.sync_from_cloud(cache_file)
pass
self.sync_from_cloud(dobj)
pass
def sync_from_cloud(self, cache_file):
s3_file = self.get_cache_file_s3_name(cache_file)
key = self._bucket.get_key(s3_file)
def sync_from_cloud(self, dobj):
key = self._bucket.get_key(dobj.cache_file_aws_key)
if not key:
raise DataSyncError('File "{}" is not exist in the cloud'.format(cache_file))
raise DataSyncError('File "{}" does not exist in the cloud'.format(dobj.cache_file_aws_key))
Logger.info('Downloading cache file "{}" from S3 {}/{}'.format(cache_file, self._bucket.name, s3_file))
key.get_contents_to_filename(cache_file, cb=percent_cb)
Logger.info('Downloading cache file from S3 "{}/{}"'.format(self._bucket.name,
dobj.cache_file_aws_key))
key.get_contents_to_filename(dobj.cache_file_relative, cb=percent_cb)
Logger.info('Downloading completed')
pass
def sync_to_cloud(self, cache_file):
target_file = self.get_cache_file_s3_name(cache_file)
key = self._bucket.get_key(target_file)
def sync_to_cloud(self, dobj):
key = self._bucket.get_key(dobj.cache_file_aws_key)
if key:
Logger.verbose('File already uploaded to the cloud. Checking checksum...')
Logger.verbose('File already uploaded to the cloud. Checksum validation...')
md5_cloud = key.etag[1:-1]
md5_local = file_md5(cache_file)
md5_local = file_md5(dobj.cache_file_relative)
if md5_cloud == md5_local:
Logger.verbose('File checksum matches. No uploading is needed.')
return
Logger.info('Checksum miss-match. Re-uploading is required.')
Logger.info('Uploading cache file "{}" to S3 {}/{}'.format(cache_file, self._bucket.name, target_file))
key = self._bucket.new_key(target_file)
key.set_contents_from_filename(cache_file, cb=percent_cb)
Logger.info('Uploading cache file "{}" to S3 "{}"'.format(dobj.cache_file_relative,
dobj.cache_file_aws_key))
key = self._bucket.new_key(dobj.cache_file_aws_key)
key.set_contents_from_filename(dobj.cache_file_relative, cb=percent_cb)
Logger.info('Uploading completed')
pass
+59 -23
View File
@@ -9,10 +9,10 @@ class DataFilePathError(NeatLynxException):
class NotInDataDirError(NeatLynxException):
def __init__(self, data_dir):
def __init__(self, file, data_dir):
NeatLynxException.__init__(self,
'Data file location error: the file has to be in the data directory "{}"'.
format(data_dir))
'Data file location error: the file "{}" has to be in the data directory "{}"'.
format(file, data_dir))
class DataFileObj(object):
@@ -20,41 +20,44 @@ class DataFileObj(object):
CACHE_FILE_SEP = '_'
def __init__(self, data_file, git, config):
'''
nlx file name - system file name without data/cache/state prefix
'''
self._git = git
self._config = config
self._curr_dir_abs = os.path.realpath(os.curdir)
data_file = os.path.realpath(data_file)
self._data_file = data_file
self._data_file_name = os.path.basename(data_file)
data_file_dir = os.path.realpath(os.path.dirname(data_file))
data_dir = os.path.join(self._git.git_dir_abs, config.data_dir)
self._rel_path = os.path.relpath(data_file_dir, data_dir)
if not data_file_dir.startswith(data_dir):
raise NotInDataDirError(config.data_dir)
if self._rel_path == '.':
self._rel_path = ''
if not self.data_file_abs.startswith(self.data_dir_abs):
raise NotInDataDirError(self._data_file, self._config.data_dir)
pass
@property
def data_dir_abs(self):
return os.path.join(self._git.git_dir_abs, self._config.data_dir)
# Data file properties
@property
def data_file_name(self):
return self._data_file_name
return os.path.basename(self._data_file)
@property
def data_file_nlx(self):
return os.path.join(self._config.data_dir, self._rel_path, self.data_file_name)
return os.path.relpath(self.data_file_abs, self.data_dir_abs)
# @property
# def data_file_abs(self):
# return os.path.realpath(self.data_file_relative)
@property
def data_file_abs(self):
return os.path.join(self._git.git_dir_abs, self.data_file_nlx)
'''Do not fully resolve file name since it is a link'''
return os.path.abspath(self.data_file_relative)
@property
def data_file_relative(self):
return os.path.relpath(self.data_file_abs, self._curr_dir_abs)
return self._data_file
# Cache file properties
@property
@@ -67,11 +70,15 @@ class DataFileObj(object):
def cache_file_nlx(self):
if not self._git.curr_commit:
return None
return os.path.join(self._config.cache_dir, self._rel_path, self.cache_file_name)
return self.data_file_nlx + self.CACHE_FILE_SEP + self._git.curr_commit
@property
def cache_file_aws_key(self):
return '{}/{}'.format(self._config.aws_storage_prefix, self.cache_file_nlx).strip('/')
@property
def cache_file_abs(self):
return os.path.join(self._git.git_dir_abs, self.cache_file_nlx)
return os.path.join(self._git.git_dir_abs, self._config.cache_dir, self.cache_file_nlx)
@property
def cache_file_relative(self):
@@ -84,12 +91,41 @@ class DataFileObj(object):
@property
def state_file_nlx(self):
return os.path.join(self._config.state_dir, self._rel_path, self.state_file_name)
return self.data_file_nlx + self.STATE_FILE_SUFFIX
@property
def state_file_abs(self):
return os.path.join(self._git.git_dir_abs, self.state_file_nlx)
return os.path.join(self._git.git_dir_abs, self._config.state_dir, self.state_file_nlx)
@property
def state_file_relative(self):
return os.path.relpath(self.state_file_abs, self._curr_dir_abs)
class DataFileObjExisting(DataFileObj):
def __init__(self, data_file, git, config):
DataFileObj.__init__(self, data_file, git, config)
if not os.path.islink(data_file):
raise DataFilePathError('Data file must be a symbolic link')
pass
@property
def cache_file_name(self):
return os.path.basename(self.cache_file_abs)
@property
def cache_file_abs(self):
return os.path.realpath(self.data_file_relative)
@staticmethod
def remove_dir_if_empty(dir):
cache_file_dir = os.path.dirname(dir)
if not os.listdir(cache_file_dir):
os.rmdir(cache_file_dir)
def remove_state_dir_if_empty(self):
self.remove_dir_if_empty(self.state_file_relative)
def remove_cache_dir_if_empty(self):
self.remove_dir_if_empty(self.cache_file_relative)
+15
View File
@@ -49,6 +49,21 @@ class TestDataFileObjBasic(BasicDataDirTest):
self.assertEqual(self._dobj.state_file_abs, os.path.join(self._test_git_dir, 'state', 'file.txt.state'))
class TestNlxNames(BasicDataDirTest):
def setUp(self):
BasicDataDirTest.setUp(self, test_dir=os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'))
git = GitWrapperI(git_dir=self._test_git_dir, commit='ad45ba8')
config = ConfigI('data', 'cache', 'state')
self._dobj = DataFileObj(os.path.join('data', 'file.txt'), git, config)
pass
def test_data_nlx(self):
self.assertEqual(self._dobj.data_file_nlx, 'file.txt')
pass
class TestDataFileObjInDataDir(BasicDataDirTest):
def setUp(self):
BasicDataDirTest.setUp(self, os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'))