mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 15:28:28 +01:00
Download files and add .gitignore
This commit is contained in:
@@ -0,0 +1,4 @@
|
||||
neatlynx.conf
|
||||
neatlynx/__pycache__
|
||||
.idea
|
||||
cache
|
||||
@@ -1,8 +1,10 @@
|
||||
import sys
|
||||
import os
|
||||
from pathlib import Path
|
||||
from shutil import copyfile
|
||||
|
||||
import re
|
||||
import requests
|
||||
|
||||
from neatlynx.cmd_base import CmdBase, Logger
|
||||
from neatlynx.data_file_obj import DataFileObj
|
||||
from neatlynx.exceptions import NeatLynxException
|
||||
@@ -24,11 +26,12 @@ class CmdDataImport(CmdBase):
|
||||
pass
|
||||
|
||||
def run(self):
|
||||
input = Path(self.args.input)
|
||||
if not input.exists():
|
||||
raise DataImportError('Input file "{}" does not exist'.format(input))
|
||||
if not input.is_file():
|
||||
raise DataImportError('Input file "{}" has to be a regular file'.format(input))
|
||||
|
||||
if not CmdDataImport.is_url(self.args.input):
|
||||
if not os.path.exists(self.args.input):
|
||||
raise DataImportError('Input file "{}" does not exist'.format(self.args.input))
|
||||
if not os.path.isfile(self.args.input):
|
||||
raise DataImportError('Input file "{}" has to be a regular file'.format(self.args.input))
|
||||
|
||||
output = self.args.output
|
||||
if os.path.isdir(self.args.output):
|
||||
@@ -43,9 +46,15 @@ class CmdDataImport(CmdBase):
|
||||
os.path.dirname(dobj.data_file_relative)))
|
||||
|
||||
os.makedirs(os.path.dirname(dobj.cache_file_relative), exist_ok=True)
|
||||
copyfile(self.args.input, dobj.cache_file_relative)
|
||||
Logger.verbose('Input file "{}" was copied to cache "{}"'.format(
|
||||
self.args.input, dobj.cache_file_relative))
|
||||
if CmdDataImport.is_url(self.args.input):
|
||||
Logger.verbose('Downloading file {} ...'.format(self.args.input))
|
||||
self.download_file(self.args.input, dobj.cache_file_relative)
|
||||
Logger.verbose('Input file "{}" was downloaded to cache "{}"'.format(
|
||||
self.args.input, dobj.cache_file_relative))
|
||||
else:
|
||||
copyfile(self.args.input, dobj.cache_file_relative)
|
||||
Logger.verbose('Input file "{}" was copied to cache "{}"'.format(
|
||||
self.args.input, dobj.cache_file_relative))
|
||||
|
||||
cache_relative_to_data = os.path.relpath(dobj.cache_file_relative, os.path.dirname(dobj.data_file_relative))
|
||||
os.symlink(cache_relative_to_data, dobj.data_file_relative)
|
||||
@@ -59,6 +68,27 @@ class CmdDataImport(CmdBase):
|
||||
Logger.verbose('State file "{}" was created'.format(dobj.state_file_relative))
|
||||
pass
|
||||
|
||||
URL_REGEX = re.compile(
|
||||
r'^(?:http|ftp)s?://' # http:// or https://
|
||||
r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain...
|
||||
r'localhost|' # localhost...
|
||||
r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip
|
||||
r'(?::\d+)?' # optional port
|
||||
r'(?:/?|[/?]\S+)$', re.IGNORECASE)
|
||||
|
||||
@staticmethod
|
||||
def is_url(url):
|
||||
return CmdDataImport.URL_REGEX.match(url) is not None
|
||||
|
||||
@staticmethod
|
||||
def download_file(from_url, to_file):
|
||||
r = requests.get(from_url, stream=True)
|
||||
with open(to_file, 'wb') as f:
|
||||
for chunk in r.iter_content(chunk_size=1024*100):
|
||||
if chunk: # filter out keep-alive new chunks
|
||||
f.write(chunk)
|
||||
return
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
|
||||
+24
-31
@@ -1,8 +1,9 @@
|
||||
import os
|
||||
from boto.s3.connection import S3Connection, Key
|
||||
from boto.s3.connection import S3Connection
|
||||
|
||||
from neatlynx.cmd_base import CmdBase, Logger
|
||||
from neatlynx.exceptions import NeatLynxException
|
||||
from neatlynx.data_file_obj import DataFileObjExisting
|
||||
|
||||
|
||||
class DataRemoveError(NeatLynxException):
|
||||
@@ -32,46 +33,42 @@ class CmdDataRemove(CmdBase):
|
||||
def run(self):
|
||||
target = self.args.target
|
||||
|
||||
if os.path.islink(target):
|
||||
return self.remove_symlink(target)
|
||||
|
||||
if os.path.isdir(target):
|
||||
if not self.args.recursive:
|
||||
raise DataRemoveError('Directory cannot be removed. Use --recurcive flag.')
|
||||
|
||||
if os.path.realpath(target) == os.path.realpath(self.config.data_dir):
|
||||
if os.path.realpath(target) == \
|
||||
os.path.realpath(os.path.join(self.git.git_dir_abs, self.config.data_dir)):
|
||||
raise DataRemoveError('data directory cannot be removed')
|
||||
|
||||
return self.remove_dir(target)
|
||||
|
||||
raise DataRemoveError('File "{}" does not exit'.format(target))
|
||||
dobj = DataFileObjExisting(target, self.git, self.config)
|
||||
if os.path.islink(dobj.data_file_relative):
|
||||
return self.remove_symlink(dobj.data_file_relative)
|
||||
|
||||
raise DataRemoveError('Cannot remove a regular file "{}"'.format(target))
|
||||
|
||||
def remove_symlink(self, file):
|
||||
if not file.startswith(self.config.data_dir):
|
||||
raise DataRemoveError('File "{}" supposes to be in data dir'.format(file))
|
||||
dobj = DataFileObjExisting(file, self.git, self.config)
|
||||
|
||||
cache_file_rel_data = os.path.join(os.path.dirname(file), os.readlink(file))
|
||||
cache_file = os.path.relpath(os.path.realpath(cache_file_rel_data), os.path.realpath(os.curdir))
|
||||
if os.path.isfile(dobj.cache_file_relative):
|
||||
os.remove(dobj.cache_file_relative)
|
||||
dobj.remove_cache_dir_if_empty()
|
||||
|
||||
rel_data_file = os.path.relpath(file, self.config.data_dir)
|
||||
state_file = os.path.join(self.config.state_dir, rel_data_file)
|
||||
if os.path.isfile(dobj.state_file_relative):
|
||||
os.remove(dobj.state_file_relative)
|
||||
dobj.remove_state_dir_if_empty()
|
||||
|
||||
if os.path.isfile(cache_file):
|
||||
os.remove(cache_file)
|
||||
os.remove(file)
|
||||
|
||||
if not os.path.isfile(state_file):
|
||||
Logger.warn('Warning: state file "{}" does not exist'.format(state_file))
|
||||
if not self.args.keep_in_cloud:
|
||||
key = self._bucket.get_key(dobj.cache_file_aws_key)
|
||||
if not key:
|
||||
Logger.warn('S3 remove warning: file "{}" does not exist in S3'.format(dobj.cache_file_aws_key))
|
||||
else:
|
||||
os.remove(state_file)
|
||||
key.delete()
|
||||
Logger.info('File "{}" was removed from S3'.format(dobj.cache_file_aws_key))
|
||||
|
||||
if not self.args.keep_in_cloud:
|
||||
s3_name = self.get_cache_file_s3_name(cache_file)
|
||||
key = self._bucket.get_key(s3_name)
|
||||
if not key:
|
||||
Logger.warn('S3 remove warning: file "{}" does not exist in S3'.format(s3_name))
|
||||
else:
|
||||
key.delete()
|
||||
Logger.info('File "{}" was removed from S3'.format(s3_name))
|
||||
os.remove(file)
|
||||
pass
|
||||
|
||||
def remove_dir(self, data_dir):
|
||||
@@ -85,10 +82,6 @@ class CmdDataRemove(CmdBase):
|
||||
raise DataRemoveError('Unsupported file type "{}"'.format(fname))
|
||||
|
||||
os.rmdir(data_dir)
|
||||
|
||||
rel_data_dir = os.path.relpath(data_dir, self.config.data_dir)
|
||||
cache_dir = os.path.join(self.config.cache_dir, rel_data_dir)
|
||||
os.rmdir(cache_dir)
|
||||
pass
|
||||
|
||||
|
||||
|
||||
+25
-34
@@ -4,6 +4,7 @@ import os
|
||||
from boto.s3.connection import S3Connection
|
||||
|
||||
from neatlynx.cmd_base import CmdBase, Logger
|
||||
from neatlynx.data_file_obj import DataFileObjExisting
|
||||
from neatlynx.exceptions import NeatLynxException
|
||||
|
||||
|
||||
@@ -50,17 +51,12 @@ class CmdDataSync(CmdBase):
|
||||
pass
|
||||
|
||||
def run(self):
|
||||
target = self.args.target
|
||||
rel_data_path = os.path.join(os.path.realpath(self.git.git_dir), self.config.data_dir)
|
||||
if not os.path.abspath(target).startswith(os.path.realpath(rel_data_path)):
|
||||
raise DataSyncError('File supposes to be in data dir - "{}"'.
|
||||
format(self.config.data_dir))
|
||||
if os.path.islink(self.args.target):
|
||||
dobj = DataFileObjExisting(self.args.target, self.git, self.config)
|
||||
return self.sync_symlink(dobj)
|
||||
|
||||
if os.path.islink(target):
|
||||
return self.sync_symlink(target)
|
||||
|
||||
if os.path.isdir(target):
|
||||
return self.sync_dir(target)
|
||||
if os.path.isdir(self.args.target):
|
||||
return self.sync_dir(self.args.target)
|
||||
|
||||
raise DataSyncError('File "{}" does not exit'.format(target))
|
||||
|
||||
@@ -70,51 +66,46 @@ class CmdDataSync(CmdBase):
|
||||
if os.path.isdir(fname):
|
||||
self.sync_dir(fname)
|
||||
elif os.path.islink(fname):
|
||||
self.sync_symlink(fname)
|
||||
self.sync_symlink(DataFileObjExisting(fname, self.git, self.config))
|
||||
else:
|
||||
raise DataSyncError('Unsupported file type "{}"'.format(fname))
|
||||
pass
|
||||
|
||||
def sync_symlink(self, file):
|
||||
cache_file_rel_data = os.path.join(os.path.dirname(file), os.readlink(file))
|
||||
cache_file = os.path.relpath(os.path.realpath(cache_file_rel_data), os.path.realpath(os.curdir))
|
||||
|
||||
if os.path.isfile(cache_file):
|
||||
self.sync_to_cloud(cache_file)
|
||||
def sync_symlink(self, dobj):
|
||||
if os.path.isfile(dobj.cache_file_relative):
|
||||
self.sync_to_cloud(dobj)
|
||||
else:
|
||||
self.sync_from_cloud(cache_file)
|
||||
pass
|
||||
self.sync_from_cloud(dobj)
|
||||
pass
|
||||
|
||||
def sync_from_cloud(self, cache_file):
|
||||
s3_file = self.get_cache_file_s3_name(cache_file)
|
||||
key = self._bucket.get_key(s3_file)
|
||||
def sync_from_cloud(self, dobj):
|
||||
key = self._bucket.get_key(dobj.cache_file_aws_key)
|
||||
if not key:
|
||||
raise DataSyncError('File "{}" is not exist in the cloud'.format(cache_file))
|
||||
raise DataSyncError('File "{}" does not exist in the cloud'.format(dobj.cache_file_aws_key))
|
||||
|
||||
Logger.info('Downloading cache file "{}" from S3 {}/{}'.format(cache_file, self._bucket.name, s3_file))
|
||||
key.get_contents_to_filename(cache_file, cb=percent_cb)
|
||||
Logger.info('Downloading cache file from S3 "{}/{}"'.format(self._bucket.name,
|
||||
dobj.cache_file_aws_key))
|
||||
key.get_contents_to_filename(dobj.cache_file_relative, cb=percent_cb)
|
||||
Logger.info('Downloading completed')
|
||||
pass
|
||||
|
||||
def sync_to_cloud(self, cache_file):
|
||||
target_file = self.get_cache_file_s3_name(cache_file)
|
||||
|
||||
key = self._bucket.get_key(target_file)
|
||||
def sync_to_cloud(self, dobj):
|
||||
key = self._bucket.get_key(dobj.cache_file_aws_key)
|
||||
if key:
|
||||
Logger.verbose('File already uploaded to the cloud. Checking checksum...')
|
||||
Logger.verbose('File already uploaded to the cloud. Checksum validation...')
|
||||
|
||||
md5_cloud = key.etag[1:-1]
|
||||
md5_local = file_md5(cache_file)
|
||||
md5_local = file_md5(dobj.cache_file_relative)
|
||||
if md5_cloud == md5_local:
|
||||
Logger.verbose('File checksum matches. No uploading is needed.')
|
||||
return
|
||||
|
||||
Logger.info('Checksum miss-match. Re-uploading is required.')
|
||||
|
||||
Logger.info('Uploading cache file "{}" to S3 {}/{}'.format(cache_file, self._bucket.name, target_file))
|
||||
key = self._bucket.new_key(target_file)
|
||||
key.set_contents_from_filename(cache_file, cb=percent_cb)
|
||||
Logger.info('Uploading cache file "{}" to S3 "{}"'.format(dobj.cache_file_relative,
|
||||
dobj.cache_file_aws_key))
|
||||
key = self._bucket.new_key(dobj.cache_file_aws_key)
|
||||
key.set_contents_from_filename(dobj.cache_file_relative, cb=percent_cb)
|
||||
Logger.info('Uploading completed')
|
||||
pass
|
||||
|
||||
|
||||
+59
-23
@@ -9,10 +9,10 @@ class DataFilePathError(NeatLynxException):
|
||||
|
||||
|
||||
class NotInDataDirError(NeatLynxException):
|
||||
def __init__(self, data_dir):
|
||||
def __init__(self, file, data_dir):
|
||||
NeatLynxException.__init__(self,
|
||||
'Data file location error: the file has to be in the data directory "{}"'.
|
||||
format(data_dir))
|
||||
'Data file location error: the file "{}" has to be in the data directory "{}"'.
|
||||
format(file, data_dir))
|
||||
|
||||
|
||||
class DataFileObj(object):
|
||||
@@ -20,41 +20,44 @@ class DataFileObj(object):
|
||||
CACHE_FILE_SEP = '_'
|
||||
|
||||
def __init__(self, data_file, git, config):
|
||||
'''
|
||||
nlx file name - system file name without data/cache/state prefix
|
||||
'''
|
||||
self._git = git
|
||||
self._config = config
|
||||
self._curr_dir_abs = os.path.realpath(os.curdir)
|
||||
|
||||
data_file = os.path.realpath(data_file)
|
||||
self._data_file = data_file
|
||||
|
||||
self._data_file_name = os.path.basename(data_file)
|
||||
data_file_dir = os.path.realpath(os.path.dirname(data_file))
|
||||
|
||||
data_dir = os.path.join(self._git.git_dir_abs, config.data_dir)
|
||||
self._rel_path = os.path.relpath(data_file_dir, data_dir)
|
||||
|
||||
if not data_file_dir.startswith(data_dir):
|
||||
raise NotInDataDirError(config.data_dir)
|
||||
|
||||
if self._rel_path == '.':
|
||||
self._rel_path = ''
|
||||
if not self.data_file_abs.startswith(self.data_dir_abs):
|
||||
raise NotInDataDirError(self._data_file, self._config.data_dir)
|
||||
pass
|
||||
|
||||
@property
|
||||
def data_dir_abs(self):
|
||||
return os.path.join(self._git.git_dir_abs, self._config.data_dir)
|
||||
|
||||
# Data file properties
|
||||
@property
|
||||
def data_file_name(self):
|
||||
return self._data_file_name
|
||||
return os.path.basename(self._data_file)
|
||||
|
||||
@property
|
||||
def data_file_nlx(self):
|
||||
return os.path.join(self._config.data_dir, self._rel_path, self.data_file_name)
|
||||
return os.path.relpath(self.data_file_abs, self.data_dir_abs)
|
||||
|
||||
# @property
|
||||
# def data_file_abs(self):
|
||||
# return os.path.realpath(self.data_file_relative)
|
||||
|
||||
@property
|
||||
def data_file_abs(self):
|
||||
return os.path.join(self._git.git_dir_abs, self.data_file_nlx)
|
||||
'''Do not fully resolve file name since it is a link'''
|
||||
return os.path.abspath(self.data_file_relative)
|
||||
|
||||
@property
|
||||
def data_file_relative(self):
|
||||
return os.path.relpath(self.data_file_abs, self._curr_dir_abs)
|
||||
return self._data_file
|
||||
|
||||
# Cache file properties
|
||||
@property
|
||||
@@ -67,11 +70,15 @@ class DataFileObj(object):
|
||||
def cache_file_nlx(self):
|
||||
if not self._git.curr_commit:
|
||||
return None
|
||||
return os.path.join(self._config.cache_dir, self._rel_path, self.cache_file_name)
|
||||
return self.data_file_nlx + self.CACHE_FILE_SEP + self._git.curr_commit
|
||||
|
||||
@property
|
||||
def cache_file_aws_key(self):
|
||||
return '{}/{}'.format(self._config.aws_storage_prefix, self.cache_file_nlx).strip('/')
|
||||
|
||||
@property
|
||||
def cache_file_abs(self):
|
||||
return os.path.join(self._git.git_dir_abs, self.cache_file_nlx)
|
||||
return os.path.join(self._git.git_dir_abs, self._config.cache_dir, self.cache_file_nlx)
|
||||
|
||||
@property
|
||||
def cache_file_relative(self):
|
||||
@@ -84,12 +91,41 @@ class DataFileObj(object):
|
||||
|
||||
@property
|
||||
def state_file_nlx(self):
|
||||
return os.path.join(self._config.state_dir, self._rel_path, self.state_file_name)
|
||||
return self.data_file_nlx + self.STATE_FILE_SUFFIX
|
||||
|
||||
@property
|
||||
def state_file_abs(self):
|
||||
return os.path.join(self._git.git_dir_abs, self.state_file_nlx)
|
||||
return os.path.join(self._git.git_dir_abs, self._config.state_dir, self.state_file_nlx)
|
||||
|
||||
@property
|
||||
def state_file_relative(self):
|
||||
return os.path.relpath(self.state_file_abs, self._curr_dir_abs)
|
||||
|
||||
|
||||
class DataFileObjExisting(DataFileObj):
|
||||
def __init__(self, data_file, git, config):
|
||||
DataFileObj.__init__(self, data_file, git, config)
|
||||
|
||||
if not os.path.islink(data_file):
|
||||
raise DataFilePathError('Data file must be a symbolic link')
|
||||
pass
|
||||
|
||||
@property
|
||||
def cache_file_name(self):
|
||||
return os.path.basename(self.cache_file_abs)
|
||||
|
||||
@property
|
||||
def cache_file_abs(self):
|
||||
return os.path.realpath(self.data_file_relative)
|
||||
|
||||
@staticmethod
|
||||
def remove_dir_if_empty(dir):
|
||||
cache_file_dir = os.path.dirname(dir)
|
||||
if not os.listdir(cache_file_dir):
|
||||
os.rmdir(cache_file_dir)
|
||||
|
||||
def remove_state_dir_if_empty(self):
|
||||
self.remove_dir_if_empty(self.state_file_relative)
|
||||
|
||||
def remove_cache_dir_if_empty(self):
|
||||
self.remove_dir_if_empty(self.cache_file_relative)
|
||||
|
||||
@@ -49,6 +49,21 @@ class TestDataFileObjBasic(BasicDataDirTest):
|
||||
self.assertEqual(self._dobj.state_file_abs, os.path.join(self._test_git_dir, 'state', 'file.txt.state'))
|
||||
|
||||
|
||||
class TestNlxNames(BasicDataDirTest):
|
||||
def setUp(self):
|
||||
BasicDataDirTest.setUp(self, test_dir=os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'))
|
||||
|
||||
git = GitWrapperI(git_dir=self._test_git_dir, commit='ad45ba8')
|
||||
config = ConfigI('data', 'cache', 'state')
|
||||
|
||||
self._dobj = DataFileObj(os.path.join('data', 'file.txt'), git, config)
|
||||
pass
|
||||
|
||||
def test_data_nlx(self):
|
||||
self.assertEqual(self._dobj.data_file_nlx, 'file.txt')
|
||||
pass
|
||||
|
||||
|
||||
class TestDataFileObjInDataDir(BasicDataDirTest):
|
||||
def setUp(self):
|
||||
BasicDataDirTest.setUp(self, os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'))
|
||||
|
||||
Reference in New Issue
Block a user