mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 07:18:27 +01:00
+79
-5
@@ -10,6 +10,15 @@ import filecmp
|
||||
import math
|
||||
|
||||
from boto.s3.connection import S3Connection
|
||||
try:
|
||||
import httplib
|
||||
except ImportError:
|
||||
# Python3 workaround for ResumableDownloadHandler.
|
||||
# See https://github.com/boto/boto/pull/3755.
|
||||
import sys
|
||||
import http.client as httplib
|
||||
sys.modules['httplib'] = httplib
|
||||
from boto.s3.resumable_download_handler import ResumableDownloadHandler
|
||||
from google.cloud import storage as gc
|
||||
|
||||
try:
|
||||
@@ -30,7 +39,6 @@ from dvc.utils import cached_property
|
||||
from dvc.system import System
|
||||
from dvc.utils import map_progress
|
||||
|
||||
|
||||
STATUS_UNKNOWN = 0
|
||||
STATUS_OK = 1
|
||||
STATUS_MODIFIED = 2
|
||||
@@ -381,6 +389,14 @@ class DataCloudAWS(DataCloudBase):
|
||||
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _upload_tracker(fname):
|
||||
return fname + '.upload'
|
||||
|
||||
@staticmethod
|
||||
def _download_tracker(fname):
|
||||
return fname + '.download'
|
||||
|
||||
def _import(self, bucket_name, key_name, fname, data_item):
|
||||
|
||||
bucket = self._get_bucket_aws(bucket_name)
|
||||
@@ -398,9 +414,9 @@ class DataCloudAWS(DataCloudBase):
|
||||
|
||||
Logger.debug('Downloading cache file from S3 "{}/{}" to "{}"'.format(bucket.name, key_name, fname))
|
||||
|
||||
temp_file = None
|
||||
res_h = ResumableDownloadHandler(tracker_file_name=self._download_tracker(tmp_file), num_retries=10)
|
||||
try:
|
||||
key.get_contents_to_filename(tmp_file, cb=create_cb(name))
|
||||
key.get_contents_to_filename(tmp_file, cb=create_cb(name), res_download_handler=res_h)
|
||||
os.rename(tmp_file, fname)
|
||||
except Exception as exc:
|
||||
Logger.error('Failed to download "{}": {}'.format(key_name, exc))
|
||||
@@ -411,12 +427,65 @@ class DataCloudAWS(DataCloudBase):
|
||||
|
||||
return data_item
|
||||
|
||||
def _push_multipart(self, key, fname):
|
||||
def _read_upload_tracker(self, fname):
|
||||
try:
|
||||
return open(self._upload_tracker(fname), 'r').read()
|
||||
except Exception as e:
|
||||
Logger.debug("Failed to read upload tracker file for {}: {}".format(fname, e))
|
||||
return None
|
||||
|
||||
def _write_upload_tracker(self, fname, mp_id):
|
||||
try:
|
||||
open(self._upload_tracker(fname), 'w+').write(mp_id)
|
||||
except Exception as e:
|
||||
Logger.debug("Failed to write upload tracker file for {}: {}".format(fname, e))
|
||||
|
||||
def _unlink_upload_tracker(self, fname):
|
||||
try:
|
||||
os.unlink(self._upload_tracker(fname))
|
||||
except Exception as e:
|
||||
Logger.debug("Failed to unlink upload tracker file for {}: {}".format(fname, e))
|
||||
|
||||
def _resume_multipart(self, key, fname):
|
||||
try:
|
||||
mp_id = open(self._upload_tracker(fname), 'r').read()
|
||||
except Exception as e:
|
||||
Logger.debug("Failed to read upload tracker file for {}: {}".format(fname, e))
|
||||
return None
|
||||
|
||||
for mp in key.bucket.get_all_multipart_uploads():
|
||||
if mp.id != mp_id:
|
||||
continue
|
||||
|
||||
Logger.debug("Found existing multipart {}".format(mp_id))
|
||||
return mp
|
||||
|
||||
return None
|
||||
|
||||
def _create_multipart(self, key, fname):
|
||||
# AWS doesn't provide easilly accessible md5 for multipart
|
||||
# objects, so we have to store our own md5 sum to use later.
|
||||
metadata = {'dvc-md5' : str(file_md5(fname)[0])}
|
||||
|
||||
mp = key.bucket.initiate_multipart_upload(key.name, metadata=metadata)
|
||||
self._write_upload_tracker(fname, mp.id)
|
||||
return mp
|
||||
|
||||
def _get_multipart(self, key, fname):
|
||||
mp = self._resume_multipart(key, fname)
|
||||
if mp != None:
|
||||
return mp
|
||||
|
||||
return self._create_multipart(key, fname)
|
||||
|
||||
def _skip_part(self, mp, fp, part_num, size):
|
||||
for p in mp.get_all_parts():
|
||||
if p.part_number == part_num and p.size == size:# and p.etag and p.last_modified
|
||||
Logger.debug("Skipping part #{}".format(str(part_num)))
|
||||
return True
|
||||
return False
|
||||
|
||||
def _push_multipart(self, key, fname):
|
||||
mp = self._get_multipart(key, fname)
|
||||
|
||||
source_size = os.stat(fname).st_size
|
||||
chunk_size = 50*1024*1024
|
||||
@@ -429,8 +498,12 @@ class DataCloudAWS(DataCloudBase):
|
||||
size = min([chunk_size, left])
|
||||
part_num = i + 1
|
||||
|
||||
if self._skip_part(mp, fp, part_num, size):
|
||||
continue
|
||||
|
||||
fp.seek(offset)
|
||||
mp.upload_part_from_file(fp=fp,
|
||||
replace=False,
|
||||
size=size,
|
||||
num_cb=100,
|
||||
part_num=part_num,
|
||||
@@ -440,6 +513,7 @@ class DataCloudAWS(DataCloudBase):
|
||||
raise Exception("Couldn't upload all file parts")
|
||||
|
||||
mp.complete_upload()
|
||||
self._unlink_upload_tracker(fname)
|
||||
|
||||
def push(self, data_item):
|
||||
""" push, aws version """
|
||||
|
||||
Reference in New Issue
Block a user