Merge pull request #178 from efiop/master

aws: use multipart to push changes

dvc/data_cloud.py

@@ -7,6 +7,7 @@ import tempfile
 import requests
 import configparser
 import filecmp
+import math
 
 from boto.s3.connection import S3Connection
 from google.cloud import storage as gc
@@ -455,6 +456,28 @@ class DataCloudAWS(DataCloudBase):
 
         return data_item
 
+    def _push_multipart(self, b, aws_key, fname):
+        mp = b.initiate_multipart_upload(aws_key)
+
+        source_size = os.stat(fname).st_size
+        chunk_size = 5000*1024*1024
+        chunk_count = int(math.ceil(source_size / float(chunk_size)))
+
+        with open(fname, 'r') as fp:
+            for i in range(chunk_count):
+                offset = i * chunk_size
+                left = source_size - offset
+                size = min([chunk_size, left])
+                part_num = i + 1
+
+                fp.seek(offset)
+                mp.upload_part_from_file(fp=fp, part_num=part_num, size=size, cb=create_cb(fname))
+
+        if len(mp.get_all_parts()) != chunk_count:
+            raise Exception("Couldn't upload all file parts")
+
+        mp.complete_upload()
+
     def push(self, data_item):
         """ push, aws version """
 
@@ -473,8 +496,7 @@ class DataCloudAWS(DataCloudBase):
         key = bucket.new_key(aws_key)
 
         try:
-            key.set_contents_from_filename(data_item.resolved_cache.relative,
-                                       cb=create_cb(data_item.resolved_cache.relative))
+            self._push_multipart(bucket, aws_key, data_item.resolved_cache.relative)
         except Exception as exc:
             Logger.error('Failed to upload "{}": {}'.format(data_item.resolved_cache.relative, exc))
             return None