google cloud installs ALL THE THINGS

fixes to single file commands
got data import to work
get path_factory from settings
cloud selector
data sync -> s3 works again
blank dvc.conf

README.md

@@ -10,6 +10,36 @@ This project is distributed under the Apache license version 2.0 (see the LICENS
 
 By submitting a pull request for this project, you agree to license your contribution under the Apache license version 2.0 to this project.
 
+# Setup / Configuration
+`dvc` supports both aws and google cloud; you will need accounts either from [aws](https://aws.amazon.com/) or [gcloud](https://cloud.google.com/).
+
+To see your available buckets, run either `TODO` or `gsutil ls`.
+
+
+## Google Cloud Setup
+To setup your google cloud credentials:
+TODO
+
+To test they are set up correctly, 
+TODO
+
+# Usage
+```
+mkdir t
+cd !$
+git init .
+dvc init
+
+Info. Directories data/, cache/ and state/ were created
+Info. File .gitignore was created
+Info. Directory cache was added to .gitignore file
+Info. [Git] A new commit 34687b2 was made in the current branch. Added files:
+Info. [Git]	A  .gitignore
+Info. [Git]	A  dvc.conf
+
+
+
+```
 
 # Development
 Create a virtualenv, for example, at `~/.environments/dvc` by `mkdir -p ~/.environments/dvc; virtualenv ~/.environments/dvc` or use `virtualenvwrapper`.

dvc.conf

@@ -2,7 +2,7 @@
 DataDir = data
 CacheDir = cache
 StateDir = state
-Cloud = AWS
+Cloud = google
 
 # Log levels: Debug, Info, Warning and Error
 LogLevel = Info
@@ -11,7 +11,7 @@ LogLevel = Info
 AccessKeyId =
 SecretAccessKey =
 
-StoragePath = dvc/tutorial
+StoragePath =
 
 
 Region = us-east-1
@@ -29,4 +29,14 @@ Storage = my-100gb-drive-io
 KeyDir = ~/.ssh
 KeyName = dvc-key
 
-SecurityGroup = dvc-group
+SecurityGroup = dvc-group
+
+
+[GC]
+ProjectName =
+
+StoragePath =
+
+
+[Data]
+StoragePath =

dvc/command/base.py

@@ -13,7 +13,6 @@ class CmdBase(object):
         parser = argparse.ArgumentParser()
         self.define_args(parser)
         self._parsed_args, self._command_args = parser.parse_known_args(args=self.args)
-        pass
 
     @property
     def settings(self):
@@ -35,8 +34,9 @@ class CmdBase(object):
     def config(self):
         return self._settings.config
 
-    def cache_file_aws_key(self, file):
-        return '{}/{}'.format(self.config.aws_storage_prefix, file).strip('/')
+    def cache_file_key(self, file):
+        print 'storage_prefix %s' % self.config.storage_prefix
+        return '{}/{}'.format(self.config.storage_prefix, file).strip('/')
 
     @property
     def dvc_home(self):

dvc/command/data_sync.py

@@ -2,6 +2,7 @@ import hashlib
 import os
 
 from boto.s3.connection import S3Connection
+from google.cloud import storage as gc
 
 from dvc.command.base import CmdBase
 from dvc.logger import Logger
@@ -39,22 +40,12 @@ class CmdDataSync(CmdBase):
     def __init__(self, settings):
         super(CmdDataSync, self).__init__(settings)
 
-        conn = S3Connection(self.config.aws_access_key_id, self.config.aws_secret_access_key)
-
-        bucket_name = self.config.aws_storage_bucket
-        self._bucket = conn.lookup(bucket_name)
-        if not self._bucket:
-            self._bucket = conn.create_bucket(bucket_name)
-            Logger.info('S3 bucket "{}" was created'.format(bucket_name))
-        pass
-
     def define_args(self, parser):
         self.add_string_arg(parser, 'target', 'Target to sync - file or directory')
-        pass
 
     def run(self):
         if System.islink(self.parsed_args.target):
-            data_item = self.path_factory.existing_data_item(self.parsed_args.target)
+            data_item = self.settings.path_factory.existing_data_item(self.parsed_args.target)
             return self.sync_symlink(data_item)
 
         if os.path.isdir(self.parsed_args.target):
@@ -68,33 +59,48 @@ class CmdDataSync(CmdBase):
             if os.path.isdir(fname):
                 self.sync_dir(fname)
             elif System.islink(fname):
-                self.sync_symlink(self.path_factory.existing_data_item(fname))
+                self.sync_symlink(self.settings.path_factory.existing_data_item(fname))
             else:
                 raise DataSyncError('Unsupported file type "{}"'.format(fname))
-        pass
 
     def sync_symlink(self, data_item):
         if os.path.isfile(data_item.cache.relative):
             self.sync_to_cloud(data_item)
         else:
             self.sync_from_cloud(data_item)
-        pass
 
-    def sync_from_cloud(self, item):
-        aws_key = self.cache_file_aws_key(item.cache.dvc)
-        key = self._bucket.get_key(aws_key)
+    def _get_bucket_aws(self):
+        """ get a bucket object, aws """
+
+        conn = S3Connection(self.config.aws_access_key_id, self.config.aws_secret_access_key, host='s3-us-west-2.amazonaws.com')
+        bucket_name = self.config.storage_bucket
+        bucket = conn.lookup(bucket_name)
+        if bucket is None:
+            raise DataSyncError('Bucket "{}" can\'t be accessed'.format(bucket_name))
+        return bucket
+
+    def _sync_from_cloud_aws(self, item):
+        """ sync from cloud, aws version """
+
+        bucket = _get_bucket_aws()
+
+        key_name = self.cache_file_key(item.cache.dvc)
+        print 'aws: key name: %s' % key_name
+        key = bucket.get_key(key_name)
         if not key:
-            raise DataSyncError('File "{}" does not exist in the cloud'.format(aws_key))
+            raise DataSyncError('File "{}" does not exist in the cloud'.format(key_name))
 
-        Logger.info('Downloading cache file from S3 "{}/{}"'.format(self._bucket.name,
-                                                                        aws_key))
+        Logger.info('Downloading cache file from S3 "{}/{}"'.format(bucket.name, key_name))
         key.get_contents_to_filename(item.cache.relative, cb=percent_cb)
         Logger.info('Downloading completed')
-        pass
 
-    def sync_to_cloud(self, data_item):
-        aws_key = self.cache_file_aws_key(data_item.cache.dvc)
-        key = self._bucket.get_key(aws_key)
+    def _sync_to_cloud_aws(self, data_item):
+        """ sync_to_cloud, aws version """
+
+        aws_key = self.cache_file_key(data_item.cache.dvc)
+        print 'aws_key is %s' % aws_key
+        bucket = self._get_bucket_aws()
+        key = bucket.get_key(aws_key)
         if key:
             Logger.debug('File already uploaded to the cloud. Checksum validation...')
 
@@ -106,12 +112,72 @@ class CmdDataSync(CmdBase):
 
             Logger.debug('Checksum miss-match. Re-uploading is required.')
 
-        Logger.info('Uploading cache file "{}" to S3 "{}"'.format(data_item.cache.relative,
-                                                                      aws_key))
-        key = self._bucket.new_key(aws_key)
+        Logger.info('Uploading cache file "{}" to S3 "{}"'.format(data_item.cache.relative, aws_key))
+        key = bucket.new_key(aws_key)
         key.set_contents_from_filename(data_item.cache.relative, cb=percent_cb)
         Logger.info('Uploading completed')
-        pass
+
+
+    def _get_bucket_gc(self):
+        """ get a bucket object, gc """
+        client = gc.Client()
+        bucket = client.bucket(self.config.storage_bucket)
+        if not bucket.exists():
+            raise DataSyncError('sync up: google cloud bucket {} doesn\'t exist'.format(self.config.storage_bucket))
+        return bucket
+
+    def _sync_from_cloud_gcp(self, item):
+        """ sync from cloud, gcp version """
+
+        bucket = self._get_bucket_gc()
+        key = self.cache_file_key(item.cache.dvc)
+        print 'key: %s' % key
+
+        blob = bucket.get_blob(key)
+        if not blob:
+            raise DataSyncError('File "{}" does not exist in the cloud'.format(key))
+
+        Logger.info('Downloading cache file from gc "{}/{}"'.format(bucket.name, key))
+
+        blob.download_to_filename(item.cache.relative)
+        Logger.info('Downloading completed')
+
+    def _sync_to_cloud_gcp(self, data_item):
+        """ sync_to_cloud, gcp version """
+
+        bucket = self._get_bucket_gc()
+        blob_name = self.cache_file_key(data_item.cache.dvc)
+        print 'blob_name: %s' % blob_name
+        return
+
+        blob = bucket.blob(blob_name)
+        if blob.exists():
+            if blob.md5_hash() == file_md5(data_item.cache.relative):
+                Logger.debug('checksum %s matches.  Skipping upload' % data_item.cache.relative)
+                return
+            Logger.debug('checksum %s mismatch.  re-uploading' % data_item.cache.relative)
+
+        Logger.info('uploading cache file "{} to gc "{}"'.format(data_item.cache.relative, blob_name))
+
+        blob.upload_from_filename(data_item.cache.relative)
+        Logger.info('uploading %s completed' % data_item.cache.relative)
+
+
+    def sync_from_cloud(self, item):
+        cloud = self.settings.config.get_cloud
+        assert cloud in ['amazon', 'google'], 'unknown cloud %s' % cloud
+        if self.settings.config.get_cloud == 'amazon':
+            return self._sync_from_cloud_aws(item)
+        elif self.settings.config.get_cloud == 'google':
+            return self._sync_from_cloud_gcp(item)
+
+    def sync_to_cloud(self, data_item):
+        cloud = self.settings.config.get_cloud
+        assert cloud in ['amazon', 'google'], 'unknown cloud %s' % cloud
+        if self.settings.config.get_cloud == 'amazon':
+            return self._sync_to_cloud_aws(data_item)
+        elif self.settings.config.get_cloud == 'google':
+            return self._sync_to_cloud_gcp(data_item)
 
 
 if __name__ == '__main__':

dvc/command/init.py

@@ -18,7 +18,7 @@ class CmdInit(CmdBase):
 DataDir = {}
 CacheDir = {}
 StateDir = {}
-Cloud = AWS
+Cloud = amazon
 
 # Log levels: Debug, Info, Warning and Error
 LogLevel = Info
@@ -45,7 +45,11 @@ Storage = my-100gb-drive-io
 KeyDir = ~/.ssh
 KeyName = dvc-key
 
-SecurityGroup = dvc-group'''
+SecurityGroup = dvc-group
+
+[GC]
+ProjectName =
+'''
 
     def __init__(self, settings):
         super(CmdInit, self).__init__(settings)

dvc/command/test.py

@@ -0,0 +1,58 @@
+import os
+import sys
+
+from pathlib import Path
+
+from google.cloud import storage
+
+from dvc.command.base import CmdBase
+from dvc.logger import Logger
+from dvc.config import Config
+from dvc.runtime import Runtime
+
+from dvc.settings import SettingsError
+
+
+
+class CmdTest(CmdBase):
+    def __init__(self, settings):
+        super(CmdTest, self).__init__(settings)
+
+    def define_args(self, parser):
+        self.set_skip_git_actions(parser)
+
+    def run(self):
+
+        cloud = self.settings._config.get_cloud
+        assert cloud in ['amazon', 'google'], 'unknown cloud: %s' % cloud
+        if cloud == 'amazon':
+            print 'TODO'
+            sys.exit(-1)
+        elif cloud == 'google':
+            Logger.info('testing gcloud')
+
+            if self.settings._config.gc_project_name == '':
+                Logger.error('Please specify the google cloud project name in dvc.conf')
+                raise SettingsError('must specify GC ProjectName')
+
+            try:
+                client = storage.Client(project=self.settings._config.gc_project_name)
+            except:
+                Logger.error('initializing storage.Client raised')
+                raise
+
+            sb = self.config.storage_bucket
+            if sb == '':
+                Logger.error('Please specificy a bucket for google cloud in dvc.conf')
+                sys.exit(-1)
+
+            bucket = client.bucket(sb)
+            if not bucket.exists():
+                Logger.error('google cloud storage: bucket %s doesn\'t exist' % sb)
+                sys.exit(-1)
+            Logger.info('bucket %s visible: google cloud seems to be configured correctly' % sb)
+            sys.exit(0)
+
+
+if __name__ == '__main__':
+    Runtime.run(CmdTest, False)

dvc/config.py

@@ -30,7 +30,7 @@ class ConfigI(object):
         return self._state_dir
 
     @property
-    def aws_storage_prefix(self):
+    def storage_prefix(self):
         return ''
 
 
@@ -66,19 +66,46 @@ class Config(ConfigI):
         return self._config['AWS']['SecretAccessKey']
 
     @property
-    def aws_storage_path(self):
-        return self._config['AWS']['StoragePath']
+    def storage_path(self):
+        """ get storage path
 
-    def _aws_storage_path_parts(self):
-        return self.aws_storage_path.strip('/').split('/', 1)
+        Precedence: Storage, then cloud specific
+        """
+
+        path = self._config['Data'].get('StoragePath', None)
+        if path:
+            return path
+
+        cloud = self.get_cloud
+        assert cloud in ['amazon', 'google'], 'unknown cloud %s' % cloud
+        if cloud == 'amazon':
+            path = self._config['AWS'].get('StoragePath', None)
+        elif cloud == 'google':
+            path = self._config['GC'].get('StoragePath', None)
+        if path is None:
+            raise ConfigError('invalid StoragePath: not set for Data or cloud specific')
+        return path
+
+    def _storage_path_parts(self):
+        return self.storage_path.strip('/').split('/', 1)
 
     @property
-    def aws_storage_bucket(self):
-        return self._aws_storage_path_parts()[0]
+    def storage_bucket(self):
+        """ Data -> StoragePath takes precedence; if doesn't exist, use cloud-specific """
+        return self._storage_path_parts()[0]
 
     @property
-    def aws_storage_prefix(self):
-        parts = self._aws_storage_path_parts()
+    def storage_prefix(self):
+        parts = self._storage_path_parts()
         if len(parts) > 1:
             return parts[1]
         return ''
+
+    @property
+    def gc_project_name(self):
+        return self._config['GC']['ProjectName']
+
+    @property
+    def get_cloud(self):
+        """ get cloud choice: currently one of ['amazon', 'google'] """
+        return self._config['Global']['Cloud']

dvc2.py

@@ -10,11 +10,13 @@ import sys
 
 from dvc.runtime import Runtime
 from dvc.command.init import CmdInit
+from dvc.command.data_import import CmdDataImport
 from dvc.command.remove import CmdDataRemove
 from dvc.command.run import CmdRun
 from dvc.command.repro import CmdRepro
 from dvc.command.data_sync import CmdDataSync
 from dvc.command.data_import import CmdDataImport
+from dvc.command.test import CmdTest
 
 
 def print_usage():
@@ -30,16 +32,24 @@ def print_usage():
             '    data sync    TODO',
             '    data remove  TODO',
             '    data import  TODO',
+            '',
+            'test credentials',
+            '    test aws TODO',
+            '    test gcloud  test gcloud and credentials are setup correctly',
             '')
     print('\n'.join(usage))
 
 if __name__ == '__main__':
-    cmds = ['init', 'run', 'sync', 'repro', 'data', 'data-sync', 'data-remove', 'data-import', 'cloud', \
-            'cloud', 'cloud-run', 'cloud-instance-create', 'cloud-instance-remove', 'cloud-instance-describe']
+    cmds = ['init', 'run', 'sync', 'data', 'data-sync', 'data-remove', 'data-import', 'cloud', \
+            'cloud', 'cloud-run', 'cloud-instance-create', 'cloud-instance-remove', 'cloud-instance-describe', \
+            'test', 'test-aws', 'test-gcloud', 'test-cloud']
     cmds_expand = {'data':  ['sync', 'remove', 'import'],
-                   'cloud': ['run', 'instance-create', 'instance-remove', 'instance-describe']
+                   'cloud': ['run', 'instance-create', 'instance-remove', 'instance-describe'],
+                   'test':  ['aws', 'cloud', 'gcloud'],
                   }
 
+    print('sys.argv: (%d) %s' % (len(sys.argv), sys.argv))
+
     if len(sys.argv) < 2 or sys.argv[1] not in cmds:
         print('Unimplemented or unrecognized command. ' + ' '.join(sys.argv[1]))
         print_usage()
@@ -76,6 +86,11 @@ if __name__ == '__main__':
         print('cloud-instance-remove unimplemented')
     elif cmd == 'cloud-instance-describe' or (cmd == 'cloud' and subcmd == 'instance-describe'):
         print('cloud-instance-describe unimplemented')
+
+    elif cmd == 'test-aws' or (cmd == 'test' and subcmd == 'aws'):
+        print('TODO: test aws credentials')
+    elif cmd == 'test-gcloud' or (cmd=='test' and subcmd in ['cloud', 'gcloud']):
+        Runtime.run(CmdTest, args_start_loc=argv_offset)
     else:
         print('Unimplemented or unrecognized command. ' + ' '.join(sys.argv[1]))
         print_usage()

requirements.txt

@@ -3,28 +3,81 @@ appdirs==1.4.3
 appnope==0.1.0
 backports.shutil-get-terminal-size==1.0.0
 boto==2.46.1
+cachetools==2.0.0
 configparser==3.5.0
 decorator==4.0.11
+dill==0.2.6
 enum34==1.1.6
 fasteners==0.14.1
+future==0.16.0
+futures==3.0.5
+gapic-google-cloud-datastore-v1==0.15.3
+gapic-google-cloud-error-reporting-v1beta1==0.15.3
+gapic-google-cloud-logging-v2==0.91.3
+gapic-google-cloud-pubsub-v1==0.15.3
+gapic-google-cloud-spanner-admin-database-v1==0.15.3
+gapic-google-cloud-spanner-admin-instance-v1==0.15.3
+gapic-google-cloud-spanner-v1==0.15.3
+gapic-google-cloud-speech-v1beta1==0.15.3
+gapic-google-cloud-vision-v1==0.90.3
+google-auth==0.10.0
+google-auth-httplib2==0.0.2
+google-cloud==0.24.0
+google-cloud-bigquery==0.24.0
+google-cloud-bigtable==0.24.0
+google-cloud-core==0.24.0
+google-cloud-datastore==1.0.0
+google-cloud-dns==0.24.0
+google-cloud-error-reporting==0.24.0
+google-cloud-language==0.24.0
+google-cloud-logging==0.24.0
+google-cloud-monitoring==0.24.0
+google-cloud-pubsub==0.24.0
+google-cloud-resource-manager==0.24.0
+google-cloud-runtimeconfig==0.24.0
+google-cloud-spanner==0.24.0
+google-cloud-speech==0.24.0
+google-cloud-storage==1.0.0
+google-cloud-translate==0.24.0
+google-cloud-vision==0.24.0
+google-gax==0.15.8
+googleapis-common-protos==1.5.2
+grpc-google-iam-v1==0.11.1
+grpcio==1.2.1
+httplib2==0.10.3
 ipython==5.3.0
 ipython-genutils==0.2.0
 macholib==1.8
 modulegraph==0.14
 monotonic==1.3
 nose==1.3.7
+oauth2client==3.0.0
 packaging==16.8
 pathlib==1.0.1
 pathlib2==2.2.1
 pexpect==4.2.1
 pickleshare==0.7.4
+ply==3.8
 prompt-toolkit==1.0.14
+proto-google-cloud-datastore-v1==0.90.3
+proto-google-cloud-error-reporting-v1beta1==0.15.3
+proto-google-cloud-logging-v2==0.91.3
+proto-google-cloud-pubsub-v1==0.15.3
+proto-google-cloud-spanner-admin-database-v1==0.15.3
+proto-google-cloud-spanner-admin-instance-v1==0.15.3
+proto-google-cloud-spanner-v1==0.15.3
+proto-google-cloud-speech-v1beta1==0.15.3
+proto-google-cloud-vision-v1==0.90.3
+protobuf==3.2.0
 ptyprocess==0.5.1
 py2app==0.12
+pyasn1==0.2.3
+pyasn1-modules==0.0.8
 Pygments==2.2.0
 PyInstaller==3.2.1
 pyparsing==2.2.0
 requests==2.13.0
+rsa==3.4.2
 scandir==1.5
 simplegeneric==0.8.1
 six==1.10.0