From a5df4758149a6d263f60e8778822cc0ed2e9948f Mon Sep 17 00:00:00 2001 From: Earl Hathaway Date: Mon, 3 Apr 2017 16:05:44 -0700 Subject: [PATCH 1/9] rebased / cleaned: add virtualenv, requirements, etc --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 3c0b61fdf..71d36e3fb 100644 --- a/README.md +++ b/README.md @@ -15,9 +15,15 @@ By submitting a pull request for this project, you agree to license your contrib Create a virtualenv, for example, at `~/.environments/dvc` by `mkdir -p ~/.environments/dvc; virtualenv ~/.environments/dvc` or use `virtualenvwrapper`. ``` +# if you use virtualenvwrapper workon dvc +# otherwise: +source ~/.environments/dvc/bin/activate + pip install -r requirements.txt + +# happy coding! ``` # Building From f84cbd0d17a9094bb8987513d4eda3e8d8f9f78a Mon Sep 17 00:00:00 2001 From: Earl Hathaway Date: Fri, 7 Apr 2017 12:11:16 -0700 Subject: [PATCH 2/9] google cloud installs ALL THE THINGS fixes to single file commands got data import to work get path_factory from settings cloud selector data sync -> s3 works again blank dvc.conf --- README.md | 30 ++++++++++ dvc.conf | 16 ++++- dvc/command/base.py | 6 +- dvc/command/data_sync.py | 122 ++++++++++++++++++++++++++++++--------- dvc/command/init.py | 8 ++- dvc/command/test.py | 58 +++++++++++++++++++ dvc/config.py | 45 ++++++++++++--- dvc2.py | 21 ++++++- requirements.txt | 53 +++++++++++++++++ 9 files changed, 311 insertions(+), 48 deletions(-) create mode 100644 dvc/command/test.py diff --git a/README.md b/README.md index 71d36e3fb..65569ab5c 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,36 @@ This project is distributed under the Apache license version 2.0 (see the LICENS By submitting a pull request for this project, you agree to license your contribution under the Apache license version 2.0 to this project. +# Setup / Configuration +`dvc` supports both aws and google cloud; you will need accounts either from [aws](https://aws.amazon.com/) or [gcloud](https://cloud.google.com/). + +To see your available buckets, run either `TODO` or `gsutil ls`. + + +## Google Cloud Setup +To setup your google cloud credentials: +TODO + +To test they are set up correctly, +TODO + +# Usage +``` +mkdir t +cd !$ +git init . +dvc init + +Info. Directories data/, cache/ and state/ were created +Info. File .gitignore was created +Info. Directory cache was added to .gitignore file +Info. [Git] A new commit 34687b2 was made in the current branch. Added files: +Info. [Git] A .gitignore +Info. [Git] A dvc.conf + + + +``` # Development Create a virtualenv, for example, at `~/.environments/dvc` by `mkdir -p ~/.environments/dvc; virtualenv ~/.environments/dvc` or use `virtualenvwrapper`. diff --git a/dvc.conf b/dvc.conf index 94afee0f5..ec634402a 100644 --- a/dvc.conf +++ b/dvc.conf @@ -2,7 +2,7 @@ DataDir = data CacheDir = cache StateDir = state -Cloud = AWS +Cloud = google # Log levels: Debug, Info, Warning and Error LogLevel = Info @@ -11,7 +11,7 @@ LogLevel = Info AccessKeyId = SecretAccessKey = -StoragePath = dvc/tutorial +StoragePath = Region = us-east-1 @@ -29,4 +29,14 @@ Storage = my-100gb-drive-io KeyDir = ~/.ssh KeyName = dvc-key -SecurityGroup = dvc-group \ No newline at end of file +SecurityGroup = dvc-group + + +[GC] +ProjectName = + +StoragePath = + + +[Data] +StoragePath = diff --git a/dvc/command/base.py b/dvc/command/base.py index 2db660051..07b67bf0d 100644 --- a/dvc/command/base.py +++ b/dvc/command/base.py @@ -13,7 +13,6 @@ class CmdBase(object): parser = argparse.ArgumentParser() self.define_args(parser) self._parsed_args, self._command_args = parser.parse_known_args(args=self.args) - pass @property def settings(self): @@ -35,8 +34,9 @@ class CmdBase(object): def config(self): return self._settings.config - def cache_file_aws_key(self, file): - return '{}/{}'.format(self.config.aws_storage_prefix, file).strip('/') + def cache_file_key(self, file): + print 'storage_prefix %s' % self.config.storage_prefix + return '{}/{}'.format(self.config.storage_prefix, file).strip('/') @property def dvc_home(self): diff --git a/dvc/command/data_sync.py b/dvc/command/data_sync.py index 5509f1a62..7ef1a7401 100644 --- a/dvc/command/data_sync.py +++ b/dvc/command/data_sync.py @@ -2,6 +2,7 @@ import hashlib import os from boto.s3.connection import S3Connection +from google.cloud import storage as gc from dvc.command.base import CmdBase from dvc.logger import Logger @@ -39,22 +40,12 @@ class CmdDataSync(CmdBase): def __init__(self, settings): super(CmdDataSync, self).__init__(settings) - conn = S3Connection(self.config.aws_access_key_id, self.config.aws_secret_access_key) - - bucket_name = self.config.aws_storage_bucket - self._bucket = conn.lookup(bucket_name) - if not self._bucket: - self._bucket = conn.create_bucket(bucket_name) - Logger.info('S3 bucket "{}" was created'.format(bucket_name)) - pass - def define_args(self, parser): self.add_string_arg(parser, 'target', 'Target to sync - file or directory') - pass def run(self): if System.islink(self.parsed_args.target): - data_item = self.path_factory.existing_data_item(self.parsed_args.target) + data_item = self.settings.path_factory.existing_data_item(self.parsed_args.target) return self.sync_symlink(data_item) if os.path.isdir(self.parsed_args.target): @@ -68,33 +59,48 @@ class CmdDataSync(CmdBase): if os.path.isdir(fname): self.sync_dir(fname) elif System.islink(fname): - self.sync_symlink(self.path_factory.existing_data_item(fname)) + self.sync_symlink(self.settings.path_factory.existing_data_item(fname)) else: raise DataSyncError('Unsupported file type "{}"'.format(fname)) - pass def sync_symlink(self, data_item): if os.path.isfile(data_item.cache.relative): self.sync_to_cloud(data_item) else: self.sync_from_cloud(data_item) - pass - def sync_from_cloud(self, item): - aws_key = self.cache_file_aws_key(item.cache.dvc) - key = self._bucket.get_key(aws_key) + def _get_bucket_aws(self): + """ get a bucket object, aws """ + + conn = S3Connection(self.config.aws_access_key_id, self.config.aws_secret_access_key, host='s3-us-west-2.amazonaws.com') + bucket_name = self.config.storage_bucket + bucket = conn.lookup(bucket_name) + if bucket is None: + raise DataSyncError('Bucket "{}" can\'t be accessed'.format(bucket_name)) + return bucket + + def _sync_from_cloud_aws(self, item): + """ sync from cloud, aws version """ + + bucket = _get_bucket_aws() + + key_name = self.cache_file_key(item.cache.dvc) + print 'aws: key name: %s' % key_name + key = bucket.get_key(key_name) if not key: - raise DataSyncError('File "{}" does not exist in the cloud'.format(aws_key)) + raise DataSyncError('File "{}" does not exist in the cloud'.format(key_name)) - Logger.info('Downloading cache file from S3 "{}/{}"'.format(self._bucket.name, - aws_key)) + Logger.info('Downloading cache file from S3 "{}/{}"'.format(bucket.name, key_name)) key.get_contents_to_filename(item.cache.relative, cb=percent_cb) Logger.info('Downloading completed') - pass - def sync_to_cloud(self, data_item): - aws_key = self.cache_file_aws_key(data_item.cache.dvc) - key = self._bucket.get_key(aws_key) + def _sync_to_cloud_aws(self, data_item): + """ sync_to_cloud, aws version """ + + aws_key = self.cache_file_key(data_item.cache.dvc) + print 'aws_key is %s' % aws_key + bucket = self._get_bucket_aws() + key = bucket.get_key(aws_key) if key: Logger.debug('File already uploaded to the cloud. Checksum validation...') @@ -106,12 +112,72 @@ class CmdDataSync(CmdBase): Logger.debug('Checksum miss-match. Re-uploading is required.') - Logger.info('Uploading cache file "{}" to S3 "{}"'.format(data_item.cache.relative, - aws_key)) - key = self._bucket.new_key(aws_key) + Logger.info('Uploading cache file "{}" to S3 "{}"'.format(data_item.cache.relative, aws_key)) + key = bucket.new_key(aws_key) key.set_contents_from_filename(data_item.cache.relative, cb=percent_cb) Logger.info('Uploading completed') - pass + + + def _get_bucket_gc(self): + """ get a bucket object, gc """ + client = gc.Client() + bucket = client.bucket(self.config.storage_bucket) + if not bucket.exists(): + raise DataSyncError('sync up: google cloud bucket {} doesn\'t exist'.format(self.config.storage_bucket)) + return bucket + + def _sync_from_cloud_gcp(self, item): + """ sync from cloud, gcp version """ + + bucket = self._get_bucket_gc() + key = self.cache_file_key(item.cache.dvc) + print 'key: %s' % key + + blob = bucket.get_blob(key) + if not blob: + raise DataSyncError('File "{}" does not exist in the cloud'.format(key)) + + Logger.info('Downloading cache file from gc "{}/{}"'.format(bucket.name, key)) + + blob.download_to_filename(item.cache.relative) + Logger.info('Downloading completed') + + def _sync_to_cloud_gcp(self, data_item): + """ sync_to_cloud, gcp version """ + + bucket = self._get_bucket_gc() + blob_name = self.cache_file_key(data_item.cache.dvc) + print 'blob_name: %s' % blob_name + return + + blob = bucket.blob(blob_name) + if blob.exists(): + if blob.md5_hash() == file_md5(data_item.cache.relative): + Logger.debug('checksum %s matches. Skipping upload' % data_item.cache.relative) + return + Logger.debug('checksum %s mismatch. re-uploading' % data_item.cache.relative) + + Logger.info('uploading cache file "{} to gc "{}"'.format(data_item.cache.relative, blob_name)) + + blob.upload_from_filename(data_item.cache.relative) + Logger.info('uploading %s completed' % data_item.cache.relative) + + + def sync_from_cloud(self, item): + cloud = self.settings.config.get_cloud + assert cloud in ['amazon', 'google'], 'unknown cloud %s' % cloud + if self.settings.config.get_cloud == 'amazon': + return self._sync_from_cloud_aws(item) + elif self.settings.config.get_cloud == 'google': + return self._sync_from_cloud_gcp(item) + + def sync_to_cloud(self, data_item): + cloud = self.settings.config.get_cloud + assert cloud in ['amazon', 'google'], 'unknown cloud %s' % cloud + if self.settings.config.get_cloud == 'amazon': + return self._sync_to_cloud_aws(data_item) + elif self.settings.config.get_cloud == 'google': + return self._sync_to_cloud_gcp(data_item) if __name__ == '__main__': diff --git a/dvc/command/init.py b/dvc/command/init.py index c2a458198..4b5c542a5 100644 --- a/dvc/command/init.py +++ b/dvc/command/init.py @@ -18,7 +18,7 @@ class CmdInit(CmdBase): DataDir = {} CacheDir = {} StateDir = {} -Cloud = AWS +Cloud = amazon # Log levels: Debug, Info, Warning and Error LogLevel = Info @@ -45,7 +45,11 @@ Storage = my-100gb-drive-io KeyDir = ~/.ssh KeyName = dvc-key -SecurityGroup = dvc-group''' +SecurityGroup = dvc-group + +[GC] +ProjectName = +''' def __init__(self, settings): super(CmdInit, self).__init__(settings) diff --git a/dvc/command/test.py b/dvc/command/test.py new file mode 100644 index 000000000..f92fbc164 --- /dev/null +++ b/dvc/command/test.py @@ -0,0 +1,58 @@ +import os +import sys + +from pathlib import Path + +from google.cloud import storage + +from dvc.command.base import CmdBase +from dvc.logger import Logger +from dvc.config import Config +from dvc.runtime import Runtime + +from dvc.settings import SettingsError + + + +class CmdTest(CmdBase): + def __init__(self, settings): + super(CmdTest, self).__init__(settings) + + def define_args(self, parser): + self.set_skip_git_actions(parser) + + def run(self): + + cloud = self.settings._config.get_cloud + assert cloud in ['amazon', 'google'], 'unknown cloud: %s' % cloud + if cloud == 'amazon': + print 'TODO' + sys.exit(-1) + elif cloud == 'google': + Logger.info('testing gcloud') + + if self.settings._config.gc_project_name == '': + Logger.error('Please specify the google cloud project name in dvc.conf') + raise SettingsError('must specify GC ProjectName') + + try: + client = storage.Client(project=self.settings._config.gc_project_name) + except: + Logger.error('initializing storage.Client raised') + raise + + sb = self.config.storage_bucket + if sb == '': + Logger.error('Please specificy a bucket for google cloud in dvc.conf') + sys.exit(-1) + + bucket = client.bucket(sb) + if not bucket.exists(): + Logger.error('google cloud storage: bucket %s doesn\'t exist' % sb) + sys.exit(-1) + Logger.info('bucket %s visible: google cloud seems to be configured correctly' % sb) + sys.exit(0) + + +if __name__ == '__main__': + Runtime.run(CmdTest, False) diff --git a/dvc/config.py b/dvc/config.py index 11a10409f..7082c0a3b 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -30,7 +30,7 @@ class ConfigI(object): return self._state_dir @property - def aws_storage_prefix(self): + def storage_prefix(self): return '' @@ -66,19 +66,46 @@ class Config(ConfigI): return self._config['AWS']['SecretAccessKey'] @property - def aws_storage_path(self): - return self._config['AWS']['StoragePath'] + def storage_path(self): + """ get storage path - def _aws_storage_path_parts(self): - return self.aws_storage_path.strip('/').split('/', 1) + Precedence: Storage, then cloud specific + """ + + path = self._config['Data'].get('StoragePath', None) + if path: + return path + + cloud = self.get_cloud + assert cloud in ['amazon', 'google'], 'unknown cloud %s' % cloud + if cloud == 'amazon': + path = self._config['AWS'].get('StoragePath', None) + elif cloud == 'google': + path = self._config['GC'].get('StoragePath', None) + if path is None: + raise ConfigError('invalid StoragePath: not set for Data or cloud specific') + return path + + def _storage_path_parts(self): + return self.storage_path.strip('/').split('/', 1) @property - def aws_storage_bucket(self): - return self._aws_storage_path_parts()[0] + def storage_bucket(self): + """ Data -> StoragePath takes precedence; if doesn't exist, use cloud-specific """ + return self._storage_path_parts()[0] @property - def aws_storage_prefix(self): - parts = self._aws_storage_path_parts() + def storage_prefix(self): + parts = self._storage_path_parts() if len(parts) > 1: return parts[1] return '' + + @property + def gc_project_name(self): + return self._config['GC']['ProjectName'] + + @property + def get_cloud(self): + """ get cloud choice: currently one of ['amazon', 'google'] """ + return self._config['Global']['Cloud'] diff --git a/dvc2.py b/dvc2.py index 8c205370e..da4649ecc 100755 --- a/dvc2.py +++ b/dvc2.py @@ -10,11 +10,13 @@ import sys from dvc.runtime import Runtime from dvc.command.init import CmdInit +from dvc.command.data_import import CmdDataImport from dvc.command.remove import CmdDataRemove from dvc.command.run import CmdRun from dvc.command.repro import CmdRepro from dvc.command.data_sync import CmdDataSync from dvc.command.data_import import CmdDataImport +from dvc.command.test import CmdTest def print_usage(): @@ -30,16 +32,24 @@ def print_usage(): ' data sync TODO', ' data remove TODO', ' data import TODO', + '', + 'test credentials', + ' test aws TODO', + ' test gcloud test gcloud and credentials are setup correctly', '') print('\n'.join(usage)) if __name__ == '__main__': - cmds = ['init', 'run', 'sync', 'repro', 'data', 'data-sync', 'data-remove', 'data-import', 'cloud', \ - 'cloud', 'cloud-run', 'cloud-instance-create', 'cloud-instance-remove', 'cloud-instance-describe'] + cmds = ['init', 'run', 'sync', 'data', 'data-sync', 'data-remove', 'data-import', 'cloud', \ + 'cloud', 'cloud-run', 'cloud-instance-create', 'cloud-instance-remove', 'cloud-instance-describe', \ + 'test', 'test-aws', 'test-gcloud', 'test-cloud'] cmds_expand = {'data': ['sync', 'remove', 'import'], - 'cloud': ['run', 'instance-create', 'instance-remove', 'instance-describe'] + 'cloud': ['run', 'instance-create', 'instance-remove', 'instance-describe'], + 'test': ['aws', 'cloud', 'gcloud'], } + print('sys.argv: (%d) %s' % (len(sys.argv), sys.argv)) + if len(sys.argv) < 2 or sys.argv[1] not in cmds: print('Unimplemented or unrecognized command. ' + ' '.join(sys.argv[1])) print_usage() @@ -76,6 +86,11 @@ if __name__ == '__main__': print('cloud-instance-remove unimplemented') elif cmd == 'cloud-instance-describe' or (cmd == 'cloud' and subcmd == 'instance-describe'): print('cloud-instance-describe unimplemented') + + elif cmd == 'test-aws' or (cmd == 'test' and subcmd == 'aws'): + print('TODO: test aws credentials') + elif cmd == 'test-gcloud' or (cmd=='test' and subcmd in ['cloud', 'gcloud']): + Runtime.run(CmdTest, args_start_loc=argv_offset) else: print('Unimplemented or unrecognized command. ' + ' '.join(sys.argv[1])) print_usage() diff --git a/requirements.txt b/requirements.txt index 349695e87..7d38fb5c0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,28 +3,81 @@ appdirs==1.4.3 appnope==0.1.0 backports.shutil-get-terminal-size==1.0.0 boto==2.46.1 +cachetools==2.0.0 configparser==3.5.0 decorator==4.0.11 +dill==0.2.6 enum34==1.1.6 fasteners==0.14.1 +future==0.16.0 +futures==3.0.5 +gapic-google-cloud-datastore-v1==0.15.3 +gapic-google-cloud-error-reporting-v1beta1==0.15.3 +gapic-google-cloud-logging-v2==0.91.3 +gapic-google-cloud-pubsub-v1==0.15.3 +gapic-google-cloud-spanner-admin-database-v1==0.15.3 +gapic-google-cloud-spanner-admin-instance-v1==0.15.3 +gapic-google-cloud-spanner-v1==0.15.3 +gapic-google-cloud-speech-v1beta1==0.15.3 +gapic-google-cloud-vision-v1==0.90.3 +google-auth==0.10.0 +google-auth-httplib2==0.0.2 +google-cloud==0.24.0 +google-cloud-bigquery==0.24.0 +google-cloud-bigtable==0.24.0 +google-cloud-core==0.24.0 +google-cloud-datastore==1.0.0 +google-cloud-dns==0.24.0 +google-cloud-error-reporting==0.24.0 +google-cloud-language==0.24.0 +google-cloud-logging==0.24.0 +google-cloud-monitoring==0.24.0 +google-cloud-pubsub==0.24.0 +google-cloud-resource-manager==0.24.0 +google-cloud-runtimeconfig==0.24.0 +google-cloud-spanner==0.24.0 +google-cloud-speech==0.24.0 +google-cloud-storage==1.0.0 +google-cloud-translate==0.24.0 +google-cloud-vision==0.24.0 +google-gax==0.15.8 +googleapis-common-protos==1.5.2 +grpc-google-iam-v1==0.11.1 +grpcio==1.2.1 +httplib2==0.10.3 ipython==5.3.0 ipython-genutils==0.2.0 macholib==1.8 modulegraph==0.14 monotonic==1.3 nose==1.3.7 +oauth2client==3.0.0 packaging==16.8 pathlib==1.0.1 pathlib2==2.2.1 pexpect==4.2.1 pickleshare==0.7.4 +ply==3.8 prompt-toolkit==1.0.14 +proto-google-cloud-datastore-v1==0.90.3 +proto-google-cloud-error-reporting-v1beta1==0.15.3 +proto-google-cloud-logging-v2==0.91.3 +proto-google-cloud-pubsub-v1==0.15.3 +proto-google-cloud-spanner-admin-database-v1==0.15.3 +proto-google-cloud-spanner-admin-instance-v1==0.15.3 +proto-google-cloud-spanner-v1==0.15.3 +proto-google-cloud-speech-v1beta1==0.15.3 +proto-google-cloud-vision-v1==0.90.3 +protobuf==3.2.0 ptyprocess==0.5.1 py2app==0.12 +pyasn1==0.2.3 +pyasn1-modules==0.0.8 Pygments==2.2.0 PyInstaller==3.2.1 pyparsing==2.2.0 requests==2.13.0 +rsa==3.4.2 scandir==1.5 simplegeneric==0.8.1 six==1.10.0 From 0772e41638bdcc05d3b0cf9c8786648064b7a828 Mon Sep 17 00:00:00 2001 From: Earl Hathaway Date: Sun, 16 Apr 2017 21:59:12 -0700 Subject: [PATCH 3/9] fix aws region bug --- dvc/command/data_sync.py | 2 +- dvc/config.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/dvc/command/data_sync.py b/dvc/command/data_sync.py index 7ef1a7401..b622ff7f1 100644 --- a/dvc/command/data_sync.py +++ b/dvc/command/data_sync.py @@ -72,7 +72,7 @@ class CmdDataSync(CmdBase): def _get_bucket_aws(self): """ get a bucket object, aws """ - conn = S3Connection(self.config.aws_access_key_id, self.config.aws_secret_access_key, host='s3-us-west-2.amazonaws.com') + conn = S3Connection(self.config.aws_access_key_id, self.config.aws_secret_access_key, host=self.config.aws_region_host) bucket_name = self.config.storage_bucket bucket = conn.lookup(bucket_name) if bucket is None: diff --git a/dvc/config.py b/dvc/config.py index 7082c0a3b..4590cfab0 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -65,6 +65,21 @@ class Config(ConfigI): def aws_secret_access_key(self): return self._config['AWS']['SecretAccessKey'] + @property + def aws_region_host(self): + """ get the region host needed for s3 access + + See notes http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region + """ + + region = self._config['AWS']['Region'] + if region is None or region == '': + return 's3.amazonaws.com' + if region == 'us-east-1': + return 's3.amazonaws.com' + return 's3.%s.amazonaws.com' % region + + @property def storage_path(self): """ get storage path From 6fa96f5406807f171cccc477d8167ee13f690c07 Mon Sep 17 00:00:00 2001 From: Earl Hathaway Date: Sun, 16 Apr 2017 21:49:55 -0700 Subject: [PATCH 4/9] more docs; remove debugging --- README.md | 25 ++++++++++++++++++++----- dvc.conf | 1 - dvc/command/base.py | 1 - dvc/command/data_sync.py | 5 ----- dvc/command/init.py | 5 ++++- 5 files changed, 24 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 65569ab5c..7f406c012 100644 --- a/README.md +++ b/README.md @@ -13,15 +13,30 @@ By submitting a pull request for this project, you agree to license your contrib # Setup / Configuration `dvc` supports both aws and google cloud; you will need accounts either from [aws](https://aws.amazon.com/) or [gcloud](https://cloud.google.com/). -To see your available buckets, run either `TODO` or `gsutil ls`. +Edit `dvc.conf`, and fill in `StoragePath` under `[Data]` with your preferred bucket and path. For example, +``` +[Data] +StoragePath = gc://dvc-demo/dvcdata +``` +## Amazon AWS Setup +NB: To see your available buckets, run `aws s3 ls` + +Configuration: +* edit `dvc.conf`, and set `Cloud = amazon` +* under `[AWS]`, fill in `AccessKeyId` and `SecretAccessKey` + +Test `dvc` is correctly configured by running `dvc test-aws` (TODO) ## Google Cloud Setup -To setup your google cloud credentials: -TODO +NB: To see your available buckets, run `gsutil ls`. + +Configuration: +* edit `dvc.conf`, and set `Cloud = google` +* under `[GC]`, set `ProjectName` to your preferred (or just your default) project name. + +Test `dvc` is correctly configured by running `dvc test-gcloud`. -To test they are set up correctly, -TODO # Usage ``` diff --git a/dvc.conf b/dvc.conf index ec634402a..94ee81888 100644 --- a/dvc.conf +++ b/dvc.conf @@ -34,7 +34,6 @@ SecurityGroup = dvc-group [GC] ProjectName = - StoragePath = diff --git a/dvc/command/base.py b/dvc/command/base.py index 07b67bf0d..c2c96695d 100644 --- a/dvc/command/base.py +++ b/dvc/command/base.py @@ -35,7 +35,6 @@ class CmdBase(object): return self._settings.config def cache_file_key(self, file): - print 'storage_prefix %s' % self.config.storage_prefix return '{}/{}'.format(self.config.storage_prefix, file).strip('/') @property diff --git a/dvc/command/data_sync.py b/dvc/command/data_sync.py index b622ff7f1..6739a54e5 100644 --- a/dvc/command/data_sync.py +++ b/dvc/command/data_sync.py @@ -85,7 +85,6 @@ class CmdDataSync(CmdBase): bucket = _get_bucket_aws() key_name = self.cache_file_key(item.cache.dvc) - print 'aws: key name: %s' % key_name key = bucket.get_key(key_name) if not key: raise DataSyncError('File "{}" does not exist in the cloud'.format(key_name)) @@ -98,7 +97,6 @@ class CmdDataSync(CmdBase): """ sync_to_cloud, aws version """ aws_key = self.cache_file_key(data_item.cache.dvc) - print 'aws_key is %s' % aws_key bucket = self._get_bucket_aws() key = bucket.get_key(aws_key) if key: @@ -131,7 +129,6 @@ class CmdDataSync(CmdBase): bucket = self._get_bucket_gc() key = self.cache_file_key(item.cache.dvc) - print 'key: %s' % key blob = bucket.get_blob(key) if not blob: @@ -147,8 +144,6 @@ class CmdDataSync(CmdBase): bucket = self._get_bucket_gc() blob_name = self.cache_file_key(data_item.cache.dvc) - print 'blob_name: %s' % blob_name - return blob = bucket.blob(blob_name) if blob.exists(): diff --git a/dvc/command/init.py b/dvc/command/init.py index 4b5c542a5..ae14452ae 100644 --- a/dvc/command/init.py +++ b/dvc/command/init.py @@ -18,7 +18,7 @@ class CmdInit(CmdBase): DataDir = {} CacheDir = {} StateDir = {} -Cloud = amazon +Cloud = # Log levels: Debug, Info, Warning and Error LogLevel = Info @@ -49,6 +49,9 @@ SecurityGroup = dvc-group [GC] ProjectName = + +[Data] +StoragePath = ''' def __init__(self, settings): From 193dc4f1a0671c547367ca456550f14cf21cc149 Mon Sep 17 00:00:00 2001 From: Earl Hathaway Date: Sun, 16 Apr 2017 22:40:53 -0700 Subject: [PATCH 5/9] forgot self --- dvc/command/data_sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/command/data_sync.py b/dvc/command/data_sync.py index 6739a54e5..f840b8ceb 100644 --- a/dvc/command/data_sync.py +++ b/dvc/command/data_sync.py @@ -82,7 +82,7 @@ class CmdDataSync(CmdBase): def _sync_from_cloud_aws(self, item): """ sync from cloud, aws version """ - bucket = _get_bucket_aws() + bucket = self._get_bucket_aws() key_name = self.cache_file_key(item.cache.dvc) key = bucket.get_key(key_name) From b15b36d95593034a58bc47d86069d8f73aec9b2e Mon Sep 17 00:00:00 2001 From: Dmitry Petrov Date: Mon, 17 Apr 2017 12:44:39 -0700 Subject: [PATCH 6/9] Fix cloud sync issue --- dvc/command/data_sync.py | 6 +++--- dvc/path/data_item.py | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/dvc/command/data_sync.py b/dvc/command/data_sync.py index f840b8ceb..268b620e6 100644 --- a/dvc/command/data_sync.py +++ b/dvc/command/data_sync.py @@ -45,7 +45,7 @@ class CmdDataSync(CmdBase): def run(self): if System.islink(self.parsed_args.target): - data_item = self.settings.path_factory.existing_data_item(self.parsed_args.target) + data_item = self.settings.path_factory.data_item(self.parsed_args.target) return self.sync_symlink(data_item) if os.path.isdir(self.parsed_args.target): @@ -83,9 +83,9 @@ class CmdDataSync(CmdBase): """ sync from cloud, aws version """ bucket = self._get_bucket_aws() - - key_name = self.cache_file_key(item.cache.dvc) + key_name = self.cache_file_key(item.resolved_cache.dvc) key = bucket.get_key(key_name) + if not key: raise DataSyncError('File "{}" does not exist in the cloud'.format(key_name)) diff --git a/dvc/path/data_item.py b/dvc/path/data_item.py index 92c4d048e..6eaf4420f 100644 --- a/dvc/path/data_item.py +++ b/dvc/path/data_item.py @@ -86,6 +86,12 @@ class DataItem(object): cache_file = os.path.join(cache_dir, file_name) return Path(cache_file, self._git) + @cached_property + def resolved_cache(self): + resolved_cache = os.path.realpath(self._data.relative) + return Path(resolved_cache, self._git) + + @cached_property def data_dir_abs(self): return os.path.join(self._git.git_dir_abs, self._config.data_dir) From fd53c0c38cbe52bae12a8fe300f9a62f3c10d948 Mon Sep 17 00:00:00 2001 From: Earl Hathaway Date: Mon, 17 Apr 2017 14:09:06 -0700 Subject: [PATCH 7/9] fix filename issues; correctly decode google cloud md5 hash; gcp sync works; aws sync works; --- dvc/command/data_sync.py | 35 +++++++++++++++++++++-------------- dvc2.py | 3 ++- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/dvc/command/data_sync.py b/dvc/command/data_sync.py index 268b620e6..ad68f85d5 100644 --- a/dvc/command/data_sync.py +++ b/dvc/command/data_sync.py @@ -1,3 +1,4 @@ +import base64 import hashlib import os @@ -29,11 +30,12 @@ def percent_cb(complete, total): def file_md5(fname): + """ get the (md5 hexdigest, md5 digest) of a file """ hash_md5 = hashlib.md5() with open(fname, "rb") as f: for chunk in iter(lambda: f.read(1024*1000), b""): hash_md5.update(chunk) - return hash_md5.hexdigest() + return (hash_md5.hexdigest(), hash_md5.digest()) class CmdDataSync(CmdBase): @@ -59,12 +61,12 @@ class CmdDataSync(CmdBase): if os.path.isdir(fname): self.sync_dir(fname) elif System.islink(fname): - self.sync_symlink(self.settings.path_factory.existing_data_item(fname)) + self.sync_symlink(self.settings.path_factory.data_item(fname)) else: raise DataSyncError('Unsupported file type "{}"'.format(fname)) def sync_symlink(self, data_item): - if os.path.isfile(data_item.cache.relative): + if os.path.isfile(data_item.resolved_cache.dvc): self.sync_to_cloud(data_item) else: self.sync_from_cloud(data_item) @@ -83,27 +85,27 @@ class CmdDataSync(CmdBase): """ sync from cloud, aws version """ bucket = self._get_bucket_aws() + key_name = self.cache_file_key(item.resolved_cache.dvc) key = bucket.get_key(key_name) - if not key: raise DataSyncError('File "{}" does not exist in the cloud'.format(key_name)) Logger.info('Downloading cache file from S3 "{}/{}"'.format(bucket.name, key_name)) - key.get_contents_to_filename(item.cache.relative, cb=percent_cb) + key.get_contents_to_filename(item.resolved_cache.relative, cb=percent_cb) Logger.info('Downloading completed') def _sync_to_cloud_aws(self, data_item): """ sync_to_cloud, aws version """ - aws_key = self.cache_file_key(data_item.cache.dvc) + aws_key = self.cache_file_key(data_item.resolved_cache.dvc) bucket = self._get_bucket_aws() key = bucket.get_key(aws_key) if key: Logger.debug('File already uploaded to the cloud. Checksum validation...') md5_cloud = key.etag[1:-1] - md5_local = file_md5(data_item.cache.relative) + md5_local = file_md5(data_item.resolved_cache.dvc)[0] if md5_cloud == md5_local: Logger.debug('File checksum matches. No uploading is needed.') return @@ -118,17 +120,20 @@ class CmdDataSync(CmdBase): def _get_bucket_gc(self): """ get a bucket object, gc """ - client = gc.Client() + client = gc.Client(project=self.config.gc_project_name) bucket = client.bucket(self.config.storage_bucket) if not bucket.exists(): raise DataSyncError('sync up: google cloud bucket {} doesn\'t exist'.format(self.config.storage_bucket)) return bucket def _sync_from_cloud_gcp(self, item): - """ sync from cloud, gcp version """ + """ sync from cloud, gcp version + Params: + TODO + """ bucket = self._get_bucket_gc() - key = self.cache_file_key(item.cache.dvc) + key = self.cache_file_key(item.resolved_cache.dvc) blob = bucket.get_blob(key) if not blob: @@ -136,18 +141,20 @@ class CmdDataSync(CmdBase): Logger.info('Downloading cache file from gc "{}/{}"'.format(bucket.name, key)) - blob.download_to_filename(item.cache.relative) + blob.download_to_filename(item.resolved_cache.dvc) Logger.info('Downloading completed') def _sync_to_cloud_gcp(self, data_item): """ sync_to_cloud, gcp version """ bucket = self._get_bucket_gc() - blob_name = self.cache_file_key(data_item.cache.dvc) + blob_name = self.cache_file_key(data_item.resolved_cache.dvc) - blob = bucket.blob(blob_name) + blob = bucket.get_blob(blob_name) if blob.exists(): - if blob.md5_hash() == file_md5(data_item.cache.relative): + b64_encoded_md5 = base64.b64encode(file_md5(data_item.resolved_cache.dvc)[1]) + + if blob.md5_hash == b64_encoded_md5: Logger.debug('checksum %s matches. Skipping upload' % data_item.cache.relative) return Logger.debug('checksum %s mismatch. re-uploading' % data_item.cache.relative) diff --git a/dvc2.py b/dvc2.py index da4649ecc..56693c835 100755 --- a/dvc2.py +++ b/dvc2.py @@ -51,7 +51,8 @@ if __name__ == '__main__': print('sys.argv: (%d) %s' % (len(sys.argv), sys.argv)) if len(sys.argv) < 2 or sys.argv[1] not in cmds: - print('Unimplemented or unrecognized command. ' + ' '.join(sys.argv[1])) + if len(sys.argv) >= 2: + print('Unimplemented or unrecognized command: ' + ' '.join(sys.argv[1:])) print_usage() sys.exit(-1) From a06ff8e8d4c87901b2f71f419845d09d42763c40 Mon Sep 17 00:00:00 2001 From: Earl Hathaway Date: Mon, 17 Apr 2017 16:46:12 -0700 Subject: [PATCH 8/9] fix paths for uploading post data-import --- dvc/command/data_sync.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/dvc/command/data_sync.py b/dvc/command/data_sync.py index ad68f85d5..28aa3c817 100644 --- a/dvc/command/data_sync.py +++ b/dvc/command/data_sync.py @@ -112,9 +112,9 @@ class CmdDataSync(CmdBase): Logger.debug('Checksum miss-match. Re-uploading is required.') - Logger.info('Uploading cache file "{}" to S3 "{}"'.format(data_item.cache.relative, aws_key)) + Logger.info('Uploading cache file "{}" to S3 "{}"'.format(data_item.resolved_cache.relative, aws_key)) key = bucket.new_key(aws_key) - key.set_contents_from_filename(data_item.cache.relative, cb=percent_cb) + key.set_contents_from_filename(data_item.resolved_cache.relative, cb=percent_cb) Logger.info('Uploading completed') @@ -151,7 +151,7 @@ class CmdDataSync(CmdBase): blob_name = self.cache_file_key(data_item.resolved_cache.dvc) blob = bucket.get_blob(blob_name) - if blob.exists(): + if blob is not None and blob.exists(): b64_encoded_md5 = base64.b64encode(file_md5(data_item.resolved_cache.dvc)[1]) if blob.md5_hash == b64_encoded_md5: @@ -161,8 +161,9 @@ class CmdDataSync(CmdBase): Logger.info('uploading cache file "{} to gc "{}"'.format(data_item.cache.relative, blob_name)) - blob.upload_from_filename(data_item.cache.relative) - Logger.info('uploading %s completed' % data_item.cache.relative) + blob = bucket.blob(blob_name) + blob.upload_from_filename(data_item.resolved_cache.relative) + Logger.info('uploading %s completed' % data_item.resolved_cache.relative) def sync_from_cloud(self, item): From 2c849da1d723aef5a3927e7d7ba15d1d010e73e0 Mon Sep 17 00:00:00 2001 From: Dmitry Petrov Date: Mon, 17 Apr 2017 17:08:17 -0700 Subject: [PATCH 9/9] Tutorial --- README.md | 94 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 93 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7f406c012..58df13015 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,99 @@ [![Test Coverage](https://codeclimate.com/github/dmpetrov/dataversioncontrol/badges/coverage.svg)](https://codeclimate.com/github/dmpetrov/dataversioncontrol/coverage) # Data version control -Git for data science projects. It streamlines your git code, your data (S3 and GCP) and the dependencies to a single reproducible environment. +Git for data science projects. It streamlines your git code, your data (S3 and GCP) and dependencies to a single reproducible environment. + + +# Tutorial + +## Install + +Please download dvc from http://dataversioncontrol.com for your platform or + install through source code from Github https://github.com/dmpetrov/dataversioncontrol. + +## Initialize dvc environment + +DVC is an extension of the git command. +It works on top of git command in existing git repositories. +Let's create a new git repository for the tutorial perpose. + +``` +$ cd ~/src +$ mkdir tag_classifier +$ cd tag_classifier +$ git init +Initialized empty Git repository in /Users/dmitry/src/tag_classifier/.git/ +``` + +Initialize dvc environment: + +``` +$ dvc init +Info. Directories data/, cache/ and state/ were created +Info. File .gitignore was created +Info. Directory cache was added to .gitignore file +Info. [Git] A new commit a5ed2b9 was made in the current branch. Added files: +Info. [Git] A .gitignore +Info. [Git] A dvc.conf +``` + +The output shows the actions that dvc has made. +Special directories `data/`, `cache/` and `state/` were created to keep track of your external data which are not stored in git but have to be under users control. +These three directories are mondatory for any dvc project. + +Names of the directory might be changed by the `dvc init` command line parameters `--data-dir`, `--cache-dir` and `--state-dir` respectively. +All these special directories names are going to be saved in `dvc.conf` config file as a result of + the init command and should not be changed over dvc project life time. + +As you might see from the output `.gitignore` file was created (or modified if you already had one). +This is result of including cache directory to this file and also `.dvc.conf.lock` file. +Meaning of these changes will be explain later in the tutorial. + +DVC commits all the chenges to the git repository at the end of successful run. +The commit step might be skipped by command line parameter `--skip-git-actions`. +This parameter might be applied to any dvc command. + +## Importing data files + +Data file has to be imported in dvc `data/` directory before using it. + +``` +$ dvc data-import https://s3-us-west-2.amazonaws.com/dataversioncontrol/functests/stackoverflow_raw_small/Badges.xml data/ +$ dvc data-import https://s3-us-west-2.amazonaws.com/dvc-doc/small/Badges.xml data/ +Info. [Git] A new commit 9f27943 was made in the current branch. Added files: +Info. [Git] A data/Badges.xml +Info. [Git] A state/Badges.xml.state +``` + +The data file was downloaded and now it is available in the data directory. +However, it is only a part of the work that dvc has done. + +If you look carefully at the file you could see that this is actually a symlink to + a cache directory file Badges.xml_a5ed2b9: +``` +$ ls -l data/Badges.xml +lrwxr-xr-x 1 dmitry staff 27B Apr 17 16:19 data/Badges.xml@ -> ../cache/Badges.xml_a5ed2b9 +``` + +Instead of importing the file to `data/Badges.xml` the system imports the file in the cache directory + `cache/Badges.xml_a5ed2b9` and creates a symlink `data/Badges.xml`. +The file in the cache directory `cache/Badges.xml_a5ed2b9` has the same name as + the file in the data directory but extended by a suffix `_a5ed2b9` which + correspondes to a git snapshot checksum on the time when the file was created. + + + +The system transparently moves the actual file to the cache directory and creates a symlink + + + + +1. `data/` directory was created to keep all your data files that not suppose to go to Git repository. +This is the directory users suppose to use for all the data files. +However, dvc transparently moves all actual files to the `cache` directory and keeps only + symlink in `data/`. +2. `cache/` directory is a directory the file content. + # Copyright