From 0c561e7f121ff69ffad2986205e175f5c3f6e209 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Thu, 14 Sep 2017 20:47:35 +0300 Subject: [PATCH 1/4] google cloud: fix cloud settings and requirements These are to allow using OAuth with your dvc, as this is the recomended way to access gcp. Fixes #203 Signed-off-by: Ruslan Kuprieiev --- dvc/data_cloud.py | 2 +- requirements.txt | 1 + setup.py | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index 5df1b28c8..ea7ef8532 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -579,7 +579,7 @@ class DataCloudGCP(DataCloudBase): """ DataCloud class for Google Cloud Platform """ @property def gc_project_name(self): - return self._cloud_config.get('ProjectName', None) + return self._cloud_settings.cloud_config.get('ProjectName', None) def sanity_check(self): project = self.gc_project_name diff --git a/requirements.txt b/requirements.txt index 8754dcc29..11f78e9b1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -88,3 +88,4 @@ colorama==0.3.9 google-compute-engine==2.4.1 configobj==5.0.6 networkx==1.11 +gcs-oauth2-boto-plugin==1.14 diff --git a/setup.py b/setup.py index 28868597e..7c2099104 100644 --- a/setup.py +++ b/setup.py @@ -94,6 +94,7 @@ install_requires = [ 'google-compute-engine==2.4.1', 'configobj==5.0.6', 'networkx==1.11', + 'gcs-oauth2-boto-plugin==1.14', ] if platform.system() == 'Darwin': From ce609c5f50dcfcad901b2bf1e1df4e9472e0ae46 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Thu, 14 Sep 2017 22:16:05 +0300 Subject: [PATCH 2/4] status: don't return 1 if len(statuses) != len(targets) Number of targets from command line doesn't have to match number of actual files that we've processed, as it may as well be a directory. Fixes #204 Signed-off-by: Ruslan Kuprieiev --- dvc/command/data_sync.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/dvc/command/data_sync.py b/dvc/command/data_sync.py index 1bd28eb8c..dff434da0 100644 --- a/dvc/command/data_sync.py +++ b/dvc/command/data_sync.py @@ -52,9 +52,6 @@ class CmdDataStatus(CmdBase): with DvcLock(self.is_locker, self.git): status = self.cloud.status(self.parsed_args.targets, self.parsed_args.jobs) - if len(list(status)) != len(self.parsed_args.targets): - return 1 - self._show(status) return 0 From cc2dff765b2f7a194ea9efb28b776c6d745b4671 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Thu, 14 Sep 2017 22:17:40 +0300 Subject: [PATCH 3/4] data: check checksum before pulling data Fixes #205 Signed-off-by: Ruslan Kuprieiev --- dvc/data_cloud.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index ea7ef8532..7a6c311d1 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -614,7 +614,11 @@ class DataCloudGCP(DataCloudBase): Logger.error('File "{}" does not exist in the cloud'.format(key)) return None - Logger.info('Downloading cache file from gc "{}/{}"'.format(bucket.name, key)) + if self._cmp_checksum(blob, fname): + Logger.debug('File "{}" matches with "{}".'.format(fname, key)) + return data_item + + Logger.debug('Downloading cache file from gc "{}/{}"'.format(bucket.name, key)) # percent_cb is not available for download_to_filename, so # lets at least update progress at keypoints(start, finish) @@ -629,7 +633,7 @@ class DataCloudGCP(DataCloudBase): progress.finish_target(name) - Logger.info('Downloading completed') + Logger.debug('Downloading completed') return data_item @@ -654,7 +658,7 @@ class DataCloudGCP(DataCloudBase): blob.upload_from_filename(data_item.resolved_cache.relative) progress.finish_target(name) - Logger.info('uploading %s completed' % data_item.resolved_cache.relative) + Logger.debug('uploading %s completed' % data_item.resolved_cache.relative) return data_item From 92ae0ba2a77822734473d879b1583c8928335c18 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Thu, 14 Sep 2017 22:15:04 +0300 Subject: [PATCH 4/4] functest: add gcp sync test Temporarily disabled for travis. Fixes #203 Signed-off-by: Ruslan Kuprieiev --- functest/common.sh | 9 +++++ functest/run.sh | 2 +- functest/{test-sync-aws.sh => test-sync.sh} | 45 ++++++++++++++++----- 3 files changed, 46 insertions(+), 10 deletions(-) rename functest/{test-sync-aws.sh => test-sync.sh} (62%) diff --git a/functest/common.sh b/functest/common.sh index 53f018231..e2aaafec3 100644 --- a/functest/common.sh +++ b/functest/common.sh @@ -19,6 +19,9 @@ CODE_S3=$BASE_DIR_S3/code TEST_REPO_S3='dvc-test/myrepo' TEST_REPO_REGION='us-east-2' +TEST_REPO_GCP='dvc-test/myrepo' +TEST_REPO_GCP_PROJECT='dvc-project' + function dvc_print() { echo -e "$1" } @@ -81,6 +84,7 @@ function dvc_create_git_repo() { dvc_prepare_cache + rm -rf $TEST_REPO mkdir $TEST_REPO cd $TEST_REPO @@ -111,6 +115,11 @@ function dvc_clean_cloud_aws() { aws s3 rm --recursive "s3://$TEST_REPO_S3/" } +function dvc_clean_cloud_gcp() { + dvc_info "Cleaning $TEST_REPO_GCP_PROJECT/$TEST_REPO_GCP" + gsutil rm -rf "gs://$TEST_REPO_GCP/" || true +} + function dvc_check_dirs() { for d in $1; do if [ ! -d "$d" ]; then diff --git a/functest/run.sh b/functest/run.sh index cc36fa836..d2626de49 100755 --- a/functest/run.sh +++ b/functest/run.sh @@ -6,7 +6,7 @@ source common.sh TESTS+=" test-init.sh" TESTS+=" test-import.sh" -TESTS+=" test-sync-aws.sh" +TESTS+=" test-sync.sh" TESTS+=" test-repro-code.sh" for t in $TESTS; do diff --git a/functest/test-sync-aws.sh b/functest/test-sync.sh similarity index 62% rename from functest/test-sync-aws.sh rename to functest/test-sync.sh index 736bf982d..389a1bcf6 100755 --- a/functest/test-sync-aws.sh +++ b/functest/test-sync.sh @@ -4,18 +4,11 @@ set -e source common.sh -if [[ "$TRAVIS_PULL_REQUEST" == "false" && "$TRAVIS_SECURE_ENV_VARS" == "true" ]]; then - dvc_create_repo - - dvc_info "Setting up AWS cloud" - dvc_clean_cloud_aws - dvc config aws.storagepath $TEST_REPO_S3 - dvc config aws.region $TEST_REPO_REGION - +function test_sync() { dvc_info "Checking status" dvc status data/xml - dvc_info "Pushing to AWS" + dvc_info "Pushing data" dvc push data/xml dvc_info "Checking status" @@ -41,8 +34,42 @@ if [[ "$TRAVIS_PULL_REQUEST" == "false" && "$TRAVIS_SECURE_ENV_VARS" == "true" ] dvc_info "Pulling data" dvc pull data/xml +} + +function test_aws() { + dvc_create_repo + + dvc_info "Setting up AWS cloud" + dvc_clean_cloud_aws + dvc config global.cloud AWS + dvc config aws.storagepath $TEST_REPO_S3 + dvc config aws.region $TEST_REPO_REGION + + test_sync dvc_clean_cloud_aws +} + +function test_gcp() { + dvc_create_repo + + dvc_info "Setting up GCP cloud" + dvc_clean_cloud_gcp + dvc config global.cloud GCP + dvc config gcp.storagepath $TEST_REPO_GCP + dvc config gcp.projectname $TEST_REPO_GCP_PROJECT + + test_sync + + dvc_clean_cloud_gcp +} + +if [[ "$TRAVIS_PULL_REQUEST" == "false" && "$TRAVIS_SECURE_ENV_VARS" == "true" ]]; then + test_aws + # Temporarily disabled for travis + if [[ "$TRAVIS" != "true" ]]; then + test_gcp + fi fi dvc_pass