diff --git a/dvc/command/data_sync.py b/dvc/command/data_sync.py index 1bd28eb8c..dff434da0 100644 --- a/dvc/command/data_sync.py +++ b/dvc/command/data_sync.py @@ -52,9 +52,6 @@ class CmdDataStatus(CmdBase): with DvcLock(self.is_locker, self.git): status = self.cloud.status(self.parsed_args.targets, self.parsed_args.jobs) - if len(list(status)) != len(self.parsed_args.targets): - return 1 - self._show(status) return 0 diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index 5df1b28c8..7a6c311d1 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -579,7 +579,7 @@ class DataCloudGCP(DataCloudBase): """ DataCloud class for Google Cloud Platform """ @property def gc_project_name(self): - return self._cloud_config.get('ProjectName', None) + return self._cloud_settings.cloud_config.get('ProjectName', None) def sanity_check(self): project = self.gc_project_name @@ -614,7 +614,11 @@ class DataCloudGCP(DataCloudBase): Logger.error('File "{}" does not exist in the cloud'.format(key)) return None - Logger.info('Downloading cache file from gc "{}/{}"'.format(bucket.name, key)) + if self._cmp_checksum(blob, fname): + Logger.debug('File "{}" matches with "{}".'.format(fname, key)) + return data_item + + Logger.debug('Downloading cache file from gc "{}/{}"'.format(bucket.name, key)) # percent_cb is not available for download_to_filename, so # lets at least update progress at keypoints(start, finish) @@ -629,7 +633,7 @@ class DataCloudGCP(DataCloudBase): progress.finish_target(name) - Logger.info('Downloading completed') + Logger.debug('Downloading completed') return data_item @@ -654,7 +658,7 @@ class DataCloudGCP(DataCloudBase): blob.upload_from_filename(data_item.resolved_cache.relative) progress.finish_target(name) - Logger.info('uploading %s completed' % data_item.resolved_cache.relative) + Logger.debug('uploading %s completed' % data_item.resolved_cache.relative) return data_item diff --git a/functest/common.sh b/functest/common.sh index 53f018231..e2aaafec3 100644 --- a/functest/common.sh +++ b/functest/common.sh @@ -19,6 +19,9 @@ CODE_S3=$BASE_DIR_S3/code TEST_REPO_S3='dvc-test/myrepo' TEST_REPO_REGION='us-east-2' +TEST_REPO_GCP='dvc-test/myrepo' +TEST_REPO_GCP_PROJECT='dvc-project' + function dvc_print() { echo -e "$1" } @@ -81,6 +84,7 @@ function dvc_create_git_repo() { dvc_prepare_cache + rm -rf $TEST_REPO mkdir $TEST_REPO cd $TEST_REPO @@ -111,6 +115,11 @@ function dvc_clean_cloud_aws() { aws s3 rm --recursive "s3://$TEST_REPO_S3/" } +function dvc_clean_cloud_gcp() { + dvc_info "Cleaning $TEST_REPO_GCP_PROJECT/$TEST_REPO_GCP" + gsutil rm -rf "gs://$TEST_REPO_GCP/" || true +} + function dvc_check_dirs() { for d in $1; do if [ ! -d "$d" ]; then diff --git a/functest/run.sh b/functest/run.sh index cc36fa836..d2626de49 100755 --- a/functest/run.sh +++ b/functest/run.sh @@ -6,7 +6,7 @@ source common.sh TESTS+=" test-init.sh" TESTS+=" test-import.sh" -TESTS+=" test-sync-aws.sh" +TESTS+=" test-sync.sh" TESTS+=" test-repro-code.sh" for t in $TESTS; do diff --git a/functest/test-sync-aws.sh b/functest/test-sync.sh similarity index 62% rename from functest/test-sync-aws.sh rename to functest/test-sync.sh index 736bf982d..389a1bcf6 100755 --- a/functest/test-sync-aws.sh +++ b/functest/test-sync.sh @@ -4,18 +4,11 @@ set -e source common.sh -if [[ "$TRAVIS_PULL_REQUEST" == "false" && "$TRAVIS_SECURE_ENV_VARS" == "true" ]]; then - dvc_create_repo - - dvc_info "Setting up AWS cloud" - dvc_clean_cloud_aws - dvc config aws.storagepath $TEST_REPO_S3 - dvc config aws.region $TEST_REPO_REGION - +function test_sync() { dvc_info "Checking status" dvc status data/xml - dvc_info "Pushing to AWS" + dvc_info "Pushing data" dvc push data/xml dvc_info "Checking status" @@ -41,8 +34,42 @@ if [[ "$TRAVIS_PULL_REQUEST" == "false" && "$TRAVIS_SECURE_ENV_VARS" == "true" ] dvc_info "Pulling data" dvc pull data/xml +} + +function test_aws() { + dvc_create_repo + + dvc_info "Setting up AWS cloud" + dvc_clean_cloud_aws + dvc config global.cloud AWS + dvc config aws.storagepath $TEST_REPO_S3 + dvc config aws.region $TEST_REPO_REGION + + test_sync dvc_clean_cloud_aws +} + +function test_gcp() { + dvc_create_repo + + dvc_info "Setting up GCP cloud" + dvc_clean_cloud_gcp + dvc config global.cloud GCP + dvc config gcp.storagepath $TEST_REPO_GCP + dvc config gcp.projectname $TEST_REPO_GCP_PROJECT + + test_sync + + dvc_clean_cloud_gcp +} + +if [[ "$TRAVIS_PULL_REQUEST" == "false" && "$TRAVIS_SECURE_ENV_VARS" == "true" ]]; then + test_aws + # Temporarily disabled for travis + if [[ "$TRAVIS" != "true" ]]; then + test_gcp + fi fi dvc_pass diff --git a/requirements.txt b/requirements.txt index 8754dcc29..11f78e9b1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -88,3 +88,4 @@ colorama==0.3.9 google-compute-engine==2.4.1 configobj==5.0.6 networkx==1.11 +gcs-oauth2-boto-plugin==1.14 diff --git a/setup.py b/setup.py index 28868597e..7c2099104 100644 --- a/setup.py +++ b/setup.py @@ -94,6 +94,7 @@ install_requires = [ 'google-compute-engine==2.4.1', 'configobj==5.0.6', 'networkx==1.11', + 'gcs-oauth2-boto-plugin==1.14', ] if platform.system() == 'Darwin':