Merge pull request #206 from efiop/master

Misc data-related changes(GCP, tests and status)

dvc/command/data_sync.py

@@ -52,9 +52,6 @@ class CmdDataStatus(CmdBase):
         with DvcLock(self.is_locker, self.git):
             status = self.cloud.status(self.parsed_args.targets, self.parsed_args.jobs)
 
-            if len(list(status)) != len(self.parsed_args.targets):
-                return 1
-
             self._show(status)
 
             return 0

dvc/data_cloud.py

@@ -579,7 +579,7 @@ class DataCloudGCP(DataCloudBase):
     """ DataCloud class for Google Cloud Platform """
     @property
     def gc_project_name(self):
-        return self._cloud_config.get('ProjectName', None)
+        return self._cloud_settings.cloud_config.get('ProjectName', None)
 
     def sanity_check(self):
         project = self.gc_project_name
@@ -614,7 +614,11 @@ class DataCloudGCP(DataCloudBase):
             Logger.error('File "{}" does not exist in the cloud'.format(key))
             return None
 
-        Logger.info('Downloading cache file from gc "{}/{}"'.format(bucket.name, key))
+        if self._cmp_checksum(blob, fname):
+            Logger.debug('File "{}" matches with "{}".'.format(fname, key))
+            return data_item
+
+        Logger.debug('Downloading cache file from gc "{}/{}"'.format(bucket.name, key))
 
         # percent_cb is not available for download_to_filename, so
         # lets at least update progress at keypoints(start, finish)
@@ -629,7 +633,7 @@ class DataCloudGCP(DataCloudBase):
 
         progress.finish_target(name)
 
-        Logger.info('Downloading completed')
+        Logger.debug('Downloading completed')
 
         return data_item
 
@@ -654,7 +658,7 @@ class DataCloudGCP(DataCloudBase):
         blob.upload_from_filename(data_item.resolved_cache.relative)
 
         progress.finish_target(name)
-        Logger.info('uploading %s completed' % data_item.resolved_cache.relative)
+        Logger.debug('uploading %s completed' % data_item.resolved_cache.relative)
 
         return data_item
 

functest/common.sh

@@ -19,6 +19,9 @@ CODE_S3=$BASE_DIR_S3/code
 TEST_REPO_S3='dvc-test/myrepo'
 TEST_REPO_REGION='us-east-2'
 
+TEST_REPO_GCP='dvc-test/myrepo'
+TEST_REPO_GCP_PROJECT='dvc-project'
+
 function dvc_print() {
 	echo -e "$1"
 }
@@ -81,6 +84,7 @@ function dvc_create_git_repo() {
 
 	dvc_prepare_cache
 
+	rm -rf $TEST_REPO
 	mkdir $TEST_REPO
         cd $TEST_REPO
 
@@ -111,6 +115,11 @@ function dvc_clean_cloud_aws() {
 	aws s3 rm --recursive "s3://$TEST_REPO_S3/"
 }
 
+function dvc_clean_cloud_gcp() {
+	dvc_info "Cleaning $TEST_REPO_GCP_PROJECT/$TEST_REPO_GCP"
+	gsutil rm -rf "gs://$TEST_REPO_GCP/" || true
+}
+
 function dvc_check_dirs() {
 	for d in $1; do
 		if [ ! -d "$d" ]; then

functest/run.sh

@@ -6,7 +6,7 @@ source common.sh
 
 TESTS+=" test-init.sh"
 TESTS+=" test-import.sh"
-TESTS+=" test-sync-aws.sh"
+TESTS+=" test-sync.sh"
 TESTS+=" test-repro-code.sh"
 
 for t in $TESTS; do

functest/test-sync.sh

@@ -4,18 +4,11 @@ set -e
 
 source common.sh
 
-if [[ "$TRAVIS_PULL_REQUEST" == "false" && "$TRAVIS_SECURE_ENV_VARS" == "true" ]]; then
-	dvc_create_repo
-
-	dvc_info "Setting up AWS cloud"
-	dvc_clean_cloud_aws
-	dvc config aws.storagepath $TEST_REPO_S3
-	dvc config aws.region $TEST_REPO_REGION
-
+function test_sync() {
 	dvc_info "Checking status"
 	dvc status data/xml
 
-	dvc_info "Pushing to AWS"
+	dvc_info "Pushing data"
 	dvc push data/xml
 
 	dvc_info "Checking status"
@@ -41,8 +34,42 @@ if [[ "$TRAVIS_PULL_REQUEST" == "false" && "$TRAVIS_SECURE_ENV_VARS" == "true" ]
 
 	dvc_info "Pulling data"
 	dvc pull data/xml
+} 
+
+function test_aws() {
+	dvc_create_repo
+
+	dvc_info "Setting up AWS cloud"
+	dvc_clean_cloud_aws
+	dvc config global.cloud AWS
+	dvc config aws.storagepath $TEST_REPO_S3
+	dvc config aws.region $TEST_REPO_REGION
+
+	test_sync
 
 	dvc_clean_cloud_aws
+}
+
+function test_gcp() {
+	dvc_create_repo
+
+	dvc_info "Setting up GCP cloud"
+	dvc_clean_cloud_gcp
+	dvc config global.cloud GCP
+	dvc config gcp.storagepath $TEST_REPO_GCP
+	dvc config gcp.projectname $TEST_REPO_GCP_PROJECT
+
+	test_sync
+
+	dvc_clean_cloud_gcp
+}
+
+if [[ "$TRAVIS_PULL_REQUEST" == "false" && "$TRAVIS_SECURE_ENV_VARS" == "true" ]]; then
+	test_aws
+	# Temporarily disabled for travis
+	if [[ "$TRAVIS" != "true" ]]; then
+		test_gcp
+	fi
 fi
 
 dvc_pass

requirements.txt

@@ -88,3 +88,4 @@ colorama==0.3.9
 google-compute-engine==2.4.1
 configobj==5.0.6
 networkx==1.11
+gcs-oauth2-boto-plugin==1.14

setup.py

@@ -94,6 +94,7 @@ install_requires = [
     'google-compute-engine==2.4.1',
     'configobj==5.0.6',
     'networkx==1.11',
+    'gcs-oauth2-boto-plugin==1.14',
 ]
 
 if platform.system() == 'Darwin':