From a58f3b8184e33ed47c551bf67a2524861a184311 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Wed, 14 Feb 2018 22:28:58 +0200 Subject: [PATCH] test: enable gcp tests Signed-off-by: Ruslan Kuprieiev --- .appveyor.yml | 7 ++++++- scripts/ci/gcp-creds.json.enc | Bin 0 -> 2336 bytes scripts/ci/install.sh | 1 + tests/basic_env.py | 1 + tests/test_data_cloud.py | 13 +++++++++---- 5 files changed, 17 insertions(+), 5 deletions(-) create mode 100644 scripts/ci/gcp-creds.json.enc diff --git a/.appveyor.yml b/.appveyor.yml index d552bb7ec..560516cfb 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -5,11 +5,13 @@ branches: platform: x86 environment: - my_variable: aws_access_key_id: secure: veTfhOrd4wUpG7NlRsvS3tufuaFfST8i7SZa2HVH6Vc= aws_secret_access_key: secure: nCQcN/Mzce0erwcNOU3yAuWgF6ZuTvPBEqTF3XFp1poGcDbSIg+s67eVxsxWCXQj + GOOGLE_APPLICATION_CREDENTIALS: gcp-creds.json + GCP_CREDS: + secure: 96fJ3r2i2GohbXHwnSs5N4EplQ7q8YmLpPWM0AC+f4s= matrix: @@ -42,6 +44,8 @@ install: - cinst graphviz - cinst wget - cinst awscli + - cinst gsutil + - cinst openssl.light - wget --no-check-certificate https://github.com/dpinney/omf/raw/master/omf/static/pygraphviz-1.3.1-cp27-none-win32.whl - pip install pygraphviz-1.3.1-cp27-none-win32.whl - pip install -r requirements.txt @@ -52,6 +56,7 @@ build: false before_test: - aws configure set aws_access_key_id "%aws_access_key_id%" - aws configure set aws_secret_access_key "%aws_secret_access_key%" + - openssl enc -d -aes-256-cbc -md md5 -k "%GCP_CREDS%" -in scripts\ci\gcp-creds.json.enc -out scripts\ci\gcp-creds.json & exit 0 - pip install -r test-requirements.txt test_script: diff --git a/scripts/ci/gcp-creds.json.enc b/scripts/ci/gcp-creds.json.enc new file mode 100644 index 0000000000000000000000000000000000000000..d18531f9da84a737a41a073cc901e661fa304f53 GIT binary patch literal 2336 zcmV+*3E%cpVQh3|WM5y{hSaV(^J)o)eXhS2PQFlcfgjXn2O+|8=|j6rZW`ht{tZ~$ z9ht+taSI!QIQN#5Q*Z38JwFEBP>8R752UZ$_8aY*#!*~+gLKP(&})Pfzj4FsJ#o5Q zPY$kTd;zRS01L+^7H>%2R-t*C3>*k-qOYaJfzAy?+$UmG_;v353jnyJ3|NGAjYCI8 zNf347K09{(7p-KQr_*$9C85;x<-|3rR)OV#&DHv@(QSdAu8-R{WwK zD+t>#dTO;Qs_93Ewmm3fs!O`x_Fh3+>fv_iYRZ$Yney%?>Uz_iX2MQD4F#^pp6C|< zV~-wkohpYh^Gh$c;KW7J3T?q`1oxBNZR~T{%AXGBRSjWy{?ECSFe$0wkU5n905TE% zT+OHdA4sa|bf0u)FXCvGp4=X%D0BRB4S@WFDae5yj&`A#Q-|8;(+Qb|?`XfHq-z%o zDlYQmdIEuO{Ln#geH6Ot$mQ%Jz5hY>Lb*7|BoDe*&27Vd?XCah*khpq#8V!8Sw|Xl zZJ${=y`uIsC#RB8IBD4ca_OJdf2O}(U466vn!-K2ZTdFiM{?|pf4yAtL{n{jz57k{ zxPlgFOw13Jtix}-76y~h4=iQPoyfavfeBxG@TRxT{^1(!FpeJhe^YsW^^L7d4d5U; zVc!~wJ z@08^5Qq_my%6E0GZVhPGcW*0#f7!z_;rEphIoI2}53h$OXPI#tH`Zj<84w`g=MjBu z)Q3oB#d$8I{x@v<#=Be$_D-;t<@qc>OSQ;2UU%XZJ3ZB;?C){%cUtk+9Z( zwbGlDso`@cROwoIDw|x>^{sUG#gK^J_TkXry`~bu0zO1Q=z&Vqk-0Fm&CegLf_CF^ zlVPHsqlbMUJx((tOn-wXn6X+Bu-L?BAYqh$$Ti|_sqF>Fc1Lnw%?uGQ_q=NBVF0{2 zMVUKXhB7+^&)Np1m^XuK8XRSPpxc9jVwhMw$o20u{&`ozrD;xT?(D487n2#aO{b=r zqRa5ZC(D z?wAIW1u>Tw4Iupx20XG}Blq^{e|JpOrTDtl=78z#oxObSp__8IEGg{*i)7rn85a5Z zLJMt;s`TDEWcNU8@%+v55z<`Dc#u-NQ}`9z^~I{**ywO&{fn*?I~q9OBQJR!Z!$=d^1B@XeNe=1b`t2dBXHo5|>0`hXLnD-xu2N7rL(flWs?UAg@Q zlr?59#QgVR5VRpc!W1TH!Pn|EWCJRnUP|DZA|roPRL(I8SFu_+-}>>!HbWJM-2nuU zl4%IfkmDaGP5VwLdfYRESnUUEo0@n)y(%$K!leArR1obt0e7>*RWlW92Afsyqp}p> z(w4ceoQo!KTma7;uuT`JE?8X=Yb=3bpF;D0WG@kAHfK{prw~%>Z{t&0bs&|lPP*_Y zy&^kEW+ej8H~>}OU1W5xFWE$5wk7%;`^V@2|LbY0?)O#%1f0B!eB-6dFbF?*HL0`u zSa-l-git`u$O-=;nGGRY5XY$HYIN)EKu2cR%lpENz*3U*LfcG4 zuHg9H7z4;MPf#V2Xh(6yy#u)I3QgO-hY%>{R)X|?pLTD9>;atG@1ktIv1pP}+7_^G z`YX4k)Fx24tut}EYs^^-uA@KkiL}ez5B1i@Zg-ns|5&pUhDfEU1j#T8bvR3RhLm5H znG}M(4pL5s_-lp6Px`ddJXdRLF1%p)I@D5ZW~?$?yr|90Y5s60JO+-QGQ1Z&Pfuu) zvWoiPh3q3?wYV=*_$w{Lo6%A|CMZKNY5PH`r~R_N^`ELb(O3Y@!!`7+H2>Eo0P_NA zVB)=V{T7YVSPm3{Bcj2-G>-sH7pr@`?NQ+A6ra@eX9lVaSFql6ODUO3$6BbJLf+xc z#%Pb8J;ReqeGj*1a{T^&52|DbsH4{Fx9mF?ukxuF=F)QS7U&Stpn@^VPpsMz+yeOJ z`P#Lx!+Uf9Vfd;jci>>@10Yn4WtBh5Uj;M;TW9oIju=4V>R%5+%SBiqsjoran7}}v zSKB|wx1d7vBt=@iiWflPR;N<0;hE&~=U#=_OX5O!4MbMpNhcF*L}-v-%*@E&l~@c; zA;Mfe@-xkr1JcaY#$nJj@u_IdnfIV`vvodsZG~=Vk4?wCc{lj3$TY=?ZHPmL3FY=e zK(ubL8dV?8``P5ix! zHC~H{27rBCP+BF~lBQ#6_NGsU;0nzpg#h0Q{B?zVtsK!3Xw#MoDF4rGG?A!9N_<%_ zUqRJ0$a-Bs?^tIZ{3htc0EFd}&7Usc)pe(zE8&l{CHwc9v~vbRiIiO{y;)tros=ry zGOc{J=Wo`eWG@?wVz*qJxu~5(ZFFvGld8@7!RRnRlZHnfsuqwsme2PCDz)si3d-zS za ~/.aws/credentials printf "aws_access_key_id = $AWS_ACCESS_KEY_ID\n" >> ~/.aws/credentials printf "aws_secret_access_key = $AWS_SECRET_ACCESS_KEY\n" >> ~/.aws/credentials printf "[default]\n" > ~/.aws/config +openssl enc -d -aes-256-cbc -md md5 -k $GCP_CREDS -in scripts/ci/gcp-creds.json.enc -out scripts/ci/gcp-creds.json || true diff --git a/tests/basic_env.py b/tests/basic_env.py index 4101b9494..68ed97a89 100644 --- a/tests/basic_env.py +++ b/tests/basic_env.py @@ -8,6 +8,7 @@ from dvc.project import Project class TestDir(TestCase): + GCP_CREDS_FILE = os.path.abspath(os.path.join('scripts', 'ci', 'gcp-creds.json')) DATA_DIR = 'data_dir' DATA_SUB_DIR = os.path.join(DATA_DIR, 'data_sub_dir') DATA = os.path.join(DATA_DIR, 'data') diff --git a/tests/test_data_cloud.py b/tests/test_data_cloud.py index d47c5f314..508fea7be 100644 --- a/tests/test_data_cloud.py +++ b/tests/test_data_cloud.py @@ -128,24 +128,29 @@ class TestDataCloudAWS(TestDataCloudBase): class TestDataCloudGCP(TestDataCloudBase): - TEST_REPO_GCP = 'dvc-test/myrepo' + TEST_REPO_GCP_BUCKET = 'dvc-test' TEST_REPO_GCP_PROJECT='dvc-project' def _should_test(self): if os.getenv("DVC_TEST_GCP") == "true": return True + if os.getenv("GOOGLE_APPLICATION_CREDENTIALS") and os.getenv("GCP_CREDS"): + return True + return False def _setup_cloud(self): if not self._should_test(): return - # Cleanup - os.system("gsutil rm -rf gs://{}/".format(self.TEST_REPO_GCP)) + if os.getenv("GOOGLE_APPLICATION_CREDENTIALS") and os.getenv("GCP_CREDS"): + shutil.copyfile(self.GCP_CREDS_FILE, os.getenv("GOOGLE_APPLICATION_CREDENTIALS")) + + repo = self.TEST_REPO_GCP_BUCKET + '/' + str(uuid.uuid4()) # Setup cloud - config = {'StoragePath': self.TEST_REPO_GCP, + config = {'StoragePath': repo, 'Region': self.TEST_REPO_GCP_PROJECT} cloud_settings = CloudSettings(self.dvc.cache.cache_dir, None, config) self.cloud = DataCloudGCP(cloud_settings)