From d54f803eb6cb6ca44eff355741a99e024d20de68 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Wed, 14 Feb 2018 18:22:19 +0200 Subject: [PATCH 1/3] system: remove unused code Signed-off-by: Ruslan Kuprieiev --- dvc/system.py | 49 ---------------------------------------------- tests/basic_env.py | 5 ++--- 2 files changed, 2 insertions(+), 52 deletions(-) diff --git a/dvc/system.py b/dvc/system.py index 6de1fa75b..174414626 100644 --- a/dvc/system.py +++ b/dvc/system.py @@ -5,7 +5,6 @@ import subprocess from builtins import str if os.name == 'nt': - from ctypes import create_unicode_buffer, windll import ntfsutils.hardlink as winlink @@ -33,51 +32,3 @@ class System(object): return os.path.samefile(path1, path2) return winlink.samefile(path1, path2) - - @staticmethod - def _get_symlink_string(path): - p = subprocess.Popen(["dir", path], shell=True) - output, _ = p.communicate() - if p.returncode != 0 or output == None: - return None - - lines = output.split('\n') - for line in lines: - if System.SYMLINK_OUTPUT in line: - return line - return None - - @staticmethod - def realpath(path): - # It is definitely not the best way to check a symlink. - - if System.is_unix(): - return os.path.realpath(path) - - output = System._get_symlink_string(path) - if output is None: - return os.path.realpath(path) - - groups = re.compile(r'\[\S+\]$').findall(output.strip()) - if len(groups) < 1: - return os.path.realpath(path) - - resolved_link = groups[0][1:-1] - return resolved_link - - @staticmethod - def get_long_path(path): - """Convert short path to a full path. It is needed for Windows.""" - if System.is_unix(): - return path - - buffer = ctypes.create_unicode_buffer(System.LONG_PATH_BUFFER_SIZE) - get_long_path_name = ctypes.windll.kernel32.GetLongPathNameW - result = get_long_path_name(u'%s' % str(path), buffer, System.LONG_PATH_BUFFER_SIZE) - if result == 0 or result > System.LONG_PATH_BUFFER_SIZE: - return path - return buffer.value - - @staticmethod - def get_cwd(): - return System.get_long_path(os.getcwd()) diff --git a/tests/basic_env.py b/tests/basic_env.py index 6dcd98280..4101b9494 100644 --- a/tests/basic_env.py +++ b/tests/basic_env.py @@ -4,7 +4,6 @@ import tempfile from git import Repo from unittest import TestCase -from dvc.system import System from dvc.project import Project @@ -23,7 +22,7 @@ class TestDir(TestCase): CODE_CONTENTS = 'import sys\nimport shutil\nshutil.copyfile(sys.argv[1], sys.argv[2])' def _pushd(self, d): - self._saved_dir = System.realpath(os.curdir) + self._saved_dir = os.path.realpath(os.curdir) os.chdir(d) def _popd(self): @@ -35,7 +34,7 @@ class TestDir(TestCase): f.write(contents) def setUp(self): - self._root_dir = System.get_long_path(tempfile.mkdtemp()) + self._root_dir = tempfile.mkdtemp() self._pushd(self._root_dir) self.create(self.FOO, self.FOO_CONTENTS) self.create(self.BAR, self.BAR_CONTENTS) From 77e61f40be189646ae39c6a4d36fca4577578d32 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Wed, 14 Feb 2018 13:57:26 +0200 Subject: [PATCH 2/3] test: test AWS on appveyor Signed-off-by: Ruslan Kuprieiev --- .appveyor.yml | 6 ++++++ tests/test_data_cloud.py | 17 +++++++++-------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 3645702c9..25fc9b22c 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -5,6 +5,12 @@ branches: platform: x86 environment: + my_variable: + aws_access_key_id: + secure: N3zfVSEvKhLSg/wvtrT1ZIVl9/egvW6xqIoRDrOcvvrkvJM0VejDHb3R3BWUUftY + aws_secret_access_key: + secure: /XGu3uYneji4fwIaFIEzDln9bx7vWZiLb6yie1temPyIRK+NQlvyfGXHcKZ5Vm4teaG68pNvQKoYulUSem4NvQ== + matrix: - PYTHON: "C:\\Python27" diff --git a/tests/test_data_cloud.py b/tests/test_data_cloud.py index a16f2f0c4..d47c5f314 100644 --- a/tests/test_data_cloud.py +++ b/tests/test_data_cloud.py @@ -99,12 +99,15 @@ class TestDataCloudAWS(TestDataCloudBase): TEST_REPO_REGION = 'us-east-2' def _should_test(self): - if os.name == 'nt': + dvc_test_aws = os.getenv("DVC_TEST_AWS") + if dvc_test_aws == "true": + return True + elif dvc_test_aws == "false": return False - if os.getenv("TRAVIS_PULL_REQUEST") == "false" and \ - os.getenv("TRAVIS_SECURE_ENV_VARS") == "true": + if os.getenv("AWS_ACCESS_KEY_ID") and os.getenv("AWS_SECRET_ACCESS_KEY"): return True + return False def _setup_cloud(self): @@ -129,12 +132,10 @@ class TestDataCloudGCP(TestDataCloudBase): TEST_REPO_GCP_PROJECT='dvc-project' def _should_test(self): - if os.name == 'nt': - return False + if os.getenv("DVC_TEST_GCP") == "true": + return True - if os.getenv("TRAVIS") == "true" or os.getenv("APPVEYOR") == "True" or os.getenv("DVC_TEST_GCP") != "true": - return False - return True + return False def _setup_cloud(self): if not self._should_test(): From ed75e6ea90861d4379b8f1d82bdd2314b5ebacb1 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Wed, 14 Feb 2018 20:05:55 +0200 Subject: [PATCH 3/3] cloud: use temp dir instead of individual files Signed-off-by: Ruslan Kuprieiev --- dvc/cloud/base.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/dvc/cloud/base.py b/dvc/cloud/base.py index 59538706f..262604783 100644 --- a/dvc/cloud/base.py +++ b/dvc/cloud/base.py @@ -216,11 +216,10 @@ class DataCloudBase(object): fname = os.path.join(path, os.path.relpath(k.name, self.cache_file_key(path))) res.append(self._status(k, fname)) - tmp = tempfile.NamedTemporaryFile() - self._pull_key(k, tmp.name) - with open(tmp.name, 'r') as fd: + tmp = os.path.join(tempfile.mkdtemp(), k.name) + self._pull_key(k, tmp) + with open(tmp, 'r') as fd: d = yaml.safe_load(fd) - tmp.close() md5 = d[Output.PARAM_MD5] cache = os.path.join(self._cloud_settings.cache_dir, md5)