Merge pull request #465 from efiop/master

Windows fixes and AWS test

.appveyor.yml

@@ -5,6 +5,12 @@ branches:
 platform: x86
 
 environment:
+  my_variable:
+  aws_access_key_id:
+    secure: N3zfVSEvKhLSg/wvtrT1ZIVl9/egvW6xqIoRDrOcvvrkvJM0VejDHb3R3BWUUftY
+  aws_secret_access_key:
+    secure: /XGu3uYneji4fwIaFIEzDln9bx7vWZiLb6yie1temPyIRK+NQlvyfGXHcKZ5Vm4teaG68pNvQKoYulUSem4NvQ==
+
   matrix:
 
     - PYTHON: "C:\\Python27"

dvc/cloud/base.py

@@ -216,11 +216,10 @@ class DataCloudBase(object):
             fname = os.path.join(path, os.path.relpath(k.name, self.cache_file_key(path)))
             res.append(self._status(k, fname))
 
-            tmp = tempfile.NamedTemporaryFile()
-            self._pull_key(k, tmp.name)
-            with open(tmp.name, 'r') as fd:
+            tmp = os.path.join(tempfile.mkdtemp(), k.name)
+            self._pull_key(k, tmp)
+            with open(tmp, 'r') as fd:
                 d = yaml.safe_load(fd)
-            tmp.close()
             md5 = d[Output.PARAM_MD5]
 
             cache = os.path.join(self._cloud_settings.cache_dir, md5)

dvc/system.py

@@ -5,7 +5,6 @@ import subprocess
 from builtins import str
 
 if os.name == 'nt':
-    from ctypes import create_unicode_buffer, windll
     import ntfsutils.hardlink as winlink
 
 
@@ -33,51 +32,3 @@ class System(object):
             return os.path.samefile(path1, path2)
 
         return winlink.samefile(path1, path2)
-
-    @staticmethod
-    def _get_symlink_string(path):
-        p = subprocess.Popen(["dir", path], shell=True)
-        output, _ = p.communicate()
-        if p.returncode != 0 or output == None:
-            return None
-
-        lines = output.split('\n')
-        for line in lines:
-            if System.SYMLINK_OUTPUT in line:
-                return line
-        return None
-
-    @staticmethod
-    def realpath(path):
-        # It is definitely not the best way to check a symlink.
-
-        if System.is_unix():
-            return os.path.realpath(path)
-
-        output = System._get_symlink_string(path)
-        if output is None:
-            return os.path.realpath(path)
-
-        groups = re.compile(r'\[\S+\]$').findall(output.strip())
-        if len(groups) < 1:
-            return os.path.realpath(path)
-
-        resolved_link = groups[0][1:-1]
-        return resolved_link
-
-    @staticmethod
-    def get_long_path(path):
-        """Convert short path to a full path. It is needed for Windows."""
-        if System.is_unix():
-            return path
-
-        buffer = ctypes.create_unicode_buffer(System.LONG_PATH_BUFFER_SIZE)
-        get_long_path_name = ctypes.windll.kernel32.GetLongPathNameW
-        result = get_long_path_name(u'%s' % str(path), buffer, System.LONG_PATH_BUFFER_SIZE)
-        if result == 0 or result > System.LONG_PATH_BUFFER_SIZE:
-            return path
-        return buffer.value
-
-    @staticmethod
-    def get_cwd():
-        return System.get_long_path(os.getcwd())

tests/basic_env.py

@@ -4,7 +4,6 @@ import tempfile
 from git import Repo
 from unittest import TestCase
 
-from dvc.system import System
 from dvc.project import Project
 
 
@@ -23,7 +22,7 @@ class TestDir(TestCase):
     CODE_CONTENTS = 'import sys\nimport shutil\nshutil.copyfile(sys.argv[1], sys.argv[2])'
 
     def _pushd(self, d):
-        self._saved_dir = System.realpath(os.curdir)
+        self._saved_dir = os.path.realpath(os.curdir)
         os.chdir(d)
 
     def _popd(self):
@@ -35,7 +34,7 @@ class TestDir(TestCase):
             f.write(contents)
 
     def setUp(self):
-        self._root_dir = System.get_long_path(tempfile.mkdtemp())
+        self._root_dir = tempfile.mkdtemp()
         self._pushd(self._root_dir)
         self.create(self.FOO, self.FOO_CONTENTS)
         self.create(self.BAR, self.BAR_CONTENTS)

tests/test_data_cloud.py

@@ -99,12 +99,15 @@ class TestDataCloudAWS(TestDataCloudBase):
     TEST_REPO_REGION = 'us-east-2'
 
     def _should_test(self):
-        if os.name == 'nt':
+        dvc_test_aws = os.getenv("DVC_TEST_AWS")
+        if dvc_test_aws == "true":
+            return True
+        elif dvc_test_aws == "false":
             return False
 
-        if os.getenv("TRAVIS_PULL_REQUEST") == "false" and \
-           os.getenv("TRAVIS_SECURE_ENV_VARS") == "true":
+        if os.getenv("AWS_ACCESS_KEY_ID") and os.getenv("AWS_SECRET_ACCESS_KEY"):
             return True
+
         return False
 
     def _setup_cloud(self):
@@ -129,12 +132,10 @@ class TestDataCloudGCP(TestDataCloudBase):
     TEST_REPO_GCP_PROJECT='dvc-project'
 
     def _should_test(self):
-        if os.name == 'nt':
-            return False
+        if os.getenv("DVC_TEST_GCP") == "true":
+            return True
 
-        if os.getenv("TRAVIS") == "true" or os.getenv("APPVEYOR") == "True" or os.getenv("DVC_TEST_GCP") != "true":
-            return False
-        return True
+        return False
 
     def _setup_cloud(self):
         if not self._should_test():