Merge pull request #1587 from mroutis/fix-unicode

py2: Add unicode support for open and os.walk

dvc/config.py

@@ -2,7 +2,7 @@
 
 from __future__ import unicode_literals
 
-from dvc.utils.compat import str
+from dvc.utils.compat import str, open
 
 import os
 import errno

dvc/istextfile.py

@@ -2,7 +2,7 @@
 
 from __future__ import unicode_literals
 
-from dvc.utils.compat import is_py3
+from dvc.utils.compat import is_py3, open
 
 # Based on https://eli.thegreenplace.net/2011/10/19/
 # perls-guess-if-file-is-text-or-binary-implemented-in-python

dvc/project.py

@@ -1,6 +1,6 @@
 from __future__ import unicode_literals
 
-from dvc.utils.compat import str, builtin_str
+from dvc.utils.compat import str, builtin_str, open
 
 import collections
 import os
@@ -200,7 +200,7 @@ class Project(object):
 
         fnames = []
         if recursive and os.path.isdir(fname):
-            for root, dirs, files in os.walk(fname):
+            for root, dirs, files in os.walk(str(fname)):
                 for f in files:
                     path = os.path.join(root, f)
                     if Stage.is_stage_file(path):
@@ -351,7 +351,7 @@ class Project(object):
         os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
 
     def _unprotect_dir(self, path):
-        for root, dirs, files in os.walk(path):
+        for root, dirs, files in os.walk(str(path)):
             for f in files:
                 path = os.path.join(root, f)
                 self._unprotect_file(path)
@@ -1335,7 +1335,7 @@ class Project(object):
 
         stages = []
         outs = []
-        for root, dirs, files in os.walk(from_directory):
+        for root, dirs, files in os.walk(str(from_directory)):
             for fname in files:
                 path = os.path.join(root, fname)
                 if not Stage.is_stage_file(path):

dvc/remote/http.py

@@ -1,4 +1,5 @@
 from __future__ import unicode_literals
+from dvc.utils.compat import open
 
 import os
 import threading

dvc/remote/local.py

@@ -200,7 +200,7 @@ class RemoteLOCAL(RemoteBase):
     def collect_dir_cache(self, dname):
         dir_info = []
 
-        for root, dirs, files in os.walk(dname):
+        for root, dirs, files in os.walk(str(dname)):
             bar = False
 
             if len(files) > LARGE_DIR_SIZE:
@@ -276,6 +276,7 @@ class RemoteLOCAL(RemoteBase):
         # NOTE: Writing first and renaming after that
         # to make sure that the operation is atomic.
         tmp = "{}.{}".format(path, str(uuid.uuid4()))
+
         with open(tmp, "w+") as fd:
             json.dump(dir_info, fd, sort_keys=True)
         move(tmp, path)
@@ -348,7 +349,7 @@ class RemoteLOCAL(RemoteBase):
     def _discard_working_directory_changes(self, path, dir_info, force=False):
         working_dir_files = set(
             os.path.join(root, file)
-            for root, _, files in os.walk(path)
+            for root, _, files in os.walk(str(path))
             for file in files
         )
 

dvc/scm/git.py

@@ -6,7 +6,7 @@ from __future__ import unicode_literals
 # (called git.py) instead of importing gitpython's git module.
 from __future__ import absolute_import
 
-from dvc.utils.compat import str
+from dvc.utils.compat import str, open
 
 import os
 

dvc/stage.py

@@ -1,6 +1,6 @@
 from __future__ import unicode_literals
 
-from dvc.utils.compat import str
+from dvc.utils.compat import str, open
 
 import os
 import yaml

dvc/state.py

@@ -299,7 +299,7 @@ class State(object):  # pylint: disable=too-many-instance-attributes
         mtime = os.path.getmtime(path)
 
         if os.path.isdir(path):
-            for root, dirs, files in os.walk(path):
+            for root, dirs, files in os.walk(str(path)):
                 for name in dirs + files:
                     entry = os.path.join(root, name)
                     stat = os.stat(entry)

dvc/system.py

@@ -1,6 +1,6 @@
 from __future__ import unicode_literals
 
-from dvc.utils.compat import str
+from dvc.utils.compat import str, open
 
 import os
 

dvc/updater.py

@@ -1,4 +1,5 @@
 from __future__ import unicode_literals
+from dvc.utils.compat import str, open
 
 import sys
 import os

dvc/utils/__init__.py

@@ -1,7 +1,7 @@
 """Helpers for other modules."""
 
 from __future__ import unicode_literals
-from dvc.utils.compat import str, builtin_str
+from dvc.utils.compat import str, builtin_str, open
 
 import os
 import sys

dvc/utils/compat.py

@@ -18,6 +18,7 @@ if is_py2:
     from BaseHTTPServer import HTTPServer  # noqa: F401
     from SimpleHTTPServer import SimpleHTTPRequestHandler  # noqa: F401
     import ConfigParser  # noqa: F401
+    from io import open  # noqa: F401
 
     builtin_str = str  # noqa: F821
     bytes = str  # noqa: F821
@@ -43,3 +44,4 @@ elif is_py3:
     numeric_types = (int, float)  # noqa: F821
     integer_types = (int,)  # noqa: F821
     input = input  # noqa: F821
+    open = open  # noqa: F821

scripts/innosetup/config_gen.py

@@ -1,6 +1,6 @@
 # This script generates config.ini for setup.iss script
 from dvc import VERSION
-from dvc.utils.compat import ConfigParser
+from dvc.utils.compat import ConfigParser, open
 
 config = ConfigParser.ConfigParser()
 config.add_section("Version")

tests/test_add.py

@@ -32,6 +32,16 @@ class TestAdd(TestDvc):
         self.assertEqual(stage.cmd, None)
         self.assertEqual(stage.outs[0].info["md5"], md5)
 
+    def test_unicode(self):
+        fname = "\xe1"
+
+        with open(fname, "w") as fobj:
+            fobj.write("something")
+
+        stage = self.dvc.add(fname)[0]
+
+        self.assertTrue(os.path.isfile(stage.path))
+
 
 class TestAddUnupportedFile(TestDvc):
     def test(self):