dvc: don't forget to convert to ospath for dir cache entries

Signed-off-by: Ruslan Kuprieiev <[email protected]>

dvc/dependency/local.py

@@ -1,7 +1,5 @@
 import os
 import schema
-import posixpath
-import ntpath
 
 try:
     from urlparse import urlparse
@@ -31,7 +29,7 @@ class DependencyLOCAL(DependencyBase):
             path = os.path.join(remote.prefix, urlparse(path).path.lstrip('/'))
 
         if not os.path.isabs(path):
-            path = self.ospath(path)
+            path = self.remote.ospath(path)
             path = os.path.join(stage.cwd, path)
         self.path = os.path.abspath(os.path.normpath(path))
 
@@ -67,26 +65,9 @@ class DependencyLOCAL(DependencyBase):
 
         self.info = self.remote.save_info(self.path_info)
 
-    def ospath(self, path):
-        if os.name == 'nt':
-            return self.ntpath(path)
-        return self.unixpath(path)
-
-    @staticmethod
-    def unixpath(path):
-        assert not ntpath.isabs(path)
-        assert not posixpath.isabs(path)
-        return path.replace('\\', '/')
-
-    @staticmethod
-    def ntpath(path):
-        assert not ntpath.isabs(path)
-        assert not posixpath.isabs(path)
-        return path.replace('/', '\\')
-
     def dumpd(self):
         if self.path.startswith(self.stage.project.root_dir):
-            path = self.unixpath(os.path.relpath(self.path, self.stage.cwd))
+            path = self.remote.unixpath(os.path.relpath(self.path, self.stage.cwd))
         else:
             path = self.path
 

dvc/remote/local.py

@@ -1,14 +1,16 @@
 import os
 import uuid
 import json
+import ntpath
 import shutil
 import filecmp
+import posixpath
+from operator import itemgetter
 
 from dvc.system import System
 from dvc.remote.base import RemoteBase, STATUS_MAP
-from dvc.state import State
 from dvc.logger import Logger
-from dvc.utils import remove, move, copyfile, file_md5, to_chunks
+from dvc.utils import remove, move, copyfile, file_md5, dict_md5, to_chunks
 from dvc.config import Config
 from dvc.exceptions import DvcException
 from dvc.progress import progress
@@ -18,8 +20,9 @@ from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
 class RemoteLOCAL(RemoteBase):
     scheme = ''
     REGEX = r'^(?P<path>(/+|.:\\+).*)$'
-    PARAM_MD5 = State.PARAM_MD5
-    PARAM_RELPATH = State.PARAM_RELPATH
+    PARAM_MD5 = 'md5'
+    PARAM_RELPATH = 'relpath'
+    MD5_DIR_SUFFIX = '.dir'
 
     CACHE_TYPES = ['reflink', 'hardlink', 'symlink', 'copy']
     CACHE_TYPE_MAP = {
@@ -104,6 +107,50 @@ class RemoteLOCAL(RemoteBase):
 
         raise DvcException('No possible cache types left to try out.')
 
+    @classmethod
+    def ospath(cls, path):
+        if os.name == 'nt':
+            return cls.ntpath(path)
+        return cls.unixpath(path)
+
+    @staticmethod
+    def unixpath(path):
+        assert not ntpath.isabs(path)
+        assert not posixpath.isabs(path)
+        return path.replace('\\', '/')
+
+    @staticmethod
+    def ntpath(path):
+        assert not ntpath.isabs(path)
+        assert not posixpath.isabs(path)
+        return path.replace('/', '\\')
+
+    def collect_dir_cache(self, dname):
+        dir_info = []
+
+        for root, dirs, files in os.walk(dname):
+            for fname in files:
+                path = os.path.join(root, fname)
+                relpath = self.unixpath(os.path.relpath(path, dname))
+
+                # FIXME: we could've used `md5 = state.update(path, dump=False)`
+                # here, but it is around twice as slow(on ssd, don't know about
+                # hdd) for a directory with small files. What we could do here
+                # is introduce some kind of a limit for file size, after which
+                # we would actually register it in our state file.
+                md5 = file_md5(path)[0]
+                dir_info.append({self.PARAM_RELPATH: relpath,
+                                 self.PARAM_MD5: md5})
+
+        # NOTE: sorting the list by path to ensure reproducibility
+        dir_info = sorted(dir_info, key=itemgetter(self.PARAM_RELPATH))
+
+        md5 = dict_md5(dir_info) + self.MD5_DIR_SUFFIX
+        if self.changed(md5):
+            self.dump_dir_cache(md5, dir_info)
+
+        return (md5, dir_info)
+
     def load_dir_cache(self, md5):
         path = self.get(md5)
 
@@ -122,6 +169,9 @@ class RemoteLOCAL(RemoteBase):
             Logger.error(msg.format(os.path.relpath(path)))
             return []
 
+        for info in d:
+            info['relpath'] = self.ospath(info['relpath'])
+
         return d
 
     def dump_dir_cache(self, md5, dir_info):
@@ -141,9 +191,9 @@ class RemoteLOCAL(RemoteBase):
             json.dump(dir_info, fd, sort_keys=True)
         move(tmp, path)
 
-    @staticmethod
-    def is_dir_cache(cache):
-        return cache.endswith(State.MD5_DIR_SUFFIX)
+    @classmethod
+    def is_dir_cache(cls, cache):
+        return cache.endswith(cls.MD5_DIR_SUFFIX)
 
     def checkout(self, path_info, checksum_info):
         path = path_info['path']
@@ -215,8 +265,8 @@ class RemoteLOCAL(RemoteBase):
         md5, dir_info = self.state.update_info(path)
 
         for entry in dir_info:
-            relpath = entry[State.PARAM_RELPATH]
-            m = entry[State.PARAM_MD5]
+            relpath = entry[self.PARAM_RELPATH]
+            m = entry[self.PARAM_MD5]
             p = os.path.join(path, relpath)
             c = self.get(m)
 

dvc/state.py

@@ -2,11 +2,10 @@ import os
 import json
 import nanotime
 import threading
-from operator import itemgetter
 
 from dvc.lock import Lock
 from dvc.system import System
-from dvc.utils import file_md5, dict_md5, remove
+from dvc.utils import file_md5, remove
 from dvc.exceptions import DvcException
 from dvc.signal_handler import SignalHandler
 from dvc.logger import Logger
@@ -45,10 +44,6 @@ class State(object):
     STATE_FILE = 'state'
     STATE_LOCK_FILE = 'state.lock'
 
-    PARAM_RELPATH = 'relpath'
-    PARAM_MD5 = 'md5'
-    MD5_DIR_SUFFIX = '.dir'
-
     def __init__(self, project):
         self.project = project
         self.dvc_dir = project.dvc_dir
@@ -65,34 +60,9 @@ class State(object):
     def init(project):
         return State(project)
 
-    def _collect_dir(self, dname):
-        dir_info = []
-
-        for root, dirs, files in os.walk(dname):
-            for fname in files:
-                path = os.path.join(root, fname)
-                relpath = os.path.relpath(path, dname)
-
-                # FIXME: we could've used `md5 = self.update(path, dump=False)` here,
-                # but it is around twice as slow(on ssd, don't know about hdd) for a
-                # directory with small files. What we could do here is introduce some
-                # kind of a limit for file size, after which we would actually register
-                # it in our state file.
-                md5 = file_md5(path)[0]
-                dir_info.append({self.PARAM_RELPATH: relpath, self.PARAM_MD5: md5})
-
-        # NOTE: sorting the list by path to ensure reproducibility
-        dir_info = sorted(dir_info, key=itemgetter(self.PARAM_RELPATH))
-
-        md5 = dict_md5(dir_info) + self.MD5_DIR_SUFFIX
-        if self.project.cache.local.changed(md5):
-            self.project.cache.local.dump_dir_cache(md5, dir_info)
-
-        return (md5, dir_info)
-
     def _collect(self, path):
         if os.path.isdir(path):
-            return self._collect_dir(path)
+            return self.project.cache.local.collect_dir_cache(path)
         else:
             return (file_md5(path)[0], None)
 

tests/test_add.py

@@ -39,7 +39,17 @@ class TestAddDirectory(TestDvc):
         dname = 'directory'
         os.mkdir(dname)
         self.create(os.path.join(dname, 'file'), 'file')
-        self.dvc.add(dname)
+        stage = self.dvc.add(dname)
+        self.assertNotEqual(stage, None)
+        self.assertEqual(len(stage.deps), 0)
+        self.assertEqual(len(stage.outs), 1)
+
+        md5 = stage.outs[0].info['md5']
+
+        dir_info = self.dvc.cache.local.load_dir_cache(md5)
+        for info in dir_info:
+            self.assertTrue('\\' not in info['relpath'])
+
 
 class TestAddDirectoryWithForwardSlash(TestDvc):
     def test(self):