cache: create cache dirs only when needed

Signed-off-by: Ruslan Kuprieiev <[email protected]>

dvc/cache.py

@@ -11,29 +11,23 @@ class Cache(object):
         if not os.path.exists(self.cache_dir):
             os.mkdir(self.cache_dir)
 
-        for subdir in self._cache_subdirs:
-            if not os.path.exists(subdir):
-                os.mkdir(subdir)
-
     @staticmethod
     def init(dvc_dir):
         cache_dir = os.path.join(dvc_dir, Cache.CACHE_DIR)
         os.mkdir(cache_dir)
         return Cache(dvc_dir)
 
-    @property
-    def _cache_subdirs(self):
-        dirs = []
-        for x in range(0, 256):
-            dirs.append(os.path.join(self.cache_dir, format(x, '02x')))
-        return dirs
-
     def all(self):
         clist = []
-        for subdir in self._cache_subdirs:
+        for entry in os.listdir(self.cache_dir):
+            subdir = os.path.join(self.cache_dir, entry)
+            if not os.path.isdir(subdir):
+                continue
+
             for cache in os.listdir(subdir):
                 path = os.path.join(subdir, cache)
                 clist.append(path)
+
         return clist
 
     def get(self, md5):

dvc/output.py

@@ -232,6 +232,11 @@ class Output(Dependency):
 
     def hardlink(self, src, link):
         self.project.logger.debug(u'creating hardlink {} -> {}'.format(src, link))
+
+        dname = os.path.dirname(link)
+        if not os.path.exists(dname):
+            os.makedirs(dname)
+
         System.hardlink(src, link)
         os.chmod(src, stat.S_IREAD)
 
@@ -304,11 +309,6 @@ class Output(Dependency):
 
         for relpath, cache in self.dir_cache().items():
             path = os.path.join(self.path, relpath)
-            dname = os.path.dirname(path)
-
-            if not os.path.exists(dname):
-                os.makedirs(dname)
-
             self.hardlink(cache, path)
 
     def _collect_dir(self):
@@ -330,6 +330,7 @@ class Output(Dependency):
         return dir_info
 
     def _save_dir(self):
+        dname = os.path.dirname(self.cache)
         dir_info = self._collect_dir()
 
         for entry in dir_info:
@@ -344,6 +345,9 @@ class Output(Dependency):
             else:
                 self.hardlink(path, cache)
 
+        if not os.path.isdir(dname):
+            os.makedirs(dname)
+
         with open(self.cache, 'w+') as fd:
             yaml.safe_dump(dir_info, fd, default_flow_style=False)
 

tests/basic_env.py

@@ -31,6 +31,10 @@ class TestDir(TestCase):
         self._saved_dir = None
 
     def create(self, name, contents):
+        dname = os.path.dirname(name)
+        if len(dname) > 0 and not os.path.isdir(dname):
+            os.makedirs(dname)
+
         with open(name, 'a') as f:
             f.write(contents)
 

tests/test_gc.py

@@ -28,7 +28,6 @@ class TestGC(TestDvc):
             self.bad_cache.append(path)
 
         path = os.path.join(self.dvc.cache.cache_dir, '45', '6', 'data')
-        os.mkdir(os.path.dirname(path))
         self.create(path, 'md5: "123"')
         self.bad_cache.append(path)