Merge pull request #531 from efiop/master

Fixes for pull/push CLI and cache

dvc/cache.py

@@ -11,29 +11,23 @@ class Cache(object):
         if not os.path.exists(self.cache_dir):
             os.mkdir(self.cache_dir)
 
-        for subdir in self._cache_subdirs:
-            if not os.path.exists(subdir):
-                os.mkdir(subdir)
-
     @staticmethod
     def init(dvc_dir):
         cache_dir = os.path.join(dvc_dir, Cache.CACHE_DIR)
         os.mkdir(cache_dir)
         return Cache(dvc_dir)
 
-    @property
-    def _cache_subdirs(self):
-        dirs = []
-        for x in range(0, 256):
-            dirs.append(os.path.join(self.cache_dir, format(x, '02x')))
-        return dirs
-
     def all(self):
         clist = []
-        for subdir in self._cache_subdirs:
+        for entry in os.listdir(self.cache_dir):
+            subdir = os.path.join(self.cache_dir, entry)
+            if not os.path.isdir(subdir):
+                continue
+
             for cache in os.listdir(subdir):
                 path = os.path.join(subdir, cache)
                 clist.append(path)
+
         return clist
 
     def get(self, md5):

dvc/cli.py

@@ -138,38 +138,29 @@ def parse_args(argv=None):
                         default=None,
                         help='DVC files.')
 
-    # Cache
-    cache_parser = subparsers.add_parser(
-                        'cache',
-                        parents=[parent_parser],
-                        help='Cache operations')
-    cache_subparsers = cache_parser.add_subparsers(
-                        dest='cmd',
-                        help='Use `dvc cache CMD` --help for command-specific help')
-
     # Pull
-    pull_parser = cache_subparsers.add_parser(
+    pull_parser = subparsers.add_parser(
                         'pull',
                         parents=[parent_cache_parser],
                         help='Pull data files from the cloud')
     pull_parser.set_defaults(func=CmdDataPull)
 
     # Push
-    push_parser = cache_subparsers.add_parser(
+    push_parser = subparsers.add_parser(
                         'push',
                         parents=[parent_cache_parser],
                         help='Push data files to the cloud')
     push_parser.set_defaults(func=CmdDataPush)
 
-    #Fetch
-    fetch_parser = cache_subparsers.add_parser(
+    # Fetch
+    fetch_parser = subparsers.add_parser(
                         'fetch',
                         parents=[parent_cache_parser],
                         help='Fetch data files from the cloud')
     fetch_parser.set_defaults(func=CmdDataFetch)
 
     # Status
-    status_parser = cache_subparsers.add_parser(
+    status_parser = subparsers.add_parser(
                         'status',
                         parents=[parent_cache_parser],
                         help='Show mismatches between local cache and cloud cache')

dvc/output.py

@@ -232,6 +232,11 @@ class Output(Dependency):
 
     def hardlink(self, src, link):
         self.project.logger.debug(u'creating hardlink {} -> {}'.format(src, link))
+
+        dname = os.path.dirname(link)
+        if not os.path.exists(dname):
+            os.makedirs(dname)
+
         System.hardlink(src, link)
         os.chmod(src, stat.S_IREAD)
 
@@ -304,11 +309,6 @@ class Output(Dependency):
 
         for relpath, cache in self.dir_cache().items():
             path = os.path.join(self.path, relpath)
-            dname = os.path.dirname(path)
-
-            if not os.path.exists(dname):
-                os.makedirs(dname)
-
             self.hardlink(cache, path)
 
     def _collect_dir(self):
@@ -330,6 +330,7 @@ class Output(Dependency):
         return dir_info
 
     def _save_dir(self):
+        dname = os.path.dirname(self.cache)
         dir_info = self._collect_dir()
 
         for entry in dir_info:
@@ -344,6 +345,9 @@ class Output(Dependency):
             else:
                 self.hardlink(path, cache)
 
+        if not os.path.isdir(dname):
+            os.makedirs(dname)
+
         with open(self.cache, 'w+') as fd:
             yaml.safe_dump(dir_info, fd, default_flow_style=False)
 

tests/basic_env.py

@@ -31,6 +31,10 @@ class TestDir(TestCase):
         self._saved_dir = None
 
     def create(self, name, contents):
+        dname = os.path.dirname(name)
+        if len(dname) > 0 and not os.path.isdir(dname):
+            os.makedirs(dname)
+
         with open(name, 'a') as f:
             f.write(contents)
 

tests/test_cli.py

@@ -62,19 +62,19 @@ class TestRun(TestDvc):
 
 class TestPull(TestDvc):
     def test(self):
-        args = parse_args(['cache', 'pull'])
+        args = parse_args(['pull'])
         self.assertIsInstance(args.func(args), CmdDataPull)
 
 
 class TestPush(TestDvc):
     def test(self):
-        args = parse_args(['cache', 'push'])
+        args = parse_args(['push'])
         self.assertIsInstance(args.func(args), CmdDataPush)
 
 
 class TestStatus(TestDvc):
     def test(self):
-        args = parse_args(['cache', 'status'])
+        args = parse_args(['status'])
         self.assertIsInstance(args.func(args), CmdDataStatus)
 
 

tests/test_data_cloud.py

@@ -204,26 +204,26 @@ class TestDataCloudLocalCli(TestDvc):
         cache_dir = stage_dir.outs[0].cache
 
         #FIXME check status output
-        self.main(['cache', 'status'])
+        self.main(['status'])
 
-        self.main(['cache', 'push'])
+        self.main(['push'])
         self.assertTrue(os.path.exists(cache))
         self.assertTrue(os.path.isfile(cache))
         self.assertTrue(os.path.isfile(cache_dir))
 
-        self.main(['cache', 'status'])
+        self.main(['status'])
 
         os.chmod(cache, 0o777)
         os.unlink(cache)
         self.dvc._remove_cache(cache_dir)
 
-        self.main(['cache', 'status'])
+        self.main(['status'])
 
-        self.main(['cache', 'pull'])
+        self.main(['pull'])
         self.assertTrue(os.path.exists(cache))
         self.assertTrue(os.path.isfile(cache))
         with open(cache, 'r') as fd:
             self.assertEqual(fd.read(), self.FOO_CONTENTS)
         self.assertTrue(os.path.isfile(cache_dir))
 
-        self.main(['cache', 'status'])
+        self.main(['status'])

tests/test_gc.py

@@ -28,7 +28,6 @@ class TestGC(TestDvc):
             self.bad_cache.append(path)
 
         path = os.path.join(self.dvc.cache.cache_dir, '45', '6', 'data')
-        os.mkdir(os.path.dirname(path))
         self.create(path, 'md5: "123"')
         self.bad_cache.append(path)