Merge pull request #538 from efiop/master

Misc fixes

dvc/cloud/local.py

@@ -15,9 +15,6 @@ class LocalKey(object):
     def path(self):
         return os.path.join(self.bucket, self.name)
 
-    def delete(self):
-        os.unlink(self.path)
-
 
 class DataCloudLOCAL(DataCloudBase):
     """
@@ -52,7 +49,7 @@ class DataCloudLOCAL(DataCloudBase):
 
         tmp_file = self.tmp_file(path)
         try:
-            copyfile(key.path, tmp_file)
+            copyfile(key.path, tmp_file, no_progress_bar=no_progress_bar)
         except Exception as exc:
             Logger.error('Failed to copy "{}": {}'.format(key.path, exc))
             return None

dvc/command/checkout.py

@@ -8,25 +8,5 @@ from dvc.logger import Logger
 
 class CmdCheckout(CmdBase):
     def run(self):
-        self.remove_untracked_hardlinks()
         self.project.checkout()
         return 0
-
-    def remove_untracked_hardlinks(self):
-        untracked = self.project.scm.untracked_files()
-        cache = dict((System.inode(c), c) for c in self.project.cache.all())
-        for file in untracked:
-            inode = System.inode(file)
-            if inode not in cache.keys():
-                continue
-
-            Logger.info(u'Remove \'{}\''.format(file))
-            os.chmod(file, stat.S_IWRITE)
-            os.remove(file)
-            os.chmod(cache[inode], stat.S_IREAD)
-
-            dir = os.path.dirname(file)
-            if len(dir) != 0 and not os.listdir(dir):
-                Logger.info(u'Remove empty directory \'{}\''.format(dir))
-                os.removedirs(dir)
-        pass

dvc/progress.py

@@ -13,6 +13,7 @@ class Progress(object):
 
     def set_n_total(self, total):
         self._n_total = total
+        self._n_finished = 0
 
     def _clearln(self):
         print('\r\x1b[K', end='')

dvc/project.py

@@ -11,6 +11,7 @@ from dvc.lock import Lock
 from dvc.scm import SCM
 from dvc.cache import Cache
 from dvc.cloud.data_cloud import DataCloud
+from dvc.system import System
 
 
 class StageNotFoundError(DvcException):
@@ -149,7 +150,26 @@ class Project(object):
                 raise ReproductionError(stages[n].relpath, ex)
         return result
 
+    def _remove_untracked_hardlinks(self):
+        untracked = self.scm.untracked_files()
+        cache = dict((System.inode(c), c) for c in self.cache.all())
+        for file in untracked:
+            inode = System.inode(file)
+            if inode not in cache.keys():
+                continue
+
+            Logger.info(u'Remove \'{}\''.format(file))
+            os.chmod(file, stat.S_IWRITE)
+            os.remove(file)
+            os.chmod(cache[inode], stat.S_IREAD)
+
+            dir = os.path.dirname(file)
+            if len(dir) != 0 and not os.listdir(dir):
+                Logger.info(u'Remove empty directory \'{}\''.format(dir))
+                os.removedirs(dir)
+
     def checkout(self):
+        self._remove_untracked_hardlinks()
         for stage in self.stages():
             stage.checkout()
 
@@ -173,29 +193,15 @@ class Project(object):
         return list(cache_set)
 
     def _remove_cache_file(self, cache):
-        os.chmod(cache, stat.S_IWRITE)
+        os.chmod(cache, stat.S_IWRITE | stat.S_IREAD)
         os.unlink(cache)
 
-    def _remove_cache(self, cache):
-        if os.path.isfile(cache):
-            self._remove_cache_file(cache)
-            return
-
-        for root, dirs, files in os.walk(cache, topdown=False):
-            for dname in dirs:
-                path = os.path.join(root, dname)
-                os.rmdir(path)
-            for fname in files:
-                path = os.path.join(root, fname)
-                self._remove_cache_file(path)
-        os.rmdir(cache)
-
     def gc(self):
         clist = self._used_cache()
         for cache in self.cache.all():
             if cache in clist:
                 continue
-            self._remove_cache(cache)
+            self._remove_cache_file(cache)
             self.logger.info(u'\'{}\' was removed'.format(self.to_dvc_path(cache)))
 
     def push(self, target=None, jobs=1):

dvc/utils.py

@@ -45,7 +45,7 @@ def cached_property(func):
     return property(get)
 
 
-def copyfile(src, dest):
+def copyfile(src, dest, no_progress_bar=False):
     '''Copy file with progress bar'''
     copied = 0
     name = os.path.basename(src)
@@ -64,9 +64,11 @@ def copyfile(src, dest):
             break
         fdest.write(buf)
         copied += len(buf)
-        progress.update_target(name, copied, total)
+        if not no_progress_bar:
+            progress.update_target(name, copied, total)
 
-    progress.finish_target(name)
+    if not no_progress_bar:
+        progress.finish_target(name)
 
     fsrc.close()
     fdest.close()

tests/test_data_cloud.py

@@ -71,11 +71,14 @@ class TestDataCloudBase(TestDvc):
         self.assertEqual(status, STATUS_MODIFIED)
 
         # Remove and check status
-        os.unlink(cache)
+        for root, dirs, files in os.walk(self.dvc.cache.cache_dir):
+            for f in files:
+                path = os.path.join(root, f)
+                self.dvc._remove_cache_file(path)
+
         status = self.cloud.status(cache)
         self.assertEqual(status, STATUS_DELETED)
 
-        self.dvc._remove_cache(cache_dir)
         status_dir = self.cloud.status(cache_dir)
         self.assertEqual(status_dir, STATUS_DELETED)
 
@@ -164,8 +167,6 @@ class TestDataCloudGCP(TestDataCloudBase):
 
 class TestDataCloudLOCAL(TestDataCloudBase):
     def _should_test(self):
-        if os.name == 'nt':
-            return False
         return True
 
     def _setup_cloud(self):
@@ -187,10 +188,6 @@ class TestDataCloudLocalCli(TestDvc):
         self.assertEqual(ret, 0)
 
     def test(self):
-        #FIXME enable this test for windows
-        if os.name == 'nt':
-            return
-        
         dname = 'cloud'
         os.mkdir(dname)
 
@@ -213,9 +210,10 @@ class TestDataCloudLocalCli(TestDvc):
 
         self.main(['status'])
 
-        os.chmod(cache, 0o777)
-        os.unlink(cache)
-        self.dvc._remove_cache(cache_dir)
+        for root, dirs, files in os.walk(self.dvc.cache.cache_dir):
+            for f in files:
+                path = os.path.join(root, f)
+                self.dvc._remove_cache_file(path)
 
         self.main(['status'])
 

tests/test_gc.py

@@ -27,10 +27,6 @@ class TestGC(TestDvc):
             self.create(path, i)
             self.bad_cache.append(path)
 
-        path = os.path.join(self.dvc.cache.cache_dir, '45', '6', 'data')
-        self.create(path, 'md5: "123"')
-        self.bad_cache.append(path)
-
     def test_api(self):
         self.dvc.gc()
         self._test_gc()