From 9b52063279f92f5acc1a8d6ed8292965c742b574 Mon Sep 17 00:00:00 2001 From: Dmitry Petrov Date: Sat, 8 Apr 2017 17:14:44 -0700 Subject: [PATCH] Windows symlink: extract rmfile() to utils --- dvc/command/remove.py | 7 ++++--- dvc/command/repro.py | 4 ++-- dvc/command/run.py | 4 ++-- dvc/logger.py | 2 +- dvc/utils.py | 24 +++++++++++++++++++++--- tests/basic_env.py | 2 +- tests/test_executor.py | 7 ++++--- tests/test_repository_change.py | 3 ++- 8 files changed, 37 insertions(+), 16 deletions(-) diff --git a/dvc/command/remove.py b/dvc/command/remove.py index 426d358f8..34aae7758 100644 --- a/dvc/command/remove.py +++ b/dvc/command/remove.py @@ -7,6 +7,7 @@ from dvc.command.base import CmdBase from dvc.exceptions import DvcException from dvc.logger import Logger from dvc.runtime import Runtime +from dvc.utils import rmfile class DataRemoveError(DvcException): @@ -97,7 +98,7 @@ class CmdDataRemove(CmdBase): self._remove_state_file(data_item) self._remove_cloud_cache(data_item) - os.remove(data_item.data.relative) + rmfile(data_item.data.relative) Logger.debug(u'[Cmd-Remove] Remove data item {}. Success.'.format(data_item.data.relative)) pass @@ -123,7 +124,7 @@ class CmdDataRemove(CmdBase): def _remove_dvc_path(self, dvc_path, name): Logger.debug(u'[Cmd-Remove] Remove {} {}.'.format(name, dvc_path.relative)) - os.remove(dvc_path.relative) + rmfile(dvc_path.relative) self.remove_dir_if_empty(dvc_path.relative) Logger.debug(u'[Cmd-Remove] Remove {}. Success.'.format(name)) @@ -150,7 +151,7 @@ class CmdDataRemove(CmdBase): else: self.remove_file(file) - os.rmdir(target) + rmfile(target) pass diff --git a/dvc/command/repro.py b/dvc/command/repro.py index 44a085f06..c62065285 100644 --- a/dvc/command/repro.py +++ b/dvc/command/repro.py @@ -1,4 +1,3 @@ -import os import fasteners from dvc.command.run import CmdRun @@ -7,6 +6,7 @@ from dvc.exceptions import DvcException from dvc.path.data_item import NotInDataDirError from dvc.runtime import Runtime from dvc.state_file import StateFile +from dvc.utils import rmfile class ReproError(DvcException): @@ -134,7 +134,7 @@ class ReproChange(object): def reproduce_data_file(self): Logger.debug('Reproducing data item "{}". Removing the file...'.format( self._data_item.data.dvc)) - os.remove(self._data_item.data.relative) + rmfile(self._data_item.data.relative) Logger.debug('Reproducing data item "{}". Re-runs cmd: {}'.format( self._data_item.data.relative, ' '.join(self._repro_argv))) diff --git a/dvc/command/run.py b/dvc/command/run.py index bc8618b0e..55fea6c7a 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -9,7 +9,7 @@ from dvc.path.data_item import NotInDataDirError from dvc.repository_change import RepositoryChange from dvc.runtime import Runtime from dvc.state_file import StateFile -from dvc.utils import cached_property +from dvc.utils import cached_property, rmfile class RunError(DvcException): @@ -126,7 +126,7 @@ class CmdRun(CmdBase): def remove_new_files(repo_change): for data_item in repo_change.new_data_items: Logger.error('Removing created file: {}'.format(data_item.data.relative)) - os.remove(data_item.data.relative) + rmfile(data_item.data.relative) pass @staticmethod diff --git a/dvc/logger.py b/dvc/logger.py index 3b6a8f207..824291868 100644 --- a/dvc/logger.py +++ b/dvc/logger.py @@ -1,5 +1,5 @@ class Logger(object): - DEFAULT_LEVEL = 2 + DEFAULT_LEVEL = 1 LEVEL = DEFAULT_LEVEL diff --git a/dvc/utils.py b/dvc/utils.py index e9f4e4276..5c448b2a3 100644 --- a/dvc/utils.py +++ b/dvc/utils.py @@ -2,6 +2,10 @@ import os import stat import shutil +from dvc.exceptions import DvcException +from dvc.logger import Logger + + def cached_property(f): def get(self): try: @@ -20,9 +24,23 @@ def cached_property(f): def rmtree(dir): '''Cross platform rmtree()''' if os.name == 'nt': - if not os.access(dir, os.W_OK): + if os.path.exists(dir) and not os.access(dir, os.W_OK): os.chmod(dir, stat.S_IWUSR) else: - raise + raise DvcException('Windows rmtree() error') - shutil.rmtree(dir, ignore_errors=True) + before = os.path.exists(dir) + + if os.path.exists(dir) and os.path.islink(dir): + os.unlink(dir) + else: + # shutil.rmtree(dir, ignore_errors=True) + shutil.rmtree(dir) + # os.remove(dir) + + after = os.path.exists(dir) + Logger.error('$$$$$$$$$$$$$$$$$$ rmtree(%s) before=%s, after=%s' % (dir, before, after)) + + +def rmfile(file): + os.remove(file) diff --git a/tests/basic_env.py b/tests/basic_env.py index 3bf6b1b3f..4f2c082c1 100644 --- a/tests/basic_env.py +++ b/tests/basic_env.py @@ -71,7 +71,7 @@ class DirHierarchyEnvironment(BasicEnvironment): self.settings = Settings([], self._git, self._config) self.dir1 = os.path.join('dir1') - self.dir11 = os.path.join('dir1/dir11') + self.dir11 = os.path.join('dir1', 'dir11') self.dir2 = os.path.join('dir2') os.mkdir('cache') diff --git a/tests/test_executor.py b/tests/test_executor.py index 2754f9376..60f0250c6 100644 --- a/tests/test_executor.py +++ b/tests/test_executor.py @@ -4,6 +4,7 @@ import tempfile from unittest import TestCase from dvc.executor import Executor, ExecutorError +from dvc.utils import rmfile, rmtree class TestExecutor(TestCase): @@ -21,7 +22,7 @@ class TestExecutor(TestCase): fd2.close() def tearDown(self): - shutil.rmtree(self.test_dir) + rmtree(self.test_dir) def test_suppress_output(self): output = Executor.exec_cmd_only_success(['ls', self.test_dir]) @@ -43,8 +44,8 @@ class TestExecutor(TestCase): expected = {self.file1, self.file2, os.path.basename(stdout_file), os.path.basename(stderr_file)} self.assertEqual(output_set, expected) - os.remove(stdout_file) - os.remove(stderr_file) + rmfile(stdout_file) + rmfile(stderr_file) def test_wrong_command(self): with self.assertRaises(ExecutorError): diff --git a/tests/test_repository_change.py b/tests/test_repository_change.py index 46209a4b6..7350c3f0f 100644 --- a/tests/test_repository_change.py +++ b/tests/test_repository_change.py @@ -9,6 +9,7 @@ from dvc.path.factory import PathFactory from dvc.git_wrapper import GitWrapperI from dvc.repository_change import RepositoryChange from dvc.settings import Settings +from dvc.utils import rmtree class BasicTestRepositoryChange(TestCase): @@ -55,7 +56,7 @@ class TestRepositoryChange(BasicTestRepositoryChange): stdout=self._devnull, stderr=None).wait() subprocess.Popen(['git', 'commit', '-m', '"Adding one file3"'], stdout=self._devnull, stderr=None).wait() - os.remove(file_to_remove) + rmtree(file_to_remove) change = RepositoryChange(['ls', '-la'], self.settings,