diff --git a/dvc/command/base.py b/dvc/command/base.py index b79de01b1..2db660051 100644 --- a/dvc/command/base.py +++ b/dvc/command/base.py @@ -3,8 +3,7 @@ import os from dvc.config import ConfigError from dvc.logger import Logger -from dvc.path.factory import PathFactory -from dvc.utils import cached_property +from dvc.system import System class CmdBase(object): @@ -89,7 +88,7 @@ class CmdBase(object): pass def get_cache_file_s3_name(self, cache_file): - cache_prefix_file_name = os.path.relpath(os.path.realpath(cache_file), os.path.realpath(self.git.git_dir)) + cache_prefix_file_name = os.path.relpath(System.realpath(cache_file), System.realpath(self.git.git_dir)) file_name = os.path.relpath(cache_prefix_file_name, self.config.cache_dir) dvc_file_path_trim = file_name.replace(os.sep, '/').strip('/') return self.config.aws_storage_prefix + '/' + dvc_file_path_trim diff --git a/dvc/command/data_sync.py b/dvc/command/data_sync.py index abdaaf923..5509f1a62 100644 --- a/dvc/command/data_sync.py +++ b/dvc/command/data_sync.py @@ -7,6 +7,7 @@ from dvc.command.base import CmdBase from dvc.logger import Logger from dvc.exceptions import DvcException from dvc.runtime import Runtime +from dvc.system import System class DataSyncError(DvcException): @@ -52,7 +53,7 @@ class CmdDataSync(CmdBase): pass def run(self): - if os.path.islink(self.parsed_args.target): + if System.islink(self.parsed_args.target): data_item = self.path_factory.existing_data_item(self.parsed_args.target) return self.sync_symlink(data_item) @@ -66,7 +67,7 @@ class CmdDataSync(CmdBase): fname = os.path.join(dir, f) if os.path.isdir(fname): self.sync_dir(fname) - elif os.path.islink(fname): + elif System.islink(fname): self.sync_symlink(self.path_factory.existing_data_item(fname)) else: raise DataSyncError('Unsupported file type "{}"'.format(fname)) diff --git a/dvc/command/run.py b/dvc/command/run.py index bc8618b0e..d6ffb29b6 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -24,10 +24,8 @@ class CmdRun(CmdBase): def define_args(self, parser): self.set_skip_git_actions(parser) - parser.add_argument('--not-repro', - help='Not reproducible', - action='store_false', - default=False) + parser.add_argument('--not-repro', help='Not reproducible', + action='store_false', default=False) parser.add_argument('--stdout', help='output std output to a file') parser.add_argument('--stderr', help='output std error to a file') parser.add_argument('--input', '-i', action='append', @@ -36,6 +34,7 @@ class CmdRun(CmdBase): help='Declare output data items for reproducible cmd') parser.add_argument('--code', '-c', action='append', help='Code dependencies which produce the output') + parser.add_argument('--shell', help='Shell command', action='store_true', default=False) pass @property @@ -68,21 +67,23 @@ class CmdRun(CmdBase): self.run_command(self.command_args, self.data_items_from_args(self.command_args), self.parsed_args.stdout, - self.parsed_args.stderr) + self.parsed_args.stderr, + self.parsed_args.shell) return self.commit_if_needed('DVC run: {}'.format(' '.join(self.args))) finally: lock.release() return 1 - def run_command(self, cmd_args, data_items_from_args, stdout=None, stderr=None): - Logger.debug('Run command with args: {}. Data items from args: {}. stdout={}, stderr={}'.format( + def run_command(self, cmd_args, data_items_from_args, stdout=None, stderr=None, shell=False): + Logger.debug('Run command with args: {}. Data items from args: {}. stdout={}, stderr={}, shell={}'.format( ' '.join(cmd_args), ', '.join([x.data.dvc for x in data_items_from_args]), stdout, - stderr)) + stderr, + shell)) - repo_change = RepositoryChange(cmd_args, self.settings, stdout, stderr) + repo_change = RepositoryChange(cmd_args, self.settings, stdout, stderr, shell=shell) if not self.skip_git_actions and not self._validate_file_states(repo_change): self.remove_new_files(repo_change) diff --git a/dvc/executor.py b/dvc/executor.py index d48257a75..e2becc48d 100644 --- a/dvc/executor.py +++ b/dvc/executor.py @@ -10,7 +10,7 @@ class ExecutorError(DvcException): class Executor: @staticmethod - def exec_cmd(cmd, stdout_file=None, stderr_file=None, cwd=None): + def exec_cmd(cmd, stdout_file=None, stderr_file=None, cwd=None, shell=False): stdout, stdout_fd = Executor.output_file(stdout_file) stderr, stderr_fd = Executor.output_file(stderr_file) @@ -18,7 +18,8 @@ class Executor: p = subprocess.Popen(cmd, cwd=cwd, stdout=stdout, - stderr=stderr) + stderr=stderr, + shell=shell) p.wait() out, err = map(lambda s: s.decode().strip('\n\r') if s else '', p.communicate()) @@ -46,8 +47,9 @@ class Executor: return output, output_fd @staticmethod - def exec_cmd_only_success(cmd, stdout_file=None, stderr_file=None, cwd=None): - code, out, err = Executor.exec_cmd(cmd, stdout_file=stdout_file, stderr_file=stderr_file, cwd=cwd) + def exec_cmd_only_success(cmd, stdout_file=None, stderr_file=None, cwd=None, shell=False): + code, out, err = Executor.exec_cmd(cmd, stdout_file=stdout_file, + stderr_file=stderr_file, cwd=cwd, shell=shell) if code != 0: raise ExecutorError('Git command error ({}):\n{}'.format(' '.join(cmd), err)) return out diff --git a/dvc/git_wrapper.py b/dvc/git_wrapper.py index 02e9eec7e..85b72d516 100644 --- a/dvc/git_wrapper.py +++ b/dvc/git_wrapper.py @@ -3,6 +3,7 @@ import os from dvc.logger import Logger from dvc.config import Config from dvc.executor import Executor, ExecutorError +from dvc.system import System class GitWrapperI(object): @@ -20,7 +21,7 @@ class GitWrapperI(object): @property def git_dir_abs(self): - return os.path.realpath(self.git_dir) + return System.realpath(self.git_dir) @property def curr_dir_abs(self): @@ -43,6 +44,12 @@ class GitWrapperI(object): return GitWrapper.parse_porcelain_files(out) + @staticmethod + def git_path_to_system_path(path): + if os.name == 'nt': + return path.replace('/', '\\') + return path + @staticmethod def parse_porcelain_files(out): result = [] @@ -50,7 +57,7 @@ class GitWrapperI(object): lines = out.split('\n') for line in lines: status = line[:2] - file = line[3:] + file = GitWrapperI.git_path_to_system_path(line[3:]) result.append((status, file)) return result @@ -134,11 +141,11 @@ class GitWrapper(GitWrapperI): @staticmethod def abs_paths_to_relative(files): - cur_dir = os.path.realpath(os.curdir) + cur_dir = System.realpath(os.curdir) result = [] for file in files: - result.append(os.path.relpath(os.path.realpath(file), cur_dir)) + result.append(os.path.relpath(System.realpath(file), cur_dir)) return result @@ -157,11 +164,12 @@ class GitWrapper(GitWrapperI): commit = Executor.exec_cmd_only_success(['git', 'log', '-1', '--pretty=format:"%h"', file]) commit = commit.strip('"') - Logger.debug('[Git] Identify changes. Success.') - changed_files = Executor.exec_cmd_only_success(['git', 'diff', '--name-only', 'HEAD', commit]) changed_files = changed_files.strip('"') + Logger.debug('[Git] Identify changes. Success. Changed files: {}'.format( + changed_files.replace('\n', ', '))) + code_files, code_dirs = self.separate_dependency_files_and_dirs(code_dependencies) code_files_set = set([path_factory.path(x).dvc for x in code_files]) for changed_file in changed_files.split('\n'): diff --git a/dvc/path/data_item.py b/dvc/path/data_item.py index d43482eb7..92c4d048e 100644 --- a/dvc/path/data_item.py +++ b/dvc/path/data_item.py @@ -1,22 +1,36 @@ import os - import shutil from dvc.path.path import Path from dvc.exceptions import DvcException +from dvc.system import System from dvc.utils import cached_property class DataItemError(DvcException): def __init__(self, msg): - DvcException.__init__(self, 'Data file error: {}'.format(msg)) + super(DataItemError, self).__init__('Data item error: {}'.format(msg)) class NotInDataDirError(DvcException): - def __init__(self, file, data_dir): - DvcException.__init__(self, - 'Data file location error: the file "{}" has to be in the data directory "{}"'. - format(file, data_dir)) + def __init__(self, msg): + super(NotInDataDirError, self).__init__(msg) + + @staticmethod + def build(file, data_dir): + msg = 'the file "{}" is not in the data directory "{}"'.format(file, data_dir) + return NotInDataDirError(msg) + + +class DataItemInStatusDirError(NotInDataDirError): + def __init__(self, msg): + super(DataItemInStatusDirError, self).__init__(msg) + + @staticmethod + def build(file, data_dir): + msg = 'the file "{}" is in state directory, not in the data directory "{}"'.format( + file, data_dir) + return DataItemInStatusDirError(msg) class DataItem(object): @@ -29,11 +43,15 @@ class DataItem(object): self._cache_file = cache_file self._data = Path(data_file, git) + + if self._data.abs.startswith(self.state_dir_abs): + raise DataItemInStatusDirError.build(data_file, self._config.data_dir) + if not self._data.abs.startswith(self.data_dir_abs): - raise NotInDataDirError(data_file, self._config.data_dir) + raise NotInDataDirError.build(data_file, self._config.data_dir) if not self._data.dvc.startswith(self._config.data_dir): - raise NotInDataDirError(data_file, self._config.data_dir) + raise NotInDataDirError.build(data_file, self._config.data_dir) pass def __hash__(self): @@ -58,25 +76,31 @@ class DataItem(object): @cached_property def cache(self): + cache_dir = os.path.join(self._git.git_dir_abs, self._config.cache_dir) + if self._cache_file: - return Path(self._cache_file, self._git) + file_name = os.path.basename(self._cache_file) else: - cache_dir = os.path.join(self._git.git_dir_abs, self._config.cache_dir) - cache_file_suffix = self.CACHE_FILE_SEP + self._git.curr_commit - cache_file = os.path.join(cache_dir, self.data_dvc_short + cache_file_suffix) - return Path(cache_file, self._git) + file_name = self.data_dvc_short + self.CACHE_FILE_SEP + self._git.curr_commit + + cache_file = os.path.join(cache_dir, file_name) + return Path(cache_file, self._git) @cached_property def data_dir_abs(self): return os.path.join(self._git.git_dir_abs, self._config.data_dir) + @cached_property + def state_dir_abs(self): + return os.path.join(self._git.git_dir_abs, self._config.state_dir) + @property def _symlink_file(self): data_file_dir = os.path.dirname(self.data.relative) return os.path.relpath(self.cache.relative, data_file_dir) def create_symlink(self): - os.symlink(self._symlink_file, self.data.relative) + System.symlink(self._symlink_file, self.data.relative) def move_data_to_cache(self): cache_dir = os.path.dirname(self.cache.relative) @@ -84,4 +108,4 @@ class DataItem(object): os.makedirs(cache_dir) shutil.move(self.data.relative, self.cache.relative) - os.symlink(self._symlink_file, self.data.relative) + System.symlink(self._symlink_file, self.data.relative) diff --git a/dvc/path/factory.py b/dvc/path/factory.py index a9615b7d9..9e70c498b 100644 --- a/dvc/path/factory.py +++ b/dvc/path/factory.py @@ -3,13 +3,14 @@ import os from dvc.path.data_item import DataItem, DataItemError, NotInDataDirError from dvc.path.path import Path from dvc.path.stated_data_item import StatedDataItem +from dvc.system import System class PathFactory(object): def __init__(self, git, config): self._git = git self._config = config - self._curr_dir_abs = os.path.realpath(os.curdir) + self._curr_dir_abs = System.realpath(os.curdir) def path(self, relative_raw): return Path(relative_raw, self._git) @@ -21,9 +22,12 @@ class PathFactory(object): return StatedDataItem(state, data_file, self._git, self._config) def existing_data_item(self, file): - if not os.path.islink(file): + if not os.path.exists(file): + raise DataItemError(u'Data file "%s" is not exist' % file) + + if not System.islink(file): raise DataItemError(u'Data file "%s" must be a symbolic link' % file) - resolved_symlink = os.path.realpath(file) + resolved_symlink = System.realpath(file) return DataItem(file, self._git, self._config, resolved_symlink) def to_data_items(self, files): diff --git a/dvc/path/path.py b/dvc/path/path.py index 21e0c049a..e129502f3 100644 --- a/dvc/path/path.py +++ b/dvc/path/path.py @@ -1,9 +1,15 @@ import os +from dvc.system import System + class Path(object): - def __init__(self, relative_raw, git): - self._abs = os.path.abspath(relative_raw) + def __init__(self, path, git): + if not os.path.isabs(path): + pwd = System.get_cwd() + path = os.path.normpath(os.path.join(pwd, path)) + + self._abs = path self._dvc = os.path.relpath(self.abs, git.git_dir_abs) self._relative = os.path.relpath(self._abs, git.curr_dir_abs) self._filename = os.path.basename(self._abs) diff --git a/dvc/repository_change.py b/dvc/repository_change.py index a1428969c..fa8879619 100644 --- a/dvc/repository_change.py +++ b/dvc/repository_change.py @@ -4,7 +4,7 @@ from dvc.exceptions import DvcException from dvc.git_wrapper import GitWrapper from dvc.executor import Executor from dvc.logger import Logger -from dvc.path.data_item import NotInDataDirError +from dvc.path.data_item import NotInDataDirError, DataItemInStatusDirError from dvc.path.stated_data_item import StatedDataItem from dvc.utils import cached_property @@ -17,17 +17,19 @@ class RepositoryChangeError(DvcException): class RepositoryChange(object): """Pre-condition: git repository has no changes""" - def __init__(self, cmd_args, settings, stdout, stderr): + def __init__(self, cmd_args, settings, stdout, stderr, shell=False): self._settings = settings - Logger.debug(u'[Repository change] Exec command: {}. stdout={}, stderr={}'.format( + Logger.debug(u'[Repository change] Exec command: {}. stdout={}, stderr={}, shell={}'.format( u' '.join(cmd_args), stdout, - stderr)) - Executor.exec_cmd_only_success(cmd_args, stdout, stderr) + stderr, + shell)) + Executor.exec_cmd_only_success(cmd_args, stdout, stderr, shell=shell) self._stated_data_items = [] self._externally_created_files = [] + self._created_status_files = [] self._init_file_states() @cached_property @@ -57,6 +59,8 @@ class RepositoryChange(object): Logger.debug('[Repository change] Add status: {} {}'.format( item.status, item.data.dvc)) + except DataItemInStatusDirError: + self._created_status_files.append(file) except NotInDataDirError: self._externally_created_files.append(file) pass @@ -90,3 +94,7 @@ class RepositoryChange(object): @property def externally_created_files(self): return self._externally_created_files + + @property + def created_status_files(self): + return self._created_status_files diff --git a/dvc/runtime.py b/dvc/runtime.py index 04aa0d66d..b3a0a0a40 100644 --- a/dvc/runtime.py +++ b/dvc/runtime.py @@ -6,14 +6,20 @@ from dvc.exceptions import DvcException from dvc.git_wrapper import GitWrapper from dvc.logger import Logger from dvc.settings import Settings +from dvc.system import System class Runtime(object): CONFIG = 'dvc.conf' + @staticmethod + def symlink_setup(): + if os.name != 'nt': + return + @staticmethod def conf_file_path(git_dir): - return os.path.realpath(os.path.join(git_dir, Runtime.CONFIG)) + return System.realpath(os.path.join(git_dir, Runtime.CONFIG)) @staticmethod def run(cmd_class, parse_config=True, args_start_loc=1): @@ -24,6 +30,7 @@ class Runtime(object): """ try: + Runtime.symlink_setup() runtime_git = GitWrapper() if parse_config: diff --git a/dvc/state_file.py b/dvc/state_file.py index 6c64160a7..d6ab87233 100644 --- a/dvc/state_file.py +++ b/dvc/state_file.py @@ -4,6 +4,7 @@ import json import time from dvc.exceptions import DvcException +from dvc.system import System class StateFileError(DvcException): @@ -126,7 +127,7 @@ class StateFile(object): return result def get_dvc_path(self): - pwd = os.path.realpath(os.curdir) + pwd = System.get_cwd() if not pwd.startswith(self.git.git_dir_abs): raise StateFileError('the file cannot be created outside of a git repository') diff --git a/dvc/system.py b/dvc/system.py new file mode 100644 index 000000000..b67f840f3 --- /dev/null +++ b/dvc/system.py @@ -0,0 +1,95 @@ +import ctypes +import os +import re + +if os.name == 'nt': + from ctypes import create_unicode_buffer, windll + +from dvc.executor import Executor + + +class System(object): + SYMLINK_OUTPUT = '' + LONG_PATH_BUFFER_SIZE = 1024 + + @staticmethod + def is_unix(): + return os.name != 'nt' + + @staticmethod + def symlink(source, link_name): + if System.is_unix(): + return os.symlink(source, link_name) + + flags = 0 + if source is not None and os.path.isdir(source): + flags = 1 + + func = ctypes.windll.kernel32.CreateSymbolicLinkW + func.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_uint32) + func.restype = ctypes.c_ubyte + + if func(link_name, source, flags) == 0: + raise ctypes.WinError() + + @staticmethod + def islink(path): + # It is definitely not the best way to check a symlink. + + if System.is_unix(): + return os.path.islink(path) + + if not os.path.exists(path): + return False + + if not os.path.isfile(path): + return False + + return System._get_symlink_string(path) is not None + + @staticmethod + def _get_symlink_string(path): + code, output, _ = Executor.exec_cmd(["dir", path], shell=True) + if code != 0: + return None + + lines = output.split('\n') + for line in lines: + if System.SYMLINK_OUTPUT in line: + return line + return None + + @staticmethod + def realpath(path): + # It is definitely not the best way to check a symlink. + + if System.is_unix(): + return os.path.realpath(path) + + output = System._get_symlink_string(path) + if output is None: + return os.path.realpath(path) + + groups = re.compile(r'\[\S+\]$').findall(output.strip()) + if len(groups) < 1: + return os.path.realpath(path) + + resolved_link = groups[0][1:-1] + return resolved_link + + @staticmethod + def get_long_path(path): + """Convert short path to a full path. It is needed for Windows.""" + if System.is_unix(): + return path + + buffer = create_unicode_buffer(System.LONG_PATH_BUFFER_SIZE) + get_long_path_name = windll.kernel32.GetLongPathNameW + result = get_long_path_name(unicode(path), buffer, System.LONG_PATH_BUFFER_SIZE) + if result == 0 or result > System.LONG_PATH_BUFFER_SIZE: + return path + return buffer.value + + @staticmethod + def get_cwd(): + return System.get_long_path(os.getcwd()) diff --git a/dvc/utils.py b/dvc/utils.py index 4528a937d..8f73b897c 100644 --- a/dvc/utils.py +++ b/dvc/utils.py @@ -1,3 +1,6 @@ +import os +import stat +import shutil def cached_property(f): def get(self): @@ -12,3 +15,12 @@ def cached_property(f): return x return property(get) + + +def rmtree(dir): + '''Cross platform rmtree()''' + if os.name == 'nt': + if os.path.exists(dir) and not os.access(dir, os.W_OK): + os.chmod(dir, stat.S_IWUSR) + + shutil.rmtree(dir, ignore_errors=True) diff --git a/tests/basic_env.py b/tests/basic_env.py index 9895a38a2..ca25df40b 100644 --- a/tests/basic_env.py +++ b/tests/basic_env.py @@ -1,5 +1,5 @@ import os -import shutil +import tempfile from unittest import TestCase from dvc.config import ConfigI @@ -7,19 +7,24 @@ from dvc.git_wrapper import GitWrapperI from dvc.path.data_item import DataItem from dvc.path.factory import PathFactory from dvc.settings import Settings +from dvc.system import System +from dvc.utils import rmtree class BasicEnvironment(TestCase): - def init_environment(self, test_dir=os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'), curr_dir=None): - self._test_dir = os.path.realpath(test_dir) + def init_environment(self, + test_dir=System.get_long_path(tempfile.mkdtemp()), + curr_dir=None): + self._test_dir = System.realpath(test_dir) self._proj_dir = 'proj' self._test_git_dir = os.path.join(self._test_dir, self._proj_dir) - self._old_curr_dir_abs = os.path.realpath(os.curdir) + self._old_curr_dir_abs = System.realpath(os.curdir) - shutil.rmtree(self._test_dir, ignore_errors=True) + if os.path.exists(self._test_dir): + rmtree(self._test_dir) if curr_dir: - self._curr_dir = os.path.realpath(curr_dir) + self._curr_dir = System.realpath(curr_dir) else: self._curr_dir = self._test_git_dir @@ -30,8 +35,12 @@ class BasicEnvironment(TestCase): os.makedirs(self._test_git_dir) data_dir = os.path.join(self._test_git_dir, 'data') - if not os.path.isdir(data_dir): - os.makedirs(data_dir) + cache_dir = os.path.join(self._test_git_dir, 'cache') + state_dir = os.path.join(self._test_git_dir, 'state') + + os.makedirs(data_dir) + os.makedirs(cache_dir) + os.makedirs(state_dir) os.chdir(self._curr_dir) @@ -39,7 +48,7 @@ class BasicEnvironment(TestCase): if self._old_curr_dir_abs: os.chdir(self._old_curr_dir_abs) if self._test_git_dir: - shutil.rmtree(self._test_dir, ignore_errors=True) + rmtree(self._test_dir) pass @@ -71,11 +80,9 @@ class DirHierarchyEnvironment(BasicEnvironment): self.settings = Settings([], self._git, self._config) self.dir1 = os.path.join('dir1') - self.dir11 = os.path.join('dir1/dir11') + self.dir11 = os.path.join('dir1', 'dir11') self.dir2 = os.path.join('dir2') - os.mkdir('cache') - os.mkdir('state') self.create_dirs(self.dir1) self.create_dirs(self.dir11) self.create_dirs(self.dir2) @@ -106,7 +113,7 @@ class DirHierarchyEnvironment(BasicEnvironment): self.create_content_file(cache_result, content) relevant_dir = self.relevant_dir(data_file) - os.symlink(os.path.join(relevant_dir, cache_result), file_result) + System.symlink(os.path.join(relevant_dir, cache_result), file_result) else: cache_result = None self.create_content_file(file_result, content) diff --git a/tests/test_cmd_run.py b/tests/test_cmd_run.py index 748b6e745..97a58572f 100644 --- a/tests/test_cmd_run.py +++ b/tests/test_cmd_run.py @@ -1,5 +1,4 @@ import os -import shutil import tempfile from unittest import TestCase @@ -10,12 +9,14 @@ from dvc.path.data_item import DataItem from dvc.path.factory import PathFactory from dvc.git_wrapper import GitWrapper from dvc.settings import Settings +from dvc.system import System +from dvc.utils import rmtree class RunBasicTest(TestCase): def setUp(self): - self.test_dir = tempfile.mkdtemp() - self._old_curr_dir_abs = os.path.realpath(os.curdir) + self.test_dir = System.get_long_path(tempfile.mkdtemp()) + self._old_curr_dir_abs = System.realpath(os.curdir) self.tearDown() os.mkdir(self.test_dir) @@ -46,7 +47,7 @@ class RunBasicTest(TestCase): fd.close() def tearDown(self): - shutil.rmtree(self.test_dir, ignore_errors=True) + rmtree(self.test_dir) os.chdir(self._old_curr_dir_abs) @@ -55,7 +56,8 @@ class TestRunOutsideData(RunBasicTest): self.settings._args = [] cmd_run = CmdRun(self.settings) with self.assertRaises(RunError): - cmd_run.run_command(['touch', 'file1', 'file2'], []) + cmd_run.run_command(['echo', 'test'], [], shell=True, + stdout='file1', stderr='file2') pass @@ -65,18 +67,20 @@ class RunTwoFilesBase(RunBasicTest): self.settings._args = [] cmd_run = CmdRun(self.settings) - self.input_param_file = 'data/extra_input' + self.input_param_file = os.path.join('data', 'extra_input') cmd_run.parsed_args.input = [self.input_param_file] - self.extra_output_file = 'data/extra_output' + self.extra_output_file = os.path.join('data', 'extra_output') cmd_run.parsed_args.output = [self.extra_output_file] - self.file_name1 = 'data/file1' - self.file_name2 = 'data/file2' - self.state_objs = cmd_run.run_command(['touch', self.file_name1, self.file_name2], []) + self.file_name1 = os.path.join('data', 'file1') + self.file_name2 = os.path.join('data', 'file2') + self.state_objs = cmd_run.run_command(['echo', 'test'], [], shell=True, + stdout=self.file_name1, + stderr=self.file_name2) - self.state_file_name1 = 'state/file1' + DataItem.STATE_FILE_SUFFIX - self.state_file_name2 = 'state/file2' + DataItem.STATE_FILE_SUFFIX + self.state_file_name1 = os.path.join('state', 'file1' + DataItem.STATE_FILE_SUFFIX) + self.state_file_name2 = os.path.join('state', 'file2' + DataItem.STATE_FILE_SUFFIX) self.state_file1 = None self.state_file2 = None diff --git a/tests/test_data_file_obj.py b/tests/test_data_file_obj.py index c8e083381..32ef904ff 100644 --- a/tests/test_data_file_obj.py +++ b/tests/test_data_file_obj.py @@ -4,6 +4,7 @@ from dvc.config import ConfigI from dvc.git_wrapper import GitWrapperI from dvc.path.data_item import NotInDataDirError, DataItemError, DataItem from dvc.path.factory import PathFactory +from dvc.system import System from tests.basic_env import BasicEnvironment @@ -43,40 +44,40 @@ class TestPathFactory(BasicEnvironment): config = ConfigI('data', 'cache', 'state') self.path_factory = PathFactory(git, config) - self.file = os.path.join('data', 'file.txt') + self.data_file = os.path.join('data', 'file.txt') + self.cache_file = os.path.join('cache', 'fsymlinc.txt') - fd = open(self.file, 'w+') + fd = open(self.cache_file, 'w+') fd.write('some text') fd.close() - self.file_symlink = os.path.join('data', 'fsymlinc.txt') os.chdir('data') - os.symlink('file.txt', 'fsymlinc.txt') + System.symlink(os.path.join('..', self.cache_file), 'file.txt') os.chdir('..') pass def test_data_file_factory(self): - data_path = self.path_factory.data_item(self.file) - self.assertEqual(data_path.data.dvc, self.file) + data_path = self.path_factory.data_item(self.data_file) + self.assertEqual(data_path.data.dvc, self.data_file) - indirect_file_path = os.path.join('..', self._proj_dir, self.file) + indirect_file_path = os.path.join('..', self._proj_dir, self.data_file) data_path_indirect = self.path_factory.data_item(indirect_file_path) - self.assertEqual(data_path_indirect.data.dvc, self.file) + self.assertEqual(data_path_indirect.data.dvc, self.data_file) pass def test_data_symlink_factory(self): - data_path = self.path_factory.existing_data_item(self.file_symlink) - self.assertEqual(data_path.data.dvc, self.file_symlink) + data_path = self.path_factory.existing_data_item(self.data_file) + self.assertEqual(data_path.cache.dvc, self.cache_file) pass def test_data_symlink_factory_cache(self): - data_path = self.path_factory.existing_data_item(self.file_symlink) - self.assertEqual(data_path.cache.dvc, self.file) + data_path = self.path_factory.existing_data_item(self.data_file) + self.assertEqual(data_path.data.dvc, self.data_file) pass def test_data_symlink_factory_exception(self): with self.assertRaises(DataItemError): - self.path_factory.existing_data_item(self.file) + self.path_factory.existing_data_item(self.cache_file) pass def test_data_path(self): @@ -132,7 +133,8 @@ class TestDataPathInDataDir(BasicEnvironment): self.assertEqual(self.data_path.state.abs, target) def test_symlink(self): - self.assertEqual(self.data_path._symlink_file, '../../../ca/dir1/d2/file.txt_eeeff8f') + expected = os.path.join('..', '..', '..', 'ca', 'dir1', 'd2', 'file.txt_eeeff8f') + self.assertEqual(self.data_path._symlink_file, expected) def test_data_dir(self): data_path = self.path_factory.data_item(self.data_dir) diff --git a/tests/test_dvc_path.py b/tests/test_dvc_path.py index 760eaa0b8..5f2cd7149 100644 --- a/tests/test_dvc_path.py +++ b/tests/test_dvc_path.py @@ -4,12 +4,13 @@ from unittest import TestCase from dvc.path.path import Path from dvc.git_wrapper import GitWrapperI +from dvc.system import System class TestDvcPathTest(TestCase): def setUp(self): - self.curr_dir = os.path.realpath('.') - self.test_dir = os.path.realpath('/tmp/ntx_unit_test/dvc_path') + self.curr_dir = System.realpath('.') + self.test_dir = System.realpath('/tmp/ntx_unit_test/dvc_path') self.tearDown() os.makedirs(os.path.join(self.test_dir, 'data')) os.makedirs(os.path.join(self.test_dir, 'code', 'lib')) @@ -25,7 +26,7 @@ class TestDvcPathTest(TestCase): self.assertEqual(path.dvc, dvc_file_name) self.assertEqual(path.filename, os.path.basename(dvc_file_name)) self.assertEqual(path.relative, relative_file_name) - self.assertEquals(path.abs[0], os.path.sep) + self.assertTrue(os.path.isabs(path.abs)) self.assertTrue(path.abs.endswith(dvc_file_name)) def basic_test(self): diff --git a/tests/test_executor.py b/tests/test_executor.py index 2754f9376..2d6f7dabb 100644 --- a/tests/test_executor.py +++ b/tests/test_executor.py @@ -4,44 +4,40 @@ import tempfile from unittest import TestCase from dvc.executor import Executor, ExecutorError +from dvc.system import System class TestExecutor(TestCase): def setUp(self): - self.test_dir = tempfile.mkdtemp() + self.test_dir = System.get_long_path(tempfile.mkdtemp()) - self.file1 = 'file1.txt' - fd1 = open(os.path.join(self.test_dir, self.file1), 'w+') - fd1.write('this is file 1') - fd1.close() - - self.file2 = 'f.csv' - fd2 = open(os.path.join(self.test_dir, self.file2), 'w+') - fd2.write('this is file 222') - fd2.close() + self.code_file = 'myfile.py' + fd = open(self.code_file, 'w') + fd.write('import sys\nsys.stdout.write("111")\nsys.stderr.write("222")\n') + fd.close() def tearDown(self): shutil.rmtree(self.test_dir) def test_suppress_output(self): - output = Executor.exec_cmd_only_success(['ls', self.test_dir]) - output_set = set(output.split()) - self.assertEqual(output_set, {self.file1, self.file2}) + output = Executor.exec_cmd_only_success(['python', self.code_file]) + self.assertEqual(output, '111') def test_output_to_std_console(self): - output = Executor.exec_cmd_only_success(['ls', self.test_dir], '-', '-') + # This output "111" in stdout and "222" are not suppressible. + # You will see the outputs when the unit-test runs. + output = Executor.exec_cmd_only_success(['python', self.code_file], '-', '-') self.assertEqual(output, '') def test_custom_file_outputs(self): stdout_file = os.path.join(self.test_dir, 'stdout.txt') stderr_file = os.path.join(self.test_dir, 'stderr.txt') - output = Executor.exec_cmd_only_success(['ls', self.test_dir], stdout_file, stderr_file) + + output = Executor.exec_cmd_only_success(['python', self.code_file], stdout_file, stderr_file) self.assertEqual(output, '') - - output_set = set(open(stdout_file).read().split()) - expected = {self.file1, self.file2, os.path.basename(stdout_file), os.path.basename(stderr_file)} - self.assertEqual(output_set, expected) + self.assertEqual(open(stdout_file).read(), "111") + self.assertEqual(open(stderr_file).read(), "222") os.remove(stdout_file) os.remove(stderr_file) diff --git a/tests/test_repository_change.py b/tests/test_repository_change.py index 46209a4b6..2425464cb 100644 --- a/tests/test_repository_change.py +++ b/tests/test_repository_change.py @@ -9,12 +9,13 @@ from dvc.path.factory import PathFactory from dvc.git_wrapper import GitWrapperI from dvc.repository_change import RepositoryChange from dvc.settings import Settings +from dvc.system import System class BasicTestRepositoryChange(TestCase): def setUp(self): - self.test_dir = tempfile.mkdtemp() - self._old_curr_dir_abs = os.path.realpath(os.curdir) + self.test_dir = System.get_long_path(tempfile.mkdtemp()) + self._old_curr_dir_abs = System.realpath(os.curdir) self.tearDown() os.mkdir(self.test_dir) @@ -57,10 +58,11 @@ class TestRepositoryChange(BasicTestRepositoryChange): stdout=self._devnull, stderr=None).wait() os.remove(file_to_remove) - change = RepositoryChange(['ls', '-la'], + change = RepositoryChange(['echo', 'test'], self.settings, file_created_by_run, - file_to_modify) + file_to_modify, + shell=True) expected_new = [file_created_by_run, file_created_before_run] new_file_abs = [x.data.dvc for x in change.new_data_items] @@ -85,10 +87,11 @@ class TestRepositoryChangeDeepInDirectory(BasicTestRepositoryChange): deep_file = os.path.join('data', 'dir32', 'file1.txt') self.create_file(deep_file) - change = RepositoryChange(['ls', '-la'], + change = RepositoryChange(['echo', 'text'], self.settings, deep_file, - None) + None, + shell=True) self.assertEqual([x.data.dvc for x in change.new_data_items], [deep_file]) @@ -98,10 +101,11 @@ class TestRepositoryChangeExternallyCreated(BasicTestRepositoryChange): not_in_data_dir_file = 'file1.txt' self.create_file(not_in_data_dir_file) - change = RepositoryChange(['ls', '-la'], + change = RepositoryChange(['echo', 'textsd'], self.settings, not_in_data_dir_file, - None) + None, + shell=True) - self.assertEqual([os.path.realpath(not_in_data_dir_file)], + self.assertEqual([System.realpath(not_in_data_dir_file)], change.externally_created_files) diff --git a/tests/test_repro.py b/tests/test_repro.py index bc05a9740..6bd78eaf4 100644 --- a/tests/test_repro.py +++ b/tests/test_repro.py @@ -1,8 +1,9 @@ +import os + from dvc.command.remove import CmdDataRemove -from dvc.command.repro import ReproChange, CmdRepro -from dvc.command.run import CmdRun, RunError +from dvc.command.repro import CmdRepro +from dvc.command.run import CmdRun from dvc.executor import Executor -from dvc.settings import Settings from tests.test_cmd_run import RunBasicTest @@ -10,33 +11,50 @@ class ReproBasicEnv(RunBasicTest): def setUp(self): super(ReproBasicEnv, self).setUp() - self.file_name1 = 'data/file1' + self.file_name1 = os.path.join('data', 'file1') self.file1_code_file = 'file1.py' - self.create_file_and_commit(self.file1_code_file, 'An awesome code...') - self.settings._args = ['printf', 'Hello\nMary', '--not-repro', + self.create_file_and_commit(self.file1_code_file, 'print("Hello")' + os.linesep + 'print("Mary")') + self.settings._args = ['python', self.file1_code_file, '--not-repro', '--stdout', self.file_name1, '--code', self.file1_code_file] cmd_file1 = CmdRun(self.settings) self.assertEqual(cmd_file1.code_dependencies, [self.file1_code_file]) cmd_file1.run() - self.file_name11 = 'data/file11' - self.settings._args = ['head', '-n', '1', self.file_name1, '--stdout', self.file_name11] + self.file_name11 = os.path.join('data', 'file11') + self.file11_code_file = 'file11.py' + self.create_file_and_commit(self.file11_code_file, + 'import sys' + os.linesep + 'print(open(sys.argv[1]).readline().strip())') + + self.settings._args = ['python', self.file11_code_file, self.file_name1, + '--stdout', self.file_name11, '--code', self.file11_code_file] CmdRun(self.settings).run() - self.file_name2 = 'data/file2' - self.settings._args = ['printf', 'Bobby', '--not-repro', '--stdout', self.file_name2] + self.file_name2 = os.path.join('data', 'file2') + self.file2_code_file = 'file2.py' + self.create_file_and_commit(self.file2_code_file, + 'print("Bobby")') + self.settings._args = ['python', self.file2_code_file, + '--not-repro', '--stdout', self.file_name2] CmdRun(self.settings).run() self.file_res_code_file = 'code_res.py' - self.create_file_and_commit(self.file_res_code_file, 'Another piece of code') - self.file_name_res = 'data/file_res' - self.settings._args = ['cat', self.file_name11, self.file_name2, '--stdout', self.file_name_res, + self.create_file_and_commit(self.file_res_code_file, + 'import sys' + os.linesep + + 'text1 = open(sys.argv[1]).read()' + os.linesep + + 'text2 = open(sys.argv[2]).read()' + os.linesep + + 'print(text1 + text2)') + self.file_name_res = os.path.join('data', 'file_res') + self.settings._args = ['python', self.file_res_code_file, + self.file_name11, + self.file_name2, + '--stdout', self.file_name_res, '--code', self.file_res_code_file] cmd_res = CmdRun(self.settings) self.assertEqual(cmd_res.code_dependencies, [self.file_res_code_file]) cmd_res.run() - self.assertEqual(open(self.file_name_res).read(), 'Hello\nBobby') + lines = list(filter(None, map(str.strip, open(self.file_name_res).readlines()))) + self.assertEqual(lines, ['Hello', 'Bobby']) def create_file_and_commit(self, file_name, content='Any', message='Just a commit'): self.create_file(file_name, content) @@ -61,7 +79,7 @@ class ReproCodeDependencyTest(ReproBasicEnv): self.settings._args = [self.file_name_res] CmdRepro(self.settings).run() - self.assertEqual(open(self.file_name_res).read(), 'Hello\nBobby') + self.assertEqual(open(self.file_name_res).read().strip(), 'Hello\nBobby') class ReproChangedDependency(ReproBasicEnv): @@ -77,7 +95,11 @@ class ReproChangedDependency(ReproBasicEnv): self.settings._args = [self.file_name1, '--keep-in-cloud'] CmdDataRemove(self.settings).run() - self.settings._args = ['printf', 'Goodbye\nJack', '--stdout', self.file_name1] + file1_code_file = 'file1_2.py' + self.create_file_and_commit(file1_code_file, 'print("Goodbye")' + os.linesep + 'print("Jack")') + self.settings._args = ['python', file1_code_file, '--not-repro', + '--stdout', self.file_name1, '--code', file1_code_file] + CmdRun(self.settings).run() @@ -88,4 +110,4 @@ class ReproChangedDeepDependency(ReproChangedDependency): self.settings._args = [self.file_name_res] CmdRepro(self.settings).run() - self.assertEqual(open(self.file_name_res).read(), 'Goodbye\nBobby') + self.assertEqual(open(self.file_name_res).read().strip(), 'Goodbye\nBobby')