Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-10 07:18:27 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #16 from dmpetrov/windows_symlink

Browse Source 
Windows symlink
This commit is contained in:
Dmitry Petrov
2017-04-11 19:40:12 -07:00
committed by GitHub
parent 12ff96f964 48c86267e7
commit 5b7977e47b
20 changed files with 342 additions and 138 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/command/base.py
+2 -3
View File
@@ -3,8 +3,7 @@ import os
from dvc.config import ConfigError
from dvc.logger import Logger
from dvc.path.factory import PathFactory
from dvc.utils import cached_property
from dvc.system import System
class CmdBase(object):
@@ -89,7 +88,7 @@ class CmdBase(object):
pass
def get_cache_file_s3_name(self, cache_file):
cache_prefix_file_name = os.path.relpath(os.path.realpath(cache_file), os.path.realpath(self.git.git_dir))
cache_prefix_file_name = os.path.relpath(System.realpath(cache_file), System.realpath(self.git.git_dir))
file_name = os.path.relpath(cache_prefix_file_name, self.config.cache_dir)
dvc_file_path_trim = file_name.replace(os.sep, '/').strip('/')
return self.config.aws_storage_prefix + '/' + dvc_file_path_trim
dvc/command/data_sync.py
+3 -2
View File
@@ -7,6 +7,7 @@ from dvc.command.base import CmdBase
from dvc.logger import Logger
from dvc.exceptions import DvcException
from dvc.runtime import Runtime
from dvc.system import System
class DataSyncError(DvcException):
@@ -52,7 +53,7 @@ class CmdDataSync(CmdBase):
pass
def run(self):
if os.path.islink(self.parsed_args.target):
if System.islink(self.parsed_args.target):
data_item = self.path_factory.existing_data_item(self.parsed_args.target)
return self.sync_symlink(data_item)
@@ -66,7 +67,7 @@ class CmdDataSync(CmdBase):
fname = os.path.join(dir, f)
if os.path.isdir(fname):
self.sync_dir(fname)
elif os.path.islink(fname):
elif System.islink(fname):
self.sync_symlink(self.path_factory.existing_data_item(fname))
else:
raise DataSyncError('Unsupported file type "{}"'.format(fname))
dvc/command/run.py
+10 -9
View File
@@ -24,10 +24,8 @@ class CmdRun(CmdBase):
def define_args(self, parser):
self.set_skip_git_actions(parser)
parser.add_argument('--not-repro',
help='Not reproducible',
action='store_false',
default=False)
parser.add_argument('--not-repro', help='Not reproducible',
action='store_false', default=False)
parser.add_argument('--stdout', help='output std output to a file')
parser.add_argument('--stderr', help='output std error to a file')
parser.add_argument('--input', '-i', action='append',
@@ -36,6 +34,7 @@ class CmdRun(CmdBase):
help='Declare output data items for reproducible cmd')
parser.add_argument('--code', '-c', action='append',
help='Code dependencies which produce the output')
parser.add_argument('--shell', help='Shell command', action='store_true', default=False)
pass
@property
@@ -68,21 +67,23 @@ class CmdRun(CmdBase):
self.run_command(self.command_args,
self.data_items_from_args(self.command_args),
self.parsed_args.stdout,
self.parsed_args.stderr)
self.parsed_args.stderr,
self.parsed_args.shell)
return self.commit_if_needed('DVC run: {}'.format(' '.join(self.args)))
finally:
lock.release()
return 1
def run_command(self, cmd_args, data_items_from_args, stdout=None, stderr=None):
Logger.debug('Run command with args: {}. Data items from args: {}. stdout={}, stderr={}'.format(
def run_command(self, cmd_args, data_items_from_args, stdout=None, stderr=None, shell=False):
Logger.debug('Run command with args: {}. Data items from args: {}. stdout={}, stderr={}, shell={}'.format(
' '.join(cmd_args),
', '.join([x.data.dvc for x in data_items_from_args]),
stdout,
stderr))
stderr,
shell))
repo_change = RepositoryChange(cmd_args, self.settings, stdout, stderr)
repo_change = RepositoryChange(cmd_args, self.settings, stdout, stderr, shell=shell)
if not self.skip_git_actions and not self._validate_file_states(repo_change):
self.remove_new_files(repo_change)
dvc/executor.py
+6 -4
View File
@@ -10,7 +10,7 @@ class ExecutorError(DvcException):
class Executor:
@staticmethod
def exec_cmd(cmd, stdout_file=None, stderr_file=None, cwd=None):
def exec_cmd(cmd, stdout_file=None, stderr_file=None, cwd=None, shell=False):
stdout, stdout_fd = Executor.output_file(stdout_file)
stderr, stderr_fd = Executor.output_file(stderr_file)
@@ -18,7 +18,8 @@ class Executor:
p = subprocess.Popen(cmd,
cwd=cwd,
stdout=stdout,
stderr=stderr)
stderr=stderr,
shell=shell)
p.wait()
out, err = map(lambda s: s.decode().strip('\n\r') if s else '', p.communicate())
@@ -46,8 +47,9 @@ class Executor:
return output, output_fd
@staticmethod
def exec_cmd_only_success(cmd, stdout_file=None, stderr_file=None, cwd=None):
code, out, err = Executor.exec_cmd(cmd, stdout_file=stdout_file, stderr_file=stderr_file, cwd=cwd)
def exec_cmd_only_success(cmd, stdout_file=None, stderr_file=None, cwd=None, shell=False):
code, out, err = Executor.exec_cmd(cmd, stdout_file=stdout_file,
stderr_file=stderr_file, cwd=cwd, shell=shell)
if code != 0:
raise ExecutorError('Git command error ({}):\n{}'.format(' '.join(cmd), err))
return out
dvc/git_wrapper.py
+14 -6
View File
@@ -3,6 +3,7 @@ import os
from dvc.logger import Logger
from dvc.config import Config
from dvc.executor import Executor, ExecutorError
from dvc.system import System
class GitWrapperI(object):
@@ -20,7 +21,7 @@ class GitWrapperI(object):
@property
def git_dir_abs(self):
return os.path.realpath(self.git_dir)
return System.realpath(self.git_dir)
@property
def curr_dir_abs(self):
@@ -43,6 +44,12 @@ class GitWrapperI(object):
return GitWrapper.parse_porcelain_files(out)
@staticmethod
def git_path_to_system_path(path):
if os.name == 'nt':
return path.replace('/', '\\')
return path
@staticmethod
def parse_porcelain_files(out):
result = []
@@ -50,7 +57,7 @@ class GitWrapperI(object):
lines = out.split('\n')
for line in lines:
status = line[:2]
file = line[3:]
file = GitWrapperI.git_path_to_system_path(line[3:])
result.append((status, file))
return result
@@ -134,11 +141,11 @@ class GitWrapper(GitWrapperI):
@staticmethod
def abs_paths_to_relative(files):
cur_dir = os.path.realpath(os.curdir)
cur_dir = System.realpath(os.curdir)
result = []
for file in files:
result.append(os.path.relpath(os.path.realpath(file), cur_dir))
result.append(os.path.relpath(System.realpath(file), cur_dir))
return result
@@ -157,11 +164,12 @@ class GitWrapper(GitWrapperI):
commit = Executor.exec_cmd_only_success(['git', 'log', '-1', '--pretty=format:"%h"', file])
commit = commit.strip('"')
Logger.debug('[Git] Identify changes. Success.')
changed_files = Executor.exec_cmd_only_success(['git', 'diff', '--name-only', 'HEAD', commit])
changed_files = changed_files.strip('"')
Logger.debug('[Git] Identify changes. Success. Changed files: {}'.format(
changed_files.replace('\n', ', ')))
code_files, code_dirs = self.separate_dependency_files_and_dirs(code_dependencies)
code_files_set = set([path_factory.path(x).dvc for x in code_files])
for changed_file in changed_files.split('\n'):
dvc/path/data_item.py
+39 -15
View File
@@ -1,22 +1,36 @@
import os
import shutil
from dvc.path.path import Path
from dvc.exceptions import DvcException
from dvc.system import System
from dvc.utils import cached_property
class DataItemError(DvcException):
def __init__(self, msg):
DvcException.__init__(self, 'Data file error: {}'.format(msg))
super(DataItemError, self).__init__('Data item error: {}'.format(msg))
class NotInDataDirError(DvcException):
def __init__(self, file, data_dir):
DvcException.__init__(self,
'Data file location error: the file "{}" has to be in the data directory "{}"'.
format(file, data_dir))
def __init__(self, msg):
super(NotInDataDirError, self).__init__(msg)
@staticmethod
def build(file, data_dir):
msg = 'the file "{}" is not in the data directory "{}"'.format(file, data_dir)
return NotInDataDirError(msg)
class DataItemInStatusDirError(NotInDataDirError):
def __init__(self, msg):
super(DataItemInStatusDirError, self).__init__(msg)
@staticmethod
def build(file, data_dir):
msg = 'the file "{}" is in state directory, not in the data directory "{}"'.format(
file, data_dir)
return DataItemInStatusDirError(msg)
class DataItem(object):
@@ -29,11 +43,15 @@ class DataItem(object):
self._cache_file = cache_file
self._data = Path(data_file, git)
if self._data.abs.startswith(self.state_dir_abs):
raise DataItemInStatusDirError.build(data_file, self._config.data_dir)
if not self._data.abs.startswith(self.data_dir_abs):
raise NotInDataDirError(data_file, self._config.data_dir)
raise NotInDataDirError.build(data_file, self._config.data_dir)
if not self._data.dvc.startswith(self._config.data_dir):
raise NotInDataDirError(data_file, self._config.data_dir)
raise NotInDataDirError.build(data_file, self._config.data_dir)
pass
def __hash__(self):
@@ -58,25 +76,31 @@ class DataItem(object):
@cached_property
def cache(self):
cache_dir = os.path.join(self._git.git_dir_abs, self._config.cache_dir)
if self._cache_file:
return Path(self._cache_file, self._git)
file_name = os.path.basename(self._cache_file)
else:
cache_dir = os.path.join(self._git.git_dir_abs, self._config.cache_dir)
cache_file_suffix = self.CACHE_FILE_SEP + self._git.curr_commit
cache_file = os.path.join(cache_dir, self.data_dvc_short + cache_file_suffix)
return Path(cache_file, self._git)
file_name = self.data_dvc_short + self.CACHE_FILE_SEP + self._git.curr_commit
cache_file = os.path.join(cache_dir, file_name)
return Path(cache_file, self._git)
@cached_property
def data_dir_abs(self):
return os.path.join(self._git.git_dir_abs, self._config.data_dir)
@cached_property
def state_dir_abs(self):
return os.path.join(self._git.git_dir_abs, self._config.state_dir)
@property
def _symlink_file(self):
data_file_dir = os.path.dirname(self.data.relative)
return os.path.relpath(self.cache.relative, data_file_dir)
def create_symlink(self):
os.symlink(self._symlink_file, self.data.relative)
System.symlink(self._symlink_file, self.data.relative)
def move_data_to_cache(self):
cache_dir = os.path.dirname(self.cache.relative)
@@ -84,4 +108,4 @@ class DataItem(object):
os.makedirs(cache_dir)
shutil.move(self.data.relative, self.cache.relative)
os.symlink(self._symlink_file, self.data.relative)
System.symlink(self._symlink_file, self.data.relative)
dvc/path/factory.py
+7 -3
View File
@@ -3,13 +3,14 @@ import os
from dvc.path.data_item import DataItem, DataItemError, NotInDataDirError
from dvc.path.path import Path
from dvc.path.stated_data_item import StatedDataItem
from dvc.system import System
class PathFactory(object):
def __init__(self, git, config):
self._git = git
self._config = config
self._curr_dir_abs = os.path.realpath(os.curdir)
self._curr_dir_abs = System.realpath(os.curdir)
def path(self, relative_raw):
return Path(relative_raw, self._git)
@@ -21,9 +22,12 @@ class PathFactory(object):
return StatedDataItem(state, data_file, self._git, self._config)
def existing_data_item(self, file):
if not os.path.islink(file):
if not os.path.exists(file):
raise DataItemError(u'Data file "%s" is not exist' % file)
if not System.islink(file):
raise DataItemError(u'Data file "%s" must be a symbolic link' % file)
resolved_symlink = os.path.realpath(file)
resolved_symlink = System.realpath(file)
return DataItem(file, self._git, self._config, resolved_symlink)
def to_data_items(self, files):
dvc/path/path.py
+8 -2
View File
@@ -1,9 +1,15 @@
import os
from dvc.system import System
class Path(object):
def __init__(self, relative_raw, git):
self._abs = os.path.abspath(relative_raw)
def __init__(self, path, git):
if not os.path.isabs(path):
pwd = System.get_cwd()
path = os.path.normpath(os.path.join(pwd, path))
self._abs = path
self._dvc = os.path.relpath(self.abs, git.git_dir_abs)
self._relative = os.path.relpath(self._abs, git.curr_dir_abs)
self._filename = os.path.basename(self._abs)
dvc/repository_change.py
+13 -5
View File
@@ -4,7 +4,7 @@ from dvc.exceptions import DvcException
from dvc.git_wrapper import GitWrapper
from dvc.executor import Executor
from dvc.logger import Logger
from dvc.path.data_item import NotInDataDirError
from dvc.path.data_item import NotInDataDirError, DataItemInStatusDirError
from dvc.path.stated_data_item import StatedDataItem
from dvc.utils import cached_property
@@ -17,17 +17,19 @@ class RepositoryChangeError(DvcException):
class RepositoryChange(object):
"""Pre-condition: git repository has no changes"""
def __init__(self, cmd_args, settings, stdout, stderr):
def __init__(self, cmd_args, settings, stdout, stderr, shell=False):
self._settings = settings
Logger.debug(u'[Repository change] Exec command: {}. stdout={}, stderr={}'.format(
Logger.debug(u'[Repository change] Exec command: {}. stdout={}, stderr={}, shell={}'.format(
u' '.join(cmd_args),
stdout,
stderr))
Executor.exec_cmd_only_success(cmd_args, stdout, stderr)
stderr,
shell))
Executor.exec_cmd_only_success(cmd_args, stdout, stderr, shell=shell)
self._stated_data_items = []
self._externally_created_files = []
self._created_status_files = []
self._init_file_states()
@cached_property
@@ -57,6 +59,8 @@ class RepositoryChange(object):
Logger.debug('[Repository change] Add status: {} {}'.format(
item.status,
item.data.dvc))
except DataItemInStatusDirError:
self._created_status_files.append(file)
except NotInDataDirError:
self._externally_created_files.append(file)
pass
@@ -90,3 +94,7 @@ class RepositoryChange(object):
@property
def externally_created_files(self):
return self._externally_created_files
@property
def created_status_files(self):
return self._created_status_files
dvc/runtime.py
+8 -1
View File
@@ -6,14 +6,20 @@ from dvc.exceptions import DvcException
from dvc.git_wrapper import GitWrapper
from dvc.logger import Logger
from dvc.settings import Settings
from dvc.system import System
class Runtime(object):
CONFIG = 'dvc.conf'
@staticmethod
def symlink_setup():
if os.name != 'nt':
return
@staticmethod
def conf_file_path(git_dir):
return os.path.realpath(os.path.join(git_dir, Runtime.CONFIG))
return System.realpath(os.path.join(git_dir, Runtime.CONFIG))
@staticmethod
def run(cmd_class, parse_config=True, args_start_loc=1):
@@ -24,6 +30,7 @@ class Runtime(object):
"""
try:
Runtime.symlink_setup()
runtime_git = GitWrapper()
if parse_config:
dvc/state_file.py
+2 -1
View File
@@ -4,6 +4,7 @@ import json
import time
from dvc.exceptions import DvcException
from dvc.system import System
class StateFileError(DvcException):
@@ -126,7 +127,7 @@ class StateFile(object):
return result
def get_dvc_path(self):
pwd = os.path.realpath(os.curdir)
pwd = System.get_cwd()
if not pwd.startswith(self.git.git_dir_abs):
raise StateFileError('the file cannot be created outside of a git repository')
dvc/system.py
+95
View File
@@ -0,0 +1,95 @@
import ctypes
import os
import re
if os.name == 'nt':
from ctypes import create_unicode_buffer, windll
from dvc.executor import Executor
class System(object):
SYMLINK_OUTPUT = '<SYMLINK>'
LONG_PATH_BUFFER_SIZE = 1024
@staticmethod
def is_unix():
return os.name != 'nt'
@staticmethod
def symlink(source, link_name):
if System.is_unix():
return os.symlink(source, link_name)
flags = 0
if source is not None and os.path.isdir(source):
flags = 1
func = ctypes.windll.kernel32.CreateSymbolicLinkW
func.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_uint32)
func.restype = ctypes.c_ubyte
if func(link_name, source, flags) == 0:
raise ctypes.WinError()
@staticmethod
def islink(path):
# It is definitely not the best way to check a symlink.
if System.is_unix():
return os.path.islink(path)
if not os.path.exists(path):
return False
if not os.path.isfile(path):
return False
return System._get_symlink_string(path) is not None
@staticmethod
def _get_symlink_string(path):
code, output, _ = Executor.exec_cmd(["dir", path], shell=True)
if code != 0:
return None
lines = output.split('\n')
for line in lines:
if System.SYMLINK_OUTPUT in line:
return line
return None
@staticmethod
def realpath(path):
# It is definitely not the best way to check a symlink.
if System.is_unix():
return os.path.realpath(path)
output = System._get_symlink_string(path)
if output is None:
return os.path.realpath(path)
groups = re.compile(r'\[\S+\]$').findall(output.strip())
if len(groups) < 1:
return os.path.realpath(path)
resolved_link = groups[0][1:-1]
return resolved_link
@staticmethod
def get_long_path(path):
"""Convert short path to a full path. It is needed for Windows."""
if System.is_unix():
return path
buffer = create_unicode_buffer(System.LONG_PATH_BUFFER_SIZE)
get_long_path_name = windll.kernel32.GetLongPathNameW
result = get_long_path_name(unicode(path), buffer, System.LONG_PATH_BUFFER_SIZE)
if result == 0 or result > System.LONG_PATH_BUFFER_SIZE:
return path
return buffer.value
@staticmethod
def get_cwd():
return System.get_long_path(os.getcwd())
dvc/utils.py
+12
View File
@@ -1,3 +1,6 @@
import os
import stat
import shutil
def cached_property(f):
def get(self):
@@ -12,3 +15,12 @@ def cached_property(f):
return x
return property(get)
def rmtree(dir):
'''Cross platform rmtree()'''
if os.name == 'nt':
if os.path.exists(dir) and not os.access(dir, os.W_OK):
os.chmod(dir, stat.S_IWUSR)
shutil.rmtree(dir, ignore_errors=True)
tests/basic_env.py
+20 -13
View File
@@ -1,5 +1,5 @@
import os
import shutil
import tempfile
from unittest import TestCase
from dvc.config import ConfigI
@@ -7,19 +7,24 @@ from dvc.git_wrapper import GitWrapperI
from dvc.path.data_item import DataItem
from dvc.path.factory import PathFactory
from dvc.settings import Settings
from dvc.system import System
from dvc.utils import rmtree
class BasicEnvironment(TestCase):
def init_environment(self, test_dir=os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'), curr_dir=None):
self._test_dir = os.path.realpath(test_dir)
def init_environment(self,
test_dir=System.get_long_path(tempfile.mkdtemp()),
curr_dir=None):
self._test_dir = System.realpath(test_dir)
self._proj_dir = 'proj'
self._test_git_dir = os.path.join(self._test_dir, self._proj_dir)
self._old_curr_dir_abs = os.path.realpath(os.curdir)
self._old_curr_dir_abs = System.realpath(os.curdir)
shutil.rmtree(self._test_dir, ignore_errors=True)
if os.path.exists(self._test_dir):
rmtree(self._test_dir)
if curr_dir:
self._curr_dir = os.path.realpath(curr_dir)
self._curr_dir = System.realpath(curr_dir)
else:
self._curr_dir = self._test_git_dir
@@ -30,8 +35,12 @@ class BasicEnvironment(TestCase):
os.makedirs(self._test_git_dir)
data_dir = os.path.join(self._test_git_dir, 'data')
if not os.path.isdir(data_dir):
os.makedirs(data_dir)
cache_dir = os.path.join(self._test_git_dir, 'cache')
state_dir = os.path.join(self._test_git_dir, 'state')
os.makedirs(data_dir)
os.makedirs(cache_dir)
os.makedirs(state_dir)
os.chdir(self._curr_dir)
@@ -39,7 +48,7 @@ class BasicEnvironment(TestCase):
if self._old_curr_dir_abs:
os.chdir(self._old_curr_dir_abs)
if self._test_git_dir:
shutil.rmtree(self._test_dir, ignore_errors=True)
rmtree(self._test_dir)
pass
@@ -71,11 +80,9 @@ class DirHierarchyEnvironment(BasicEnvironment):
self.settings = Settings([], self._git, self._config)
self.dir1 = os.path.join('dir1')
self.dir11 = os.path.join('dir1/dir11')
self.dir11 = os.path.join('dir1', 'dir11')
self.dir2 = os.path.join('dir2')
os.mkdir('cache')
os.mkdir('state')
self.create_dirs(self.dir1)
self.create_dirs(self.dir11)
self.create_dirs(self.dir2)
@@ -106,7 +113,7 @@ class DirHierarchyEnvironment(BasicEnvironment):
self.create_content_file(cache_result, content)
relevant_dir = self.relevant_dir(data_file)
os.symlink(os.path.join(relevant_dir, cache_result), file_result)
System.symlink(os.path.join(relevant_dir, cache_result), file_result)
else:
cache_result = None
self.create_content_file(file_result, content)
tests/test_cmd_run.py
+16 -12
View File
@@ -1,5 +1,4 @@
import os
import shutil
import tempfile
from unittest import TestCase
@@ -10,12 +9,14 @@ from dvc.path.data_item import DataItem
from dvc.path.factory import PathFactory
from dvc.git_wrapper import GitWrapper
from dvc.settings import Settings
from dvc.system import System
from dvc.utils import rmtree
class RunBasicTest(TestCase):
def setUp(self):
self.test_dir = tempfile.mkdtemp()
self._old_curr_dir_abs = os.path.realpath(os.curdir)
self.test_dir = System.get_long_path(tempfile.mkdtemp())
self._old_curr_dir_abs = System.realpath(os.curdir)
self.tearDown()
os.mkdir(self.test_dir)
@@ -46,7 +47,7 @@ class RunBasicTest(TestCase):
fd.close()
def tearDown(self):
shutil.rmtree(self.test_dir, ignore_errors=True)
rmtree(self.test_dir)
os.chdir(self._old_curr_dir_abs)
@@ -55,7 +56,8 @@ class TestRunOutsideData(RunBasicTest):
self.settings._args = []
cmd_run = CmdRun(self.settings)
with self.assertRaises(RunError):
cmd_run.run_command(['touch', 'file1', 'file2'], [])
cmd_run.run_command(['echo', 'test'], [], shell=True,
stdout='file1', stderr='file2')
pass
@@ -65,18 +67,20 @@ class RunTwoFilesBase(RunBasicTest):
self.settings._args = []
cmd_run = CmdRun(self.settings)
self.input_param_file = 'data/extra_input'
self.input_param_file = os.path.join('data', 'extra_input')
cmd_run.parsed_args.input = [self.input_param_file]
self.extra_output_file = 'data/extra_output'
self.extra_output_file = os.path.join('data', 'extra_output')
cmd_run.parsed_args.output = [self.extra_output_file]
self.file_name1 = 'data/file1'
self.file_name2 = 'data/file2'
self.state_objs = cmd_run.run_command(['touch', self.file_name1, self.file_name2], [])
self.file_name1 = os.path.join('data', 'file1')
self.file_name2 = os.path.join('data', 'file2')
self.state_objs = cmd_run.run_command(['echo', 'test'], [], shell=True,
stdout=self.file_name1,
stderr=self.file_name2)
self.state_file_name1 = 'state/file1' + DataItem.STATE_FILE_SUFFIX
self.state_file_name2 = 'state/file2' + DataItem.STATE_FILE_SUFFIX
self.state_file_name1 = os.path.join('state', 'file1' + DataItem.STATE_FILE_SUFFIX)
self.state_file_name2 = os.path.join('state', 'file2' + DataItem.STATE_FILE_SUFFIX)
self.state_file1 = None
self.state_file2 = None
tests/test_data_file_obj.py
+16 -14
View File
@@ -4,6 +4,7 @@ from dvc.config import ConfigI
from dvc.git_wrapper import GitWrapperI
from dvc.path.data_item import NotInDataDirError, DataItemError, DataItem
from dvc.path.factory import PathFactory
from dvc.system import System
from tests.basic_env import BasicEnvironment
@@ -43,40 +44,40 @@ class TestPathFactory(BasicEnvironment):
config = ConfigI('data', 'cache', 'state')
self.path_factory = PathFactory(git, config)
self.file = os.path.join('data', 'file.txt')
self.data_file = os.path.join('data', 'file.txt')
self.cache_file = os.path.join('cache', 'fsymlinc.txt')
fd = open(self.file, 'w+')
fd = open(self.cache_file, 'w+')
fd.write('some text')
fd.close()
self.file_symlink = os.path.join('data', 'fsymlinc.txt')
os.chdir('data')
os.symlink('file.txt', 'fsymlinc.txt')
System.symlink(os.path.join('..', self.cache_file), 'file.txt')
os.chdir('..')
pass
def test_data_file_factory(self):
data_path = self.path_factory.data_item(self.file)
self.assertEqual(data_path.data.dvc, self.file)
data_path = self.path_factory.data_item(self.data_file)
self.assertEqual(data_path.data.dvc, self.data_file)
indirect_file_path = os.path.join('..', self._proj_dir, self.file)
indirect_file_path = os.path.join('..', self._proj_dir, self.data_file)
data_path_indirect = self.path_factory.data_item(indirect_file_path)
self.assertEqual(data_path_indirect.data.dvc, self.file)
self.assertEqual(data_path_indirect.data.dvc, self.data_file)
pass
def test_data_symlink_factory(self):
data_path = self.path_factory.existing_data_item(self.file_symlink)
self.assertEqual(data_path.data.dvc, self.file_symlink)
data_path = self.path_factory.existing_data_item(self.data_file)
self.assertEqual(data_path.cache.dvc, self.cache_file)
pass
def test_data_symlink_factory_cache(self):
data_path = self.path_factory.existing_data_item(self.file_symlink)
self.assertEqual(data_path.cache.dvc, self.file)
data_path = self.path_factory.existing_data_item(self.data_file)
self.assertEqual(data_path.data.dvc, self.data_file)
pass
def test_data_symlink_factory_exception(self):
with self.assertRaises(DataItemError):
self.path_factory.existing_data_item(self.file)
self.path_factory.existing_data_item(self.cache_file)
pass
def test_data_path(self):
@@ -132,7 +133,8 @@ class TestDataPathInDataDir(BasicEnvironment):
self.assertEqual(self.data_path.state.abs, target)
def test_symlink(self):
self.assertEqual(self.data_path._symlink_file, '../../../ca/dir1/d2/file.txt_eeeff8f')
expected = os.path.join('..', '..', '..', 'ca', 'dir1', 'd2', 'file.txt_eeeff8f')
self.assertEqual(self.data_path._symlink_file, expected)
def test_data_dir(self):
data_path = self.path_factory.data_item(self.data_dir)
tests/test_dvc_path.py
+4 -3
View File
@@ -4,12 +4,13 @@ from unittest import TestCase
from dvc.path.path import Path
from dvc.git_wrapper import GitWrapperI
from dvc.system import System
class TestDvcPathTest(TestCase):
def setUp(self):
self.curr_dir = os.path.realpath('.')
self.test_dir = os.path.realpath('/tmp/ntx_unit_test/dvc_path')
self.curr_dir = System.realpath('.')
self.test_dir = System.realpath('/tmp/ntx_unit_test/dvc_path')
self.tearDown()
os.makedirs(os.path.join(self.test_dir, 'data'))
os.makedirs(os.path.join(self.test_dir, 'code', 'lib'))
@@ -25,7 +26,7 @@ class TestDvcPathTest(TestCase):
self.assertEqual(path.dvc, dvc_file_name)
self.assertEqual(path.filename, os.path.basename(dvc_file_name))
self.assertEqual(path.relative, relative_file_name)
self.assertEquals(path.abs[0], os.path.sep)
self.assertTrue(os.path.isabs(path.abs))
self.assertTrue(path.abs.endswith(dvc_file_name))
def basic_test(self):
tests/test_executor.py
+15 -19
View File
@@ -4,44 +4,40 @@ import tempfile
from unittest import TestCase
from dvc.executor import Executor, ExecutorError
from dvc.system import System
class TestExecutor(TestCase):
def setUp(self):
self.test_dir = tempfile.mkdtemp()
self.test_dir = System.get_long_path(tempfile.mkdtemp())
self.file1 = 'file1.txt'
fd1 = open(os.path.join(self.test_dir, self.file1), 'w+')
fd1.write('this is file 1')
fd1.close()
self.file2 = 'f.csv'
fd2 = open(os.path.join(self.test_dir, self.file2), 'w+')
fd2.write('this is file 222')
fd2.close()
self.code_file = 'myfile.py'
fd = open(self.code_file, 'w')
fd.write('import sys\nsys.stdout.write("111")\nsys.stderr.write("222")\n')
fd.close()
def tearDown(self):
shutil.rmtree(self.test_dir)
def test_suppress_output(self):
output = Executor.exec_cmd_only_success(['ls', self.test_dir])
output_set = set(output.split())
self.assertEqual(output_set, {self.file1, self.file2})
output = Executor.exec_cmd_only_success(['python', self.code_file])
self.assertEqual(output, '111')
def test_output_to_std_console(self):
output = Executor.exec_cmd_only_success(['ls', self.test_dir], '-', '-')
# This output "111" in stdout and "222" are not suppressible.
# You will see the outputs when the unit-test runs.
output = Executor.exec_cmd_only_success(['python', self.code_file], '-', '-')
self.assertEqual(output, '')
def test_custom_file_outputs(self):
stdout_file = os.path.join(self.test_dir, 'stdout.txt')
stderr_file = os.path.join(self.test_dir, 'stderr.txt')
output = Executor.exec_cmd_only_success(['ls', self.test_dir], stdout_file, stderr_file)
output = Executor.exec_cmd_only_success(['python', self.code_file], stdout_file, stderr_file)
self.assertEqual(output, '')
output_set = set(open(stdout_file).read().split())
expected = {self.file1, self.file2, os.path.basename(stdout_file), os.path.basename(stderr_file)}
self.assertEqual(output_set, expected)
self.assertEqual(open(stdout_file).read(), "111")
self.assertEqual(open(stderr_file).read(), "222")
os.remove(stdout_file)
os.remove(stderr_file)
tests/test_repository_change.py
+13 -9
View File
@@ -9,12 +9,13 @@ from dvc.path.factory import PathFactory
from dvc.git_wrapper import GitWrapperI
from dvc.repository_change import RepositoryChange
from dvc.settings import Settings
from dvc.system import System
class BasicTestRepositoryChange(TestCase):
def setUp(self):
self.test_dir = tempfile.mkdtemp()
self._old_curr_dir_abs = os.path.realpath(os.curdir)
self.test_dir = System.get_long_path(tempfile.mkdtemp())
self._old_curr_dir_abs = System.realpath(os.curdir)
self.tearDown()
os.mkdir(self.test_dir)
@@ -57,10 +58,11 @@ class TestRepositoryChange(BasicTestRepositoryChange):
stdout=self._devnull, stderr=None).wait()
os.remove(file_to_remove)
change = RepositoryChange(['ls', '-la'],
change = RepositoryChange(['echo', 'test'],
self.settings,
file_created_by_run,
file_to_modify)
file_to_modify,
shell=True)
expected_new = [file_created_by_run, file_created_before_run]
new_file_abs = [x.data.dvc for x in change.new_data_items]
@@ -85,10 +87,11 @@ class TestRepositoryChangeDeepInDirectory(BasicTestRepositoryChange):
deep_file = os.path.join('data', 'dir32', 'file1.txt')
self.create_file(deep_file)
change = RepositoryChange(['ls', '-la'],
change = RepositoryChange(['echo', 'text'],
self.settings,
deep_file,
None)
None,
shell=True)
self.assertEqual([x.data.dvc for x in change.new_data_items], [deep_file])
@@ -98,10 +101,11 @@ class TestRepositoryChangeExternallyCreated(BasicTestRepositoryChange):
not_in_data_dir_file = 'file1.txt'
self.create_file(not_in_data_dir_file)
change = RepositoryChange(['ls', '-la'],
change = RepositoryChange(['echo', 'textsd'],
self.settings,
not_in_data_dir_file,
None)
None,
shell=True)
self.assertEqual([os.path.realpath(not_in_data_dir_file)],
self.assertEqual([System.realpath(not_in_data_dir_file)],
change.externally_created_files)
tests/test_repro.py
+39 -17
View File
@@ -1,8 +1,9 @@
import os
from dvc.command.remove import CmdDataRemove
from dvc.command.repro import ReproChange, CmdRepro
from dvc.command.run import CmdRun, RunError
from dvc.command.repro import CmdRepro
from dvc.command.run import CmdRun
from dvc.executor import Executor
from dvc.settings import Settings
from tests.test_cmd_run import RunBasicTest
@@ -10,33 +11,50 @@ class ReproBasicEnv(RunBasicTest):
def setUp(self):
super(ReproBasicEnv, self).setUp()
self.file_name1 = 'data/file1'
self.file_name1 = os.path.join('data', 'file1')
self.file1_code_file = 'file1.py'
self.create_file_and_commit(self.file1_code_file, 'An awesome code...')
self.settings._args = ['printf', 'Hello\nMary', '--not-repro',
self.create_file_and_commit(self.file1_code_file, 'print("Hello")' + os.linesep + 'print("Mary")')
self.settings._args = ['python', self.file1_code_file, '--not-repro',
'--stdout', self.file_name1, '--code', self.file1_code_file]
cmd_file1 = CmdRun(self.settings)
self.assertEqual(cmd_file1.code_dependencies, [self.file1_code_file])
cmd_file1.run()
self.file_name11 = 'data/file11'
self.settings._args = ['head', '-n', '1', self.file_name1, '--stdout', self.file_name11]
self.file_name11 = os.path.join('data', 'file11')
self.file11_code_file = 'file11.py'
self.create_file_and_commit(self.file11_code_file,
'import sys' + os.linesep + 'print(open(sys.argv[1]).readline().strip())')
self.settings._args = ['python', self.file11_code_file, self.file_name1,
'--stdout', self.file_name11, '--code', self.file11_code_file]
CmdRun(self.settings).run()
self.file_name2 = 'data/file2'
self.settings._args = ['printf', 'Bobby', '--not-repro', '--stdout', self.file_name2]
self.file_name2 = os.path.join('data', 'file2')
self.file2_code_file = 'file2.py'
self.create_file_and_commit(self.file2_code_file,
'print("Bobby")')
self.settings._args = ['python', self.file2_code_file,
'--not-repro', '--stdout', self.file_name2]
CmdRun(self.settings).run()
self.file_res_code_file = 'code_res.py'
self.create_file_and_commit(self.file_res_code_file, 'Another piece of code')
self.file_name_res = 'data/file_res'
self.settings._args = ['cat', self.file_name11, self.file_name2, '--stdout', self.file_name_res,
self.create_file_and_commit(self.file_res_code_file,
'import sys' + os.linesep +
'text1 = open(sys.argv[1]).read()' + os.linesep +
'text2 = open(sys.argv[2]).read()' + os.linesep +
'print(text1 + text2)')
self.file_name_res = os.path.join('data', 'file_res')
self.settings._args = ['python', self.file_res_code_file,
self.file_name11,
self.file_name2,
'--stdout', self.file_name_res,
'--code', self.file_res_code_file]
cmd_res = CmdRun(self.settings)
self.assertEqual(cmd_res.code_dependencies, [self.file_res_code_file])
cmd_res.run()
self.assertEqual(open(self.file_name_res).read(), 'Hello\nBobby')
lines = list(filter(None, map(str.strip, open(self.file_name_res).readlines())))
self.assertEqual(lines, ['Hello', 'Bobby'])
def create_file_and_commit(self, file_name, content='Any', message='Just a commit'):
self.create_file(file_name, content)
@@ -61,7 +79,7 @@ class ReproCodeDependencyTest(ReproBasicEnv):
self.settings._args = [self.file_name_res]
CmdRepro(self.settings).run()
self.assertEqual(open(self.file_name_res).read(), 'Hello\nBobby')
self.assertEqual(open(self.file_name_res).read().strip(), 'Hello\nBobby')
class ReproChangedDependency(ReproBasicEnv):
@@ -77,7 +95,11 @@ class ReproChangedDependency(ReproBasicEnv):
self.settings._args = [self.file_name1, '--keep-in-cloud']
CmdDataRemove(self.settings).run()
self.settings._args = ['printf', 'Goodbye\nJack', '--stdout', self.file_name1]
file1_code_file = 'file1_2.py'
self.create_file_and_commit(file1_code_file, 'print("Goodbye")' + os.linesep + 'print("Jack")')
self.settings._args = ['python', file1_code_file, '--not-repro',
'--stdout', self.file_name1, '--code', file1_code_file]
CmdRun(self.settings).run()
@@ -88,4 +110,4 @@ class ReproChangedDeepDependency(ReproChangedDependency):
self.settings._args = [self.file_name_res]
CmdRepro(self.settings).run()
self.assertEqual(open(self.file_name_res).read(), 'Goodbye\nBobby')
self.assertEqual(open(self.file_name_res).read().strip(), 'Goodbye\nBobby')