mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-10 07:18:27 +01:00
dvc: initial unittests
Initial unittests for new design. Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
Ruslan Kuprieiev
+38
-107
@@ -1,124 +1,55 @@
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
from git import Repo
|
||||
from unittest import TestCase
|
||||
|
||||
from dvc.config import ConfigI
|
||||
from dvc.git_wrapper import GitWrapperI
|
||||
from dvc.path.data_item import DataItem
|
||||
from dvc.path.factory import PathFactory
|
||||
from dvc.settings import Settings
|
||||
from dvc.system import System
|
||||
from dvc.utils import rmtree
|
||||
from dvc.project import Project
|
||||
|
||||
class BasicEnvironment(TestCase):
|
||||
def init_environment(self,
|
||||
test_dir=System.get_long_path(tempfile.mkdtemp()),
|
||||
curr_dir=None):
|
||||
self._test_dir = System.realpath(test_dir)
|
||||
self._proj_dir = 'proj'
|
||||
self._test_git_dir = os.path.join(self._test_dir, self._proj_dir)
|
||||
self._old_curr_dir_abs = System.realpath(os.curdir)
|
||||
|
||||
if os.path.exists(self._test_dir):
|
||||
rmtree(self._test_dir)
|
||||
class TestDir(TestCase):
|
||||
FOO = 'foo'
|
||||
FOO_CONTENTS = FOO
|
||||
BAR = 'bar'
|
||||
BAR_CONTENTS = BAR
|
||||
CODE = 'code.py'
|
||||
CODE_CONTENTS = 'import sys\nimport shutil\nshutil.copyfile(sys.argv[1], sys.argv[2])'
|
||||
|
||||
if curr_dir:
|
||||
self._curr_dir = System.realpath(curr_dir)
|
||||
else:
|
||||
self._curr_dir = self._test_git_dir
|
||||
def _pushd(self, d):
|
||||
self._saved_dir = System.realpath(os.curdir)
|
||||
os.chdir(d)
|
||||
|
||||
if not os.path.exists(self._curr_dir):
|
||||
os.makedirs(self._curr_dir)
|
||||
def _popd(self):
|
||||
os.chdir(self._saved_dir)
|
||||
self._saved_dir = None
|
||||
|
||||
if not os.path.isdir(self._test_git_dir):
|
||||
os.makedirs(self._test_git_dir)
|
||||
def create(self, name, contents):
|
||||
with open(name, 'a') as f:
|
||||
f.write(contents)
|
||||
|
||||
data_dir = os.path.join(self._test_git_dir, 'data')
|
||||
cache_dir = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME)
|
||||
state_dir = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, 'data')
|
||||
|
||||
os.makedirs(data_dir)
|
||||
os.makedirs(cache_dir)
|
||||
os.makedirs(state_dir)
|
||||
|
||||
os.chdir(self._curr_dir)
|
||||
def setUp(self):
|
||||
self._root_dir = System.get_long_path(tempfile.mkdtemp())
|
||||
self._pushd(self._root_dir)
|
||||
self.create(self.FOO, self.FOO_CONTENTS)
|
||||
self.create(self.BAR, self.BAR_CONTENTS)
|
||||
self.create(self.CODE, self.CODE_CONTENTS)
|
||||
|
||||
def tearDown(self):
|
||||
if self._old_curr_dir_abs:
|
||||
os.chdir(self._old_curr_dir_abs)
|
||||
if self._test_git_dir:
|
||||
rmtree(self._test_dir)
|
||||
pass
|
||||
self._popd()
|
||||
shutil.rmtree(self._root_dir)
|
||||
|
||||
|
||||
class DirHierarchyEnvironment(BasicEnvironment):
|
||||
def init_environment(self,
|
||||
test_dir=os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'),
|
||||
curr_dir=None,
|
||||
commit='abc12345'):
|
||||
''' Creates data environment with data, cache and state dirs.
|
||||
data/
|
||||
file1.txt --> ../cache/file1.txt_abc123
|
||||
dir1/
|
||||
file2.txt --> ../../cache/dir1/file2.txt_abc123
|
||||
file3.txt --> ../../cache/dir1/file3.txt_abc123
|
||||
dir11/
|
||||
file4.txt --> ../../../cache/dir1/dir11/file4.txt_abc123
|
||||
dir2/
|
||||
file5.txt --> ../../cache/dir2/file5.txt_abc123
|
||||
file6.txt --> an actual file
|
||||
'''
|
||||
class TestGit(TestDir):
|
||||
def setUp(self):
|
||||
super(TestGit, self).setUp()
|
||||
self.git = Repo.init()
|
||||
self.git.index.add([self.CODE])
|
||||
self.git.index.commit('add code')
|
||||
|
||||
BasicEnvironment.init_environment(self, test_dir, curr_dir)
|
||||
|
||||
self._commit = commit
|
||||
|
||||
self._git = GitWrapperI(git_dir=self._test_git_dir, commit=self._commit)
|
||||
self._config = ConfigI('data')
|
||||
self.path_factory = PathFactory(self._git, self._config)
|
||||
self.settings = Settings([], self._git, self._config)
|
||||
|
||||
self.dir1 = 'dir1'
|
||||
self.dir11 = os.path.join('dir1', 'dir11')
|
||||
self.dir2 = 'dir2'
|
||||
|
||||
self.create_dirs(self.dir1)
|
||||
self.create_dirs(self.dir11)
|
||||
self.create_dirs(self.dir2)
|
||||
|
||||
self.file1, self.cache1, self.state1 = self.crate_data_item('file1.txt')
|
||||
self.file2, self.cache2, self.state2 = self.crate_data_item(os.path.join(self.dir1, 'file2.txt'))
|
||||
self.file3, self.cache3, self.state3 = self.crate_data_item(os.path.join(self.dir1, 'file3.txt'))
|
||||
self.file4, self.cache4, self.state4 = self.crate_data_item(os.path.join(self.dir11, 'file4.txt'))
|
||||
self.file5, self.cache5, self.state5 = self.crate_data_item(os.path.join(self.dir2, 'file5.txt'))
|
||||
self.file6, self.cache6, self.state6 = self.crate_data_item(os.path.join(self.dir2, 'file6.txt'),
|
||||
cache_file=False)
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def create_content_file(file, content='some test'):
|
||||
fd = open(file, 'w+')
|
||||
fd.write(content)
|
||||
fd.close()
|
||||
|
||||
def crate_data_item(self, data_file, cache_file=True, content='random text'):
|
||||
file_result = os.path.join('data', data_file)
|
||||
state_result = os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, file_result + DataItem.STATE_FILE_SUFFIX)
|
||||
|
||||
dummy_md5 = os.path.basename(data_file) + DataItem.CACHE_FILE_SEP + self._commit
|
||||
self.create_content_file(state_result, '{"Md5" : "' + dummy_md5 + '", "Command" : "run", "Cwd" : "dir"}')
|
||||
|
||||
if cache_file:
|
||||
cache_result = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME, dummy_md5)
|
||||
self.create_content_file(cache_result, content)
|
||||
|
||||
System.hardlink(cache_result, file_result)
|
||||
else:
|
||||
cache_result = None
|
||||
self.create_content_file(file_result, content)
|
||||
|
||||
return file_result, cache_result, state_result
|
||||
|
||||
def create_dirs(self, dir):
|
||||
os.makedirs(os.path.join('data', dir))
|
||||
os.makedirs(os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, 'data', dir))
|
||||
class TestDvc(TestGit):
|
||||
def setUp(self):
|
||||
super(TestDvc, self).setUp()
|
||||
self.dvc = Project.init(self._root_dir)
|
||||
self.dvc.logger.be_verbose()
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
import os
|
||||
import shutil
|
||||
import filecmp
|
||||
|
||||
from dvc.data_cloud import file_md5
|
||||
from dvc.stage import Stage, OutputNoCacheError, OutputOutsideOfRepoError
|
||||
from dvc.stage import OutputDoesNotExistError, OutputIsNotFileError
|
||||
from dvc.project import StageNotFoundError
|
||||
|
||||
from tests.basic_env import TestDvc
|
||||
|
||||
|
||||
class TestAdd(TestDvc):
|
||||
def test(self):
|
||||
md5 = file_md5(self.FOO)[0]
|
||||
|
||||
stage = self.dvc.add(self.FOO)
|
||||
|
||||
self.assertIsInstance(stage, Stage)
|
||||
self.assertTrue(os.path.isfile(stage.path))
|
||||
self.assertEqual(len(stage.outs), 1)
|
||||
self.assertEqual(len(stage.deps), 0)
|
||||
self.assertTrue(stage.locked)
|
||||
self.assertEqual(stage.cmd, None)
|
||||
self.assertEqual(stage.outs[0].md5, md5)
|
||||
|
||||
|
||||
class TestAddNonExistentFile(TestDvc):
|
||||
def test(self):
|
||||
with self.assertRaises(OutputDoesNotExistError) as cx:
|
||||
self.dvc.add('non_existent_file')
|
||||
|
||||
|
||||
class TestAddFileOutsideOfRepo(TestDvc):
|
||||
def test(self):
|
||||
with self.assertRaises(OutputOutsideOfRepoError) as cx:
|
||||
self.dvc.add(os.path.join(os.path.dirname(self.dvc.root_dir), self.FOO))
|
||||
|
||||
|
||||
class TestAddDirectory(TestDvc):
|
||||
def test(self):
|
||||
dname = 'directory'
|
||||
os.mkdir(dname)
|
||||
with self.assertRaises(OutputIsNotFileError) as cx:
|
||||
self.dvc.add(dname)
|
||||
@@ -0,0 +1,24 @@
|
||||
import os
|
||||
|
||||
from dvc.cache import Cache
|
||||
|
||||
from tests.basic_env import TestDvc
|
||||
|
||||
|
||||
class TestCache(TestDvc):
|
||||
def setUp(self):
|
||||
super(TestCache, self).setUp()
|
||||
self.cache1 = os.path.join(self.dvc.cache.cache_dir, '1')
|
||||
self.cache2 = os.path.join(self.dvc.cache.cache_dir, '2')
|
||||
self.create(self.cache1, '1')
|
||||
self.create(self.cache2, '2')
|
||||
|
||||
def test_all(self):
|
||||
flist = Cache(self.dvc.dvc_dir).all()
|
||||
self.assertEquals(len(flist), 2)
|
||||
self.assertTrue(self.cache1 in flist)
|
||||
self.assertTrue(self.cache2 in flist)
|
||||
|
||||
def test_get(self):
|
||||
cache = Cache(self.dvc.dvc_dir).get(os.path.basename(self.cache1))
|
||||
self.assertEquals(cache, self.cache1)
|
||||
@@ -0,0 +1,30 @@
|
||||
import os
|
||||
import shutil
|
||||
import filecmp
|
||||
|
||||
from dvc.data_cloud import file_md5
|
||||
from dvc.stage import Stage, OutputNoCacheError, OutputOutsideOfRepoError
|
||||
from dvc.stage import OutputDoesNotExistError, OutputIsNotFileError
|
||||
from dvc.project import StageNotFoundError
|
||||
|
||||
from tests.basic_env import TestDvc
|
||||
|
||||
|
||||
class TestCheckout(TestDvc):
|
||||
def test_checkout(self):
|
||||
cmd = 'python {} {} {}'.format(self.CODE, self.FOO, 'out')
|
||||
deps = [self.FOO]
|
||||
deps_no_cache = [self.CODE]
|
||||
outs = [os.path.join(self.dvc.root_dir, 'out')]
|
||||
outs_no_cache = []
|
||||
locked = False
|
||||
fname = os.path.join(self.dvc.root_dir, 'out.dvc')
|
||||
cwd = os.curdir
|
||||
|
||||
stage = self.dvc.run(cmd, deps, deps_no_cache, outs, outs_no_cache, locked, fname, cwd)
|
||||
|
||||
shutil.copy(self.FOO, 'orig')
|
||||
os.unlink(self.FOO)
|
||||
self.dvc.checkout()
|
||||
self.assertTrue(os.path.isfile(self.FOO))
|
||||
self.assertTrue(filecmp.cmp(self.FOO, 'orig'))
|
||||
@@ -1,159 +0,0 @@
|
||||
import os
|
||||
|
||||
from dvc.command.remove import CmdRemove
|
||||
from tests.basic_env import DirHierarchyEnvironment
|
||||
|
||||
class TestCmdRemove(DirHierarchyEnvironment):
|
||||
def setUp(self):
|
||||
DirHierarchyEnvironment.init_environment(self)
|
||||
|
||||
def test_file_by_file_removal(self):
|
||||
self.settings.parse_args('remove --keep-in-cloud')
|
||||
cmd = CmdRemove(self.settings)
|
||||
|
||||
dir1_dvc_name = os.path.join('data', self.dir1)
|
||||
self.assertTrue(os.path.exists(dir1_dvc_name))
|
||||
|
||||
cmd.remove_dir(dir1_dvc_name)
|
||||
self.assertFalse(os.path.exists(dir1_dvc_name))
|
||||
|
||||
def test_remove_deep_dir(self):
|
||||
self.settings.parse_args('remove --keep-in-cloud --recursive')
|
||||
cmd = CmdRemove(self.settings)
|
||||
|
||||
dir1_dvc_name = os.path.join('data', self.dir1)
|
||||
self.assertTrue(os.path.exists(dir1_dvc_name))
|
||||
|
||||
cmd.remove_dir(dir1_dvc_name)
|
||||
self.assertFalse(os.path.exists(dir1_dvc_name))
|
||||
|
||||
def test_not_recursive_removal(self):
|
||||
self.settings.parse_args('remove --keep-in-cloud')
|
||||
cmd = CmdRemove(self.settings)
|
||||
|
||||
dir1_dvc_name = os.path.join('data', self.dir1)
|
||||
self.assertTrue(os.path.exists(dir1_dvc_name))
|
||||
|
||||
cmd.remove_dir(dir1_dvc_name)
|
||||
pass
|
||||
|
||||
def test_data_dir_removal(self):
|
||||
self.settings.parse_args('remove --keep-in-cloud --recursive')
|
||||
cmd = CmdRemove(self.settings)
|
||||
|
||||
data_dir = 'data'
|
||||
self.assertTrue(os.path.exists(data_dir))
|
||||
|
||||
cmd.remove_dir(data_dir)
|
||||
pass
|
||||
|
||||
|
||||
class TestRemoveDataItem(DirHierarchyEnvironment):
|
||||
def setUp(self):
|
||||
DirHierarchyEnvironment.init_environment(self)
|
||||
|
||||
def test_remove_data_instance(self):
|
||||
self.settings.parse_args('remove --keep-in-cloud')
|
||||
cmd = CmdRemove(self.settings)
|
||||
|
||||
self.assertTrue(os.path.isfile(self.file1))
|
||||
self.assertTrue(os.path.isfile(self.cache1))
|
||||
self.assertTrue(os.path.isfile(self.state1))
|
||||
|
||||
cmd.remove_file(self.file1)
|
||||
self.assertFalse(os.path.exists(self.file1))
|
||||
self.assertFalse(os.path.exists(self.cache1))
|
||||
self.assertFalse(os.path.exists(self.state1))
|
||||
|
||||
def test_keep_cache(self):
|
||||
self.settings.parse_args('remove --keep-in-cloud --keep-in-cache')
|
||||
cmd = CmdRemove(self.settings)
|
||||
|
||||
self.assertTrue(os.path.isfile(self.file1))
|
||||
self.assertTrue(os.path.isfile(self.cache1))
|
||||
self.assertTrue(os.path.isfile(self.state1))
|
||||
|
||||
cmd.remove_file(self.file1)
|
||||
self.assertFalse(os.path.exists(self.file1))
|
||||
self.assertTrue(os.path.exists(self.cache1))
|
||||
self.assertFalse(os.path.exists(self.state1))
|
||||
|
||||
def test_remove_data_instance_without_cache(self):
|
||||
self.settings.parse_args('remove --keep-in-cloud')
|
||||
cmd = CmdRemove(self.settings)
|
||||
|
||||
self.assertTrue(os.path.isfile(self.file6))
|
||||
self.assertIsNone(self.cache6)
|
||||
self.assertTrue(os.path.exists(self.state6))
|
||||
self.assertTrue(os.path.isfile(self.state6))
|
||||
|
||||
cmd.remove_file(self.file6)
|
||||
self.assertFalse(os.path.exists(self.file6))
|
||||
|
||||
|
||||
class TestRemoveEndToEnd(DirHierarchyEnvironment):
|
||||
def setUp(self):
|
||||
DirHierarchyEnvironment.init_environment(self)
|
||||
|
||||
def test_end_to_end_with_no_error(self):
|
||||
dir11_full = os.path.join('data', self.dir11)
|
||||
dir2_full = os.path.join('data', self.dir2)
|
||||
|
||||
self.settings.parse_args('remove --no-git-actions --keep-in-cloud --recursive {} {}'.format(dir11_full, self.file5))
|
||||
cmd = CmdRemove(self.settings)
|
||||
|
||||
self.assertTrue(os.path.exists(dir11_full))
|
||||
self.assertTrue(os.path.exists(self.file5))
|
||||
self.assertTrue(os.path.exists(dir2_full))
|
||||
self.assertTrue(os.path.exists(self.file1))
|
||||
|
||||
self.assertTrue(cmd.remove_all())
|
||||
|
||||
self.assertFalse(os.path.exists(dir11_full))
|
||||
self.assertFalse(os.path.exists(self.file5))
|
||||
self.assertTrue(os.path.exists(dir2_full))
|
||||
self.assertTrue(os.path.exists(self.file1))
|
||||
|
||||
def test_run(self):
|
||||
dir11_full = os.path.join('data', self.dir11)
|
||||
dir2_full = os.path.join('data', self.dir2)
|
||||
|
||||
self.settings.parse_args('remove --no-git-actions --keep-in-cloud --recursive {} {}'.format(dir11_full, self.file5))
|
||||
cmd = CmdRemove(self.settings)
|
||||
|
||||
self.assertTrue(os.path.exists(dir11_full))
|
||||
self.assertTrue(os.path.exists(self.file5))
|
||||
self.assertTrue(os.path.exists(dir2_full))
|
||||
self.assertTrue(os.path.exists(self.file1))
|
||||
|
||||
self.assertEqual(cmd.run(), 0)
|
||||
|
||||
self.assertFalse(os.path.exists(dir11_full))
|
||||
self.assertFalse(os.path.exists(self.file5))
|
||||
self.assertTrue(os.path.exists(dir2_full))
|
||||
self.assertTrue(os.path.exists(self.file1))
|
||||
|
||||
|
||||
class TestRemoveEndToEndWithAnError(DirHierarchyEnvironment):
|
||||
def setUp(self):
|
||||
DirHierarchyEnvironment.init_environment(self)
|
||||
|
||||
def test(self):
|
||||
dir11_full = os.path.join('data', self.dir11)
|
||||
dir2_full = os.path.join('data', self.dir2)
|
||||
|
||||
self.settings.parse_args('remove --no-git-actions --keep-in-cloud --recursive {} {}'.format(dir11_full, self.file5))
|
||||
cmd = CmdRemove(self.settings)
|
||||
|
||||
self.assertTrue(os.path.exists(dir11_full))
|
||||
self.assertTrue(os.path.exists(self.file5))
|
||||
self.assertTrue(os.path.exists(dir2_full))
|
||||
self.assertTrue(os.path.exists(self.file1))
|
||||
self.assertTrue(os.path.exists(self.file6))
|
||||
|
||||
self.assertTrue(cmd.remove_all())
|
||||
|
||||
self.assertFalse(os.path.exists(dir11_full))
|
||||
self.assertFalse(os.path.exists(self.file5))
|
||||
self.assertTrue(os.path.exists(dir2_full))
|
||||
self.assertTrue(os.path.exists(self.file1))
|
||||
@@ -1,100 +0,0 @@
|
||||
import os
|
||||
import tempfile
|
||||
from unittest import TestCase
|
||||
|
||||
from dvc.command.run import CmdRun, RunError
|
||||
from dvc.config import ConfigI
|
||||
from dvc.executor import Executor
|
||||
from dvc.path.data_item import DataItem
|
||||
from dvc.path.factory import PathFactory
|
||||
from dvc.git_wrapper import GitWrapper
|
||||
from dvc.settings import Settings
|
||||
from dvc.system import System
|
||||
from dvc.utils import rmtree
|
||||
|
||||
|
||||
class RunBasicTest(TestCase):
|
||||
def setUp(self):
|
||||
self.test_dir = System.get_long_path(tempfile.mkdtemp())
|
||||
self._old_curr_dir_abs = System.realpath(os.curdir)
|
||||
|
||||
self.tearDown()
|
||||
os.mkdir(self.test_dir)
|
||||
os.chdir(self.test_dir)
|
||||
os.mkdir('data')
|
||||
os.mkdir(ConfigI.CONFIG_DIR)
|
||||
os.mkdir(os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME))
|
||||
os.mkdir(os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME))
|
||||
|
||||
self.init_git_repo()
|
||||
self.git = GitWrapper()
|
||||
|
||||
self.config = ConfigI('data')
|
||||
self.path_factory = PathFactory(self.git, self.config)
|
||||
|
||||
self.settings = Settings('run cmd', self.git, self.config)
|
||||
pass
|
||||
|
||||
def init_git_repo(self):
|
||||
Executor.exec_cmd_only_success(['git', 'init'])
|
||||
msg = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME)
|
||||
msg += '\n'
|
||||
msg += os.path.join(ConfigI.CONFIG_DIR, '.' + ConfigI.CONFIG + '.lock')
|
||||
self.create_file('.gitignore', msg)
|
||||
Executor.exec_cmd_only_success(['git', 'add', '.gitignore'])
|
||||
Executor.exec_cmd_only_success(['git', 'commit', '-m', '"Init test repo"'])
|
||||
|
||||
@staticmethod
|
||||
def create_file(file2, content='random text'):
|
||||
fd = open(file2, 'w+')
|
||||
fd.write(content)
|
||||
fd.close()
|
||||
|
||||
def tearDown(self):
|
||||
rmtree(self.test_dir)
|
||||
os.chdir(self._old_curr_dir_abs)
|
||||
|
||||
|
||||
class RunTwoFilesBase(RunBasicTest):
|
||||
def setUp(self):
|
||||
super(RunTwoFilesBase, self).setUp()
|
||||
|
||||
self.input_param_file = os.path.join('data', 'extra_input')
|
||||
self.extra_output_file = os.path.join('data', 'extra_output')
|
||||
|
||||
self.settings.parse_args('run --input {} --output {} cmd'.format(self.input_param_file, self.extra_output_file))
|
||||
self.settings._args = []
|
||||
cmd_run = CmdRun(self.settings)
|
||||
|
||||
self.file_name1 = os.path.join('data', 'file1')
|
||||
self.file_name2 = os.path.join('data', 'file2')
|
||||
self.state_objs = cmd_run.run_command(['echo', 'test'], [], [], shell=True,
|
||||
stdout=self.file_name1,
|
||||
stderr=self.file_name2)
|
||||
|
||||
self.state_file_name1 = os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, self.file_name1 + DataItem.STATE_FILE_SUFFIX)
|
||||
self.state_file_name2 = os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, self.file_name2 + DataItem.STATE_FILE_SUFFIX)
|
||||
|
||||
self.state_file1 = None
|
||||
self.state_file2 = None
|
||||
for s in self.state_objs:
|
||||
if s.file == self.state_file_name1:
|
||||
self.state_file1 = s
|
||||
elif s.file == self.state_file_name2:
|
||||
self.state_file2 = s
|
||||
pass
|
||||
|
||||
|
||||
class TestRunStateFiles(RunTwoFilesBase):
|
||||
def test(self):
|
||||
names = [x.file for x in self.state_objs]
|
||||
self.assertEqual({self.state_file_name1, self.state_file_name2}, set(names))
|
||||
|
||||
|
||||
class TestRunExtraInExtraOutFiles(RunTwoFilesBase):
|
||||
def test(self):
|
||||
self.assertIsNotNone(self.state_file1)
|
||||
self.assertEqual(self.state_file1.input_files, [self.input_param_file])
|
||||
|
||||
output_set = {self.file_name1, self.file_name2, self.extra_output_file}
|
||||
self.assertEqual(set(self.state_file1.output_files), output_set)
|
||||
+14
-11
@@ -1,4 +1,5 @@
|
||||
import os
|
||||
import configparser
|
||||
from unittest import TestCase
|
||||
import mock
|
||||
|
||||
@@ -15,15 +16,18 @@ except ImportError:
|
||||
# Python 3
|
||||
builtin_module_name = 'builtins'
|
||||
|
||||
from dvc.config import Config
|
||||
from dvc.data_cloud import DataCloud
|
||||
from dvc.settings import Settings
|
||||
|
||||
|
||||
class TestConfigTest(TestCase):
|
||||
def setUp(self):
|
||||
pass
|
||||
|
||||
def _conf(self, s):
|
||||
c = configparser.SafeConfigParser()
|
||||
c.readfp(s)
|
||||
return c
|
||||
|
||||
def storage_path_hierarchy_test(self):
|
||||
""" StoragePath should be read first from global and then from cloud section """
|
||||
|
||||
@@ -41,9 +45,8 @@ class TestConfigTest(TestCase):
|
||||
"StoragePath = googlesb/google_storage_path"
|
||||
)
|
||||
s = StringIO('\n'.join(c))
|
||||
conf = Config(s, conf_pseudo_file = s)
|
||||
settings = Settings(None, None, conf)
|
||||
cloud = DataCloud(settings)
|
||||
conf = self._conf(s)
|
||||
cloud = DataCloud(conf)
|
||||
self.assertEqual(cloud.typ, 'AWS')
|
||||
self.assertEqual(cloud._cloud.storage_bucket, 'globalsb')
|
||||
self.assertEqual(cloud._cloud.storage_prefix, 'global_storage_path')
|
||||
@@ -61,8 +64,8 @@ class TestConfigTest(TestCase):
|
||||
"StoragePath = googlesb/google_storage_path"
|
||||
)
|
||||
s = StringIO('\n'.join(c))
|
||||
conf = Config(s, conf_pseudo_file=s)
|
||||
cloud = DataCloud(Settings(None, None, conf))
|
||||
conf = self._conf(s)
|
||||
cloud = DataCloud(conf)
|
||||
self.assertEqual(cloud.typ, 'AWS')
|
||||
self.assertEqual(cloud._cloud.storage_bucket, 'awssb')
|
||||
self.assertEqual(cloud._cloud.storage_prefix, 'aws_storage_path')
|
||||
@@ -100,8 +103,8 @@ class TestConfigTest(TestCase):
|
||||
patcher = mock.patch(builtin_module_name + '.open', side_effect=self.mocked_open_aws_default_credentials)
|
||||
|
||||
# patcher.start()
|
||||
conf = Config(s, conf_pseudo_file=s)
|
||||
cloud = DataCloud(Settings(None, None, conf))
|
||||
conf = self._conf(s)
|
||||
cloud = DataCloud(conf)
|
||||
aws_creds = cloud._cloud._get_credentials()
|
||||
patcher.stop()
|
||||
|
||||
@@ -131,8 +134,8 @@ class TestConfigTest(TestCase):
|
||||
patcher = mock.patch(builtin_module_name + '.open', side_effect=self.mocked_open_aws_default_credentials)
|
||||
|
||||
patcher.start()
|
||||
conf = Config(s, conf_pseudo_file=s)
|
||||
cloud = DataCloud(Settings(None, None, conf))
|
||||
conf = self._conf(s)
|
||||
cloud = DataCloud(conf)
|
||||
aws_creds = cloud._cloud._get_credentials()
|
||||
patcher.stop()
|
||||
|
||||
|
||||
@@ -1,196 +0,0 @@
|
||||
import os
|
||||
|
||||
from dvc.config import ConfigI
|
||||
from dvc.git_wrapper import GitWrapperI
|
||||
from dvc.path.data_item import DataDirError, DataItemError, DataItem
|
||||
from dvc.path.factory import PathFactory
|
||||
from dvc.system import System
|
||||
from tests.basic_env import BasicEnvironment
|
||||
|
||||
|
||||
class TestDataFileObjBasic(BasicEnvironment):
|
||||
def setUp(self):
|
||||
BasicEnvironment.init_environment(self, test_dir=os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'))
|
||||
|
||||
git = GitWrapperI(git_dir=self._test_git_dir, commit='ad45ba8')
|
||||
config = ConfigI()
|
||||
|
||||
file = os.path.join('data', 'file.txt')
|
||||
self.cache_file_full_path = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME, 'file.txt_ad45ba8')
|
||||
self._data_path = DataItem(file, git, config, cache_file=self.cache_file_full_path)
|
||||
pass
|
||||
|
||||
def test_data_file(self):
|
||||
data_file_full_path = os.path.join(self._test_git_dir, 'data', 'file.txt')
|
||||
self.assertEqual(self._data_path.data.abs, data_file_full_path)
|
||||
|
||||
def test_cache_file(self):
|
||||
self.assertEqual(self._data_path.cache.abs, self.cache_file_full_path)
|
||||
|
||||
def test_state_file(self):
|
||||
state_file_full_path = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, 'data', 'file.txt.state')
|
||||
self.assertEqual(self._data_path.state.abs, state_file_full_path)
|
||||
|
||||
|
||||
class TestDataItemWithGivenCache(BasicEnvironment):
|
||||
def setUp(self):
|
||||
BasicEnvironment.init_environment(self, test_dir=os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'))
|
||||
|
||||
self._git = GitWrapperI(git_dir=self._test_git_dir, commit='ad45ba8')
|
||||
self._config = ConfigI('data')
|
||||
|
||||
self._file = os.path.join('data', 'file.txt')
|
||||
self._cache = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME, 'file.txt_abcd')
|
||||
|
||||
self._data_item = DataItem(self._file, self._git, self._config, self._cache)
|
||||
pass
|
||||
|
||||
def test_basic(self):
|
||||
self.assertEqual(self._data_item.data.relative, self._file)
|
||||
self.assertEqual(self._data_item.cache.relative, self._cache)
|
||||
|
||||
|
||||
class TestDeepDataItemWithGivenCache(BasicEnvironment):
|
||||
def setUp(self):
|
||||
BasicEnvironment.init_environment(self, test_dir=os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'))
|
||||
|
||||
self._git = GitWrapperI(git_dir=self._test_git_dir, commit='ad45ba8')
|
||||
self._config = ConfigI('data')
|
||||
|
||||
self._file = os.path.join('data', 'dir1', 'file.txt')
|
||||
self._cache = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME, 'dir1', 'file.txt_abcd')
|
||||
|
||||
self._data_item = DataItem(self._file, self._git, self._config, self._cache)
|
||||
pass
|
||||
|
||||
def test_basic(self):
|
||||
self.assertEqual(self._data_item.data.relative, self._file)
|
||||
self.assertEqual(self._data_item.cache.relative, self._cache)
|
||||
|
||||
|
||||
class TestPathFactory(BasicEnvironment):
|
||||
def setUp(self):
|
||||
BasicEnvironment.init_environment(self, test_dir=os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'))
|
||||
|
||||
git = GitWrapperI(git_dir=self._test_git_dir, commit='ad45ba8')
|
||||
config = ConfigI()
|
||||
self.path_factory = PathFactory(git, config)
|
||||
|
||||
self.data_file = os.path.join('data', 'file.txt')
|
||||
dummy_md5 = 'fsymlinc.txt'
|
||||
self.cache_file = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME, dummy_md5)
|
||||
self.state_file = os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, 'data', 'file.txt' + DataItem.STATE_FILE_SUFFIX)
|
||||
|
||||
with open(self.cache_file, 'w+') as fd:
|
||||
fd.write('some text')
|
||||
|
||||
with open(self.state_file, 'w+') as fd:
|
||||
fd.write('{"Md5" : "' + dummy_md5 + '", "Command" : "run", "Cwd" : "dir"}')
|
||||
|
||||
os.chdir('data')
|
||||
System.hardlink(os.path.join('..', self.cache_file), 'file.txt')
|
||||
os.chdir('..')
|
||||
pass
|
||||
|
||||
def test_data_file_factory(self):
|
||||
data_path = self.path_factory.data_item(self.data_file)
|
||||
self.assertEqual(data_path.data.dvc, self.data_file)
|
||||
|
||||
indirect_file_path = os.path.join('..', self._proj_dir, self.data_file)
|
||||
data_path_indirect = self.path_factory.data_item(indirect_file_path)
|
||||
self.assertEqual(data_path_indirect.data.dvc, self.data_file)
|
||||
pass
|
||||
|
||||
def test_data_hardlink_factory(self):
|
||||
data_path = self.path_factory.existing_data_item(self.data_file)
|
||||
self.assertEqual(data_path.cache.dvc, self.cache_file)
|
||||
pass
|
||||
|
||||
def test_data_hardlink_factory_cache(self):
|
||||
data_path = self.path_factory.existing_data_item(self.data_file)
|
||||
self.assertEqual(data_path.data.dvc, self.data_file)
|
||||
pass
|
||||
|
||||
def test_data_hardlink_factory_exception(self):
|
||||
with self.assertRaises(DataItemError):
|
||||
self.path_factory.existing_data_item(self.cache_file)
|
||||
pass
|
||||
|
||||
def test_data_path(self):
|
||||
file = os.path.join('data', 'file1')
|
||||
path = self.path_factory.path(file)
|
||||
self.assertEqual(path.dvc, file)
|
||||
self.assertEqual(path.relative, file)
|
||||
self.assertTrue(path.abs.endswith(file))
|
||||
|
||||
|
||||
class TestDataPathInDataDir(BasicEnvironment):
|
||||
def setUp(self):
|
||||
BasicEnvironment.init_environment(self, os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'))
|
||||
|
||||
git = GitWrapperI(git_dir=self._test_git_dir, commit='eeeff8f')
|
||||
self.data_dir = 'da'
|
||||
config = ConfigI()
|
||||
self.path_factory = PathFactory(git, config)
|
||||
|
||||
deep_path = os.path.join(self.data_dir, 'dir1', 'd2', 'file.txt')
|
||||
self.cache_file = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME, 'dir1', 'd2', 'file.txt_eeeff8f')
|
||||
self.data_path = self.path_factory.data_item(deep_path, cache_file=self.cache_file)
|
||||
pass
|
||||
|
||||
def test_data_file(self):
|
||||
target = os.path.join(self._test_git_dir, 'da', 'dir1', 'd2', 'file.txt')
|
||||
self.assertEqual(self.data_path.data.abs, target)
|
||||
|
||||
def test_cache_file(self):
|
||||
self.assertEqual(self.data_path.cache.abs, self.cache_file)
|
||||
|
||||
def test_state_file(self):
|
||||
target = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, 'da', 'dir1', 'd2', 'file.txt.state')
|
||||
self.assertEqual(self.data_path.state.abs, target)
|
||||
|
||||
|
||||
class TestDataFileObjLongPath(BasicEnvironment):
|
||||
def setUp(self):
|
||||
curr_dir = os.path.join(os.path.sep, 'tmp', 'ntx_unit_test', 'proj', 'mydir', 'dd3')
|
||||
|
||||
BasicEnvironment.init_environment(self,
|
||||
os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'),
|
||||
curr_dir)
|
||||
|
||||
self._git = GitWrapperI(git_dir=self._test_git_dir, commit='123ed8')
|
||||
self._config = ConfigI('data')
|
||||
self.path_factory = PathFactory(self._git, self._config)
|
||||
|
||||
self.cache_file = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME, 'file1.txt_123ed8')
|
||||
self.data_path = self.path_factory.data_item(os.path.join('..', '..', 'data', 'file1.txt'), cache_file=self.cache_file)
|
||||
pass
|
||||
|
||||
def test_data_file(self):
|
||||
target = os.path.join(self._test_git_dir, 'data', 'file1.txt')
|
||||
self.assertEqual(self.data_path.data.abs, target)
|
||||
|
||||
def test_cache_file(self):
|
||||
self.assertEqual(self.data_path.cache.abs, self.cache_file)
|
||||
|
||||
def test_state_file(self):
|
||||
target = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, 'data', 'file1.txt.state')
|
||||
self.assertEqual(self.data_path.state.abs, target)
|
||||
|
||||
|
||||
class RunOutsideGitRepoTest(BasicEnvironment):
|
||||
def setUp(self):
|
||||
curr_dir = os.path.join(os.path.sep, 'tmp')
|
||||
|
||||
BasicEnvironment.init_environment(self,
|
||||
os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'),
|
||||
curr_dir)
|
||||
|
||||
def test_outside_git_dir(self):
|
||||
git = GitWrapperI(git_dir=self._test_git_dir, commit='123ed8')
|
||||
config = ConfigI('data')
|
||||
path_factory = PathFactory(git, config)
|
||||
|
||||
with self.assertRaises(DataDirError):
|
||||
path_factory.data_item('file.txt')
|
||||
pass
|
||||
@@ -1,70 +0,0 @@
|
||||
import os
|
||||
import shutil
|
||||
from unittest import TestCase
|
||||
|
||||
from dvc.path.path import Path
|
||||
from dvc.git_wrapper import GitWrapperI
|
||||
from dvc.system import System
|
||||
|
||||
|
||||
class TestDvcPathTest(TestCase):
|
||||
def setUp(self):
|
||||
self.curr_dir = System.realpath('.')
|
||||
self.test_dir = System.realpath('/tmp/ntx_unit_test/dvc_path')
|
||||
self.tearDown()
|
||||
os.makedirs(os.path.join(self.test_dir, 'data'))
|
||||
os.makedirs(os.path.join(self.test_dir, 'code', 'lib'))
|
||||
os.makedirs(os.path.join(self.test_dir, 'd1', 'd2', 'dir3', 'd4', 'dir5'))
|
||||
|
||||
self._git = GitWrapperI(self.test_dir)
|
||||
|
||||
def tearDown(self):
|
||||
os.chdir(self.curr_dir)
|
||||
shutil.rmtree(self.test_dir, ignore_errors=True)
|
||||
|
||||
def _validate_dvc_path(self, path, dvc_file_name, relative_file_name):
|
||||
self.assertEqual(path.dvc, dvc_file_name)
|
||||
self.assertEqual(path.filename, os.path.basename(dvc_file_name))
|
||||
self.assertEqual(path.relative, relative_file_name)
|
||||
self.assertTrue(os.path.isabs(path.abs))
|
||||
self.assertTrue(path.abs.endswith(dvc_file_name))
|
||||
|
||||
def basic_test(self):
|
||||
os.chdir(self.test_dir)
|
||||
|
||||
file = os.path.join('data', 'file.txt')
|
||||
path = Path(file, self._git)
|
||||
self._validate_dvc_path(path, file, file)
|
||||
pass
|
||||
|
||||
def from_dir_test(self):
|
||||
os.chdir(os.path.join(self.test_dir, 'code'))
|
||||
|
||||
file_dvc_path = os.path.join('data', 'file1.txt')
|
||||
file_relative_path = os.path.join('..', file_dvc_path)
|
||||
|
||||
path = Path(file_relative_path, self._git)
|
||||
self._validate_dvc_path(path, file_dvc_path, file_relative_path)
|
||||
pass
|
||||
|
||||
def from_deep_dirs_test(self):
|
||||
deep_dir = os.path.join('d1', 'd2', 'dir3')
|
||||
os.chdir(os.path.join(self.test_dir, deep_dir))
|
||||
|
||||
file_dvc = os.path.join('code', 'lib', 'context_switcher_structs.asm')
|
||||
file_relative = os.path.join('..', '..', '..', file_dvc)
|
||||
|
||||
path = Path(file_relative, self._git)
|
||||
self._validate_dvc_path(path, file_dvc, file_relative)
|
||||
pass
|
||||
|
||||
def go_deeper_test(self):
|
||||
deep_dir = os.path.join('d1', 'd2', 'dir3')
|
||||
os.chdir(os.path.join(self.test_dir, deep_dir))
|
||||
|
||||
file_relative_path = os.path.join(deep_dir, 'd4', 'dir5', 'rawdata.tsv')
|
||||
file_dvc_path = os.path.join(deep_dir, file_relative_path)
|
||||
|
||||
path = Path(file_relative_path, self._git)
|
||||
self._validate_dvc_path(path, file_dvc_path, file_relative_path)
|
||||
pass
|
||||
@@ -0,0 +1,32 @@
|
||||
import os
|
||||
import shutil
|
||||
import filecmp
|
||||
|
||||
from dvc.data_cloud import file_md5
|
||||
from dvc.stage import Stage, OutputNoCacheError, OutputOutsideOfRepoError
|
||||
from dvc.stage import OutputDoesNotExistError, OutputIsNotFileError
|
||||
from dvc.project import StageNotFoundError
|
||||
|
||||
from tests.basic_env import TestDvc
|
||||
|
||||
|
||||
class TestGC(TestDvc):
|
||||
def test_gc(self):
|
||||
stage = self.dvc.add(self.FOO)
|
||||
good_cache = self.dvc.cache.all()
|
||||
|
||||
bad_cache = []
|
||||
for i in ['1', '2', '3']:
|
||||
path = os.path.join(self.dvc.cache.cache_dir, i)
|
||||
self.create(path, i)
|
||||
bad_cache.append(path)
|
||||
|
||||
self.dvc.gc()
|
||||
|
||||
self.assertTrue(os.path.isdir(self.dvc.cache.cache_dir))
|
||||
for c in bad_cache:
|
||||
self.assertFalse(os.path.exists(c))
|
||||
|
||||
for c in good_cache:
|
||||
self.assertTrue(os.path.exists(c))
|
||||
self.assertTrue(os.path.isfile(c))
|
||||
@@ -1,119 +0,0 @@
|
||||
import os
|
||||
import tempfile
|
||||
import filecmp
|
||||
from copy import copy
|
||||
|
||||
from dvc.command.import_file import CmdImportFile, ImportFileError
|
||||
from tests.basic_env import DirHierarchyEnvironment
|
||||
|
||||
|
||||
class TestCmdRemove(DirHierarchyEnvironment):
|
||||
def setUp(self):
|
||||
DirHierarchyEnvironment.init_environment(self)
|
||||
|
||||
def test_import_file_to_dir(self):
|
||||
with tempfile.NamedTemporaryFile(mode='w') as temp:
|
||||
content = 'Some data'
|
||||
temp.write(content)
|
||||
temp.flush()
|
||||
|
||||
dir = os.path.join('data', self.dir1)
|
||||
settings = copy(self.settings)
|
||||
settings.parse_args('import {} {}'.format(temp.name, dir))
|
||||
cmd = CmdImportFile(settings)
|
||||
cmd.run()
|
||||
|
||||
dvc_name = os.path.join(dir, os.path.basename(temp.name))
|
||||
data_item = self.path_factory.existing_data_item(dvc_name)
|
||||
|
||||
self.assertTrue(os.path.exists(data_item.data.relative))
|
||||
self.assertTrue(filecmp.cmp(data_item.data.relative, data_item.cache.relative))
|
||||
self.assertEqual(open(data_item.data.relative).read(), content)
|
||||
|
||||
self.assertTrue(os.path.exists(data_item.cache.relative))
|
||||
self.assertEqual(open(data_item.cache.relative).read(), content)
|
||||
|
||||
self.assertTrue(os.path.exists(data_item.state.relative))
|
||||
pass
|
||||
|
||||
def test_import_file_to_file(self):
|
||||
with tempfile.NamedTemporaryFile(mode='w') as temp:
|
||||
content = 'Some data'
|
||||
temp.write(content)
|
||||
temp.flush()
|
||||
|
||||
target_file_name = 'targ.txt'
|
||||
target_file = os.path.join('data', self.dir1, target_file_name)
|
||||
settings = copy(self.settings)
|
||||
settings.parse_args('import {} {}'.format(temp.name, target_file))
|
||||
cmd = CmdImportFile(settings)
|
||||
cmd.run()
|
||||
|
||||
self.assertTrue(os.path.exists(target_file))
|
||||
self.assertTrue(open(target_file).read(), content)
|
||||
pass
|
||||
|
||||
def test_import_to_existing_dir(self):
|
||||
with tempfile.NamedTemporaryFile(mode='w') as temp:
|
||||
content = 'Some data'
|
||||
temp.write(content)
|
||||
temp.flush()
|
||||
|
||||
dir = os.path.join('data', self.dir11)
|
||||
settings = copy(self.settings)
|
||||
settings.parse_args('import {} {}'.format(temp.name, dir))
|
||||
cmd = CmdImportFile(settings)
|
||||
cmd.run()
|
||||
|
||||
dvc_name = os.path.join(dir, os.path.basename(temp.name))
|
||||
|
||||
self.assertTrue(os.path.exists(dvc_name))
|
||||
self.assertTrue(open(dvc_name).read(), content)
|
||||
pass
|
||||
|
||||
def test_import_to_not_existing_dir(self):
|
||||
with tempfile.NamedTemporaryFile(mode='w') as temp:
|
||||
content = 'Some data'
|
||||
temp.write(content)
|
||||
temp.flush()
|
||||
|
||||
dir = os.path.join('data', self.dir1, 'not_existing_dir/')
|
||||
settings = copy(self.settings)
|
||||
settings.parse_args('import {} {}'.format(temp.name, dir))
|
||||
cmd = CmdImportFile(settings)
|
||||
|
||||
with self.assertRaises(ImportFileError):
|
||||
cmd.run()
|
||||
|
||||
dvc_name = os.path.join(dir, os.path.basename(temp.name))
|
||||
|
||||
self.assertFalse(os.path.exists(dir))
|
||||
self.assertFalse(os.path.exists(dvc_name))
|
||||
pass
|
||||
|
||||
def test_multiple_import(self):
|
||||
with tempfile.NamedTemporaryFile(mode='w') as temp1,\
|
||||
tempfile.NamedTemporaryFile(mode='w') as temp2:
|
||||
content = 'Some data'
|
||||
|
||||
temp1.write(content)
|
||||
temp1.flush()
|
||||
|
||||
temp2.write(content)
|
||||
temp2.flush()
|
||||
|
||||
dir = os.path.join('data', self.dir11)
|
||||
settings = copy(self.settings)
|
||||
settings.parse_args('import {} {} {}'.format(temp1.name, temp2.name, dir))
|
||||
cmd = CmdImportFile(settings)
|
||||
cmd.run()
|
||||
|
||||
dvc_name1 = os.path.join(dir, os.path.basename(temp1.name))
|
||||
dvc_name2 = os.path.join(dir, os.path.basename(temp2.name))
|
||||
|
||||
self.assertTrue(os.path.exists(dvc_name1))
|
||||
self.assertTrue(open(dvc_name1).read(), content)
|
||||
|
||||
self.assertTrue(os.path.exists(dvc_name2))
|
||||
self.assertTrue(open(dvc_name2).read(), content)
|
||||
pass
|
||||
@@ -0,0 +1,19 @@
|
||||
from dvc.lock import Lock
|
||||
|
||||
from tests.basic_env import TestDvc
|
||||
|
||||
|
||||
class TestLock(TestDvc):
|
||||
def test_lock(self):
|
||||
lock = Lock(self.dvc.dvc_dir)
|
||||
lock.lock()
|
||||
self.assertTrue(lock._lock.exists())
|
||||
self.assertTrue(lock._lock.acquired)
|
||||
lock.unlock()
|
||||
self.assertFalse(lock._lock.acquired)
|
||||
|
||||
def test_with(self):
|
||||
lock = Lock(self.dvc.dvc_dir)
|
||||
with lock:
|
||||
self.assertTrue(lock._lock.acquired)
|
||||
self.assertFalse(lock._lock.acquired)
|
||||
@@ -0,0 +1,28 @@
|
||||
import os
|
||||
|
||||
from dvc.stage import Stage, OutputOutsideOfRepoError
|
||||
from dvc.project import StageNotFoundError
|
||||
|
||||
from tests.basic_env import TestDvc
|
||||
|
||||
|
||||
class TestRemove(TestDvc):
|
||||
def test(self):
|
||||
stage = self.dvc.add(self.FOO)
|
||||
stages_removed = self.dvc.remove(self.FOO)
|
||||
|
||||
self.assertEqual(len(stages_removed), 1)
|
||||
self.assertIsInstance(stages_removed[0], Stage)
|
||||
self.assertEqual(stage.path, stages_removed[0].path)
|
||||
|
||||
|
||||
class TestRemoveNonExistentFile(TestDvc):
|
||||
def test(self):
|
||||
with self.assertRaises(StageNotFoundError) as cx:
|
||||
self.dvc.remove('non_existent_file')
|
||||
|
||||
|
||||
class TestRemoveFileOutsideOfRepo(TestDvc):
|
||||
def test(self):
|
||||
with self.assertRaises(OutputOutsideOfRepoError) as cx:
|
||||
self.dvc.remove(os.path.join(os.path.dirname(self.dvc.root_dir), self.FOO))
|
||||
@@ -1,98 +0,0 @@
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
from unittest import TestCase
|
||||
|
||||
from dvc.config import ConfigI
|
||||
from dvc.path.factory import PathFactory
|
||||
from dvc.git_wrapper import GitWrapperI
|
||||
from dvc.repository_change import RepositoryChange
|
||||
from dvc.settings import Settings
|
||||
from dvc.system import System
|
||||
|
||||
|
||||
class BasicTestRepositoryChange(TestCase):
|
||||
def setUp(self):
|
||||
self.test_dir = System.get_long_path(tempfile.mkdtemp())
|
||||
self._old_curr_dir_abs = System.realpath(os.curdir)
|
||||
|
||||
self.tearDown()
|
||||
os.mkdir(self.test_dir)
|
||||
os.chdir(self.test_dir)
|
||||
os.mkdir('data')
|
||||
|
||||
self._devnull = open(os.devnull, 'w')
|
||||
subprocess.Popen(['git', 'init'], stdout=self._devnull, stderr=None).wait()
|
||||
|
||||
self.git = GitWrapperI(self.test_dir)
|
||||
self.config = ConfigI()
|
||||
self.path_factory = PathFactory(self.git, self.config)
|
||||
self.settings = Settings([], self.git, self.config)
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.test_dir, ignore_errors=True)
|
||||
os.chdir(self._old_curr_dir_abs)
|
||||
|
||||
@staticmethod
|
||||
def create_file(file2):
|
||||
fd = open(file2, 'w+')
|
||||
fd.write('random text')
|
||||
fd.close()
|
||||
|
||||
|
||||
class TestRepositoryChange(BasicTestRepositoryChange):
|
||||
def test_all(self):
|
||||
file_to_modify = os.path.join('data', 'file1.txt')
|
||||
file_created_before_run = os.path.join('data', 'file2.txt')
|
||||
file_created_by_run = os.path.join('data', 'out.txt')
|
||||
file_to_remove = os.path.join('data', 'file3.txt')
|
||||
|
||||
self.create_file(file_to_modify)
|
||||
self.create_file(file_created_before_run)
|
||||
self.create_file(file_to_remove)
|
||||
|
||||
subprocess.Popen(['git', 'add', file_to_modify, file_to_remove],
|
||||
stdout=self._devnull, stderr=None).wait()
|
||||
subprocess.Popen(['git', 'commit', '-m', '"Adding one file3"'],
|
||||
stdout=self._devnull, stderr=None).wait()
|
||||
os.remove(file_to_remove)
|
||||
|
||||
change = RepositoryChange(['echo', 'test'],
|
||||
self.settings,
|
||||
file_created_by_run,
|
||||
file_to_modify,
|
||||
shell=True)
|
||||
|
||||
expected_new = [file_created_by_run, file_created_before_run]
|
||||
new_file_abs = [x.data.dvc for x in change.new_data_items]
|
||||
self.assertEqual(set(new_file_abs), set(expected_new))
|
||||
|
||||
self.assertEqual(len(change.removed_data_items), 1)
|
||||
self.assertEqual(change.removed_data_items[0].data.dvc, file_to_remove)
|
||||
self.assertEqual(len(change.modified_data_items), 1)
|
||||
self.assertEqual(change.modified_data_items[0].data.dvc, file_to_modify)
|
||||
|
||||
expected_to_change_set = set(expected_new + [file_to_modify])
|
||||
changed_file_names = [x.data.dvc for x in change.changed_data_items]
|
||||
self.assertEqual(set(changed_file_names), expected_to_change_set)
|
||||
|
||||
self.assertEqual([], change.externally_created_files)
|
||||
pass
|
||||
|
||||
|
||||
class TestRepositoryChangeDeepInDirectory(BasicTestRepositoryChange):
|
||||
def test(self):
|
||||
os.mkdir(os.path.join('data', 'dir32'))
|
||||
deep_file = os.path.join('data', 'dir32', 'file1.txt')
|
||||
self.create_file(deep_file)
|
||||
|
||||
change = RepositoryChange(['echo', 'text'],
|
||||
self.settings,
|
||||
deep_file,
|
||||
None,
|
||||
shell=True)
|
||||
|
||||
self.assertEqual([x.data.dvc for x in change.new_data_items], [deep_file])
|
||||
|
||||
|
||||
+50
-72
@@ -1,90 +1,73 @@
|
||||
import os
|
||||
|
||||
from dvc.command.remove import CmdRemove
|
||||
from dvc.command.repro import CmdRepro
|
||||
from dvc.command.run import CmdRun
|
||||
from dvc.executor import Executor
|
||||
from tests.test_cmd_run import RunBasicTest
|
||||
from tests.basic_env import TestDvc
|
||||
|
||||
|
||||
class ReproBasicEnv(RunBasicTest):
|
||||
class ReproBasicEnv(TestDvc):
|
||||
def setUp(self):
|
||||
super(ReproBasicEnv, self).setUp()
|
||||
|
||||
os.mkdir('data')
|
||||
|
||||
self.file_name1 = os.path.join('data', 'file1')
|
||||
self.file1_code_file = 'file1.py'
|
||||
self.create_file_and_commit(self.file1_code_file, 'print("Hello")' + os.linesep + 'print("Mary")')
|
||||
self.settings.parse_args(['run',
|
||||
'--stdout', self.file_name1,
|
||||
'--code', self.file1_code_file,
|
||||
'python', self.file1_code_file, '--not-repro'])
|
||||
cmd_file1 = CmdRun(self.settings)
|
||||
self.assertEqual(cmd_file1.code_dependencies, [self.file1_code_file])
|
||||
cmd_file1.run()
|
||||
self.create(self.file1_code_file, 'print("Hello")' + os.linesep + 'print("Mary")')
|
||||
stage = self.dvc.run(fname='file1.dvc',
|
||||
outs=[self.file_name1],
|
||||
deps_no_cache=[self.file1_code_file],
|
||||
cmd='python {} --not-repro > {}'.format(self.file1_code_file,
|
||||
self.file_name1))
|
||||
|
||||
self.file_name11 = os.path.join('data', 'file11')
|
||||
self.file11_code_file = 'file11.py'
|
||||
self.create_file_and_commit(self.file11_code_file,
|
||||
'import sys' + os.linesep + 'print(open(sys.argv[1]).readline().strip())')
|
||||
self.create(self.file11_code_file,
|
||||
'import sys' + os.linesep + 'print(open(sys.argv[1]).readline().strip())')
|
||||
|
||||
self.settings.parse_args(['run',
|
||||
'--stdout', self.file_name11,
|
||||
'--code', self.file11_code_file,
|
||||
'python', self.file11_code_file, self.file_name1])
|
||||
CmdRun(self.settings).run()
|
||||
stage = self.dvc.run(fname='file11.dvc',
|
||||
outs=[self.file_name11],
|
||||
deps=[self.file_name1],
|
||||
deps_no_cache=[self.file11_code_file],
|
||||
cmd='python {} {} > {}'.format(self.file11_code_file,
|
||||
self.file_name1,
|
||||
self.file_name11))
|
||||
self.file_name11_stage_name = stage.path
|
||||
|
||||
self.file_name2 = os.path.join('data', 'file2')
|
||||
self.file2_code_file = 'file2.py'
|
||||
self.create_file_and_commit(self.file2_code_file,
|
||||
'print("Bobby")')
|
||||
self.settings.parse_args(['run',
|
||||
'--stdout', self.file_name2,
|
||||
'python', self.file2_code_file, '--not-repro'])
|
||||
CmdRun(self.settings).run()
|
||||
self.create(self.file2_code_file, 'print("Bobby")')
|
||||
self.dvc.run(fname='file2.dvc',
|
||||
outs=[self.file_name2],
|
||||
cmd='python {} --not-repro > {}'.format(self.file2_code_file,
|
||||
self.file_name2))
|
||||
|
||||
self.file_res_code_file = 'code_res.py'
|
||||
self.create_file_and_commit(self.file_res_code_file,
|
||||
'import sys' + os.linesep +
|
||||
'text1 = open(sys.argv[1]).read()' + os.linesep +
|
||||
'text2 = open(sys.argv[2]).read()' + os.linesep +
|
||||
'print(text1 + text2)')
|
||||
self.create(self.file_res_code_file,
|
||||
'import sys' + os.linesep +
|
||||
'text1 = open(sys.argv[1]).read()' + os.linesep +
|
||||
'text2 = open(sys.argv[2]).read()' + os.linesep +
|
||||
'print(text1 + text2)')
|
||||
self.file_name_res = os.path.join('data', 'file_res')
|
||||
self.settings.parse_args(['run',
|
||||
'--stdout', self.file_name_res,
|
||||
'--code', self.file_res_code_file,
|
||||
'python', self.file_res_code_file,
|
||||
self.file_name11,
|
||||
self.file_name2])
|
||||
stage = self.dvc.run(fname='file_res.dvc',
|
||||
outs=[self.file_name_res],
|
||||
deps_no_cache=[self.file_res_code_file],
|
||||
deps=[self.file_name11, self.file_name2],
|
||||
cmd='python {} {} {} > {}'.format(self.file_res_code_file,
|
||||
self.file_name11,
|
||||
self.file_name2,
|
||||
self.file_name_res))
|
||||
self.file_name_res_stage_name = stage.path
|
||||
|
||||
cmd_res = CmdRun(self.settings)
|
||||
self.assertEqual(cmd_res.code_dependencies, [self.file_res_code_file])
|
||||
cmd_res.run()
|
||||
|
||||
lines = list(filter(None, map(str.strip, open(self.file_name_res).readlines())))
|
||||
self.assertEqual(lines, ['Hello', 'Bobby'])
|
||||
|
||||
def create_file_and_commit(self, file_name, content='Any', message='Just a commit'):
|
||||
self.create_file(file_name, content)
|
||||
self.commit_file(file_name, message)
|
||||
|
||||
@staticmethod
|
||||
def commit_file(file_name, message='Just a commit'):
|
||||
Executor.exec_cmd_only_success(['git', 'add', file_name])
|
||||
Executor.exec_cmd_only_success(['git', 'commit', '-m', message])
|
||||
|
||||
def modify_file_and_commit(self, filename, content_to_add=' '):
|
||||
def modify_file(self, filename, content_to_add=' '):
|
||||
fd = open(filename, 'a')
|
||||
fd.write(content_to_add)
|
||||
fd.close()
|
||||
self.commit_file(filename)
|
||||
|
||||
|
||||
class ReproCodeDependencyTest(ReproBasicEnv):
|
||||
def test(self):
|
||||
self.modify_file_and_commit(self.file_res_code_file)
|
||||
self.modify_file(self.file_res_code_file)
|
||||
|
||||
self.settings.parse_args('repro {}'.format(self.file_name_res))
|
||||
CmdRepro(self.settings).run()
|
||||
self.dvc.reproduce([self.file_name_res_stage_name])
|
||||
|
||||
self.assertEqual(open(self.file_name_res).read().strip(), 'Hello\nBobby')
|
||||
|
||||
@@ -93,30 +76,25 @@ class ReproChangedDependency(ReproBasicEnv):
|
||||
def test(self):
|
||||
self.recreate_file1()
|
||||
|
||||
self.settings.parse_args('repro {}'.format(self.file_name11))
|
||||
CmdRepro(self.settings).run()
|
||||
self.dvc.reproduce([self.file_name11_stage_name])
|
||||
|
||||
self.assertEqual(open(self.file_name11).read(), 'Goodbye\n')
|
||||
|
||||
def recreate_file1(self):
|
||||
self.settings.parse_args('remove {} --keep-in-cloud'.format(self.file_name1))
|
||||
CmdRemove(self.settings).run()
|
||||
self.dvc.remove(self.file_name1)
|
||||
|
||||
file1_code_file = 'file1_2.py'
|
||||
self.create_file_and_commit(file1_code_file, 'print("Goodbye")' + os.linesep + 'print("Jack")')
|
||||
self.settings.parse_args(['run',
|
||||
'--stdout', self.file_name1,
|
||||
'--code', file1_code_file,
|
||||
'python', file1_code_file, '--not-repro'])
|
||||
|
||||
CmdRun(self.settings).run()
|
||||
self.create(file1_code_file, 'print("Goodbye")' + os.linesep + 'print("Jack")')
|
||||
self.dvc.run(fname='file1.dvc',
|
||||
outs=[self.file_name1],
|
||||
deps_no_cache=[file1_code_file],
|
||||
cmd='python {} --not-repro > {}'.format(file1_code_file, self.file_name1))
|
||||
|
||||
|
||||
class ReproChangedDeepDependency(ReproChangedDependency):
|
||||
def test(self):
|
||||
self.recreate_file1()
|
||||
|
||||
self.settings.parse_args('repro {}'.format(self.file_name_res))
|
||||
CmdRepro(self.settings).run()
|
||||
self.dvc.reproduce([self.file_name_res_stage_name])
|
||||
|
||||
self.assertEqual(open(self.file_name_res).read().strip(), 'Goodbye\nBobby')
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
import os
|
||||
import filecmp
|
||||
|
||||
from dvc.data_cloud import file_md5
|
||||
|
||||
from tests.basic_env import TestDvc
|
||||
|
||||
|
||||
class TestRun(TestDvc):
|
||||
def test(self):
|
||||
cmd = 'python {} {} {}'.format(self.CODE, self.FOO, 'out')
|
||||
deps = [self.FOO]
|
||||
deps_no_cache = [self.CODE]
|
||||
outs = [os.path.join(self.dvc.root_dir, 'out')]
|
||||
outs_no_cache = []
|
||||
locked = False
|
||||
fname = os.path.join(self.dvc.root_dir, 'out.dvc')
|
||||
cwd = os.curdir
|
||||
|
||||
stage = self.dvc.run(cmd, deps, deps_no_cache, outs, outs_no_cache, locked, fname, cwd)
|
||||
|
||||
self.assertTrue(filecmp.cmp(self.FOO, 'out'))
|
||||
self.assertTrue(os.path.isfile(stage.path))
|
||||
self.assertEqual(stage.cmd, cmd)
|
||||
self.assertEqual(len(stage.deps), len(deps + deps_no_cache))
|
||||
self.assertEqual(len(stage.outs), len(outs + outs_no_cache))
|
||||
self.assertEqual(stage.outs[0].path, outs[0])
|
||||
self.assertEqual(stage.outs[0].md5, file_md5(self.FOO)[0])
|
||||
self.assertEqual(stage.locked, locked)
|
||||
self.assertTrue(stage.path, fname)
|
||||
|
||||
|
||||
class TestRunEmpty(TestDvc):
|
||||
def test(self):
|
||||
self.dvc.run('', [], [], [], [], False, 'empty.dvc', os.curdir)
|
||||
@@ -0,0 +1,32 @@
|
||||
import os
|
||||
|
||||
from git import Repo
|
||||
|
||||
from dvc.scm import SCM, Base, Git
|
||||
|
||||
from tests.basic_env import TestDir, TestGit
|
||||
|
||||
|
||||
class TestSCM(TestDir):
|
||||
def test_none(self):
|
||||
self.assertIsInstance(SCM(self._root_dir), Base)
|
||||
|
||||
def test_git(self):
|
||||
Repo.init(os.curdir)
|
||||
self.assertIsInstance(SCM(self._root_dir), Git)
|
||||
|
||||
|
||||
class TestSCMGit(TestGit):
|
||||
def test_is_repo(self):
|
||||
self.assertTrue(Git.is_repo(os.curdir))
|
||||
|
||||
def test_ignore(self):
|
||||
Git(self._root_dir).ignore('foo')
|
||||
self.assertTrue(os.path.isfile(Git.GITIGNORE))
|
||||
self.assertTrue('foo' in open(Git.GITIGNORE, 'r').readlines())
|
||||
|
||||
def test_commit(self):
|
||||
G = Git(self._root_dir)
|
||||
G.add(['foo'])
|
||||
G.commit('add')
|
||||
self.assertTrue('foo' in self.git.git.ls_files())
|
||||
@@ -0,0 +1,31 @@
|
||||
import os
|
||||
|
||||
from dvc.state import State, StateEntry
|
||||
|
||||
from tests.basic_env import TestDvc
|
||||
|
||||
|
||||
class TestState(TestDvc):
|
||||
def test_update(self):
|
||||
path = os.path.join(self.dvc.root_dir, '1')
|
||||
md5_1 = '1'
|
||||
md5_2 = '2'
|
||||
mtime_1 = 1
|
||||
mtime_2 = 2
|
||||
|
||||
state = State(self.dvc.root_dir, self.dvc.dvc_dir)
|
||||
self.assertIsNone(state.get(path))
|
||||
|
||||
state.add(path, md5_1, mtime_1)
|
||||
entry = state.get(path)
|
||||
self.assertIsInstance(entry, StateEntry)
|
||||
self.assertEqual(entry.path, path)
|
||||
self.assertEqual(entry.md5, md5_1)
|
||||
self.assertEqual(entry.mtime, mtime_1)
|
||||
|
||||
state.update(path, md5_2, mtime_2)
|
||||
entry = state.get(path)
|
||||
self.assertIsInstance(entry, StateEntry)
|
||||
self.assertEqual(entry.path, path)
|
||||
self.assertEqual(entry.md5, md5_2)
|
||||
self.assertEqual(entry.mtime, mtime_2)
|
||||
+2
-19
@@ -3,27 +3,10 @@ import shutil
|
||||
import filecmp
|
||||
|
||||
from dvc import utils
|
||||
from tests.basic_env import DirHierarchyEnvironment
|
||||
from tests.basic_env import TestDvc
|
||||
|
||||
class TestExecutor(DirHierarchyEnvironment):
|
||||
def setUp(self):
|
||||
DirHierarchyEnvironment.init_environment(self)
|
||||
|
||||
def tearDown(self):
|
||||
pass
|
||||
|
||||
def test_rmtree(self):
|
||||
root = 'testdir'
|
||||
|
||||
os.makedirs(root + '/subdir')
|
||||
with open(root + '/file1', 'w+') as f:
|
||||
f.write('file1contents')
|
||||
with open(root + '/subdir/file2', 'w+') as f:
|
||||
f.write('file2contents')
|
||||
|
||||
utils.rmtree(root)
|
||||
self.assertFalse(os.path.exists(root))
|
||||
|
||||
class TestUtils(TestDvc):
|
||||
def test_copyfile(self):
|
||||
src = 'file1'
|
||||
dest = 'file2'
|
||||
|
||||
Reference in New Issue
Block a user