mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-04 15:28:27 +01:00
491 lines
16 KiB
Python
491 lines
16 KiB
Python
import os
|
|
import mock
|
|
import shutil
|
|
import filecmp
|
|
import subprocess
|
|
|
|
from dvc.main import main
|
|
from dvc.utils import file_md5
|
|
from dvc.system import System
|
|
from dvc.stage import Stage
|
|
from dvc.stage import StageFileBadNameError, MissingDep
|
|
from dvc.stage import StageBadCwdError, StageFileAlreadyExistsError
|
|
from dvc.exceptions import (OutputDuplicationError,
|
|
CircularDependencyError,
|
|
CyclicGraphError,
|
|
ArgumentDuplicationError,
|
|
WorkingDirectoryAsOutputError)
|
|
|
|
from tests.basic_env import TestDvc
|
|
|
|
|
|
class TestRun(TestDvc):
|
|
def test(self):
|
|
cmd = 'python {} {} {}'.format(self.CODE, self.FOO, 'out')
|
|
deps = [self.FOO, self.CODE]
|
|
outs = [os.path.join(self.dvc.root_dir, 'out')]
|
|
outs_no_cache = []
|
|
fname = 'out.dvc'
|
|
cwd = os.curdir
|
|
|
|
self.dvc.add(self.FOO)
|
|
stage = self.dvc.run(cmd=cmd,
|
|
deps=deps,
|
|
outs=outs,
|
|
outs_no_cache=outs_no_cache,
|
|
fname=fname,
|
|
cwd=cwd)
|
|
|
|
self.assertTrue(filecmp.cmp(self.FOO, 'out', shallow=False))
|
|
self.assertTrue(os.path.isfile(stage.path))
|
|
self.assertEqual(stage.cmd, cmd)
|
|
self.assertEqual(len(stage.deps), len(deps))
|
|
self.assertEqual(len(stage.outs), len(outs + outs_no_cache))
|
|
self.assertEqual(stage.outs[0].path, outs[0])
|
|
self.assertEqual(stage.outs[0].checksum, file_md5(self.FOO)[0])
|
|
self.assertTrue(stage.path, fname)
|
|
|
|
with self.assertRaises(OutputDuplicationError):
|
|
stage = self.dvc.run(cmd=cmd,
|
|
deps=deps,
|
|
outs=outs,
|
|
outs_no_cache=outs_no_cache,
|
|
fname='duplicate' + fname,
|
|
cwd=cwd)
|
|
|
|
|
|
class TestRunEmpty(TestDvc):
|
|
def test(self):
|
|
self.dvc.run(cmd='',
|
|
deps=[],
|
|
outs=[],
|
|
outs_no_cache=[],
|
|
fname='empty.dvc',
|
|
cwd=os.curdir)
|
|
|
|
|
|
class TestRunMissingDep(TestDvc):
|
|
def test(self):
|
|
with self.assertRaises(MissingDep):
|
|
self.dvc.run(cmd='',
|
|
deps=['non-existing-dep'],
|
|
outs=[],
|
|
outs_no_cache=[],
|
|
fname='empty.dvc',
|
|
cwd=os.curdir)
|
|
|
|
|
|
class TestRunBadStageFilename(TestDvc):
|
|
def test(self):
|
|
with self.assertRaises(StageFileBadNameError):
|
|
self.dvc.run(cmd='',
|
|
deps=[],
|
|
outs=[],
|
|
outs_no_cache=[],
|
|
fname='empty',
|
|
cwd=os.curdir)
|
|
|
|
with self.assertRaises(StageFileBadNameError):
|
|
self.dvc.run(cmd='',
|
|
deps=[],
|
|
outs=[],
|
|
outs_no_cache=[],
|
|
fname=os.path.join(self.DATA_DIR, 'empty.dvc'),
|
|
cwd=os.curdir)
|
|
|
|
|
|
class TestRunNoExec(TestDvc):
|
|
def test(self):
|
|
self.dvc.run(cmd='python {} {} {}'.format(self.CODE, self.FOO, 'out'),
|
|
no_exec=True)
|
|
self.assertFalse(os.path.exists('out'))
|
|
|
|
|
|
class TestRunCircularDependency(TestDvc):
|
|
def test(self):
|
|
with self.assertRaises(CircularDependencyError):
|
|
self.dvc.run(cmd='',
|
|
deps=[self.FOO],
|
|
outs=[self.FOO],
|
|
fname='circular-dependency.dvc')
|
|
|
|
def test_outs_no_cache(self):
|
|
with self.assertRaises(CircularDependencyError):
|
|
self.dvc.run(cmd='',
|
|
deps=[self.FOO],
|
|
outs_no_cache=[self.FOO],
|
|
fname='circular-dependency.dvc')
|
|
|
|
def test_non_normalized_paths(self):
|
|
with self.assertRaises(CircularDependencyError):
|
|
self.dvc.run(cmd='',
|
|
deps=['./foo'],
|
|
outs=['foo'],
|
|
fname='circular-dependency.dvc')
|
|
|
|
def test_graph(self):
|
|
self.dvc.run(deps=[self.FOO],
|
|
outs=['bar.txt'],
|
|
cmd='echo bar > bar.txt')
|
|
|
|
self.dvc.run(deps=['bar.txt'],
|
|
outs=['baz.txt'],
|
|
cmd='echo baz > baz.txt')
|
|
|
|
with self.assertRaises(CyclicGraphError):
|
|
self.dvc.run(deps=['baz.txt'],
|
|
outs=[self.FOO],
|
|
cmd='echo baz > foo')
|
|
|
|
|
|
class TestRunDuplicatedArguments(TestDvc):
|
|
def test(self):
|
|
with self.assertRaises(ArgumentDuplicationError):
|
|
self.dvc.run(cmd='',
|
|
deps=[],
|
|
outs=[self.FOO, self.FOO],
|
|
fname='circular-dependency.dvc')
|
|
|
|
def test_outs_no_cache(self):
|
|
with self.assertRaises(ArgumentDuplicationError):
|
|
self.dvc.run(cmd='',
|
|
outs=[self.FOO],
|
|
outs_no_cache=[self.FOO],
|
|
fname='circular-dependency.dvc')
|
|
|
|
def test_non_normalized_paths(self):
|
|
with self.assertRaises(ArgumentDuplicationError):
|
|
self.dvc.run(cmd='',
|
|
deps=[],
|
|
outs=['foo', './foo'],
|
|
fname='circular-dependency.dvc')
|
|
|
|
|
|
class TestRunWorkingDirectoryAsOutput(TestDvc):
|
|
def test(self):
|
|
self.dvc.run(cmd='',
|
|
deps=[],
|
|
outs=[self.DATA_DIR])
|
|
|
|
with self.assertRaises(WorkingDirectoryAsOutputError):
|
|
self.dvc.run(cmd='',
|
|
cwd=self.DATA_DIR,
|
|
outs=[self.FOO],
|
|
fname='inside-cwd.dvc')
|
|
|
|
|
|
class TestRunBadCwd(TestDvc):
|
|
def test(self):
|
|
with self.assertRaises(StageBadCwdError):
|
|
self.dvc.run(cmd='',
|
|
cwd=self.mkdtemp())
|
|
|
|
|
|
class TestCmdRun(TestDvc):
|
|
def test_run(self):
|
|
ret = main(['run',
|
|
'-d', self.FOO,
|
|
'-d', self.CODE,
|
|
'-o', 'out',
|
|
'-f', 'out.dvc',
|
|
'python', self.CODE, self.FOO, 'out'])
|
|
|
|
stage = Stage.load(self.dvc, fname='out.dvc')
|
|
|
|
self.assertEqual(ret, 0)
|
|
self.assertTrue(os.path.isfile('out'))
|
|
self.assertTrue(os.path.isfile('out.dvc'))
|
|
self.assertTrue(filecmp.cmp(self.FOO, 'out', shallow=False))
|
|
self.assertEqual(stage.cmd, 'python code.py foo out')
|
|
|
|
def test_run_args_from_cli(self):
|
|
ret = main(['run', 'echo', 'foo'])
|
|
stage = Stage.load(self.dvc, fname='Dvcfile')
|
|
self.assertEqual(ret, 0)
|
|
self.assertEqual(stage.cmd, 'echo foo')
|
|
|
|
def test_run_bad_command(self):
|
|
ret = main(['run', 'non-existing-command'])
|
|
self.assertNotEqual(ret, 0)
|
|
|
|
def test_run_args_with_spaces(self):
|
|
ret = main(['run', 'echo', 'foo bar'])
|
|
stage = Stage.load(self.dvc, fname='Dvcfile')
|
|
self.assertEqual(ret, 0)
|
|
self.assertEqual(stage.cmd, 'echo "foo bar"')
|
|
|
|
@mock.patch.object(subprocess, 'Popen', side_effect=KeyboardInterrupt)
|
|
def test_keyboard_interrupt(self, mock_popen):
|
|
ret = main(['run', 'mycmd'])
|
|
self.assertEqual(ret, 252)
|
|
|
|
|
|
class TestRunRemoveOuts(TestDvc):
|
|
def test(self):
|
|
with open(self.CODE, 'w+') as fobj:
|
|
fobj.write("import sys\n")
|
|
fobj.write("import os\n")
|
|
fobj.write("if os.path.exists(sys.argv[1]):\n")
|
|
fobj.write(" sys.exit(1)\n")
|
|
fobj.write("open(sys.argv[1], 'w+').close()\n")
|
|
|
|
ret = main(['run',
|
|
'--remove-outs',
|
|
'-d', self.CODE,
|
|
'-o', self.FOO,
|
|
'python', self.CODE, self.FOO])
|
|
self.assertEqual(ret, 0)
|
|
|
|
|
|
class TestRunUnprotectOutsCopy(TestDvc):
|
|
def test(self):
|
|
with open(self.CODE, 'w+') as fobj:
|
|
fobj.write("import sys\n")
|
|
fobj.write("with open(sys.argv[1], 'a+') as fobj:\n")
|
|
fobj.write(" fobj.write('foo')\n")
|
|
|
|
ret = main(['config', 'cache.protected', 'true'])
|
|
self.assertEqual(ret, 0)
|
|
|
|
ret = main(['config', 'cache.type', 'copy'])
|
|
self.assertEqual(ret, 0)
|
|
|
|
ret = main(['run',
|
|
'-d', self.CODE,
|
|
'-o', self.FOO,
|
|
'python', self.CODE, self.FOO])
|
|
self.assertEqual(ret, 0)
|
|
self.assertFalse(os.access(self.FOO, os.W_OK))
|
|
self.assertEqual(open(self.FOO, 'r').read(), 'foofoo')
|
|
|
|
ret = main(['run',
|
|
'--overwrite-dvcfile',
|
|
'--ignore-build-cache',
|
|
'-d', self.CODE,
|
|
'-o', self.FOO,
|
|
'python', self.CODE, self.FOO])
|
|
self.assertEqual(ret, 0)
|
|
self.assertFalse(os.access(self.FOO, os.W_OK))
|
|
self.assertEqual(open(self.FOO, 'r').read(), 'foofoofoo')
|
|
|
|
|
|
class TestRunUnprotectOutsSymlink(TestDvc):
|
|
def test(self):
|
|
with open(self.CODE, 'w+') as fobj:
|
|
fobj.write("import sys\n")
|
|
fobj.write("import os\n")
|
|
fobj.write("assert os.path.exists(sys.argv[1])\n")
|
|
fobj.write("with open(sys.argv[1], 'a+') as fobj:\n")
|
|
fobj.write(" fobj.write('foo')\n")
|
|
|
|
ret = main(['config', 'cache.protected', 'true'])
|
|
self.assertEqual(ret, 0)
|
|
|
|
ret = main(['config', 'cache.type', 'symlink'])
|
|
self.assertEqual(ret, 0)
|
|
|
|
self.assertEqual(ret, 0)
|
|
ret = main(['run',
|
|
'-d', self.CODE,
|
|
'-o', self.FOO,
|
|
'python', self.CODE, self.FOO])
|
|
self.assertEqual(ret, 0)
|
|
self.assertFalse(os.access(self.FOO, os.W_OK))
|
|
self.assertTrue(System.is_symlink(self.FOO))
|
|
self.assertEqual(open(self.FOO, 'r').read(), 'foofoo')
|
|
|
|
ret = main(['run',
|
|
'--overwrite-dvcfile',
|
|
'--ignore-build-cache',
|
|
'-d', self.CODE,
|
|
'-o', self.FOO,
|
|
'python', self.CODE, self.FOO])
|
|
self.assertEqual(ret, 0)
|
|
self.assertFalse(os.access(self.FOO, os.W_OK))
|
|
self.assertTrue(System.is_symlink(self.FOO))
|
|
self.assertEqual(open(self.FOO, 'r').read(), 'foofoofoo')
|
|
|
|
|
|
class TestRunUnprotectOutsHardlink(TestDvc):
|
|
def test(self):
|
|
with open(self.CODE, 'w+') as fobj:
|
|
fobj.write("import sys\n")
|
|
fobj.write("import os\n")
|
|
fobj.write("assert os.path.exists(sys.argv[1])\n")
|
|
fobj.write("with open(sys.argv[1], 'a+') as fobj:\n")
|
|
fobj.write(" fobj.write('foo')\n")
|
|
|
|
ret = main(['config', 'cache.protected', 'true'])
|
|
self.assertEqual(ret, 0)
|
|
|
|
ret = main(['config', 'cache.type', 'hardlink'])
|
|
self.assertEqual(ret, 0)
|
|
|
|
self.assertEqual(ret, 0)
|
|
ret = main(['run',
|
|
'-d', self.CODE,
|
|
'-o', self.FOO,
|
|
'python', self.CODE, self.FOO])
|
|
self.assertEqual(ret, 0)
|
|
self.assertFalse(os.access(self.FOO, os.W_OK))
|
|
self.assertTrue(System.is_hardlink(self.FOO))
|
|
self.assertEqual(open(self.FOO, 'r').read(), 'foofoo')
|
|
|
|
ret = main(['run',
|
|
'--overwrite-dvcfile',
|
|
'--ignore-build-cache',
|
|
'-d', self.CODE,
|
|
'-o', self.FOO,
|
|
'python', self.CODE, self.FOO])
|
|
self.assertEqual(ret, 0)
|
|
self.assertFalse(os.access(self.FOO, os.W_OK))
|
|
self.assertTrue(System.is_hardlink(self.FOO))
|
|
self.assertEqual(open(self.FOO, 'r').read(), 'foofoofoo')
|
|
|
|
|
|
class TestCmdRunOverwrite(TestDvc):
|
|
def test(self):
|
|
# NOTE: using sleep() is a workaround for filesystems
|
|
# with low mtime resolution. We have to use mtime since
|
|
# comparing mtime's is the only way to check that the stage
|
|
# file didn't change(size and inode in the first test down
|
|
# below don't change).
|
|
import time
|
|
|
|
ret = main(['run',
|
|
'-d', self.FOO,
|
|
'-d', self.CODE,
|
|
'-o', 'out',
|
|
'-f', 'out.dvc',
|
|
'python', self.CODE, self.FOO, 'out'])
|
|
self.assertEqual(ret, 0)
|
|
|
|
stage_mtime = os.path.getmtime('out.dvc')
|
|
|
|
time.sleep(1)
|
|
|
|
ret = main(['run',
|
|
'-d', self.FOO,
|
|
'-d', self.CODE,
|
|
'-o', 'out',
|
|
'-f', 'out.dvc',
|
|
'python', self.CODE, self.FOO, 'out'])
|
|
self.assertEqual(ret, 0)
|
|
|
|
# NOTE: check that dvcfile was NOT overwritten
|
|
self.assertEqual(stage_mtime, os.path.getmtime('out.dvc'))
|
|
stage_mtime = os.path.getmtime('out.dvc')
|
|
|
|
time.sleep(1)
|
|
|
|
ret = main(['run',
|
|
'-d', self.FOO,
|
|
'-d', self.CODE,
|
|
'--overwrite-dvcfile',
|
|
'--ignore-build-cache',
|
|
'-o', 'out',
|
|
'-f', 'out.dvc',
|
|
'python', self.CODE, self.FOO, 'out'])
|
|
self.assertEqual(ret, 0)
|
|
|
|
# NOTE: check that dvcfile was overwritten
|
|
self.assertNotEqual(stage_mtime, os.path.getmtime('out.dvc'))
|
|
stage_mtime = os.path.getmtime('out.dvc')
|
|
|
|
time.sleep(1)
|
|
|
|
ret = main(['run',
|
|
'--overwrite-dvcfile',
|
|
'-f', 'out.dvc',
|
|
'-d', self.BAR])
|
|
self.assertEqual(ret, 0)
|
|
|
|
# NOTE: check that dvcfile was overwritten
|
|
self.assertNotEqual(stage_mtime, os.path.getmtime('out.dvc'))
|
|
|
|
|
|
class TestRunDeterministicBase(TestDvc):
|
|
def setUp(self):
|
|
super(TestRunDeterministicBase, self).setUp()
|
|
self.out_file = 'out'
|
|
self.stage_file = self.out_file + '.dvc'
|
|
self.cmd = 'python {} {} {}'.format(self.CODE, self.FOO, self.out_file)
|
|
self.deps = [self.FOO, self.CODE]
|
|
self.outs = [self.out_file]
|
|
self.overwrite = False
|
|
self.ignore_build_cache = False
|
|
|
|
self._run()
|
|
|
|
def _run(self):
|
|
self.stage = self.dvc.run(cmd=self.cmd,
|
|
fname=self.stage_file,
|
|
overwrite=self.overwrite,
|
|
ignore_build_cache=self.ignore_build_cache,
|
|
deps=self.deps,
|
|
outs=self.outs)
|
|
|
|
|
|
class TestRunDeterministic(TestRunDeterministicBase):
|
|
def test(self):
|
|
self._run()
|
|
|
|
|
|
class TestRunDeterministicOverwrite(TestRunDeterministicBase):
|
|
def test(self):
|
|
self.overwrite = True
|
|
self.ignore_build_cache = True
|
|
self._run()
|
|
|
|
|
|
class TestRunDeterministicCallback(TestRunDeterministicBase):
|
|
def test(self):
|
|
self.stage.remove()
|
|
self.deps = []
|
|
self._run()
|
|
self._run()
|
|
|
|
|
|
class TestRunDeterministicChangedDep(TestRunDeterministicBase):
|
|
def test(self):
|
|
os.unlink(self.FOO)
|
|
shutil.copy(self.BAR, self.FOO)
|
|
with self.assertRaises(StageFileAlreadyExistsError):
|
|
self._run()
|
|
|
|
|
|
class TestRunDeterministicChangedDepsList(TestRunDeterministicBase):
|
|
def test(self):
|
|
self.deps = [self.BAR, self.CODE]
|
|
with self.assertRaises(StageFileAlreadyExistsError):
|
|
self._run()
|
|
|
|
|
|
class TestRunDeterministicNewDep(TestRunDeterministicBase):
|
|
def test(self):
|
|
self.deps = [self.FOO, self.BAR, self.CODE]
|
|
with self.assertRaises(StageFileAlreadyExistsError):
|
|
self._run()
|
|
|
|
|
|
class TestRunDeterministicRemoveDep(TestRunDeterministicBase):
|
|
def test(self):
|
|
self.deps = [self.CODE]
|
|
with self.assertRaises(StageFileAlreadyExistsError):
|
|
self._run()
|
|
|
|
|
|
class TestRunDeterministicChangedOut(TestRunDeterministicBase):
|
|
def test(self):
|
|
os.unlink(self.out_file)
|
|
self.out_file_mtime = None
|
|
with self.assertRaises(StageFileAlreadyExistsError):
|
|
self._run()
|
|
|
|
|
|
class TestRunDeterministicChangedCmd(TestRunDeterministicBase):
|
|
def test(self):
|
|
self.cmd += ' arg'
|
|
with self.assertRaises(StageFileAlreadyExistsError):
|
|
self._run()
|