import os import mock import shutil import filecmp import subprocess from dvc.main import main from dvc.utils import file_md5 from dvc.system import System from dvc.stage import Stage from dvc.stage import StageFileBadNameError, MissingDep from dvc.stage import StageBadCwdError, StageFileAlreadyExistsError from dvc.exceptions import ( OutputDuplicationError, CircularDependencyError, CyclicGraphError, ArgumentDuplicationError, WorkingDirectoryAsOutputError, ) from tests.basic_env import TestDvc class TestRun(TestDvc): def test(self): cmd = "python {} {} {}".format(self.CODE, self.FOO, "out") deps = [self.FOO, self.CODE] outs = [os.path.join(self.dvc.root_dir, "out")] outs_no_cache = [] fname = "out.dvc" cwd = os.curdir self.dvc.add(self.FOO) stage = self.dvc.run( cmd=cmd, deps=deps, outs=outs, outs_no_cache=outs_no_cache, fname=fname, cwd=cwd, ) self.assertTrue(filecmp.cmp(self.FOO, "out", shallow=False)) self.assertTrue(os.path.isfile(stage.path)) self.assertEqual(stage.cmd, cmd) self.assertEqual(len(stage.deps), len(deps)) self.assertEqual(len(stage.outs), len(outs + outs_no_cache)) self.assertEqual(stage.outs[0].path, outs[0]) self.assertEqual(stage.outs[0].checksum, file_md5(self.FOO)[0]) self.assertTrue(stage.path, fname) with self.assertRaises(OutputDuplicationError): stage = self.dvc.run( cmd=cmd, deps=deps, outs=outs, outs_no_cache=outs_no_cache, fname="duplicate" + fname, cwd=cwd, ) class TestRunEmpty(TestDvc): def test(self): self.dvc.run( cmd="", deps=[], outs=[], outs_no_cache=[], fname="empty.dvc", cwd=os.curdir ) class TestRunMissingDep(TestDvc): def test(self): with self.assertRaises(MissingDep): self.dvc.run( cmd="", deps=["non-existing-dep"], outs=[], outs_no_cache=[], fname="empty.dvc", cwd=os.curdir, ) class TestRunBadStageFilename(TestDvc): def test(self): with self.assertRaises(StageFileBadNameError): self.dvc.run( cmd="", deps=[], outs=[], outs_no_cache=[], fname="empty", cwd=os.curdir ) with self.assertRaises(StageFileBadNameError): self.dvc.run( cmd="", deps=[], outs=[], outs_no_cache=[], fname=os.path.join(self.DATA_DIR, "empty.dvc"), cwd=os.curdir, ) class TestRunNoExec(TestDvc): def test(self): self.dvc.run( cmd="python {} {} {}".format(self.CODE, self.FOO, "out"), no_exec=True ) self.assertFalse(os.path.exists("out")) class TestRunCircularDependency(TestDvc): def test(self): with self.assertRaises(CircularDependencyError): self.dvc.run( cmd="", deps=[self.FOO], outs=[self.FOO], fname="circular-dependency.dvc", ) def test_outs_no_cache(self): with self.assertRaises(CircularDependencyError): self.dvc.run( cmd="", deps=[self.FOO], outs_no_cache=[self.FOO], fname="circular-dependency.dvc", ) def test_non_normalized_paths(self): with self.assertRaises(CircularDependencyError): self.dvc.run( cmd="", deps=["./foo"], outs=["foo"], fname="circular-dependency.dvc" ) def test_graph(self): self.dvc.run(deps=[self.FOO], outs=["bar.txt"], cmd="echo bar > bar.txt") self.dvc.run(deps=["bar.txt"], outs=["baz.txt"], cmd="echo baz > baz.txt") with self.assertRaises(CyclicGraphError): self.dvc.run(deps=["baz.txt"], outs=[self.FOO], cmd="echo baz > foo") class TestRunDuplicatedArguments(TestDvc): def test(self): with self.assertRaises(ArgumentDuplicationError): self.dvc.run( cmd="", deps=[], outs=[self.FOO, self.FOO], fname="circular-dependency.dvc", ) def test_outs_no_cache(self): with self.assertRaises(ArgumentDuplicationError): self.dvc.run( cmd="", outs=[self.FOO], outs_no_cache=[self.FOO], fname="circular-dependency.dvc", ) def test_non_normalized_paths(self): with self.assertRaises(ArgumentDuplicationError): self.dvc.run( cmd="", deps=[], outs=["foo", "./foo"], fname="circular-dependency.dvc" ) class TestRunWorkingDirectoryAsOutput(TestDvc): def test(self): self.dvc.run(cmd="", deps=[], outs=[self.DATA_DIR]) with self.assertRaises(WorkingDirectoryAsOutputError): self.dvc.run( cmd="", cwd=self.DATA_DIR, outs=[self.FOO], fname="inside-cwd.dvc" ) class TestRunBadCwd(TestDvc): def test(self): with self.assertRaises(StageBadCwdError): self.dvc.run(cmd="", cwd=self.mkdtemp()) class TestCmdRun(TestDvc): def test_run(self): ret = main( [ "run", "-d", self.FOO, "-d", self.CODE, "-o", "out", "-f", "out.dvc", "python", self.CODE, self.FOO, "out", ] ) stage = Stage.load(self.dvc, fname="out.dvc") self.assertEqual(ret, 0) self.assertTrue(os.path.isfile("out")) self.assertTrue(os.path.isfile("out.dvc")) self.assertTrue(filecmp.cmp(self.FOO, "out", shallow=False)) self.assertEqual(stage.cmd, "python code.py foo out") def test_run_args_from_cli(self): ret = main(["run", "echo", "foo"]) stage = Stage.load(self.dvc, fname="Dvcfile") self.assertEqual(ret, 0) self.assertEqual(stage.cmd, "echo foo") def test_run_bad_command(self): ret = main(["run", "non-existing-command"]) self.assertNotEqual(ret, 0) def test_run_args_with_spaces(self): ret = main(["run", "echo", "foo bar"]) stage = Stage.load(self.dvc, fname="Dvcfile") self.assertEqual(ret, 0) self.assertEqual(stage.cmd, 'echo "foo bar"') @mock.patch.object(subprocess, "Popen", side_effect=KeyboardInterrupt) def test_keyboard_interrupt(self, mock_popen): ret = main(["run", "mycmd"]) self.assertEqual(ret, 252) class TestRunRemoveOuts(TestDvc): def test(self): with open(self.CODE, "w+") as fobj: fobj.write("import sys\n") fobj.write("import os\n") fobj.write("if os.path.exists(sys.argv[1]):\n") fobj.write(" sys.exit(1)\n") fobj.write("open(sys.argv[1], 'w+').close()\n") ret = main( [ "run", "--remove-outs", "-d", self.CODE, "-o", self.FOO, "python", self.CODE, self.FOO, ] ) self.assertEqual(ret, 0) class TestRunUnprotectOutsCopy(TestDvc): def test(self): with open(self.CODE, "w+") as fobj: fobj.write("import sys\n") fobj.write("with open(sys.argv[1], 'a+') as fobj:\n") fobj.write(" fobj.write('foo')\n") ret = main(["config", "cache.protected", "true"]) self.assertEqual(ret, 0) ret = main(["config", "cache.type", "copy"]) self.assertEqual(ret, 0) ret = main( ["run", "-d", self.CODE, "-o", self.FOO, "python", self.CODE, self.FOO] ) self.assertEqual(ret, 0) self.assertFalse(os.access(self.FOO, os.W_OK)) self.assertEqual(open(self.FOO, "r").read(), "foofoo") ret = main( [ "run", "--overwrite-dvcfile", "--ignore-build-cache", "-d", self.CODE, "-o", self.FOO, "python", self.CODE, self.FOO, ] ) self.assertEqual(ret, 0) self.assertFalse(os.access(self.FOO, os.W_OK)) self.assertEqual(open(self.FOO, "r").read(), "foofoofoo") class TestRunUnprotectOutsSymlink(TestDvc): def test(self): with open(self.CODE, "w+") as fobj: fobj.write("import sys\n") fobj.write("import os\n") fobj.write("assert os.path.exists(sys.argv[1])\n") fobj.write("with open(sys.argv[1], 'a+') as fobj:\n") fobj.write(" fobj.write('foo')\n") ret = main(["config", "cache.protected", "true"]) self.assertEqual(ret, 0) ret = main(["config", "cache.type", "symlink"]) self.assertEqual(ret, 0) self.assertEqual(ret, 0) ret = main( ["run", "-d", self.CODE, "-o", self.FOO, "python", self.CODE, self.FOO] ) self.assertEqual(ret, 0) self.assertFalse(os.access(self.FOO, os.W_OK)) self.assertTrue(System.is_symlink(self.FOO)) self.assertEqual(open(self.FOO, "r").read(), "foofoo") ret = main( [ "run", "--overwrite-dvcfile", "--ignore-build-cache", "-d", self.CODE, "-o", self.FOO, "python", self.CODE, self.FOO, ] ) self.assertEqual(ret, 0) self.assertFalse(os.access(self.FOO, os.W_OK)) self.assertTrue(System.is_symlink(self.FOO)) self.assertEqual(open(self.FOO, "r").read(), "foofoofoo") class TestRunUnprotectOutsHardlink(TestDvc): def test(self): with open(self.CODE, "w+") as fobj: fobj.write("import sys\n") fobj.write("import os\n") fobj.write("assert os.path.exists(sys.argv[1])\n") fobj.write("with open(sys.argv[1], 'a+') as fobj:\n") fobj.write(" fobj.write('foo')\n") ret = main(["config", "cache.protected", "true"]) self.assertEqual(ret, 0) ret = main(["config", "cache.type", "hardlink"]) self.assertEqual(ret, 0) self.assertEqual(ret, 0) ret = main( ["run", "-d", self.CODE, "-o", self.FOO, "python", self.CODE, self.FOO] ) self.assertEqual(ret, 0) self.assertFalse(os.access(self.FOO, os.W_OK)) self.assertTrue(System.is_hardlink(self.FOO)) self.assertEqual(open(self.FOO, "r").read(), "foofoo") ret = main( [ "run", "--overwrite-dvcfile", "--ignore-build-cache", "-d", self.CODE, "-o", self.FOO, "python", self.CODE, self.FOO, ] ) self.assertEqual(ret, 0) self.assertFalse(os.access(self.FOO, os.W_OK)) self.assertTrue(System.is_hardlink(self.FOO)) self.assertEqual(open(self.FOO, "r").read(), "foofoofoo") class TestCmdRunOverwrite(TestDvc): def test(self): # NOTE: using sleep() is a workaround for filesystems # with low mtime resolution. We have to use mtime since # comparing mtime's is the only way to check that the stage # file didn't change(size and inode in the first test down # below don't change). import time ret = main( [ "run", "-d", self.FOO, "-d", self.CODE, "-o", "out", "-f", "out.dvc", "python", self.CODE, self.FOO, "out", ] ) self.assertEqual(ret, 0) stage_mtime = os.path.getmtime("out.dvc") time.sleep(1) ret = main( [ "run", "-d", self.FOO, "-d", self.CODE, "-o", "out", "-f", "out.dvc", "python", self.CODE, self.FOO, "out", ] ) self.assertEqual(ret, 0) # NOTE: check that dvcfile was NOT overwritten self.assertEqual(stage_mtime, os.path.getmtime("out.dvc")) stage_mtime = os.path.getmtime("out.dvc") time.sleep(1) ret = main( [ "run", "-d", self.FOO, "-d", self.CODE, "--overwrite-dvcfile", "--ignore-build-cache", "-o", "out", "-f", "out.dvc", "python", self.CODE, self.FOO, "out", ] ) self.assertEqual(ret, 0) # NOTE: check that dvcfile was overwritten self.assertNotEqual(stage_mtime, os.path.getmtime("out.dvc")) stage_mtime = os.path.getmtime("out.dvc") time.sleep(1) ret = main(["run", "--overwrite-dvcfile", "-f", "out.dvc", "-d", self.BAR]) self.assertEqual(ret, 0) # NOTE: check that dvcfile was overwritten self.assertNotEqual(stage_mtime, os.path.getmtime("out.dvc")) class TestRunDeterministicBase(TestDvc): def setUp(self): super(TestRunDeterministicBase, self).setUp() self.out_file = "out" self.stage_file = self.out_file + ".dvc" self.cmd = "python {} {} {}".format(self.CODE, self.FOO, self.out_file) self.deps = [self.FOO, self.CODE] self.outs = [self.out_file] self.overwrite = False self.ignore_build_cache = False self._run() def _run(self): self.stage = self.dvc.run( cmd=self.cmd, fname=self.stage_file, overwrite=self.overwrite, ignore_build_cache=self.ignore_build_cache, deps=self.deps, outs=self.outs, ) class TestRunDeterministic(TestRunDeterministicBase): def test(self): self._run() class TestRunDeterministicOverwrite(TestRunDeterministicBase): def test(self): self.overwrite = True self.ignore_build_cache = True self._run() class TestRunDeterministicCallback(TestRunDeterministicBase): def test(self): self.stage.remove() self.deps = [] self._run() self._run() class TestRunDeterministicChangedDep(TestRunDeterministicBase): def test(self): os.unlink(self.FOO) shutil.copy(self.BAR, self.FOO) with self.assertRaises(StageFileAlreadyExistsError): self._run() class TestRunDeterministicChangedDepsList(TestRunDeterministicBase): def test(self): self.deps = [self.BAR, self.CODE] with self.assertRaises(StageFileAlreadyExistsError): self._run() class TestRunDeterministicNewDep(TestRunDeterministicBase): def test(self): self.deps = [self.FOO, self.BAR, self.CODE] with self.assertRaises(StageFileAlreadyExistsError): self._run() class TestRunDeterministicRemoveDep(TestRunDeterministicBase): def test(self): self.deps = [self.CODE] with self.assertRaises(StageFileAlreadyExistsError): self._run() class TestRunDeterministicChangedOut(TestRunDeterministicBase): def test(self): os.unlink(self.out_file) self.out_file_mtime = None with self.assertRaises(StageFileAlreadyExistsError): self._run() class TestRunDeterministicChangedCmd(TestRunDeterministicBase): def test(self): self.cmd += " arg" with self.assertRaises(StageFileAlreadyExistsError): self._run()