diff --git a/tests/conftest.py b/tests/conftest.py index 4c46ba448..8846ece54 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,7 +9,7 @@ from dvc.testing.fixtures import * # noqa, pylint: disable=wildcard-import from .dir_helpers import * # noqa, pylint: disable=wildcard-import from .remotes import * # noqa, pylint: disable=wildcard-import -from .utils.scriptify import scriptify +from .scripts import * # noqa, pylint: disable=wildcard-import # Prevent updater and analytics from running their processes os.environ["DVC_TEST"] = "true" @@ -172,9 +172,6 @@ def custom_template(tmp_dir, dvc): return template -scriptify_fixture = pytest.fixture(lambda: scriptify, name="scriptify") - - @pytest.fixture(autouse=True) def mocked_webbrowser_open(mocker): mocker.patch("webbrowser.open") @@ -218,7 +215,7 @@ defaultBranch=master @pytest.fixture -def run_copy_metrics(tmp_dir, run_copy): +def run_copy_metrics(tmp_dir, copy_script): def run( file1, file2, diff --git a/tests/dir_helpers.py b/tests/dir_helpers.py index b9eb0dce4..b010d29a5 100644 --- a/tests/dir_helpers.py +++ b/tests/dir_helpers.py @@ -47,7 +47,6 @@ from global repo template to creating everything inplace, which: import os import pathlib -from textwrap import dedent import pytest @@ -67,17 +66,7 @@ disable_other_loggers() @pytest.fixture -def run_copy(tmp_dir, dvc): - tmp_dir.gen( - "copy.py", - ( - "import sys, shutil, os\n" - "shutil.copyfile(sys.argv[1], sys.argv[2]) " - "if os.path.isfile(sys.argv[1]) " - "else shutil.copytree(sys.argv[1], sys.argv[2])" - ), - ) - +def run_copy(tmp_dir, copy_script, dvc): def run_copy(src, dst, **run_kwargs): wdir = pathlib.Path(run_kwargs.get("wdir", ".")) wdir = pathlib.Path("../" * len(wdir.parts)) @@ -94,23 +83,7 @@ def run_copy(tmp_dir, dvc): @pytest.fixture -def run_head(tmp_dir, dvc): - """Output first line of each file to different file with '-1' appended. - Useful for tracking multiple outputs/dependencies which are not a copy - of each others. - """ - tmp_dir.gen( - { - "head.py": dedent( - """ - import sys - for file in sys.argv[1:]: - with open(file) as f, open(file +"-1","w+") as w: - w.write(f.readline()) - """ - ) - } - ) +def run_head(tmp_dir, head_script, dvc): script = os.path.abspath(tmp_dir / "head.py") def run(*args, **run_kwargs): diff --git a/tests/func/experiments/test_experiments.py b/tests/func/experiments/test_experiments.py index 1291e06f8..c0b2665c7 100644 --- a/tests/func/experiments/test_experiments.py +++ b/tests/func/experiments/test_experiments.py @@ -13,7 +13,7 @@ from dvc.repo.experiments.utils import exp_refs_by_rev from dvc.scm import resolve_rev from dvc.stage.exceptions import StageFileDoesNotExistError from dvc.utils.serialize import PythonFileCorruptedError -from tests.func.test_repro_multistage import COPY_SCRIPT +from tests.scripts import COPY_SCRIPT @pytest.mark.parametrize("name", [None, "foo"]) @@ -209,8 +209,7 @@ def test_get_baseline(tmp_dir, scm, dvc, exp_stage): assert dvc.experiments.get_baseline(f"{CELERY_STASH}@{{1}}") == init_rev -def test_update_py_params(tmp_dir, scm, dvc, test_queue): - tmp_dir.gen("copy.py", COPY_SCRIPT) +def test_update_py_params(tmp_dir, scm, dvc, test_queue, copy_script): tmp_dir.gen("params.py", "INT = 1\n") stage = dvc.run( cmd="python copy.py params.py metrics.py", @@ -371,8 +370,7 @@ def test_no_scm(tmp_dir): getattr(dvc.experiments, cmd)() -def test_untracked(tmp_dir, scm, dvc, caplog, workspace): - tmp_dir.gen("copy.py", COPY_SCRIPT) +def test_untracked(tmp_dir, scm, dvc, caplog, workspace, copy_script): tmp_dir.scm_gen("params.yaml", "foo: 1", commit="track params") stage = dvc.run( cmd="python copy.py params.yaml metrics.yaml", @@ -600,9 +598,10 @@ def test_fix_exp_head(tmp_dir, scm, tail): "params, target", itertools.product(("foo: 1", "foo: 2"), (True, False)), ) -def test_modified_data_dep(tmp_dir, scm, dvc, workspace, params, target): +def test_modified_data_dep( + tmp_dir, scm, dvc, workspace, params, target, copy_script +): tmp_dir.dvc_gen("data", "data") - tmp_dir.gen("copy.py", COPY_SCRIPT) tmp_dir.gen("params.yaml", "foo: 1") exp_stage = dvc.run( cmd="python copy.py params.yaml metrics.yaml", diff --git a/tests/func/experiments/test_gc.py b/tests/func/experiments/test_gc.py index e5f9069ca..4837952bb 100644 --- a/tests/func/experiments/test_gc.py +++ b/tests/func/experiments/test_gc.py @@ -1,12 +1,9 @@ import pytest from funcy import first -from tests.func.test_repro_multistage import COPY_SCRIPT - @pytest.mark.parametrize("queued, expected", [(True, 0), (False, 1)]) -def test_workspace(tmp_dir, scm, dvc, queued, expected): - tmp_dir.gen("copy.py", COPY_SCRIPT) +def test_workspace(tmp_dir, scm, dvc, queued, expected, copy_script): tmp_dir.gen("params.yaml", "foo: 1") stage = dvc.run( @@ -26,8 +23,7 @@ def test_workspace(tmp_dir, scm, dvc, queued, expected): @pytest.mark.parametrize("queued, expected", [(True, 0), (False, 2)]) -def test_all_commits(tmp_dir, scm, dvc, queued, expected): - tmp_dir.gen("copy.py", COPY_SCRIPT) +def test_all_commits(tmp_dir, scm, dvc, queued, expected, copy_script): tmp_dir.gen("params.yaml", "foo: 1") stage = dvc.run( @@ -55,8 +51,7 @@ def test_all_commits(tmp_dir, scm, dvc, queued, expected): @pytest.mark.parametrize("queued, expected", [(True, 2), (False, 3)]) -def test_all_branches(tmp_dir, scm, dvc, queued, expected): - tmp_dir.gen("copy.py", COPY_SCRIPT) +def test_all_branches(tmp_dir, scm, dvc, queued, expected, copy_script): tmp_dir.gen("params.yaml", "foo: 1") stage = dvc.run( @@ -91,8 +86,7 @@ def test_all_branches(tmp_dir, scm, dvc, queued, expected): @pytest.mark.parametrize("queued, expected", [(True, 2), (False, 3)]) -def test_all_tags(tmp_dir, scm, dvc, queued, expected): - tmp_dir.gen("copy.py", COPY_SCRIPT) +def test_all_tags(tmp_dir, scm, dvc, queued, expected, copy_script): tmp_dir.gen("params.yaml", "foo: 1") stage = dvc.run( diff --git a/tests/func/experiments/test_show.py b/tests/func/experiments/test_show.py index 2869053fe..59efcdd97 100644 --- a/tests/func/experiments/test_show.py +++ b/tests/func/experiments/test_show.py @@ -20,7 +20,6 @@ from dvc.repo.experiments.utils import exp_refs_by_rev from dvc.utils import relpath from dvc.utils.fs import makedirs from dvc.utils.serialize import YAMLFileCorruptedError -from tests.func.test_repro_multistage import COPY_SCRIPT LOCK_CONTENTS = { "read": { @@ -313,10 +312,10 @@ def test_show_filter( scm, dvc, capsys, + copy_script, ): capsys.readouterr() - tmp_dir.gen("copy.py", COPY_SCRIPT) params_file = tmp_dir / "params.yaml" params_data = { "foo": 1, @@ -679,8 +678,7 @@ def test_show_csv(tmp_dir, scm, dvc, exp_stage, capsys): ) -def test_show_only_changed(tmp_dir, dvc, scm, capsys): - tmp_dir.gen("copy.py", COPY_SCRIPT) +def test_show_only_changed(tmp_dir, dvc, scm, capsys, copy_script): params_file = tmp_dir / "params.yaml" params_data = { "foo": 1, @@ -726,13 +724,14 @@ def test_show_only_changed(tmp_dir, dvc, scm, capsys): assert "metrics.yaml:bar" in cap.out -def test_show_parallel_coordinates(tmp_dir, dvc, scm, mocker, capsys): +def test_show_parallel_coordinates( + tmp_dir, dvc, scm, mocker, capsys, copy_script +): from dvc.commands.experiments import show webbroser_open = mocker.patch("webbrowser.open") show_experiments = mocker.spy(show, "show_experiments") - tmp_dir.gen("copy.py", COPY_SCRIPT) params_file = tmp_dir / "params.yaml" params_data = { "foo": 1, @@ -807,8 +806,7 @@ def test_show_parallel_coordinates(tmp_dir, dvc, scm, mocker, capsys): @pytest.mark.vscode -def test_show_outs(tmp_dir, dvc, scm, erepo_dir): - tmp_dir.gen("copy.py", COPY_SCRIPT) +def test_show_outs(tmp_dir, dvc, scm, erepo_dir, copy_script): params_file = tmp_dir / "params.yaml" params_data = { "foo": 1, @@ -889,8 +887,7 @@ def test_show_outs(tmp_dir, dvc, scm, erepo_dir): } -def test_metrics_renaming(tmp_dir, dvc, scm, capsys): - tmp_dir.gen("copy.py", COPY_SCRIPT) +def test_metrics_renaming(tmp_dir, dvc, scm, capsys, copy_script): params_file = tmp_dir / "params.yaml" params_data = { "foo": 1, diff --git a/tests/func/test_import_url.py b/tests/func/test_import_url.py index 8417f9200..55cdf9f5f 100644 --- a/tests/func/test_import_url.py +++ b/tests/func/test_import_url.py @@ -13,37 +13,32 @@ from dvc.exceptions import InvalidArgumentError from dvc.stage import Stage from dvc.testing.workspace_tests import TestImport as _TestImport from dvc.utils.fs import makedirs -from tests.basic_env import TestDvc -class TestCmdImport(TestDvc): - def test(self): - ret = main(["import-url", self.FOO, "import"]) - self.assertEqual(ret, 0) - self.assertTrue(os.path.exists("import.dvc")) +def test_cmd_import(tmp_dir, dvc): + tmp_dir.gen("foo", "foo") + ret = main(["import-url", "foo", "import"]) + assert ret == 0 + assert os.path.exists("import.dvc") - ret = main(["import-url", "non-existing-file", "import"]) - self.assertNotEqual(ret, 0) - - def test_unsupported(self): - ret = main(["import-url", "unsupported://path", "import_unsupported"]) - self.assertNotEqual(ret, 0) + ret = main(["import-url", "non-existing-file", "import"]) + assert ret != 0 -class TestDefaultOutput(TestDvc): - def test(self): - tmpdir = self.mkdtemp() - filename = str(uuid4()) - tmpfile = os.path.join(tmpdir, filename) +def test_cmd_unsupported_scheme(dvc): + ret = main(["import-url", "unsupported://path", "import_unsupported"]) + assert ret != 0 - with open(tmpfile, "w", encoding="utf-8") as fd: - fd.write("content") - ret = main(["import-url", tmpfile]) - self.assertEqual(ret, 0) - self.assertTrue(os.path.exists(filename)) - with open(filename, encoding="utf-8") as fd: - self.assertEqual(fd.read(), "content") +def test_default_output(tmp_dir, dvc, cloud): + filename = str(uuid4()) + tmpfile = cloud / filename + tmpfile.write_bytes(b"content") + cloud.gen(filename, "content") + + ret = main(["import-url", tmpfile.fs_path]) + assert ret == 0 + assert (tmp_dir / filename).read_bytes() == b"content" def test_should_remove_outs_before_import(tmp_dir, dvc, mocker, erepo_dir): @@ -56,26 +51,22 @@ def test_should_remove_outs_before_import(tmp_dir, dvc, mocker, erepo_dir): assert remove_outs_call_counter.mock.call_count == 1 -class TestImportFilename(TestDvc): - def setUp(self): - super().setUp() - tmp_dir = self.mkdtemp() - self.external_source = os.path.join(tmp_dir, "file") - with open(self.external_source, "w", encoding="utf-8") as fobj: - fobj.write("content") +def test_import_filename(tmp_dir, dvc, cloud): + external_source = cloud / "file" + (cloud / "file").write_text("content", encoding="utf-8") + ret = main(["import-url", "--file", "bar.dvc", external_source.fs_path]) + assert ret == 0 + assert (tmp_dir / "bar.dvc").exists() - def test(self): - ret = main(["import-url", "--file", "bar.dvc", self.external_source]) - self.assertEqual(0, ret) - self.assertTrue(os.path.exists("bar.dvc")) + (tmp_dir / "bar.dvc").unlink() + (tmp_dir / "sub").mkdir() - os.remove("bar.dvc") - os.mkdir("sub") - - path = os.path.join("sub", "bar.dvc") - ret = main(["import-url", "--file", path, self.external_source, "out"]) - self.assertEqual(0, ret) - self.assertTrue(os.path.exists(path)) + path = tmp_dir / "sub" / "bar.dvc" + ret = main( + ["import-url", "--file", path.fs_path, external_source.fs_path, "out"] + ) + assert ret == 0 + assert path.exists() @pytest.mark.parametrize("dname", [".", "dir", "dir/subdir"]) diff --git a/tests/func/test_remote.py b/tests/func/test_remote.py index cd08ec9f3..dce848a26 100644 --- a/tests/func/test_remote.py +++ b/tests/func/test_remote.py @@ -10,58 +10,51 @@ from dvc.cli import main from dvc.config import Config from dvc.exceptions import DownloadError, RemoteCacheRequiredError, UploadError from dvc.utils.fs import remove -from tests.basic_env import TestDvc -class TestRemote(TestDvc): - def test(self): - remotes = ["a", "b", "c"] +def test_remote(dvc): + remotes = ["a", "b", "c"] - self.assertEqual(main(["remote", "list"]), 0) - self.assertNotEqual(main(["remote", "remove", remotes[0]]), 0) + assert main(["remote", "list"]) == 0 + assert main(["remote", "remove", remotes[0]]) != 0 - for r in remotes: - self.assertEqual( - main(["remote", "add", "--default", r, "s3://bucket/name"]), 0 - ) + for r in remotes: + assert main(["remote", "add", "--default", r, "s3://bucket/name"]) == 0 - self.assertEqual(main(["remote", "list"]), 0) + assert main(["remote", "list"]) == 0 - self.assertEqual( - main(["remote", "modify", remotes[0], "checksum_jobs", "1"]), 0 - ) - self.assertEqual(main(["remote", "remove", remotes[0]]), 0) + assert main(["remote", "modify", remotes[0], "checksum_jobs", "1"]) == 0 + assert main(["remote", "remove", remotes[0]]) == 0 - self.assertEqual(main(["remote", "list"]), 0) + assert main(["remote", "list"]) == 0 - def test_relative_path(self): - dname = os.path.join("..", "path", "to", "dir") - ret = main(["remote", "add", "mylocal", dname]) - self.assertEqual(ret, 0) - # NOTE: we are in the repo's root and config is in .dvc/, so - # dir path written to config should be just one level above. - rel = os.path.join("..", dname) - config = configobj.ConfigObj(self.dvc.config.files["repo"]) - self.assertEqual( - config['remote "mylocal"']["url"], rel.replace("\\", "/") - ) +def test_remote_add_relative_path(dvc): + dname = os.path.join("..", "path", "to", "dir") + ret = main(["remote", "add", "mylocal", dname]) + assert ret == 0 - def test_overwrite(self): - remote_name = "a" - remote_url = "s3://bucket/name" - self.assertEqual(main(["remote", "add", remote_name, remote_url]), 0) - self.assertEqual(main(["remote", "add", remote_name, remote_url]), 251) - self.assertEqual( - main(["remote", "add", "-f", remote_name, remote_url]), 0 - ) + # NOTE: we are in the repo's root and config is in .dvc/, so + # dir path written to config should be just one level above. + rel = os.path.join("..", dname) + config = configobj.ConfigObj(dvc.config.files["repo"]) + assert config['remote "mylocal"']["url"] == rel.replace("\\", "/") - def test_referencing_other_remotes(self): - assert main(["remote", "add", "foo", "ssh://localhost/"]) == 0 - assert main(["remote", "add", "bar", "remote://foo/dvc-storage"]) == 0 - config = configobj.ConfigObj(self.dvc.config.files["repo"]) - assert config['remote "bar"']["url"] == "remote://foo/dvc-storage" +def test_remote_overwrite(dvc): + remote_name = "a" + remote_url = "s3://bucket/name" + assert main(["remote", "add", remote_name, remote_url]) == 0 + assert main(["remote", "add", remote_name, remote_url]) == 251 + assert main(["remote", "add", "-f", remote_name, remote_url]) == 0 + + +def test_referencing_other_remotes(dvc): + assert main(["remote", "add", "foo", "ssh://localhost/"]) == 0 + assert main(["remote", "add", "bar", "remote://foo/dvc-storage"]) == 0 + + config = configobj.ConfigObj(dvc.config.files["repo"]) + assert config['remote "bar"']["url"] == "remote://foo/dvc-storage" def test_remove_default(tmp_dir, dvc): @@ -85,37 +78,35 @@ def test_remove_default(tmp_dir, dvc): assert local_config.get("core", {}).get("remote") is None -class TestRemoteRemove(TestDvc): - def test(self): - ret = main(["config", "core.checksum_jobs", "1"]) - self.assertEqual(ret, 0) +def test_remote_remove(dvc): + ret = main(["config", "core.checksum_jobs", "1"]) + assert ret == 0 - remote = "mys3" - ret = main(["remote", "add", remote, "s3://bucket/name"]) - self.assertEqual(ret, 0) + remote = "mys3" + ret = main(["remote", "add", remote, "s3://bucket/name"]) + assert ret == 0 - ret = main(["remote", "remove", remote]) - self.assertEqual(ret, 0) + ret = main(["remote", "remove", remote]) + assert ret == 0 -class TestRemoteDefault(TestDvc): - def test(self): - remote = "mys3" - ret = main(["remote", "add", "mys3", "s3://bucket/path"]) - self.assertEqual(ret, 0) +def test_remote_default_cmd(dvc): + remote = "mys3" + ret = main(["remote", "add", "mys3", "s3://bucket/path"]) + assert ret == 0 - ret = main(["remote", "default", "mys3"]) - self.assertEqual(ret, 0) - config_file = os.path.join(self.dvc.dvc_dir, Config.CONFIG) - config = configobj.ConfigObj(config_file) - default = config["core"]["remote"] - self.assertEqual(default, remote) + ret = main(["remote", "default", "mys3"]) + assert ret == 0 + config_file = os.path.join(dvc.dvc_dir, Config.CONFIG) + config = configobj.ConfigObj(config_file) + default = config["core"]["remote"] + assert default == remote - ret = main(["remote", "default", "--unset"]) - self.assertEqual(ret, 0) - config = configobj.ConfigObj(config_file) - default = config.get("core", {}).get("remote") - self.assertEqual(default, None) + ret = main(["remote", "default", "--unset"]) + assert ret == 0 + config = configobj.ConfigObj(config_file) + default = config.get("core", {}).get("remote") + assert default is None def test_show_default(dvc, capsys): diff --git a/tests/func/test_repo_index.py b/tests/func/test_repo_index.py index c419d88fc..66f5c0cf3 100644 --- a/tests/func/test_repo_index.py +++ b/tests/func/test_repo_index.py @@ -220,7 +220,7 @@ def test_unique_identifier(tmp_dir, dvc, scm, run_copy): # we get "workspace" as well from the brancher by default assert len(revs) + 1 == len(ids) possible_ids = { - True: "2ba7c7c5b395d4211348d6274b869fc7", + True: "dd9cf3a3dc2caac5835e64f258ac2d7b", False: "8406970ad2fcafaa84d9310330a67576", } assert set(ids) == {possible_ids[os.name == "posix"]} diff --git a/tests/func/test_stage.py b/tests/func/test_stage.py index 73e0699e9..ba0a0ec60 100644 --- a/tests/func/test_stage.py +++ b/tests/func/test_stage.py @@ -1,10 +1,8 @@ import os -import tempfile import pytest from dvc.annotations import Annotation -from dvc.cli import main from dvc.dvcfile import SingleStageFile from dvc.fs import LocalFileSystem from dvc.output import Output @@ -15,7 +13,6 @@ from dvc.stage.utils import compute_md5 from dvc.utils import dict_md5 from dvc.utils.serialize import dump_yaml, load_yaml from dvc.utils.strictyaml import YAMLValidationError -from tests.basic_env import TestDvc def test_cmd_obj(): @@ -71,90 +68,74 @@ def test_list(): SingleStageFile.validate(d) -class TestReload(TestDvc): - def test(self): - stages = self.dvc.add(self.FOO) - self.assertEqual(len(stages), 1) - stage = stages[0] - self.assertTrue(stage is not None) +def test_reload(tmp_dir, dvc): + (stage,) = tmp_dir.dvc_gen("foo", "foo") + d = load_yaml(stage.relpath) - d = load_yaml(stage.relpath) + # NOTE: checking that reloaded stage didn't change its checksum + md5 = "11111111111111111111111111111111" + d[stage.PARAM_MD5] = md5 + dump_yaml(stage.relpath, d) - # NOTE: checking that reloaded stage didn't change its checksum - md5 = "11111111111111111111111111111111" - d[stage.PARAM_MD5] = md5 - dump_yaml(stage.relpath, d) + dvcfile = SingleStageFile(dvc, stage.relpath) + stage = dvcfile.stage - dvcfile = SingleStageFile(self.dvc, stage.relpath) - stage = dvcfile.stage + assert stage is not None + dvcfile.dump(stage) - self.assertTrue(stage is not None) - dvcfile.dump(stage) - - d = load_yaml(stage.relpath) - self.assertEqual(d[stage.PARAM_MD5], md5) + d = load_yaml(stage.relpath) + assert d[stage.PARAM_MD5] == md5 -class TestDefaultWorkingDirectory(TestDvc): - def test_ignored_in_checksum(self): - stage = self.dvc.run( - cmd=f"echo test > {self.FOO}", - deps=[self.BAR], - outs=[self.FOO], - single_stage=True, - ) +def test_default_wdir_ignored_in_checksum(tmp_dir, dvc): + tmp_dir.gen("bar", "bar") + stage = dvc.run( + cmd="cp bar foo", + deps=["bar"], + outs=["foo"], + single_stage=True, + ) - d = stage.dumpd() - self.assertNotIn(Stage.PARAM_WDIR, d.keys()) + d = stage.dumpd() + assert Stage.PARAM_WDIR not in d.keys() - d = load_yaml(stage.relpath) - self.assertNotIn(Stage.PARAM_WDIR, d.keys()) + d = load_yaml(stage.relpath) + assert Stage.PARAM_WDIR not in d.keys() - with self.dvc.lock: - stage = SingleStageFile(self.dvc, stage.relpath).stage - self.assertFalse(stage.changed()) + with dvc.lock: + stage = SingleStageFile(dvc, stage.relpath).stage + assert not stage.changed() -class TestExternalRemoteResolution(TestDvc): - def test_remote_output(self): - tmp_path = tempfile.mkdtemp() - storage = os.path.join(tmp_path, "storage") - file_path = os.path.join(storage, "file") +def test_external_remote_output_resolution(tmp_dir, dvc, make_remote): + tmp_path = make_remote("tmp", default=False) + tmp_dir.add_remote( + url="remote://tmp/storage", name="storage", default=False + ) + storage = tmp_path / "storage" + storage.mkdir() + file_path = storage / "file" - os.makedirs(storage) + dvc.run( + cmd=f"echo file > {file_path}", + outs_no_cache=["remote://storage/file"], + single_stage=True, + ) + assert os.path.exists(file_path) - assert main(["remote", "add", "tmp", tmp_path]) == 0 - assert main(["remote", "add", "storage", "remote://tmp/storage"]) == 0 - assert ( - main( - [ - "run", - "--single-stage", - "-O", - "remote://storage/file", - f"echo file > {file_path}", - ] - ) - == 0 - ) - assert os.path.exists(file_path) +def test_external_remote_dependency_resolution(tmp_dir, dvc, make_remote): + tmp_path = make_remote("tmp", default=False) + tmp_dir.add_remote( + url="remote://tmp/storage", name="storage", default=False + ) + storage = tmp_path / "storage" + storage.mkdir() + file_path = storage / "file" + file_path.write_text("Isle of Dogs", encoding="utf-8") - def test_remote_dependency(self): - tmp_path = tempfile.mkdtemp() - storage = os.path.join(tmp_path, "storage") - file_path = os.path.join(storage, "file") - - os.makedirs(storage) - - with open(file_path, "w", encoding="utf-8") as fobj: - fobj.write("Isle of Dogs") - - assert main(["remote", "add", "tmp", tmp_path]) == 0 - assert main(["remote", "add", "storage", "remote://tmp/storage"]) == 0 - assert main(["import-url", "remote://storage/file", "movie.txt"]) == 0 - - assert os.path.exists("movie.txt") + dvc.imp_url("remote://storage/file", "movie.txt") + assert (tmp_dir / "movie.txt").read_text() == "Isle of Dogs" def test_md5_ignores_comments(tmp_dir, dvc): diff --git a/tests/scripts.py b/tests/scripts.py new file mode 100644 index 000000000..5d8e4242c --- /dev/null +++ b/tests/scripts.py @@ -0,0 +1,59 @@ +import pytest + +COPY_SCRIPT = """ +import os +import shutil +import sys + +if os.path.isfile(sys.argv[1]): + shutil.copyfile(sys.argv[1], sys.argv[2]) +else: + shutil.copytree(sys.argv[1], sys.argv[2]) +""" + + +def _add_script(tmp_dir, path, contents=""): + script, *_ = tmp_dir.gen(path, contents) + if hasattr(tmp_dir, "scm"): + tmp_dir.scm_add(path, commit=f"add {path}") + return script.fs_path + + +@pytest.fixture +def append_foo_script(tmp_dir): + return _add_script( + tmp_dir, + "append_foo.py", + """ +import sys +from pathlib import Path + +with Path(sys.argv[1]).open("a+", encoding="utf-8") as f: + f.write("foo") +""", + ) + + +@pytest.fixture +def copy_script( + tmp_dir, +): + return _add_script(tmp_dir, "copy.py", COPY_SCRIPT) + + +@pytest.fixture +def head_script(tmp_dir): + """Output first line of each file to different file with '-1' appended. + Useful for tracking multiple outputs/dependencies which are not a copy + of each others. + """ + return _add_script( + tmp_dir, + "head.py", + """ +import sys +for file in sys.argv[1:]: + with open(file) as f, open(file +"-1","w+") as w: + w.write(f.readline()) +""", + ) diff --git a/tests/unit/command/test_diff.py b/tests/unit/command/test_diff.py index c1c44c0c1..4dd6a1aad 100644 --- a/tests/unit/command/test_diff.py +++ b/tests/unit/command/test_diff.py @@ -39,7 +39,7 @@ def test_default(mocker, capsys): } mocker.patch("dvc.repo.Repo.diff", return_value=diff) - assert 0 == cmd.run() + assert cmd.run() == 0 assert ( "Added:\n" " file\n" @@ -77,7 +77,7 @@ def test_show_hash(mocker, capsys): "not in cache": [], } mocker.patch("dvc.repo.Repo.diff", return_value=diff) - assert 0 == cmd.run() + assert cmd.run() == 0 out, _ = capsys.readouterr() assert ( @@ -115,7 +115,7 @@ def test_show_json(mocker, capsys): } mocker.patch("dvc.repo.Repo.diff", return_value=diff) - assert 0 == cmd.run() + assert cmd.run() == 0 out, _ = capsys.readouterr() assert '"added": [{"path": "file1"}, {"path": "file2"}]' in out assert '"deleted": []' in out @@ -145,7 +145,7 @@ def test_show_json_and_hash(mocker, capsys): } mocker.patch("dvc.repo.Repo.diff", return_value=diff) - assert 0 == cmd.run() + assert cmd.run() == 0 out, _ = capsys.readouterr() assert ( '"added": [{"path": "file1", "hash": "11111111"}, ' @@ -180,7 +180,7 @@ def test_show_json_hide_missing(mocker, capsys): } mocker.patch("dvc.repo.Repo.diff", return_value=diff) - assert 0 == cmd.run() + assert cmd.run() == 0 out, _ = capsys.readouterr() assert '"added": [{"path": "file1"}, {"path": "file2"}]' in out assert '"deleted": []' in out @@ -200,7 +200,7 @@ def test_diff_show_markdown_and_hash(mocker, show_hash): mock_show_markdown = mocker.patch("dvc.commands.diff._show_markdown") mocker.patch("dvc.repo.Repo.diff", return_value=diff.copy()) - assert 0 == cmd.run() + assert cmd.run() == 0 mock_show_markdown.assert_called_once_with(diff, show_hash, False) @@ -226,7 +226,7 @@ def test_no_changes(mocker, capsys, opts, show, expected): cmd = args.func(args) mocker.patch("dvc.repo.Repo.diff", return_value={}) - assert 0 == cmd.run() + assert cmd.run() == 0 out, _ = capsys.readouterr() assert expected == out.strip() @@ -349,7 +349,7 @@ def test_hide_missing(mocker, capsys): } mocker.patch("dvc.repo.Repo.diff", return_value=diff) - assert 0 == cmd.run() + assert cmd.run() == 0 out, _ = capsys.readouterr() assert ( "Added:\n" diff --git a/tests/unit/repo/experiments/conftest.py b/tests/unit/repo/experiments/conftest.py index a4968ca7d..3c5c43f6d 100644 --- a/tests/unit/repo/experiments/conftest.py +++ b/tests/unit/repo/experiments/conftest.py @@ -4,7 +4,6 @@ from textwrap import dedent import pytest from dvc.repo.experiments.queue.celery import LocalCeleryQueue -from tests.func.test_repro_multistage import COPY_SCRIPT DEFAULT_ITERATIONS = 2 CHECKPOINT_SCRIPT_FORMAT = dedent( @@ -44,8 +43,7 @@ CHECKPOINT_SCRIPT = CHECKPOINT_SCRIPT_FORMAT.format( @pytest.fixture -def exp_stage(tmp_dir, scm, dvc): - tmp_dir.gen("copy.py", COPY_SCRIPT) +def exp_stage(tmp_dir, scm, dvc, copy_script): tmp_dir.gen("params.yaml", "foo: 1") stage = dvc.run( cmd="python copy.py params.yaml metrics.yaml", @@ -93,8 +91,7 @@ def checkpoint_stage(tmp_dir, scm, dvc, mocker): @pytest.fixture -def failed_exp_stage(tmp_dir, scm, dvc): - tmp_dir.gen("copy.py", COPY_SCRIPT) +def failed_exp_stage(tmp_dir, scm, dvc, copy_script): tmp_dir.gen("params.yaml", "foo: 1") stage = dvc.stage.add( cmd="python -c 'import sys; sys.exit(1)'", diff --git a/tests/unit/stage/test_cache.py b/tests/unit/stage/test_cache.py index a08d073f6..31c448609 100644 --- a/tests/unit/stage/test_cache.py +++ b/tests/unit/stage/test_cache.py @@ -180,14 +180,14 @@ def test_shared_stage_cache(tmp_dir, dvc, run_copy): run_copy("foo", "bar", name="copy-foo-bar") - parent_cache_dir = os.path.join(dvc.stage_cache.cache_dir, "88") + parent_cache_dir = os.path.join(dvc.stage_cache.cache_dir, "fd") cache_dir = os.path.join( parent_cache_dir, - "883395068439203a9de3d1e1649a16e9027bfd1ab5dab4f438d321c4a928b328", + "fdbe7847136ebe88f59a29ee5a568f893cab031f74f7d3ab050828b53fd0033a", ) cache_file = os.path.join( cache_dir, - "e42b7ebb9bc5ac4bccab769c8d1338914dad25d7ffecc8671dbd4581bad4aa15", + "8716052b2074f5acf1a379529d47d6ef1c1f50c2281a7489b8d7ce251f234f86", ) # sanity check diff --git a/tests/utils/scriptify.py b/tests/utils/scriptify.py deleted file mode 100644 index ee2d73f1d..000000000 --- a/tests/utils/scriptify.py +++ /dev/null @@ -1,36 +0,0 @@ -import inspect -import sys -import textwrap -from pathlib import Path - -SCRIPT_TEMPLATE = """\ -#! {executable} -# AUTOGENERATED USING SCRIPTIFY FOR USE IN TESTS FOR DVC - - -{code} - - -if __name__ == "__main__": - import sys - - _, *argv = sys.argv - func = {func!s} - func(*argv) -""" - - -def scriptify(func, path=None): - # you can use `breakpoint()` inside functions too - # just remember to run the test in `-s` mode. - code = textwrap.dedent(inspect.getsource(func)) - executable = sys.executable - source = SCRIPT_TEMPLATE.format( - executable=executable, code=code, func=func.__name__ - ) - - if not path: - return source, None - - Path(path).write_text(source, encoding="utf-8") - return source, f"{executable} {path}"