tests serializing of stage to lockfile (#3988)

* tests: serialize stage for lockfile

* config: make isort/black compatible

* convert class based tests to pytest

tests/unit/stage/test_serialize_pipeline_lock.py

@@ -0,0 +1,208 @@
+from collections import OrderedDict
+
+import pytest
+from voluptuous import Schema as _Schema
+
+from dvc.dvcfile import PIPELINE_FILE
+from dvc.schema import LOCK_FILE_STAGE_SCHEMA, LOCKFILE_SCHEMA
+from dvc.stage import PipelineStage, create_stage
+from dvc.stage.serialize import DEFAULT_PARAMS_FILE, to_lockfile
+from dvc.stage.serialize import (
+    to_single_stage_lockfile as _to_single_stage_lockfile,
+)
+from dvc.stage.utils import split_params_deps
+
+kwargs = {"name": "something", "cmd": "command", "path": PIPELINE_FILE}
+Schema = _Schema(LOCK_FILE_STAGE_SCHEMA)
+
+
+def to_single_stage_lockfile(stage):
+    """Validate schema on each serialization."""
+    e = _to_single_stage_lockfile(stage)
+    assert Schema(e)
+    return e
+
+
+def test_lock(dvc):
+    stage = create_stage(PipelineStage, dvc, **kwargs)
+    assert to_single_stage_lockfile(stage) == {"cmd": "command"}
+
+
+def test_lock_deps(dvc):
+    stage = create_stage(PipelineStage, dvc, deps=["input"], **kwargs)
+    stage.deps[0].info = {"md5": "md-five"}
+    assert to_single_stage_lockfile(stage) == OrderedDict(
+        [
+            ("cmd", "command"),
+            ("deps", [OrderedDict([("path", "input"), ("md5", "md-five")])]),
+        ]
+    )
+
+
+def test_lock_deps_order(dvc):
+    stage = create_stage(
+        PipelineStage, dvc, deps=["input1", "input0"], **kwargs
+    )
+    stage.deps[0].info = {"md5": "md-one1"}
+    stage.deps[1].info = {"md5": "md-zer0"}
+    assert to_single_stage_lockfile(stage) == OrderedDict(
+        [
+            ("cmd", "command"),
+            (
+                "deps",
+                [
+                    OrderedDict([("path", "input0"), ("md5", "md-zer0")]),
+                    OrderedDict([("path", "input1"), ("md5", "md-one1")]),
+                ],
+            ),
+        ]
+    )
+
+
+def test_lock_params(dvc):
+    stage = create_stage(
+        PipelineStage, dvc, params=["lorem.ipsum", "abc"], **kwargs
+    )
+    stage.deps[0].info = {"lorem.ipsum": {"lorem1": 1, "lorem2": 2}, "abc": 3}
+    assert to_single_stage_lockfile(stage)["params"][
+        DEFAULT_PARAMS_FILE
+    ] == OrderedDict([("abc", 3), ("lorem.ipsum", {"lorem1": 1, "lorem2": 2})])
+
+
+def test_lock_params_file_sorted(dvc):
+    stage = create_stage(
+        PipelineStage,
+        dvc,
+        params=[
+            "lorem.ipsum",
+            "abc",
+            {"myparams.yaml": ["foo", "foobar"]},
+            {"a-params-file.yaml": ["bar", "barr"]},
+        ],
+        **kwargs
+    )
+    stage.deps[0].info = {"lorem.ipsum": {"lorem1": 1, "lorem2": 2}, "abc": 3}
+    stage.deps[1].info = {"foo": ["f", "o", "o"], "foobar": "foobar"}
+    stage.deps[2].info = {"bar": ["b", "a", "r"], "barr": "barr"}
+    assert to_single_stage_lockfile(stage)["params"] == OrderedDict(
+        [
+            (
+                DEFAULT_PARAMS_FILE,
+                OrderedDict(
+                    [("abc", 3), ("lorem.ipsum", {"lorem1": 1, "lorem2": 2})]
+                ),
+            ),
+            (
+                "a-params-file.yaml",
+                OrderedDict([("bar", ["b", "a", "r"]), ("barr", "barr")]),
+            ),
+            (
+                "myparams.yaml",
+                OrderedDict([("foo", ["f", "o", "o"]), ("foobar", "foobar")]),
+            ),
+        ]
+    )
+
+
+def test_lock_params_no_values_filled(dvc):
+    stage = create_stage(
+        PipelineStage, dvc, params=["lorem.ipsum", "abc"], **kwargs
+    )
+    assert to_single_stage_lockfile(stage) == {"cmd": "command"}
+
+
+@pytest.mark.parametrize("typ", ["plots", "metrics", "outs"])
+def test_lock_outs(dvc, typ):
+    stage = create_stage(PipelineStage, dvc, **{typ: ["input"]}, **kwargs)
+    stage.outs[0].info = {"md5": "md-five"}
+    assert to_single_stage_lockfile(stage) == OrderedDict(
+        [
+            ("cmd", "command"),
+            ("outs", [OrderedDict([("path", "input"), ("md5", "md-five")])]),
+        ]
+    )
+
+
+@pytest.mark.parametrize("typ", ["plots", "metrics", "outs"])
+def test_lock_outs_order(dvc, typ):
+    stage = create_stage(
+        PipelineStage, dvc, **{typ: ["input1", "input0"]}, **kwargs
+    )
+    stage.outs[0].info = {"md5": "md-one1"}
+    stage.outs[1].info = {"md5": "md-zer0"}
+    assert to_single_stage_lockfile(stage) == OrderedDict(
+        [
+            ("cmd", "command"),
+            (
+                "outs",
+                [
+                    OrderedDict([("path", "input0"), ("md5", "md-zer0")]),
+                    OrderedDict([("path", "input1"), ("md5", "md-one1")]),
+                ],
+            ),
+        ]
+    )
+
+
+def test_dump_appropriate_checksums(dvc):
+    stage = create_stage(
+        PipelineStage, dvc, deps=["s3://dvc-temp/file"], **kwargs
+    )
+    stage.deps[0].info = {"etag": "is-it-etag", "md5": "or-md5?"}
+    assert to_single_stage_lockfile(stage) == OrderedDict(
+        [
+            ("cmd", "command"),
+            (
+                "deps",
+                [
+                    OrderedDict(
+                        [
+                            ("path", "s3://dvc-temp/file"),
+                            ("etag", "is-it-etag"),
+                        ]
+                    )
+                ],
+            ),
+        ]
+    )
+
+
+def test_order(dvc):
+    stage = create_stage(
+        PipelineStage,
+        dvc,
+        deps=["input"],
+        outs=["output"],
+        params=["foo-param"],
+        **kwargs
+    )
+    params, deps = split_params_deps(stage)
+
+    deps[0].info = {"md5": "md-five"}
+    params[0].info = {"foo-param": "value"}
+    stage.outs[0].info = {"md5": "md5-output"}
+
+    assert to_single_stage_lockfile(stage) == OrderedDict(
+        [
+            ("cmd", "command"),
+            ("deps", [{"path": "input", "md5": "md-five"}]),
+            ("params", {"params.yaml": {"foo-param": "value"}}),
+            ("outs", [{"path": "output", "md5": "md5-output"}]),
+        ]
+    )
+
+
+def test_to_lockfile(dvc):
+    stage = create_stage(PipelineStage, dvc, deps=["input"], **kwargs)
+    stage.deps[0].info = {"md5": "md-five"}
+    entry = to_lockfile(stage)
+    assert len(entry) == 1
+    _Schema(LOCKFILE_SCHEMA)(entry)
+    assert entry == {
+        "something": OrderedDict(
+            [
+                ("cmd", "command"),
+                ("deps", [{"path": "input", "md5": "md-five"}]),
+            ]
+        )
+    }

tests/unit/stage/test_stage.py

@@ -2,7 +2,6 @@ import os
 import signal
 import subprocess
 import threading
-from unittest import TestCase
 
 import mock
 import pytest
@@ -50,12 +49,11 @@ def test_meta_ignored():
         assert stage.compute_md5() == "e9521a22111493406ea64a88cda63e0b"
 
 
-class TestPathConversion(TestCase):
-    def test(self):
-        stage = Stage(None, "path")
+def test_path_conversion(dvc):
+    stage = Stage(dvc, "path")
 
-        stage.wdir = os.path.join("..", "..")
-        self.assertEqual(stage.dumpd()["wdir"], "../..")
+    stage.wdir = os.path.join("..", "..")
+    assert stage.dumpd()["wdir"] == "../.."
 
 
 def test_stage_update(mocker):