mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 07:18:25 +01:00
Peter Rowlands (변기호)
114a07e7a4
* objects.status: use object IDs instead of naive objects * objects.transfer: use object IDs instead of naive objects * objects: add ReferenceHashFile * odb: add ReferenceObjectDB * repo: collect used object IDs instead of naive objs * support basic/naive fs serialization * objects.stage: stage file objects in addition to trees * objects: remove save(), update transfer() * output/dep: update for osave removal * path_info: support both url and path attributes * status/gc: use object IDs instead of naive objs * diff: update staging usage * commit: update staging usage * tests: clean staging when needed * tests: update for save/transfer/stage changes * dvcfs: don't allow config serialization (repofs should be used instead) * utils: return mtime as str, size as int * tests: update unit tests for save/stage/transfer changes * objects: use Repo.open when needed in ReferenceHashFile * fix ls test staging * objects.stage: don't skip hash check when loading from state - individual hashes will still be loaded from state if they are still valid * ReferenceHashFile: serialize repo fs paths relative to repo root * preserve state when staging * objects.tree: skip hash computation in digest() when we have already loaded hash from state * always check file existence in referencehashfile * update repofs test since stage no longer allows nonexistent files * odb: only protect on check() if hash was verified * staging: use unique memfs namespace for staging per dest ODB * tests: clean full staging namespace * refodb: preserve repofs cache dir * repodependency: stage imports to local ODB and strict check circular imports * fetch: fetch staging last * staging: make dest odb mandatory and preserve state * refodb: cache nonlocal fs's * fix rebase errors * repofs: support full instantiation from fs config * repofs: preserve subrepo factory for erepos * tests: clarify repofs dirty file behavior
220 lines
6.1 KiB
Python
220 lines
6.1 KiB
Python
import os
|
|
|
|
import pytest
|
|
|
|
from dvc.dependency.base import DependencyDoesNotExistError
|
|
from dvc.dvcfile import PIPELINE_FILE
|
|
from dvc.output import OutputDoesNotExistError
|
|
from dvc.stage.exceptions import StageCommitError
|
|
from dvc.utils.serialize import dump_yaml, load_yaml
|
|
from tests.utils import clean_staging
|
|
|
|
|
|
def test_commit_recursive(tmp_dir, dvc):
|
|
tmp_dir.gen({"dir": {"file": "text1", "subdir": {"file2": "text2"}}})
|
|
stages = dvc.add("dir", recursive=True, no_commit=True)
|
|
|
|
assert len(stages) == 2
|
|
assert dvc.status() != {}
|
|
|
|
dvc.commit("dir", recursive=True)
|
|
assert dvc.status() == {}
|
|
|
|
|
|
def test_commit_force(tmp_dir, dvc):
|
|
tmp_dir.gen({"dir": {"file": "text1", "file2": "text2"}})
|
|
(stage,) = dvc.add("dir", no_commit=True)
|
|
|
|
assert stage.outs[0].changed_cache()
|
|
|
|
tmp_dir.gen("dir/file", "file content modified")
|
|
|
|
assert stage.outs[0].changed_cache()
|
|
|
|
with pytest.raises(StageCommitError):
|
|
dvc.commit(stage.path)
|
|
|
|
assert stage.outs[0].changed_cache()
|
|
|
|
dvc.commit(stage.path, force=True)
|
|
assert dvc.status([stage.path]) == {}
|
|
|
|
|
|
@pytest.mark.parametrize("run_kw", [{"single_stage": True}, {"name": "copy"}])
|
|
def test_commit_with_deps(tmp_dir, dvc, run_copy, run_kw):
|
|
tmp_dir.gen("foo", "foo")
|
|
(foo_stage,) = dvc.add("foo", no_commit=True)
|
|
assert foo_stage is not None
|
|
assert len(foo_stage.outs) == 1
|
|
|
|
stage = run_copy("foo", "file", no_commit=True, **run_kw)
|
|
assert stage is not None
|
|
assert len(stage.outs) == 1
|
|
|
|
assert foo_stage.outs[0].changed_cache()
|
|
assert stage.outs[0].changed_cache()
|
|
|
|
clean_staging()
|
|
|
|
dvc.commit(stage.path, with_deps=True)
|
|
assert not foo_stage.outs[0].changed_cache()
|
|
assert not stage.outs[0].changed_cache()
|
|
|
|
|
|
def test_commit_changed_md5(tmp_dir, dvc):
|
|
tmp_dir.gen({"file": "file content"})
|
|
(stage,) = dvc.add("file", no_commit=True)
|
|
|
|
stage_file_content = load_yaml(stage.path)
|
|
stage_file_content["md5"] = "1111111111"
|
|
dump_yaml(stage.path, stage_file_content)
|
|
|
|
clean_staging()
|
|
|
|
with pytest.raises(StageCommitError):
|
|
dvc.commit(stage.path)
|
|
|
|
dvc.commit(stage.path, force=True)
|
|
assert "md5" not in load_yaml(stage.path)
|
|
|
|
|
|
def test_commit_no_exec(tmp_dir, dvc):
|
|
tmp_dir.gen({"dep": "dep", "out": "out"})
|
|
stage = dvc.run(
|
|
name="my", cmd="mycmd", deps=["dep"], outs=["out"], no_exec=True
|
|
)
|
|
clean_staging()
|
|
|
|
assert dvc.status(stage.path)
|
|
dvc.commit(stage.path, force=True)
|
|
assert dvc.status(stage.path) == {}
|
|
|
|
|
|
def test_commit_granular_output(tmp_dir, dvc):
|
|
dvc.run(
|
|
name="mystage",
|
|
cmd=["echo foo>foo", "echo bar>bar"],
|
|
outs=["foo", "bar"],
|
|
no_commit=True,
|
|
)
|
|
clean_staging()
|
|
|
|
cache = tmp_dir / ".dvc" / "cache"
|
|
assert not list(cache.glob("*/*"))
|
|
|
|
dvc.commit("foo")
|
|
assert list(cache.glob("*/*")) == [
|
|
cache / "d3" / "b07384d113edec49eaa6238ad5ff00"
|
|
]
|
|
|
|
|
|
def test_commit_granular_output_file(tmp_dir, dvc):
|
|
tmp_dir.gen("foo", "foo")
|
|
dvc.add("foo", no_commit=True)
|
|
clean_staging()
|
|
dvc.commit("foo")
|
|
assert dvc.status() == {}
|
|
|
|
|
|
def test_commit_granular_output_dir(tmp_dir, dvc):
|
|
tmp_dir.gen(
|
|
{
|
|
"data": {
|
|
"foo": "foo",
|
|
"bar": "bar",
|
|
"subdir": {"subfoo": "subfoo", "subbar": "subbar"},
|
|
}
|
|
}
|
|
)
|
|
dvc.add("data", no_commit=True)
|
|
clean_staging()
|
|
dvc.commit("data")
|
|
assert dvc.status() == {}
|
|
|
|
|
|
def test_commit_granular_dir(tmp_dir, dvc):
|
|
tmp_dir.gen(
|
|
{
|
|
"data": {
|
|
"foo": "foo",
|
|
"bar": "bar",
|
|
"subdir": {"subfoo": "subfoo", "subbar": "subbar"},
|
|
}
|
|
}
|
|
)
|
|
dvc.add("data", no_commit=True)
|
|
clean_staging()
|
|
|
|
cache = tmp_dir / ".dvc" / "cache"
|
|
|
|
assert set(cache.glob("*/*")) == set()
|
|
|
|
dvc.commit(os.path.join("data", "foo"))
|
|
assert set(cache.glob("*/*")) == {
|
|
cache / "1a" / "ca2c799df82929bbdd976557975546.dir",
|
|
cache / "ac" / "bd18db4cc2f85cedef654fccc4a4d8",
|
|
}
|
|
clean_staging()
|
|
|
|
dvc.commit(os.path.join("data", "subdir"))
|
|
assert set(cache.glob("*/*")) == {
|
|
cache / "1a" / "ca2c799df82929bbdd976557975546.dir",
|
|
cache / "ac" / "bd18db4cc2f85cedef654fccc4a4d8",
|
|
cache / "4c" / "e8d2a2cf314a52fa7f315ca37ca445",
|
|
cache / "68" / "dde2c3c4e7953c2290f176bbdc9a54",
|
|
}
|
|
clean_staging()
|
|
|
|
dvc.commit(os.path.join("data"))
|
|
assert set(cache.glob("*/*")) == {
|
|
cache / "1a" / "ca2c799df82929bbdd976557975546.dir",
|
|
cache / "ac" / "bd18db4cc2f85cedef654fccc4a4d8",
|
|
cache / "4c" / "e8d2a2cf314a52fa7f315ca37ca445",
|
|
cache / "68" / "dde2c3c4e7953c2290f176bbdc9a54",
|
|
cache / "37" / "b51d194a7513e45b56f6524f2d51f2",
|
|
}
|
|
|
|
|
|
def test_commit_no_exec_missing_dep(tmp_dir, dvc):
|
|
stage = dvc.run(
|
|
name="my", cmd="mycmd", deps=["dep"], outs=["out"], no_exec=True
|
|
)
|
|
clean_staging()
|
|
assert dvc.status(stage.path)
|
|
|
|
with pytest.raises(DependencyDoesNotExistError):
|
|
dvc.commit(stage.path, force=True)
|
|
|
|
|
|
def test_commit_no_exec_missing_out(tmp_dir, dvc):
|
|
stage = dvc.run(name="my", cmd="mycmd", outs=["out"], no_exec=True)
|
|
clean_staging()
|
|
assert dvc.status(stage.path)
|
|
|
|
with pytest.raises(OutputDoesNotExistError):
|
|
dvc.commit(stage.path, force=True)
|
|
|
|
|
|
def test_commit_pipeline_stage(tmp_dir, dvc, run_copy):
|
|
tmp_dir.gen("foo", "foo")
|
|
stage = run_copy("foo", "bar", no_commit=True, name="copy-foo-bar")
|
|
clean_staging()
|
|
assert dvc.status(stage.addressing)
|
|
assert dvc.commit(stage.addressing, force=True) == [stage]
|
|
assert not dvc.status(stage.addressing)
|
|
|
|
# just to confirm different variants work
|
|
assert dvc.commit(f":{stage.addressing}") == [stage]
|
|
assert dvc.commit(f"{PIPELINE_FILE}:{stage.addressing}") == [stage]
|
|
assert dvc.commit(PIPELINE_FILE) == [stage]
|
|
|
|
|
|
def test_imported_entries_unchanged(tmp_dir, dvc, erepo_dir):
|
|
with erepo_dir.chdir():
|
|
erepo_dir.dvc_gen("file", "file content", "initial commit")
|
|
clean_staging()
|
|
|
|
stage = dvc.imp(os.fspath(erepo_dir), "file")
|
|
|
|
assert stage.changed_entries() == ([], [], None)
|