mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 07:18:25 +01:00
6dd435eefe
* Push running checkpoint to remote Fix #6182 * Update dvc/repo/experiments/executor/base.py Co-authored-by: Peter Rowlands (변기호) <[email protected]> * Get the branch name * Finish this PR 1. move env name to dvc/env.py. 2. add some tests for it. * Update after finally commit * Some problems found in the review 1. change the behavior of self remote 2. do not use string DVC_EXP_AUTO_PUSH 3. downgrade the logger level in auto push 4. use full branch ref * Update dvc/repo/experiments/executor/base.py Co-authored-by: Peter Rowlands (변기호) <[email protected]> * add a remote check before auto push * Split current one env into two * Rename DVC_EXP_CHECKPOINT_PUSH to DVC_EXP_AUTO_PUSH * value error * Remove getenv_bool switch to env2bool * Better on handling Dulwich exceptions * Name changing removing prints * Update dvc/env.py Co-authored-by: Peter Rowlands (변기호) <[email protected]> * Some changes in review * Add validation to repo URL * Add new type of SCMError * Update dvc/repo/experiments/executor/base.py Co-authored-by: Peter Rowlands (변기호) <[email protected]> * Use spy tests call counts and args * Update dvc/scm/git/backend/base.py Co-authored-by: Peter Rowlands (변기호) <[email protected]> * Update dvc/scm/git/backend/dulwich.py Co-authored-by: Peter Rowlands (변기호) <[email protected]> * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Remove validate_git_repo to validate_git_remote Co-authored-by: Peter Rowlands (변기호) <[email protected]> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
109 lines
2.7 KiB
Python
109 lines
2.7 KiB
Python
from textwrap import dedent
|
|
|
|
import pytest
|
|
|
|
from tests.func.test_repro_multistage import COPY_SCRIPT
|
|
|
|
DEFAULT_ITERATIONS = 2
|
|
CHECKPOINT_SCRIPT_FORMAT = dedent(
|
|
"""\
|
|
import os
|
|
import sys
|
|
import shutil
|
|
|
|
from dvc.api import make_checkpoint
|
|
|
|
checkpoint_file = {}
|
|
checkpoint_iterations = int({})
|
|
if os.path.exists(checkpoint_file):
|
|
with open(checkpoint_file) as fobj:
|
|
try:
|
|
value = int(fobj.read())
|
|
except ValueError:
|
|
value = 0
|
|
else:
|
|
with open(checkpoint_file, "w"):
|
|
pass
|
|
value = 0
|
|
|
|
shutil.copyfile({}, {})
|
|
|
|
if os.getenv("DVC_CHECKPOINT"):
|
|
for _ in range(checkpoint_iterations):
|
|
value += 1
|
|
with open(checkpoint_file, "w") as fobj:
|
|
fobj.write(str(value))
|
|
make_checkpoint()
|
|
"""
|
|
)
|
|
CHECKPOINT_SCRIPT = CHECKPOINT_SCRIPT_FORMAT.format(
|
|
"sys.argv[1]", "sys.argv[2]", "sys.argv[3]", "sys.argv[4]"
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def exp_stage(tmp_dir, scm, dvc):
|
|
tmp_dir.gen("copy.py", COPY_SCRIPT)
|
|
tmp_dir.gen("params.yaml", "foo: 1")
|
|
stage = dvc.run(
|
|
cmd="python copy.py params.yaml metrics.yaml",
|
|
metrics_no_cache=["metrics.yaml"],
|
|
params=["foo"],
|
|
name="copy-file",
|
|
deps=["copy.py"],
|
|
)
|
|
scm.add(
|
|
[
|
|
"dvc.yaml",
|
|
"dvc.lock",
|
|
"copy.py",
|
|
"params.yaml",
|
|
"metrics.yaml",
|
|
".gitignore",
|
|
]
|
|
)
|
|
scm.commit("init")
|
|
return stage
|
|
|
|
|
|
@pytest.fixture
|
|
def checkpoint_stage(tmp_dir, scm, dvc, mocker):
|
|
mocker.patch("dvc.stage.run.Monitor.AWAIT", 0.01)
|
|
|
|
tmp_dir.gen("checkpoint.py", CHECKPOINT_SCRIPT)
|
|
tmp_dir.gen("params.yaml", "foo: 1")
|
|
stage = dvc.run(
|
|
cmd=(
|
|
f"python checkpoint.py foo {DEFAULT_ITERATIONS} "
|
|
"params.yaml metrics.yaml"
|
|
),
|
|
metrics_no_cache=["metrics.yaml"],
|
|
params=["foo"],
|
|
checkpoints=["foo"],
|
|
deps=["checkpoint.py"],
|
|
no_exec=True,
|
|
name="checkpoint-file",
|
|
)
|
|
scm.add(["dvc.yaml", "checkpoint.py", "params.yaml", ".gitignore"])
|
|
scm.commit("init")
|
|
stage.iterations = DEFAULT_ITERATIONS
|
|
return stage
|
|
|
|
|
|
@pytest.fixture
|
|
def git_upstream(tmp_dir, erepo_dir):
|
|
url = "file://{}".format(erepo_dir.resolve().as_posix())
|
|
tmp_dir.scm.gitpython.repo.create_remote("upstream", url)
|
|
erepo_dir.remote = "upstream"
|
|
erepo_dir.url = url
|
|
return erepo_dir
|
|
|
|
|
|
@pytest.fixture
|
|
def git_downstream(tmp_dir, erepo_dir):
|
|
url = "file://{}".format(tmp_dir.resolve().as_posix())
|
|
erepo_dir.scm.gitpython.repo.create_remote("upstream", url)
|
|
erepo_dir.remote = "upstream"
|
|
erepo_dir.url = url
|
|
return erepo_dir
|