Compare commits

...
8 Commits
Author SHA1 Message Date
dberenbaum 1aeb8750b6 drop debugging comment 2024-03-07 07:39:53 -05:00
dberenbaum 5aae934379 test auto push with tmp dir 2024-03-07 07:39:53 -05:00
dberenbaum de1338caaf write git config to exp tmp repos 2024-03-07 07:39:53 -05:00
Ivan Shchekleinandskshetry 9957410998 fix(plots): handle empty directories (#10344)
* fix(plots): handle empty directories

* Update tests/integration/plots/test_plots.py

Co-authored-by: skshetry <[email protected]>

* cleanup after review

---------

Co-authored-by: skshetry <[email protected]>
2024-03-07 12:50:49 +05:45
dberenbaum cf9855af45 handle auto push errors 2024-03-05 09:44:48 -05:00
skshetry 0001cc3a4b drop 2.x branch (#10339)
* drop 2.x branch

* Update tests.yaml
2024-03-05 14:39:51 +05:45
skshetry 440e1491f0 Delete .github/mergify.yml (#10338) 2024-03-05 14:37:44 +05:45
AlexandreKempfandSaugat Pachhai b0384f8f51 dvc studio login automatically push experiments by default (#10331)
* studio login set auto_push + add logs for the user

* set a separate global config dir for studio login tests

The `isolate` fixture sets a separate, _isolated_ global config directory.
But it is shared among other tests. So any mutations to it affects other
tests too.

We should force `studio login` tests to use a separate tmp_dir for global config.

* cleanup message

---------

Co-authored-by: Saugat Pachhai (सौगात) <[email protected]>
2024-03-05 11:48:16 +05:45
10 changed files with 100 additions and 19 deletions
-10
View File
@@ -1,10 +0,0 @@
pull_request_rules:
- name: backport patches to 2.x
conditions:
- base=main
- label=backport
actions:
backport:
branches:
- "2.x"
title: ({{ destination_branch }}) {{ title }}
+1 -1
View File
@@ -2,7 +2,7 @@ name: "CodeQL"
on:
push:
branches: ["main", "2.x"]
branches: ["main"]
pull_request:
branches: ["main"]
schedule:
+1 -1
View File
@@ -2,7 +2,7 @@ name: Tests
on:
push:
branches: [main, 2.x]
branches: [main]
pull_request:
schedule:
- cron: '5 1 * * *' # every day at 01:05
+11
View File
@@ -46,6 +46,15 @@ class CmdStudioLogin(CmdConfig):
self.save_config(hostname, access_token)
if not self.config["exp"].get("auto_push", True):
from dvc.ui import ui
ui.warn(
"exp.auto_push is disabled. \n"
"Enable with 'dvc config exp.auto_push true' "
"to automatically push experiments to Studio."
)
config_path = self.config.files["global"]
ui.write(f"Authentication complete. Saved token to {config_path}.")
return 0
@@ -54,6 +63,8 @@ class CmdStudioLogin(CmdConfig):
with self.config.edit("global") as conf:
conf["studio"]["token"] = token
conf["studio"]["url"] = hostname
if "auto_push" not in conf["exp"]:
conf["exp"]["auto_push"] = True
class CmdStudioLogout(CmdConfig):
+18 -2
View File
@@ -677,7 +677,13 @@ class BaseExecutor(ABC):
git_remote = os.getenv(
DVC_EXP_GIT_REMOTE, exp_config.get("git_remote", "origin")
)
cls._validate_remotes(dvc, git_remote)
try:
cls._validate_remotes(dvc, git_remote)
except DvcException as exc:
logger.warning("Failed to validate remotes. Disabling auto push: %s", exc)
yield
return
yield
cls._auto_push(dvc, git_remote)
@@ -688,7 +694,17 @@ class BaseExecutor(ABC):
push_cache=True,
run_cache=True,
):
from dvc.ui import ui
from dvc.utils import format_link
branch = dvc.scm.get_ref(EXEC_BRANCH, follow=False)
link = format_link(
"https://dvc.org/doc/user-guide/experiment-management/sharing-experiments"
)
ui.write(
f"Pushing experiment to '{git_remote}'. Cancel with CTRL+C. "
f"See {link} for more info."
)
try:
dvc.experiments.push(
git_remote,
@@ -696,7 +712,7 @@ class BaseExecutor(ABC):
push_cache=push_cache,
run_cache=run_cache,
)
except BaseException as exc: # noqa: BLE001
except DvcException as exc:
logger.warning(
(
"Something went wrong while auto pushing experiment "
+14
View File
@@ -130,6 +130,8 @@ class TempDirExecutor(BaseLocalExecutor):
self.scm.stash.apply(merge_rev)
self._update_config(repo.config.read("local"))
local_git_config = os.path.join(repo.scm.root_dir, ".git", "config")
self._update_git_config(ConfigObj(local_git_config, list_values=False))
def _update_config(self, update):
local_config = os.path.join(self.root_dir, self.dvc_dir, "config.local")
@@ -143,6 +145,18 @@ class TempDirExecutor(BaseLocalExecutor):
with open(local_config, "wb") as fobj:
conf_obj.write(fobj)
def _update_git_config(self, update):
local_config = os.path.join(self.scm.root_dir, ".git", "config")
logger.debug("Writing experiments local Git config '%s'", local_config)
if os.path.exists(local_config):
conf_obj = ConfigObj(local_config, list_values=False)
conf_obj.merge(update)
else:
conf_obj = ConfigObj(update, list_values=False)
if conf_obj:
with open(local_config, "wb") as fobj:
conf_obj.write(fobj)
def init_cache(
self,
repo: "Repo",
+6 -5
View File
@@ -462,11 +462,12 @@ def _resolve_definitions(
fs, data_path, props=plot_props | props, onerror=onerror
)
# use config for parent directory with most specific definition
unpacked["data"] = {
k: v
for k, v in unpacked["data"].items()
if _closest_parent(fs, k, plot_ids_parents) == data_path
}
if unpacked.get("data"):
unpacked["data"] = {
k: v
for k, v in unpacked["data"].items()
if _closest_parent(fs, k, plot_ids_parents) == data_path
}
dpath.merge(result, unpacked)
elif _matches(targets, config_path, plot_id):
adjusted_props = _adjust_sources(fs, plot_props, config_dir)
+28
View File
@@ -409,3 +409,31 @@ def test_auto_push_on_save(
dvc.experiments.save(name=exp_name, force=True)
assert first(dvc.experiments.push(name=exp_name, git_remote=remote)) == expected_key
def test_auto_push_misconfigured(
tmp_dir, scm, dvc, git_upstream, local_remote, exp_stage, caplog
):
with dvc.config.edit() as conf:
conf["exp"]["auto_push"] = True
conf["exp"]["git_remote"] = "notfound"
exp_name = "foo"
with caplog.at_level(logging.WARNING, logger="dvc"):
dvc.experiments.run(exp_stage.addressing, params=["foo=2"], name=exp_name)
assert "Failed to validate remotes" in caplog.text
def test_auto_push_tmp_dir(tmp_dir, scm, dvc, git_upstream, local_remote, exp_stage):
remote = git_upstream.remote
with dvc.config.edit() as conf:
conf["exp"]["auto_push"] = True
conf["exp"]["git_remote"] = remote
exp_name = "foo"
dvc.experiments.run(
exp_stage.addressing, params=["foo=2"], name=exp_name, tmp_dir=True
)
assert first(dvc.experiments.push(name=exp_name, git_remote=remote)) == "up_to_date"
+12
View File
@@ -634,3 +634,15 @@ def test_nested_x_defn_collection(tmp_dir, dvc, scm, capsys):
{"step": 1, "Max_Leaf_Nodes": "50", "rev": "workspace"},
{"step": 2, "Max_Leaf_Nodes": "500", "rev": "workspace"},
]
def test_plots_empty_directory(tmp_dir, dvc, scm, capsys):
(tmp_dir / "empty").mkdir()
(tmp_dir / "dvc.yaml").dump({"plots": [{"empty": {}}]})
scm.add(["dvc.yaml"])
scm.commit("commit dvc files")
html_path, _, split_json_result = call(capsys)
assert split_json_result == {}
assert html_path == ""
+9
View File
@@ -1,9 +1,18 @@
import pytest
from dvc_studio_client.auth import AuthorizationExpiredError
from dvc import env
from dvc.cli import main
from dvc.utils.studio import STUDIO_URL
@pytest.fixture(autouse=True)
def global_config_dir(monkeypatch, tmp_dir_factory):
monkeypatch.setenv(
env.DVC_GLOBAL_CONFIG_DIR, str(tmp_dir_factory.mktemp("studio-login"))
)
def test_studio_login_token_check_failed(mocker):
mocker.patch(
"dvc_studio_client.auth.get_access_token",