From f8467cf3753d09e6c24ac303cd1bbe6aa1138afc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Saugat=20Pachhai=20=28=E0=A4=B8=E0=A5=8C=E0=A4=97=E0=A4=BE?= =?UTF-8?q?=E0=A4=A4=29?= Date: Wed, 9 Feb 2022 18:24:00 +0545 Subject: [PATCH] exp init: log created dependencies in non-interactive mode --- dvc/repo/experiments/init.py | 44 +++++++++++++++-------------- tests/func/experiments/test_init.py | 6 ++-- 2 files changed, 27 insertions(+), 23 deletions(-) diff --git a/dvc/repo/experiments/init.py b/dvc/repo/experiments/init.py index 7a0ea5a72..3d14003d2 100644 --- a/dvc/repo/experiments/init.py +++ b/dvc/repo/experiments/init.py @@ -8,7 +8,6 @@ from typing import ( Callable, Dict, Iterable, - List, Optional, TextIO, Tuple, @@ -16,7 +15,7 @@ from typing import ( cast, ) -from funcy import compact, lremove +from funcy import compact, lremove, lsplit from rich.rule import Rule from rich.syntax import Syntax @@ -24,13 +23,13 @@ from dvc.exceptions import DvcException from dvc.stage import PipelineStage from dvc.stage.serialize import to_pipeline_file from dvc.types import OptStr +from dvc.utils import humanize from dvc.utils.serialize import dumps_yaml if TYPE_CHECKING: from dvc.repo import Repo from dvc.dvcfile import DVCFile from rich.tree import Tree - from dvc.dependency import Dependency from dvc.ui import ui @@ -155,7 +154,7 @@ def init_interactive( styled=True, ) - tree_label = "DVC assumes the following workspace structure:" + tree_label = "Experiment project structure:" display_workspace_tree(workspace, tree_label, stderr=True) ret.update( compact( @@ -216,27 +215,30 @@ def is_file(path: str) -> bool: return bool(ext) -def init_deps(deps: Iterable["Dependency"]) -> List[str]: +def init_deps(stage: PipelineStage, log: bool = False) -> None: + from funcy import rpartial + from dvc.dependency import ParamsDependency from dvc.fs.local import localfs - new_deps = [ - dep - for dep in deps - if not isinstance(dep, ParamsDependency) and not dep.exists - ] - for dep in new_deps: - fs_path = dep.fs_path - if not is_file(fs_path): - localfs.makedirs(fs_path) - continue + new_deps = [dep for dep in stage.deps if not dep.exists] + params, deps = lsplit(rpartial(isinstance, ParamsDependency), new_deps) - localfs.makedirs(localfs.path.parent(fs_path), exist_ok=True) - with localfs.open(fs_path, "w", encoding="utf-8"): + if log: + paths = map("[green]{0}[/]".format, new_deps) + ui.write(f"Creating {humanize.join(paths)}", styled=True) + + # always create a file for params, detect file/folder based on extension + # for other dependencies + dirs = [dep.fs_path for dep in deps if not is_file(dep.fs_path)] + files = [dep.fs_path for dep in deps + params if is_file(dep.fs_path)] + for path in dirs: + localfs.makedirs(path) + for path in files: + localfs.makedirs(localfs.path.parent(path), exist_ok=True) + with localfs.open(path, "w", encoding="utf-8"): pass - return [dep.def_path for dep in new_deps] - def init( repo: "Repo", @@ -327,9 +329,9 @@ def init( stage.dump(update_lock=False) stage.ignore_outs() if not interactive: - label = "Creating experiment project structure:" + label = "Experiment project structure:" display_workspace_tree(context, label) - init_deps(stage.deps) + init_deps(stage, log=not interactive) if params: repo.scm_context.track_file(params) else: diff --git a/tests/func/experiments/test_init.py b/tests/func/experiments/test_init.py index cc78508de..6639b1ac4 100644 --- a/tests/func/experiments/test_init.py +++ b/tests/func/experiments/test_init.py @@ -298,7 +298,8 @@ def test_init_default(tmp_dir, scm, dvc, interactive, overrides, inp, capsys): assert "'data' does not exist, the directory will be created." in err assert not out return - assert "Creating experiment project structure: " in out + assert "Experiment project structure: " in out + assert "Creating script.py and data" in out @pytest.mark.timeout(5, func_only=True) @@ -384,7 +385,8 @@ def test_init_interactive_live( assert "'data' does not exist, the directory will be created." in err assert not out return - assert "Creating experiment project structure: " in out + assert "Experiment project structure: " in out + assert "Creating script.py and data" in out @pytest.mark.parametrize(