addressing: allow stage name addressing without ":" (#3842)

* addressing: allow stage name addressing without colon

* repo: reorganize imports

* refactor/reorganize collect_granular

* add tests for stage addressing

* fix ds and cc issues

* fix unpack error in test

* tests: stage string representations

* exception: add error message

Also remove `cause` when throwing NoOutputOrStage, as
CheckoutErrorSuggestGit will start chaining all the messages
in cause of the exception.

* rename NoOutputOrStage NoOutputOrStageError

Co-authored-by: Ruslan Kuprieiev <[email protected]>

dvc/command/pipeline.py

@@ -10,11 +10,10 @@ logger = logging.getLogger(__name__)
 class CmdPipelineShow(CmdBase):
     def _show(self, target, commands, outs, locked):
         import networkx
-        from dvc import dvcfile
         from dvc.utils import parse_target
 
         path, name = parse_target(target)
-        stage = dvcfile.Dvcfile(self.repo, path).stages[name]
+        stage = self.repo.get_stage(path, name)
         G = self.repo.graph
         stages = networkx.dfs_postorder_nodes(G, stage)
         if locked:
@@ -58,12 +57,11 @@ class CmdPipelineShow(CmdBase):
 
     def _build_graph(self, target, commands=False, outs=False):
         import networkx
-        from dvc import dvcfile
         from dvc.repo.graph import get_pipeline
         from dvc.utils import parse_target
 
         path, name = parse_target(target)
-        target_stage = dvcfile.Dvcfile(self.repo, path).stages[name]
+        target_stage = self.repo.get_stage(path, name)
         G = get_pipeline(self.repo.pipelines, target_stage)
 
         nodes = set()

dvc/exceptions.py

@@ -322,3 +322,15 @@ class RemoteCacheRequiredError(DvcException):
 
 class IsADirectoryError(DvcException):
     """Raised when a file operation is requested on a directory."""
+
+
+class NoOutputOrStageError(DvcException):
+    """
+    Raised when the target is neither an output nor a stage name in dvc.yaml
+    """
+
+    def __init__(self, target, file):
+        super().__init__(
+            f"'{target}' "
+            f"does not exist as an output or a stage name in '{file}'"
+        )

dvc/repo/__init__.py

@@ -1,3 +1,4 @@
+import logging
 import os
 from contextlib import contextmanager
 from functools import wraps
@@ -5,17 +6,25 @@ from functools import wraps
 from funcy import cached_property, cat, first
 
 from dvc.config import Config
+from dvc.dvcfile import PIPELINE_FILE, Dvcfile, is_valid_filename
 from dvc.exceptions import FileMissingError
 from dvc.exceptions import IsADirectoryError as DvcIsADirectoryError
-from dvc.exceptions import NotDvcRepoError, OutputNotFoundError
+from dvc.exceptions import (
+    NoOutputOrStageError,
+    NotDvcRepoError,
+    OutputNotFoundError,
+)
 from dvc.ignore import CleanTree
 from dvc.path_info import PathInfo
 from dvc.repo.tree import RepoTree
 from dvc.utils.fs import path_isin
 
+from ..stage.exceptions import StageFileDoesNotExistError, StageNotFound
 from ..utils import parse_target
 from .graph import check_acyclic, get_pipeline, get_pipelines
 
+logger = logging.getLogger(__name__)
+
 
 def locked(f):
     @wraps(f)
@@ -181,6 +190,25 @@ class Repo:
 
         self.scm.ignore_list(flist)
 
+    def get_stage(self, path=None, name=None):
+        if not path:
+            path = PIPELINE_FILE
+            logger.debug("Assuming '%s' to be a stage inside '%s'", name, path)
+
+        dvcfile = Dvcfile(self, path)
+        return dvcfile.stages[name]
+
+    def get_stages(self, path=None, name=None):
+        if not path:
+            path = PIPELINE_FILE
+            logger.debug("Assuming '%s' to be a stage inside '%s'", name, path)
+
+        if name:
+            return [self.get_stage(path, name)]
+
+        dvcfile = Dvcfile(self, path)
+        return list(dvcfile.stages.values())
+
     def check_modified_graph(self, new_stages):
         """Generate graph including the new stage to check for errors"""
         # Building graph might be costly for the ones with many DVC-files,
@@ -204,10 +232,9 @@ class Repo:
         stages = nx.dfs_postorder_nodes(graph)
         return [stage for stage in stages if path_isin(stage.path, path)]
 
-    def collect(self, target, with_deps=False, recursive=False, graph=None):
+    def collect(
-        import networkx as nx
+        self, target=None, with_deps=False, recursive=False, graph=None
-        from ..dvcfile import Dvcfile
+    ):
-
         if not target:
             return list(graph) if graph else self.stages
 
@@ -217,36 +244,81 @@ class Repo:
             )
 
         path, name = parse_target(target)
-        dvcfile = Dvcfile(self, path)
+        stages = self.get_stages(path, name)
-        stages = list(dvcfile.stages.filter(name).values())
         if not with_deps:
             return stages
 
         res = set()
         for stage in stages:
-            pipeline = get_pipeline(get_pipelines(graph or self.graph), stage)
+            res.update(self._collect_pipeline(stage, graph=graph))
-            res.update(nx.dfs_postorder_nodes(pipeline, stage))
         return res
 
-    def collect_granular(self, target, *args, **kwargs):
+    def _collect_pipeline(self, stage, graph=None):
-        from ..dvcfile import Dvcfile, is_valid_filename
+        import networkx as nx
 
+        pipeline = get_pipeline(get_pipelines(graph or self.graph), stage)
+        return nx.dfs_postorder_nodes(pipeline, stage)
+
+    def _collect_from_default_dvcfile(self, target):
+        dvcfile = Dvcfile(self, PIPELINE_FILE)
+        if dvcfile.exists():
+            return dvcfile.stages.get(target)
+
+    def collect_granular(
+        self, target=None, with_deps=False, recursive=False, graph=None
+    ):
+        """
+        Priority is in the order of following in case of ambiguity:
+            - .dvc file or .yaml file
+            - dir if recursive and directory exists
+            - stage_name
+            - output file
+        """
         if not target:
             return [(stage, None) for stage in self.stages]
 
         file, name = parse_target(target)
-        if is_valid_filename(file) and not kwargs.get("with_deps"):
+        stages = []
-            # Optimization: do not collect the graph for a specific .dvc target
-            stages = Dvcfile(self, file).stages.filter(name)
-            return [(stage, None) for stage in stages.values()]
 
-        try:
+        # Optimization: do not collect the graph for a specific target
-            (out,) = self.find_outs_by_path(file, strict=False)
+        if not file:
-            filter_info = PathInfo(os.path.abspath(file))
+            # parsing is ambiguous when it does not have a colon
-            return [(out.stage, filter_info)]
+            # or if it's not a dvcfile, as it can be a stage name
-        except OutputNotFoundError:
+            # in `dvc.yaml` or, an output in a stage.
-            stages = self.collect(target, *args, **kwargs)
+            logger.debug(
-            return [(stage, None) for stage in stages]
+                "Checking if stage '%s' is in '%s'", target, PIPELINE_FILE
+            )
+            if not (recursive and os.path.isdir(target)):
+                stage = self._collect_from_default_dvcfile(target)
+                if stage:
+                    stages = (
+                        self._collect_pipeline(stage) if with_deps else [stage]
+                    )
+        elif not with_deps and is_valid_filename(file):
+            stages = self.get_stages(file, name)
+
+        if not stages:
+            if not (recursive and os.path.isdir(target)):
+                try:
+                    (out,) = self.find_outs_by_path(target, strict=False)
+                    filter_info = PathInfo(os.path.abspath(target))
+                    return [(out.stage, filter_info)]
+                except OutputNotFoundError:
+                    pass
+
+            try:
+                stages = self.collect(target, with_deps, recursive, graph)
+            except StageFileDoesNotExistError as exc:
+                # collect() might try to use `target` as a stage name
+                # and throw error that dvc.yaml does not exist, whereas it
+                # should say that both stage name and file does not exist.
+                if file and is_valid_filename(file):
+                    raise
+                raise NoOutputOrStageError(target, exc.file) from exc
+            except StageNotFound as exc:
+                raise NoOutputOrStageError(target, exc.file) from exc
+
+        return [(stage, None) for stage in stages]
 
     def used_cache(
         self,
@@ -443,16 +515,13 @@ class Repo:
         return PlotTemplates(self.dvc_dir)
 
     def _collect_stages(self):
-        from dvc.dvcfile import Dvcfile, is_valid_filename
-
         stages = []
         outs = set()
 
         for root, dirs, files in self.tree.walk(self.root_dir):
             for file_name in filter(is_valid_filename, files):
                 path = os.path.join(root, file_name)
-                stage_loader = Dvcfile(self, path).stages
+                stages.extend(self.get_stages(path))
-                stages.extend(stage_loader.values())
                 outs.update(
                     out.fspath
                     for stage in stages

dvc/repo/checkout.py

@@ -1,7 +1,11 @@
 import logging
 import os
 
-from dvc.exceptions import CheckoutError, CheckoutErrorSuggestGit
+from dvc.exceptions import (
+    CheckoutError,
+    CheckoutErrorSuggestGit,
+    NoOutputOrStageError,
+)
 from dvc.progress import Tqdm
 from dvc.utils import relpath
 
@@ -18,11 +22,11 @@ def _get_unused_links(repo):
     return repo.state.get_unused_links(used)
 
 
-def _fspath_dir(path, root):
+def _fspath_dir(path):
     if not os.path.exists(str(path)):
         return str(path)
 
-    path = relpath(path, root)
+    path = relpath(path)
     return os.path.join(path, "") if os.path.isdir(path) else path
 
 
@@ -56,7 +60,7 @@ def _checkout(
         targets = [None]
         unused = _get_unused_links(self)
 
-    stats["deleted"] = [_fspath_dir(u, self.root_dir) for u in unused]
+    stats["deleted"] = [_fspath_dir(u) for u in unused]
     self.state.remove_links(unused)
 
     if isinstance(targets, str):
@@ -70,7 +74,11 @@ def _checkout(
                     target, with_deps=with_deps, recursive=recursive
                 )
             )
-        except (StageFileDoesNotExistError, StageFileBadNameError) as exc:
+        except (
+            StageFileDoesNotExistError,
+            StageFileBadNameError,
+            NoOutputOrStageError,
+        ) as exc:
             if not target:
                 raise
             raise CheckoutErrorSuggestGit(target) from exc
@@ -87,9 +95,7 @@ def _checkout(
                 filter_info=filter_info,
             )
             for key, items in result.items():
-                stats[key].extend(
+                stats[key].extend(_fspath_dir(path) for path in items)
-                    _fspath_dir(path, self.root_dir) for path in items
-                )
 
     if stats.get("failed"):
         raise CheckoutError(stats["failed"], stats)

dvc/repo/commit.py

@@ -46,3 +46,4 @@ def commit(self, target, with_deps=False, recursive=False, force=False):
         stage.commit()
 
         Dvcfile(self, stage.path).dump(stage)
+    return stages

dvc/repo/lock.py

@@ -3,13 +3,11 @@ from . import locked
 
 @locked
 def lock(self, target, unlock=False):
-    from .. import dvcfile
     from dvc.utils import parse_target
 
     path, name = parse_target(target)
-    dvcfile = dvcfile.Dvcfile(self, path)
+    stage = self.get_stage(path, name)
-    stage = dvcfile.stages[name]
     stage.locked = False if unlock else True
-    dvcfile.dump(stage, update_pipeline=True)
+    stage.dvcfile.dump(stage, update_pipeline=True)
 
     return stage

dvc/repo/remove.py

@@ -8,15 +8,14 @@ logger = logging.getLogger(__name__)
 
 @locked
 def remove(self, target, dvc_only=False):
-    from ..dvcfile import Dvcfile
+    from ..dvcfile import Dvcfile, is_valid_filename
 
     path, name = parse_target(target)
-    dvcfile = Dvcfile(self, path)
+    stages = self.get_stages(path, name)
-    stages = list(dvcfile.stages.filter(name).values())
     for stage in stages:
         stage.remove_outs(force=True)
 
-    if not dvc_only:
+    if path and is_valid_filename(path) and not dvc_only:
-        dvcfile.remove()
+        Dvcfile(self, path).remove()
 
     return stages

dvc/repo/reproduce.py

@@ -61,7 +61,6 @@ def reproduce(
     all_pipelines=False,
     **kwargs
 ):
-    from ..dvcfile import Dvcfile
     from dvc.utils import parse_target
 
     if not target and not all_pipelines:
@@ -81,8 +80,7 @@ def reproduce(
         if all_pipelines:
             pipelines = active_pipelines
         else:
-            dvcfile = Dvcfile(self, path)
+            stage = self.get_stage(path, name)
-            stage = dvcfile.stages[name]
             pipelines = [get_pipeline(active_pipelines, stage)]
 
         targets = []

dvc/stage/__init__.py

@@ -144,14 +144,10 @@ class Stage(params.StageParams):
         self._dvcfile = dvcfile
 
     def __repr__(self):
-        return "Stage: '{path}'".format(
+        return f"Stage: '{self.addressing}'"
-            path=self.path_in_repo if self.path else "No path"
-        )
 
     def __str__(self):
-        return "stage: '{path}'".format(
+        return f"stage: '{self.addressing}'"
-            path=self.relpath if self.path else "No path"
-        )
 
     @property
     def addressing(self):
@@ -159,7 +155,7 @@ class Stage(params.StageParams):
         Useful for alternative presentations where we don't need
         `Stage:` prefix.
         """
-        return self.relpath
+        return self.relpath if self.path else "No path"
 
     def __hash__(self):
         return hash(self.path_in_repo)
@@ -530,19 +526,13 @@ class PipelineStage(Stage):
     def __hash__(self):
         return hash((self.path_in_repo, self.name))
 
-    def __repr__(self):
-        return "Stage: '{path}:{name}'".format(
-            path=self.relpath if self.path else "No path", name=self.name
-        )
-
-    def __str__(self):
-        return "stage: '{path}:{name}'".format(
-            path=self.relpath if self.path else "No path", name=self.name
-        )
-
     @property
     def addressing(self):
-        return super().addressing + ":" + self.name
+        from dvc.dvcfile import PIPELINE_FILE
+
+        if self.path and self.relpath == PIPELINE_FILE:
+            return self.name
+        return f"{super().addressing}:{self.name}"
 
     def reload(self):
         return self.dvcfile.stages[self.name]

dvc/stage/exceptions.py

@@ -15,15 +15,8 @@ class StageFileFormatError(DvcException):
 
 class StageFileDoesNotExistError(DvcException):
     def __init__(self, fname):
-        from dvc.dvcfile import DVC_FILE_SUFFIX, is_dvc_file
+        self.file = fname
-
+        super().__init__(f"'{self.file}' does not exist.")
-        msg = f"'{fname}' does not exist."
-
-        sname = fname + DVC_FILE_SUFFIX
-        if is_dvc_file(sname):
-            msg += f" Do you mean '{sname}'?"
-
-        super().__init__(msg)
 
 
 class StageFileAlreadyExistsError(DvcException):
@@ -87,8 +80,10 @@ class MissingDataSource(DvcException):
 
 class StageNotFound(KeyError, DvcException):
     def __init__(self, file, name):
+        self.file = file.relpath
+        self.name = name
         super().__init__(
-            f"Stage '{name}' not found inside '{file.relpath}' file"
+            f"Stage '{self.name}' not found inside '{self.file}' file"
         )
 
 

dvc/stage/loader.py

@@ -30,16 +30,6 @@ class StageLoader(Mapping):
         self.stages_data = stages_data or {}
         self.lockfile_data = lockfile_data or {}
 
-    def filter(self, item=None):
-        if not item:
-            return self
-
-        if item not in self:
-            raise StageNotFound(self.dvcfile, item)
-
-        data = {item: self.stages_data[item]} if item in self else {}
-        return self.__class__(self.dvcfile, data, self.lockfile_data)
-
     @staticmethod
     def fill_from_lock(stage, lock_data):
         from .params import StageParams
@@ -224,9 +214,6 @@ class SingleStageLoader(Mapping):
         self.stage_data = stage_data or {}
         self.stage_text = stage_text
 
-    def filter(self, item=None):
-        return self
-
     def __getitem__(self, item):
         if item:
             logger.warning(

dvc/utils/__init__.py

@@ -395,7 +395,7 @@ def format_link(link):
 
 
 def parse_target(target, default=None):
-    from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
+    from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK, is_valid_filename
     from dvc.exceptions import DvcException
 
     if not target:
@@ -409,13 +409,19 @@ def parse_target(target, default=None):
         match.group("path"),
         match.group("name"),
     )
+    if path:
+        if os.path.basename(path) == PIPELINE_LOCK:
+            raise DvcException(
+                "Did you mean: `{}`?".format(
+                    target.replace(".lock", ".yaml", 1)
+                )
+            )
+        if not name:
+            ret = (target, None)
+            return ret if is_valid_filename(target) else ret[::-1]
+
     if not path:
         path = default or PIPELINE_FILE
         logger.debug("Assuming file to be '%s'", path)
 
-    if os.path.basename(path) == PIPELINE_LOCK:
-        raise DvcException(
-            "Did you mean: `{}`?".format(target.replace(".lock", ".yaml", 1))
-        )
-
     return path, name

tests/func/test_checkout.py

@@ -7,22 +7,20 @@ import shutil
 import pytest
 from mock import patch
 
-from dvc.dvcfile import DVC_FILE_SUFFIX, Dvcfile
+from dvc.dvcfile import DVC_FILE_SUFFIX, PIPELINE_FILE, Dvcfile
 from dvc.exceptions import (
     CheckoutError,
     CheckoutErrorSuggestGit,
     ConfirmRemoveError,
     DvcException,
+    NoOutputOrStageError,
 )
 from dvc.main import main
 from dvc.remote import S3Remote
 from dvc.remote.local import LocalRemote
 from dvc.repo import Repo as DvcRepo
 from dvc.stage import Stage
-from dvc.stage.exceptions import (
+from dvc.stage.exceptions import StageFileDoesNotExistError
-    StageFileBadNameError,
-    StageFileDoesNotExistError,
-)
 from dvc.system import System
 from dvc.utils import relpath
 from dvc.utils.fs import walk_files
@@ -403,17 +401,26 @@ class TestCheckoutSuggestGit(TestRepro):
     def test(self):
 
         try:
-            self.dvc.checkout(targets=["gitbranch"])
+            self.dvc.checkout(targets="gitbranch")
         except DvcException as exc:
             self.assertIsInstance(exc, CheckoutErrorSuggestGit)
-            self.assertIsInstance(exc.__cause__, StageFileDoesNotExistError)
+            self.assertIsInstance(exc.__cause__, NoOutputOrStageError)
+            self.assertIsInstance(
+                exc.__cause__.__cause__, StageFileDoesNotExistError
+            )
+
+        try:
+            self.dvc.checkout(targets=self.FOO)
+        except DvcException as exc:
+            self.assertIsInstance(exc, CheckoutErrorSuggestGit)
+            self.assertIsInstance(exc.__cause__, NoOutputOrStageError)
             self.assertIsNone(exc.__cause__.__cause__)
 
         try:
-            self.dvc.checkout(targets=[self.FOO])
+            self.dvc.checkout(targets="looks-like-dvcfile.dvc")
         except DvcException as exc:
             self.assertIsInstance(exc, CheckoutErrorSuggestGit)
-            self.assertIsInstance(exc.__cause__, StageFileBadNameError)
+            self.assertIsInstance(exc.__cause__, StageFileDoesNotExistError)
             self.assertIsNone(exc.__cause__.__cause__)
 
 
@@ -770,9 +777,51 @@ def test_checkout_for_external_outputs(tmp_dir, dvc):
     assert stats == {**empty_checkout, "modified": [str(file_path)]}
 
 
-def test_checkouts_for_pipeline_tracked_outs(tmp_dir, dvc, scm, run_copy):
+def test_checkouts_with_different_addressing(tmp_dir, dvc, run_copy):
-    from dvc.dvcfile import PIPELINE_FILE
+    tmp_dir.gen({"foo": "foo", "lorem": "lorem"})
+    run_copy("foo", "bar", name="copy-foo-bar")
+    run_copy("lorem", "ipsum", name="copy-lorem-ipsum")
 
+    (tmp_dir / "bar").unlink()
+    (tmp_dir / "ipsum").unlink()
+    assert set(dvc.checkout(PIPELINE_FILE)["added"]) == {"bar", "ipsum"}
+
+    (tmp_dir / "bar").unlink()
+    (tmp_dir / "ipsum").unlink()
+    assert set(dvc.checkout(":")["added"]) == {"bar", "ipsum"}
+
+    (tmp_dir / "bar").unlink()
+    assert dvc.checkout("copy-foo-bar")["added"] == ["bar"]
+
+    (tmp_dir / "bar").unlink()
+    assert dvc.checkout("dvc.yaml:copy-foo-bar")["added"] == ["bar"]
+
+    (tmp_dir / "bar").unlink()
+    assert dvc.checkout(":copy-foo-bar")["added"] == ["bar"]
+
+    (tmp_dir / "bar").unlink()
+    (tmp_dir / "data").mkdir()
+    with (tmp_dir / "data").chdir():
+        assert dvc.checkout(relpath(tmp_dir / "dvc.yaml") + ":copy-foo-bar")[
+            "added"
+        ] == [relpath(tmp_dir / "bar")]
+
+    (tmp_dir / "bar").unlink()
+    assert dvc.checkout("bar")["added"] == ["bar"]
+
+
+def test_checkouts_on_same_stage_name_and_output_name(tmp_dir, dvc, run_copy):
+    tmp_dir.gen("foo", "foo")
+    run_copy("foo", "bar", name="copy-foo-bar")
+    run_copy("foo", "copy-foo-bar", name="make_collision")
+
+    (tmp_dir / "bar").unlink()
+    (tmp_dir / "copy-foo-bar").unlink()
+    assert dvc.checkout("copy-foo-bar")["added"] == ["bar"]
+    assert dvc.checkout("./copy-foo-bar")["added"] == ["copy-foo-bar"]
+
+
+def test_checkouts_for_pipeline_tracked_outs(tmp_dir, dvc, scm, run_copy):
     tmp_dir.gen("foo", "foo")
     stage1 = run_copy("foo", "bar", name="copy-foo-bar")
     tmp_dir.gen("lorem", "lorem")

tests/func/test_commit.py

@@ -1,5 +1,6 @@
 import pytest
 
+from dvc.dvcfile import PIPELINE_FILE
 from dvc.stage.exceptions import StageCommitError
 from dvc.utils.stage import dump_stage_file, load_stage_file
 
@@ -81,3 +82,16 @@ def test_commit_no_exec(tmp_dir, dvc):
     assert dvc.status(stage.path)
     dvc.commit(stage.path, force=True)
     assert dvc.status(stage.path) == {}
+
+
+def test_commit_pipeline_stage(tmp_dir, dvc, run_copy):
+    tmp_dir.gen("foo", "foo")
+    stage = run_copy("foo", "bar", no_commit=True, name="copy-foo-bar")
+    assert dvc.status(stage.addressing)
+    assert dvc.commit(stage.addressing, force=True) == [stage]
+    assert not dvc.status(stage.addressing)
+
+    # just to confirm different variants work
+    assert dvc.commit(f":{stage.addressing}") == [stage]
+    assert dvc.commit(f"{PIPELINE_FILE}:{stage.addressing}") == [stage]
+    assert dvc.commit(PIPELINE_FILE) == [stage]

tests/func/test_data_cloud.py

@@ -932,3 +932,24 @@ def test_push_pull_all(tmp_dir, scm, dvc, setup_remote, key, expected):
 
     clean(["foo", "bar", "baz"], dvc)
     assert dvc.pull(**{key: True})["fetched"] == expected
+
+
+def test_push_pull_fetch_pipeline_stages(tmp_dir, dvc, run_copy, setup_remote):
+    remote_path = setup_remote(dvc)
+    tmp_dir.dvc_gen("foo", "foo")
+    run_copy("foo", "bar", no_commit=True, name="copy-foo-bar")
+
+    dvc.push("copy-foo-bar")
+    assert len(recurse_list_dir(remote_path)) == 1
+    # pushing everything so as we can check pull/fetch only downloads
+    # from specified targets
+    dvc.push()
+    clean(["foo", "bar"], dvc)
+
+    dvc.pull("copy-foo-bar")
+    assert (tmp_dir / "bar").exists()
+    assert len(recurse_list_dir(dvc.cache.local.cache_dir)) == 1
+    clean(["bar"], dvc)
+
+    dvc.fetch("copy-foo-bar")
+    assert len(recurse_list_dir(dvc.cache.local.cache_dir)) == 1

tests/func/test_dvcfile.py

@@ -1,10 +1,7 @@
 import pytest
 
 from dvc.dvcfile import PIPELINE_FILE, Dvcfile
-from dvc.stage.exceptions import (
+from dvc.stage.exceptions import StageFileDoesNotExistError
-    StageFileDoesNotExistError,
-    StageNameUnspecified,
-)
 from dvc.stage.loader import StageNotFound
 
 
@@ -163,37 +160,3 @@ def test_stage_collection(tmp_dir, dvc):
         single_stage=True,
     )
     assert {s for s in dvc.stages} == {stage1, stage3, stage2}
-
-
-def test_stage_filter(tmp_dir, dvc, run_copy):
-    tmp_dir.gen("foo", "foo")
-    stage1 = run_copy("foo", "bar", name="copy-foo-bar")
-    stage2 = run_copy("bar", "bax", name="copy-bar-bax")
-    stage3 = run_copy("bax", "baz", name="copy-bax-baz")
-    stages = Dvcfile(dvc, PIPELINE_FILE).stages
-    assert set(stages.filter(None).values()) == {
-        stage1,
-        stage2,
-        stage3,
-    }
-    assert set(stages.filter("copy-bar-bax").values()) == {stage2}
-    assert stages.filter("copy-bar-bax").get("copy-bar-bax") == stage2
-    with pytest.raises(StageNameUnspecified):
-        stages.get(None)
-
-    with pytest.raises(StageNotFound):
-        assert stages["unknown"]
-
-    with pytest.raises(StageNotFound):
-        assert stages.filter("unknown")
-
-
-def test_stage_filter_in_singlestage_file(tmp_dir, dvc, run_copy):
-    tmp_dir.gen("foo", "foo")
-    stage = run_copy("foo", "bar", single_stage=True)
-    dvcfile = Dvcfile(dvc, stage.path)
-    assert set(dvcfile.stages.filter(None).values()) == {stage}
-    assert dvcfile.stages.filter(None).get(None) == stage
-    assert dvcfile.stages.filter("copy-bar-bax").get("copy-bar-bax") == stage
-    assert dvcfile.stages.filter("copy-bar-bax").get(None) == stage
-    assert set(dvcfile.stages.filter("unknown").values()) == {stage}

tests/func/test_pipeline.py

@@ -296,20 +296,20 @@ def test_pipeline_list_show_multistage(tmp_dir, dvc, run_copy, caplog):
     with caplog.at_level(logging.INFO, "dvc"):
         command._show("foobar.dvc", False, False, False)
         output = caplog.text.splitlines()
-        assert "dvc.yaml:copy-foo-bar" in output[0]
+        assert "copy-foo-bar" in output[0]
         assert "foobar.dvc" in output[1]
 
     caplog.clear()
     with caplog.at_level(logging.INFO, "dvc"):
-        command._show("dvc.yaml:copy-foo-bar", False, False, False)
+        command._show("copy-foo-bar", False, False, False)
-        assert "dvc.yaml:copy-foo-bar" in caplog.text
+        assert "copy-foo-bar" in caplog.text
         assert "foobar.dvc" not in caplog.text
 
     command = CmdPipelineList([])
     caplog.clear()
     with caplog.at_level(logging.INFO, "dvc"):
         command.run()
-        assert "dvc.yaml:copy-foo-bar" in caplog.text
+        assert "copy-foo-bar" in caplog.text
         assert "foobar.dvc" in caplog.text
         assert "1 pipelines in total"
 
@@ -320,13 +320,13 @@ def test_pipeline_ascii_multistage(tmp_dir, dvc, run_copy):
     run_copy("bar", "foobar", single_stage=True)
     command = CmdPipelineShow([])
     nodes, edges, _ = command._build_graph("foobar.dvc")
-    assert set(nodes) == {"dvc.yaml:copy-foo-bar", "foobar.dvc"}
+    assert set(nodes) == {"copy-foo-bar", "foobar.dvc"}
     assert set(edges) == {
-        ("foobar.dvc", "dvc.yaml:copy-foo-bar"),
+        ("foobar.dvc", "copy-foo-bar"),
     }
 
-    nodes, *_ = command._build_graph("dvc.yaml:copy-foo-bar")
+    nodes, *_ = command._build_graph("copy-foo-bar")
-    assert set(nodes) == {"dvc.yaml:copy-foo-bar"}
+    assert set(nodes) == {"copy-foo-bar"}
 
 
 def test_pipeline_multi_outputs_stages(dvc):

tests/func/test_repo.py

@@ -1,13 +1,24 @@
 import os
+from operator import itemgetter
+
+import pytest
 
 from dvc.cache import Cache
+from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
+from dvc.exceptions import NoOutputOrStageError
+from dvc.path_info import PathInfo
 from dvc.repo import Repo
+from dvc.stage.exceptions import (
+    StageFileDoesNotExistError,
+    StageNameUnspecified,
+    StageNotFound,
+)
 from dvc.system import System
+from dvc.utils import relpath
+from dvc.utils.fs import remove
 
 
 def test_destroy(tmp_dir, dvc, run_copy):
-    from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
-
     dvc.config["cache"]["type"] = ["symlink"]
     dvc.cache = Cache(dvc)
 
@@ -88,6 +99,12 @@ def test_collect(tmp_dir, scm, dvc, run_copy):
     assert collect_outs("dvc.yaml:copy-foo-foobar", recursive=True) == {
         "foobar"
     }
+    assert collect_outs("copy-foo-foobar") == {"foobar"}
+    assert collect_outs("copy-foo-foobar", with_deps=True) == {
+        "foobar",
+        "foo",
+    }
+    assert collect_outs("copy-foo-foobar", recursive=True) == {"foobar"}
 
     run_copy("foobar", "baz", name="copy-foobar-baz")
     assert collect_outs("dvc.yaml") == {"foobar", "baz"}
@@ -98,6 +115,26 @@ def test_collect(tmp_dir, scm, dvc, run_copy):
     }
 
 
+def test_collect_dir_recursive(tmp_dir, dvc, run_head):
+    tmp_dir.gen({"dir": {"foo": "foo"}})
+    (stage1,) = dvc.add("dir", recursive=True)
+    with (tmp_dir / "dir").chdir():
+        stage2 = run_head("foo", name="copy-foo-bar")
+        stage3 = run_head("foo-1", single_stage=True)
+    assert set(dvc.collect("dir", recursive=True)) == {stage1, stage2, stage3}
+
+
+def test_collect_with_not_existing_output_or_stage_name(
+    tmp_dir, dvc, run_copy
+):
+    with pytest.raises(StageFileDoesNotExistError):
+        dvc.collect("some_file")
+    tmp_dir.dvc_gen("foo", "foo")
+    run_copy("foo", "bar", name="copy-foo-bar")
+    with pytest.raises(StageNotFound):
+        dvc.collect("some_file")
+
+
 def test_stages(tmp_dir, dvc):
     def stages():
         return {stage.relpath for stage in Repo(os.fspath(tmp_dir)).stages}
@@ -113,3 +150,215 @@ def test_stages(tmp_dir, dvc):
     tmp_dir.gen(".dvcignore", "dir")
 
     assert stages() == {"file.dvc"}
+
+
+@pytest.fixture
+def stages(tmp_dir, run_copy):
+    stage1, stage2 = tmp_dir.dvc_gen({"foo": "foo", "lorem": "lorem"})
+    return {
+        "foo-generate": stage1,
+        "lorem-generate": stage2,
+        "copy-foo-bar": run_copy("foo", "bar", single_stage=True),
+        "copy-bar-foobar": run_copy("bar", "foobar", name="copy-bar-foobar"),
+        "copy-lorem-ipsum": run_copy("lorem", "ipsum", name="lorem-ipsum"),
+    }
+
+
+def test_collect_granular_with_no_target(tmp_dir, dvc, stages):
+    assert set(map(itemgetter(0), dvc.collect_granular())) == set(
+        stages.values()
+    )
+    assert list(map(itemgetter(1), dvc.collect_granular())) == [None] * len(
+        stages
+    )
+
+
+def test_collect_granular_with_target(tmp_dir, dvc, stages):
+    assert dvc.collect_granular("bar.dvc") == [(stages["copy-foo-bar"], None)]
+    assert dvc.collect_granular(PIPELINE_FILE) == [
+        (stages["copy-bar-foobar"], None),
+        (stages["copy-lorem-ipsum"], None),
+    ]
+    assert dvc.collect_granular(":") == [
+        (stages["copy-bar-foobar"], None),
+        (stages["copy-lorem-ipsum"], None),
+    ]
+    assert dvc.collect_granular("copy-bar-foobar") == [
+        (stages["copy-bar-foobar"], None)
+    ]
+    assert dvc.collect_granular(":copy-bar-foobar") == [
+        (stages["copy-bar-foobar"], None)
+    ]
+    assert dvc.collect_granular("dvc.yaml:copy-bar-foobar") == [
+        (stages["copy-bar-foobar"], None)
+    ]
+
+    with (tmp_dir / dvc.DVC_DIR).chdir():
+        assert dvc.collect_granular(
+            relpath(tmp_dir / PIPELINE_FILE) + ":copy-bar-foobar"
+        ) == [(stages["copy-bar-foobar"], None)]
+
+    assert dvc.collect_granular("foobar") == [
+        (stages["copy-bar-foobar"], PathInfo(tmp_dir / "foobar"))
+    ]
+
+
+@pytest.mark.parametrize(
+    "target",
+    [
+        "not_existing.dvc",
+        "not_existing.dvc:stage_name",
+        "not_existing/dvc.yaml",
+        "not_existing/dvc.yaml:stage_name",
+    ],
+)
+def test_collect_with_not_existing_dvcfile(tmp_dir, dvc, target):
+    with pytest.raises(StageFileDoesNotExistError):
+        dvc.collect_granular(target)
+    with pytest.raises(StageFileDoesNotExistError):
+        dvc.collect(target)
+
+
+def test_collect_granular_with_not_existing_output_or_stage_name(tmp_dir, dvc):
+    with pytest.raises(NoOutputOrStageError):
+        dvc.collect_granular("some_file")
+    with pytest.raises(NoOutputOrStageError):
+        dvc.collect_granular("some_file", recursive=True)
+
+
+def test_collect_granular_with_deps(tmp_dir, dvc, stages):
+    assert set(
+        map(itemgetter(0), dvc.collect_granular("bar.dvc", with_deps=True))
+    ) == {stages["copy-foo-bar"], stages["foo-generate"]}
+    assert set(
+        map(
+            itemgetter(0),
+            dvc.collect_granular("copy-bar-foobar", with_deps=True),
+        )
+    ) == {
+        stages["copy-bar-foobar"],
+        stages["copy-foo-bar"],
+        stages["foo-generate"],
+    }
+    assert set(
+        map(
+            itemgetter(0), dvc.collect_granular(PIPELINE_FILE, with_deps=True),
+        )
+    ) == set(stages.values())
+
+
+def test_collect_granular_same_output_name_stage_name(tmp_dir, dvc, run_copy):
+    (stage1,) = tmp_dir.dvc_gen("foo", "foo")
+    (stage2,) = tmp_dir.dvc_gen("copy-foo-bar", "copy-foo-bar")
+    stage3 = run_copy("foo", "bar", name="copy-foo-bar")
+
+    assert dvc.collect_granular("copy-foo-bar") == [(stage3, None)]
+
+    coll = dvc.collect_granular("copy-foo-bar", with_deps=True)
+    assert set(map(itemgetter(0), coll)) == {stage3, stage1}
+    assert list(map(itemgetter(1), coll)) == [None] * 2
+
+    assert dvc.collect_granular("./copy-foo-bar") == [
+        (stage2, PathInfo(tmp_dir / "copy-foo-bar"))
+    ]
+    assert dvc.collect_granular("./copy-foo-bar", with_deps=True) == [
+        (stage2, PathInfo(tmp_dir / "copy-foo-bar"))
+    ]
+
+
+def test_collect_granular_priority_on_collision(tmp_dir, dvc, run_copy):
+    tmp_dir.gen({"dir": {"foo": "foo"}, "foo": "foo"})
+    (stage1,) = dvc.add("dir", recursive=True)
+    stage2 = run_copy("foo", "bar", name="dir")
+
+    assert dvc.collect_granular("dir") == [(stage2, None)]
+    assert dvc.collect_granular("dir", recursive=True) == [(stage1, None)]
+
+    remove(tmp_dir / "dir")
+
+    assert dvc.collect_granular("dir") == [(stage2, None)]
+    assert dvc.collect_granular("dir", recursive=True) == [(stage2, None)]
+
+
+def test_collect_granular_collision_output_dir_stage_name(
+    tmp_dir, dvc, run_copy
+):
+    stage1, *_ = tmp_dir.dvc_gen({"dir": {"foo": "foo"}, "foo": "foo"})
+    stage3 = run_copy("foo", "bar", name="dir")
+
+    assert dvc.collect_granular("dir") == [(stage3, None)]
+    assert not dvc.collect_granular("dir", recursive=True)
+    assert dvc.collect_granular("./dir") == [
+        (stage1, PathInfo(tmp_dir / "dir"))
+    ]
+
+
+def test_collect_granular_not_existing_stage_name(tmp_dir, dvc, run_copy):
+    tmp_dir.dvc_gen("foo", "foo")
+    (stage,) = tmp_dir.dvc_gen("copy-foo-bar", "copy-foo-bar")
+    run_copy("foo", "bar", name="copy-foo-bar")
+
+    assert dvc.collect_granular("copy-foo-bar.dvc:stage_name_not_needed") == [
+        (stage, None)
+    ]
+    with pytest.raises(StageNotFound):
+        dvc.collect_granular("dvc.yaml:does-not-exist")
+
+
+def test_get_stages(tmp_dir, dvc, run_copy):
+    with pytest.raises(StageFileDoesNotExistError):
+        dvc.get_stages()
+
+    tmp_dir.gen("foo", "foo")
+    stage1 = run_copy("foo", "bar", name="copy-foo-bar")
+    stage2 = run_copy("bar", "foobar", name="copy-bar-foobar")
+
+    assert set(dvc.get_stages()) == {stage1, stage2}
+    assert set(dvc.get_stages(path=PIPELINE_FILE)) == {stage1, stage2}
+    assert set(dvc.get_stages(name="copy-bar-foobar")) == {stage2}
+    assert set(dvc.get_stages(path=PIPELINE_FILE, name="copy-bar-foobar")) == {
+        stage2
+    }
+
+    with pytest.raises(StageFileDoesNotExistError):
+        dvc.get_stages(path=relpath(tmp_dir / ".." / PIPELINE_FILE))
+
+    with pytest.raises(StageNotFound):
+        dvc.get_stages(path=PIPELINE_FILE, name="copy")
+
+
+def test_get_stages_old_dvcfile(tmp_dir, dvc):
+    (stage1,) = tmp_dir.dvc_gen("foo", "foo")
+    assert set(dvc.get_stages("foo.dvc")) == {stage1}
+    assert set(dvc.get_stages("foo.dvc", name="foo-generate")) == {stage1}
+
+    with pytest.raises(StageFileDoesNotExistError):
+        dvc.get_stages(path=relpath(tmp_dir / ".." / "foo.dvc"))
+
+
+def test_get_stage(tmp_dir, dvc, run_copy):
+    tmp_dir.gen("foo", "foo")
+    stage1 = run_copy("foo", "bar", name="copy-foo-bar")
+
+    with pytest.raises(StageNameUnspecified):
+        dvc.get_stage()
+
+    with pytest.raises(StageNameUnspecified):
+        dvc.get_stage(path=PIPELINE_FILE)
+
+    assert dvc.get_stage(path=PIPELINE_FILE, name="copy-foo-bar") == stage1
+    assert dvc.get_stage(name="copy-foo-bar") == stage1
+
+    with pytest.raises(StageFileDoesNotExistError):
+        dvc.get_stage(path="something.yaml", name="name")
+
+    with pytest.raises(StageNotFound):
+        dvc.get_stage(name="random_name")
+
+
+def test_get_stage_single_stage_dvcfile(tmp_dir, dvc):
+    (stage1,) = tmp_dir.dvc_gen("foo", "foo")
+    assert dvc.get_stage("foo.dvc") == stage1
+    assert dvc.get_stage("foo.dvc", name="jpt") == stage1
+    with pytest.raises(StageFileDoesNotExistError):
+        dvc.get_stage(path="bar.dvc", name="name")

tests/func/test_repro.py

@@ -56,7 +56,7 @@ class SingleStageRun:
 
     @staticmethod
     def _get_stage_target(stage):
-        return stage.path
+        return stage.addressing
 
 
 class TestRepro(SingleStageRun, TestDvc):

tests/func/test_repro_multistage.py

@@ -32,7 +32,7 @@ class MultiStageRun:
 
     @staticmethod
     def _get_stage_target(stage):
-        return stage.path + ":" + stage.name
+        return stage.addressing
 
 
 class TestReproFailMultiStage(MultiStageRun, test_repro.TestReproFail):
@@ -312,15 +312,13 @@ def test_repro_when_cmd_changes(tmp_dir, dvc, run_copy):
 
     tmp_dir.gen("foo", "foo")
     stage = run_copy("foo", "bar", name="copy-file")
-    target = ":copy-file"
+    target = "copy-file"
     assert not dvc.reproduce(target)
 
     stage.cmd = "  ".join(stage.cmd.split())  # change cmd spacing by two
     PipelineFile(dvc, PIPELINE_FILE)._dump_pipeline_file(stage)
 
-    assert dvc.status([target]) == {
+    assert dvc.status([target]) == {target: ["changed command"]}
-        PIPELINE_FILE + target: ["changed command"]
-    }
     assert dvc.reproduce(target)[0] == stage
 
 

tests/func/test_stage.py

@@ -8,7 +8,7 @@ from dvc.main import main
 from dvc.output.local import LocalOutput
 from dvc.remote.local import LocalRemote
 from dvc.repo import Repo
-from dvc.stage import Stage
+from dvc.stage import PipelineStage, Stage
 from dvc.stage.exceptions import StageFileFormatError
 from dvc.utils.stage import dump_stage_file, load_stage_file
 from tests.basic_env import TestDvc
@@ -196,19 +196,39 @@ def test_parent_repo_collect_stages(tmp_dir, scm, dvc):
     assert subrepo_stages != []
 
 
-def test_stage_addressing(tmp_dir, dvc, run_copy):
+def test_stage_strings_representation(tmp_dir, dvc, run_copy):
     tmp_dir.dvc_gen("foo", "foo")
     stage1 = run_copy("foo", "bar", single_stage=True)
     assert stage1.addressing == "bar.dvc"
+    assert repr(stage1) == "Stage: 'bar.dvc'"
+    assert str(stage1) == "stage: 'bar.dvc'"
 
     stage2 = run_copy("bar", "baz", name="copy-bar-baz")
-    assert stage2.addressing == "dvc.yaml:copy-bar-baz"
+    assert stage2.addressing == "copy-bar-baz"
+    assert repr(stage2) == "Stage: 'copy-bar-baz'"
+    assert str(stage2) == "stage: 'copy-bar-baz'"
 
     folder = tmp_dir / "dir"
     folder.mkdir()
     with folder.chdir():
-        assert stage1.addressing == os.path.relpath(stage1.path)
+        rel_path = os.path.relpath(stage1.path)
-        assert (
+        assert stage1.addressing == rel_path
-            stage2.addressing
+        assert repr(stage1) == f"Stage: '{rel_path}'"
-            == os.path.relpath(stage2.path) + ":" + stage2.name
+        assert str(stage1) == f"stage: '{rel_path}'"
-        )
+
+        rel_path = os.path.relpath(stage2.path)
+        assert stage2.addressing == f"{rel_path}:{stage2.name}"
+        assert repr(stage2) == f"Stage: '{rel_path}:{stage2.name}'"
+        assert str(stage2) == f"stage: '{rel_path}:{stage2.name}'"
+
+
+def test_stage_on_no_path_string_repr(tmp_dir, dvc):
+    s = Stage(dvc)
+    assert s.addressing == "No path"
+    assert repr(s) == "Stage: 'No path'"
+    assert str(s) == "stage: 'No path'"
+
+    p = PipelineStage(dvc, name="stage_name")
+    assert p.addressing == "No path:stage_name"
+    assert repr(p) == "Stage: 'No path:stage_name'"
+    assert str(p) == "stage: 'No path:stage_name'"

tests/func/test_status.py

@@ -73,12 +73,12 @@ def test_status_on_pipeline_stages(tmp_dir, dvc, run_copy):
 
     stage.cmd = "  ".join(stage.cmd.split())
     stage.dvcfile._dump_pipeline_file(stage)
-    assert dvc.status() == {"dvc.yaml:copy-foo-bar": ["changed command"]}
+    assert dvc.status("copy-foo-bar") == {"copy-foo-bar": ["changed command"]}
 
     # delete outputs
     (tmp_dir / "bar").unlink()
     assert dvc.status() == {
-        "dvc.yaml:copy-foo-bar": [
+        "copy-foo-bar": [
             {"changed outs": {"bar": "deleted"}},
             "changed command",
         ]
@@ -86,7 +86,7 @@ def test_status_on_pipeline_stages(tmp_dir, dvc, run_copy):
     (tmp_dir / "foo").unlink()
     assert dvc.status() == {
         "foo.dvc": [{"changed outs": {"foo": "deleted"}}],
-        "dvc.yaml:copy-foo-bar": [
+        "copy-foo-bar": [
             {"changed deps": {"foo": "deleted"}},
             {"changed outs": {"bar": "deleted"}},
             "changed command",

tests/unit/repo/test_repo.py

@@ -87,6 +87,21 @@ def test_collect_optimization(tmp_dir, dvc, mocker):
     dvc.collect_granular(stage.path)
 
 
+def test_collect_optimization_on_stage_name(tmp_dir, dvc, mocker, run_copy):
+    tmp_dir.dvc_gen("foo", "foo")
+    stage = run_copy("foo", "bar", name="copy-foo-bar")
+    # Forget cached stages and graph and error out on collection
+    dvc._reset()
+    mocker.patch(
+        "dvc.repo.Repo.stages",
+        property(raiser(Exception("Should not collect"))),
+    )
+
+    # Should read stage directly instead of collecting the whole graph
+    assert dvc.collect("copy-foo-bar") == [stage]
+    assert dvc.collect_granular("copy-foo-bar") == [(stage, None)]
+
+
 def test_skip_graph_checks(tmp_dir, dvc, mocker, run_copy):
     # See https://github.com/iterative/dvc/issues/2671 for more info
     mock_collect_graph = mocker.patch("dvc.repo.Repo._collect_graph")

tests/unit/utils/test_utils.py

@@ -146,6 +146,9 @@ def test_resolve_output(inp, out, is_dir, expected, mocker):
         ["../models/stage.dvc", ("../models/stage.dvc", None), "def"],
         [":name", ("default", "name"), "default"],
         [":name", ("default", "name"), "default"],
+        ["something.dvc:name", ("something.dvc", "name"), None],
+        ["../something.dvc:name", ("../something.dvc", "name"), None],
+        ["file", (None, "file"), None],
     ],
 )
 def test_parse_target(inp, out, default):