output.get_obj: catch ObjectCorruptedError

data status: fix quoting on command hints

.github/CODEOWNERS

@@ -1 +0,0 @@
-*       @iterative/dvc

.github/workflows/tests.yaml

@@ -43,6 +43,9 @@ jobs:
       matrix:
         os: [ubuntu-20.04, windows-latest, macos-latest]
         pyv: ["3.8", "3.9", "3.10"]
+        include:
+        - {os: ubuntu-latest, pyv: "3.11-dev"}
+
     steps:
     - uses: actions/checkout@v3
       with:
@@ -68,6 +71,8 @@ jobs:
       run: |
         pip install --upgrade pip setuptools wheel
         pip install -e ".[dev]"
+      env:
+        AIOHTTP_NO_EXTENSIONS: ${{ matrix.pyv == '3.11-dev' && '1' }}
     - name: run tests
       timeout-minutes: 40
       run: >-

dvc/api/params.py

@@ -37,6 +37,10 @@ def params_show(
             stages to retrieve parameters from.
             Defaults to `None`.
             If `None`, all parameters from all stages will be retrieved.
+            If this method is called from a different location to the one where
+            the `dvc.yaml` is found, the relative path to the `dvc.yaml` must
+            be provided as a prefix with the syntax `{relpath}:{stage}`.
+            For example: `subdir/dvc.yaml:stage-0` or `../dvc.yaml:stage-1`.
         rev (str, optional): Name of the `Git revision`_ to retrieve parameters
             from.
             Defaults to `None`.

dvc/commands/data.py

@@ -31,15 +31,22 @@ class CmdDataStatus(CmdBase):
         "unchanged": "DVC unchanged files",
     }
     HINTS = {
-        "not_in_cache": 'use "dvc pull <file>..." '
-        "to update your local storage",
-        "committed": "git commit the corresponding dvc files "
-        "to update the repo",
-        "uncommitted": 'use "dvc commit <file>..." to track changes',
-        "untracked": 'use "git add <file> ..." or '
-        'dvc add <file>..." to commit to git or to dvc',
-        "git_dirty": "there are {}changes not tracked by dvc, "
-        'use "git status" to see',
+        "not_in_cache": ('use "dvc fetch <file>..." to download files',),
+        "committed": (
+            "git commit the corresponding dvc files to update the repo",
+        ),
+        "uncommitted": (
+            'use "dvc commit <file>..." to track changes',
+            'use "dvc checkout <file>..." to discard changes',
+        ),
+        "untracked": (
+            'use "git add <file> ..." or '
+            '"dvc add <file>..." to commit to git or to dvc',
+        ),
+        "git_dirty": (
+            "there are {}changes not tracked by dvc, "
+            'use "git status" to see',
+        ),
     }
 
     @staticmethod
@@ -76,8 +83,9 @@ class CmdDataStatus(CmdBase):
             color = cls.COLORS.get(stage, None)
 
             ui.write(header)
-            if hint := cls.HINTS.get(stage):
-                ui.write(f"  ({hint})")
+            if hints := cls.HINTS.get(stage):
+                for hint in hints:
+                    ui.write(f"  ({hint})")
 
             if isinstance(stage_status, dict):
                 items = [
@@ -92,9 +100,10 @@ class CmdDataStatus(CmdBase):
                 out = "\n".join(tabs + item for item in chunk)
                 ui.write(colorize(out, color))
 
-        if (hint := cls.HINTS.get("git_dirty")) and git_info.get("is_dirty"):
-            message = hint.format("other " if result else "")
-            ui.write(f"[blue]({message})[/]", styled=True)
+        if (hints := cls.HINTS.get("git_dirty")) and git_info.get("is_dirty"):
+            for hint in hints:
+                message = hint.format("other " if result else "")
+                ui.write(f"[blue]({message})[/]", styled=True)
         return 0
 
     def run(self) -> int:
@@ -102,7 +111,6 @@ class CmdDataStatus(CmdBase):
             status = self.repo.data_status(
                 granular=self.args.granular,
                 untracked_files=self.args.untracked_files,
-                with_dirs=self.args.with_dirs,
             )
 
         if not self.args.unchanged:
@@ -172,10 +180,4 @@ def add_parser(subparsers, parent_parser):
         nargs="?",
         help="Show untracked files.",
     )
-    data_status_parser.add_argument(
-        "--with-dirs",
-        action="store_true",
-        default=False,
-        help=argparse.SUPPRESS,
-    )
     data_status_parser.set_defaults(func=CmdDataStatus)

dvc/commands/experiments/diff.py

@@ -27,7 +27,7 @@ class CmdExperimentsDiff(CmdBase):
 
         if self.args.json:
             ui.write_json(diff)
-        else:
+        elif diff:
             from dvc.compare import show_diff
 
             precision = self.args.precision or DEFAULT_PRECISION

dvc/commands/plots.py

@@ -223,31 +223,22 @@ class CmdPlotsModify(CmdPlots):
 
 
 class CmdPlotsTemplates(CmdBase):
-    TEMPLATES_CHOICES = [
-        "simple",
-        "linear",
-        "confusion",
-        "confusion_normalized",
-        "scatter",
-        "smooth",
-        "bar_horizontal_sorted",
-        "bar_horizontal",
-    ]
-
     def run(self):
-        from dvc_render.vega_templates import dump_templates
+        from dvc.exceptions import InvalidArgumentError
+        from dvc_render.vega_templates import TEMPLATES
 
         try:
-            out = (
-                os.path.join(os.getcwd(), self.args.out)
-                if self.args.out
-                else self.repo.plots.templates_dir
-            )
+            target = self.args.template
+            if target:
+                for template in TEMPLATES:
+                    if target == template.DEFAULT_NAME:
+                        ui.write_json(template.DEFAULT_CONTENT)
+                        return 0
+                raise InvalidArgumentError(f"Unexpected template: {target}.")
 
-            targets = [self.args.target] if self.args.target else None
-            dump_templates(output=out, targets=targets)
-            templates_path = os.path.relpath(out, os.getcwd())
-            ui.write(f"Templates have been written into '{templates_path}'.")
+            else:
+                for template in TEMPLATES:
+                    ui.write(template.DEFAULT_NAME)
 
             return 0
         except DvcException:
@@ -357,8 +348,7 @@ def add_parser(subparsers, parent_parser):
     plots_modify_parser.set_defaults(func=CmdPlotsModify)
 
     TEMPLATES_HELP = (
-        "Write built-in plots templates to a directory "
-        "(.dvc/plots by default)."
+        "List built-in plots templates or show JSON specification for one."
     )
     plots_templates_parser = plots_subparsers.add_parser(
         "templates",
@@ -368,13 +358,14 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     plots_templates_parser.add_argument(
-        "target",
+        "template",
         default=None,
         nargs="?",
-        choices=CmdPlotsTemplates.TEMPLATES_CHOICES,
-        help="Template to write. Writes all templates by default.",
+        help=(
+            "Template for which to show JSON specification. "
+            "List all template names by default."
+        ),
     )
-    _add_output_argument(plots_templates_parser, typ="templates")
     plots_templates_parser.set_defaults(func=CmdPlotsTemplates)
 
 

dvc/info.py

@@ -14,6 +14,14 @@ from dvc.scm import SCMError
 from dvc.utils import error_link
 from dvc.utils.pkg import PKG
 
+SUBPROJECTS = (
+    "dvc_data",
+    "dvc_objects",
+    "dvc_render",
+    "dvc_task",
+    "dvclive",
+    "scmrepo",
+)
 package = "" if PKG is None else f"({PKG})"
 
 
@@ -23,6 +31,7 @@ def get_dvc_info():
         "---------------------------------",
         f"Platform: Python {platform.python_version()} on "
         f"{platform.platform()}",
+        f"Subprojects:{_get_subprojects()}",
         f"Supports:{_get_supported_remotes()}",
     ]
 
@@ -88,6 +97,18 @@ def _get_linktype_support_info(repo):
     return ", ".join(links)
 
 
+def _get_subprojects():
+    subprojects = []
+    for subproject in SUBPROJECTS:
+        try:
+            version = importlib_metadata.version(subproject)
+            subprojects.append(f"{subproject} = {version}")
+        except ImportError:
+            pass
+
+    return "\n\t" + "\n\t".join(subprojects)
+
+
 def _get_supported_remotes():
     supported_remotes = []
     for scheme in registry:

dvc/output.py

@@ -737,7 +737,7 @@ class Output:
         elif self.hash_info:
             try:
                 obj = oload(self.odb, self.hash_info)
-            except FileNotFoundError:
+            except (FileNotFoundError, ObjectFormatError):
                 return None
         else:
             return None

dvc/repo/data.py

@@ -1,6 +1,15 @@
 import os
 from collections import defaultdict
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, TypedDict, cast
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    Iterable,
+    List,
+    Optional,
+    TypedDict,
+    cast,
+)
 
 from dvc.ui import ui
 
@@ -10,19 +19,16 @@ if TYPE_CHECKING:
     from dvc.output import Output
     from dvc.repo import Repo
     from dvc_data.hashfile.db import HashFileDB
+    from dvc_data.hashfile.hash_info import HashInfo
     from dvc_data.hashfile.obj import HashFile
 
 
-def _in_cache(obj: Optional["HashFile"], cache: "HashFileDB") -> bool:
+def _in_cache(obj: "HashInfo", cache: "HashFileDB") -> bool:
     from dvc_objects.errors import ObjectFormatError
 
-    if not obj:
-        return False
-    if not obj.hash_info.value:
-        return False
-
+    assert obj.value
     try:
-        cache.check(obj.hash_info.value)
+        cache.check(obj.value)
         return True
     except (FileNotFoundError, ObjectFormatError):
         return False
@@ -30,31 +36,30 @@ def _in_cache(obj: Optional["HashFile"], cache: "HashFileDB") -> bool:
 
 def _shallow_diff(
     root: str,
-    old_obj: Optional["HashFile"],
-    new_obj: Optional["HashFile"],
+    old: Optional["HashInfo"],
+    new: Optional["HashInfo"],
     cache: "HashFileDB",
 ) -> Dict[str, List[str]]:
-    # TODO: add support for shallow diff in dvc-data
-    # TODO: we may want to recursively do in_cache check
     d = {}
 
-    from dvc_data.objects.tree import Tree
+    old_root = new_root = root
+    if old and old.isdir:
+        old_root = os.path.sep.join([root, ""])
+    if new and new.isdir:
+        new_root = os.path.sep.join([root, ""])
 
-    if isinstance(new_obj, Tree):
-        root = os.path.sep.join([root, ""])
+    if old and not _in_cache(old, cache):
+        d["not_in_cache"] = [old_root]
 
-    if not _in_cache(old_obj, cache):
-        d["not_in_cache"] = [root]
-
-    if old_obj is None and new_obj is None:
+    if not old and not new:
         return d
-    if old_obj is None:
-        return {"added": [root], **d}
-    if new_obj is None:
-        return {"deleted": [root], **d}
-    if old_obj.hash_info != new_obj.hash_info:
-        return {"modified": [root], **d}
-    return {"unchanged": [root], **d}
+    if not new:
+        return {"deleted": [old_root], **d}
+    if not old:
+        return {"added": [new_root], **d}
+    if old != new:
+        return {"modified": [new_root], **d}
+    return {"unchanged": [new_root], **d}
 
 
 def _granular_diff(
@@ -62,19 +67,12 @@ def _granular_diff(
     old_obj: Optional["HashFile"],
     new_obj: Optional["HashFile"],
     cache: "HashFileDB",
-    with_dirs: bool = False,
 ) -> Dict[str, List[str]]:
     from dvc_data.diff import ROOT
     from dvc_data.diff import diff as odiff
-    from dvc_data.objects.tree import Tree
 
-    drop_root = False
-    trees = isinstance(old_obj, Tree) or isinstance(new_obj, Tree)
-    if trees:
-        drop_root = not with_dirs
-
-    def path_join(root: str, *paths: str) -> str:
-        if not trees and paths == ROOT:
+    def path_join(root: str, *paths: str, isdir: bool = False) -> str:
+        if not isdir and paths == ROOT:
             return root
         return os.path.sep.join([root, *paths])
 
@@ -83,38 +81,46 @@ def _granular_diff(
     output: Dict[str, List[str]] = defaultdict(list)
     for state in ("added", "deleted", "modified", "unchanged"):
         items = getattr(diff_data, state)
-        output[state].extend(
-            path_join(root, *item.new.key)
-            for item in items  # pylint: disable=not-an-iterable
-            if not (drop_root and item.new.key == ROOT)
-        )
-        # TODO: PERF: diff is checking not_in_cache for each even if we only
-        # need it for the index.
-        # BUG: not_in_cache file also shows up as modified in staged and
-        # unstaged. We currently don't know if it is really modified.
-        output["not_in_cache"].extend(
-            path_join(root, *item.new.key)
-            for item in items  # pylint: disable=not-an-iterable
-            if not item.old.in_cache
-            and not (drop_root and item.new.key == ROOT)
-            and state != "added"
-        )
-    return output
+        for item in items:  # pylint: disable=not-an-iterable
+            entry = item.old if state == "deleted" else item.new
+            isdir = entry.oid.isdir if entry.oid else False
+
+            path = path_join(root, *entry.key, isdir=isdir)
+            output[state].append(path)
+            if not item.old.in_cache and state != "added":
+                output["not_in_cache"].append(path)
+    return dict(output)
+
+
+def _get_obj_items(root: str, obj: Optional["HashFile"]) -> List[str]:
+    if not obj:
+        return []
+
+    from dvc_data.objects.tree import Tree
+
+    sep = os.path.sep
+    if isinstance(obj, Tree):
+        return [sep.join([root, *key]) for key, _, _ in obj]
+    return [root]
 
 
 def _diff(
     root: str,
+    old_oid: Optional["HashInfo"],
     old_obj: Optional["HashFile"],
+    new_oid: Optional["HashInfo"],
     new_obj: Optional["HashFile"],
-    cache: "HashFileDB",
+    odb: "HashFileDB",
     granular: bool = False,
-    with_dirs: bool = False,
 ) -> Dict[str, List[str]]:
-    if granular:
-        return _granular_diff(
-            root, old_obj, new_obj, cache, with_dirs=with_dirs
-        )
-    return _shallow_diff(root, old_obj, new_obj, cache)
+    if not granular:
+        return _shallow_diff(root, old_oid, new_oid, odb)
+    if (old_oid and not old_obj) or (new_oid and not new_obj):
+        # we don't have enough information to give full details
+        unknown = _get_obj_items(root, new_obj)
+        shallow_diff = _shallow_diff(root, old_oid, new_oid, odb)
+        return {**shallow_diff, "unknown": unknown}
+    return _granular_diff(root, old_obj, new_obj, odb)
 
 
 class GitInfo(TypedDict, total=False):
@@ -175,8 +181,18 @@ def _diff_index_to_wtree(repo: "Repo", **kwargs: Any) -> Dict[str, List[str]]:
         cache = repo.odb.repo
         root = str(out)
         old = out.get_obj()
+
         with ui.status(f"Calculating diff for {root} between index/workspace"):
-            d = _diff(root, old, new, cache, **kwargs)
+            d = _diff(
+                root,
+                out.hash_info,
+                old,
+                new.hash_info if new else None,
+                new,
+                cache,
+                **kwargs,
+            )
+
         for state, items in d.items():
             if not items:
                 continue
@@ -198,14 +214,17 @@ def _diff_head_to_index(
 
             root = str(out)
             typ = "index" if rev == "workspace" else head
-            objs[root][typ] = out.get_obj()
+            objs[root][typ] = (out.get_obj(), out.hash_info)
 
     cache = repo.odb.repo
     for root, obj_d in objs.items():
-        old = obj_d.get(head, None)
-        new = obj_d.get("index", None)
+        old_obj, old_oid = obj_d.get(head, (None, None))
+        new_obj, new_oid = obj_d.get("index", (None, None))
         with ui.status(f"Calculating diff for {root} between head/index"):
-            d = _diff(root, old, new, cache, **kwargs)
+            d = _diff(
+                root, old_oid, old_obj, new_oid, new_obj, cache, **kwargs
+            )
+
         for state, items in d.items():
             if not items:
                 continue
@@ -216,28 +235,34 @@ def _diff_head_to_index(
 
 class Status(TypedDict):
     not_in_cache: List[str]
-    committed: Dict[str, Any]
-    uncommitted: Dict[str, Any]
+    committed: Dict[str, List[str]]
+    uncommitted: Dict[str, List[str]]
     untracked: List[str]
     unchanged: List[str]
     git: GitInfo
 
 
-def _transform_git_paths_to_dvc(repo: "Repo", files: List[str]):
+def _transform_git_paths_to_dvc(repo: "Repo", files: Iterable[str]):
     """Transform files rel. to Git root to DVC root, and drop outside files."""
     rel = repo.fs.path.relpath(repo.root_dir, repo.scm.root_dir).rstrip("/")
-    if rel in (os.curdir, ""):
-        return files
 
-    prefix = rel + os.sep
-    length = len(prefix)
-    return [file[length:] for file in files if file.startswith(prefix)]
+    # if we have repo root in a different location than scm's root,
+    # i.e. subdir repo, all git_paths need to be transformed rel. to the DVC
+    # repo root and anything outside need to be filtered out.
+    if rel not in (os.curdir, ""):
+        prefix = rel + os.sep
+        length = len(prefix)
+        files = (file[length:] for file in files if file.startswith(prefix))
+
+    start = repo.fs.path.relpath(repo.fs.path.getcwd(), repo.root_dir)
+    if start in (os.curdir, ""):
+        return list(files)
+    # we need to convert repo relative paths to curdir relative.
+    return [repo.fs.path.relpath(file, start) for file in files]
 
 
 def status(repo: "Repo", untracked_files: str = "no", **kwargs: Any) -> Status:
-    from scmrepo.exceptions import SCMError
-
-    from dvc.scm import NoSCMError
+    from dvc.scm import NoSCMError, SCMError
 
     head = kwargs.pop("head", "HEAD")
     uncommitted_diff = _diff_index_to_wtree(repo, **kwargs)

dvc/repo/experiments/queue/base.py

@@ -1,5 +1,6 @@
 import logging
 import os
+import sys
 from abc import ABC, abstractmethod
 from dataclasses import asdict, dataclass
 from typing import (
@@ -524,7 +525,14 @@ class BaseStashQueue(ABC):
         """
         logger.debug("Using experiment params '%s'", params)
 
-        from dvc.utils.hydra import apply_overrides
+        try:
+            from dvc.utils.hydra import apply_overrides
+        except ValueError:
+            if sys.version_info >= (3, 11):
+                raise DvcException(
+                    "--set-param is not supported in Python >= 3.11"
+                )
+            raise
 
         for path, overrides in params.items():
             apply_overrides(path, overrides)

dvc/repo/experiments/show.py

@@ -142,6 +142,9 @@ def show(
     fetch_running: bool = True,
 ):
 
+    if repo.scm.no_commits:
+        return {}
+
     if onerror is None:
         onerror = onerror_collect
 

dvc/repo/fetch.py

@@ -127,8 +127,16 @@ def _fetch_partial_imports(repo, targets, **kwargs):
     downloaded = 0
     failed = 0
     for stage in repo.partial_imports(targets, **kwargs):
+        dep = stage.deps[0]
+        out = stage.outs[0]
         try:
-            stage.run()
+            if dep.changed_checksum():
+                raise DataSourceChanged(f"{stage} ({dep})")
+
+            logger.info("Importing '%s' -> '%s'", dep, out)
+            out.transfer(
+                dep.def_path, jobs=kwargs.get("jobs"), odb=repo.odb.local
+            )
         except DataSourceChanged as exc:
             logger.warning(f"{exc}")
             failed += 1

dvc/repo/metrics/show.py

@@ -85,12 +85,14 @@ def _read_metrics(repo, metrics, rev, onerror=None):
 
     res = {}
     for metric in metrics:
+        rel_metric_path = os.path.join(relpath, *fs.path.parts(metric))
         if not fs.isfile(metric):
-            continue
+            if fs.isfile(rel_metric_path):
+                metric = rel_metric_path
+            else:
+                continue
 
-        res[os.path.join(relpath, *fs.path.parts(metric))] = _read_metric(
-            metric, fs, rev, onerror=onerror
-        )
+        res[rel_metric_path] = _read_metric(metric, fs, rev, onerror=onerror)
 
     return res
 

dvc/repo/params/show.py

@@ -56,7 +56,9 @@ def _collect_configs(
             default_params
         ):
             fs_paths.append(default_params)
-
+    if targets and (deps or stages) and not params:
+        # A target has been provided but it is not used in the stages
+        fs_paths = []
     return params, fs_paths
 
 

dvc/repo/plots/__init__.py

@@ -386,30 +386,56 @@ def _collect_output_plots(
     return result
 
 
-def _adjust_definitions_to_cwd(fs, config_relpath, plots_definitions):
-    result = defaultdict(dict)
+def _id_is_path(plot_props=None):
+    if not plot_props:
+        return True
 
-    config_dirname = fs.path.dirname(config_relpath)
+    y_def = plot_props.get("y")
+    return not isinstance(y_def, dict)
 
-    for plot_id, plot_def in plots_definitions.items():
 
-        y_def = plot_def.get("y", None) if plot_def else None
-        if y_def is None or not isinstance(y_def, dict):
-            # plot_id is filename
-            new_plot_id = _normpath(fs.path.join(config_dirname, plot_id))
-            result[new_plot_id] = plot_def or {}
+def _adjust_sources(fs, plot_props, config_dir):
+    new_plot_props = deepcopy(plot_props)
+    old_y = new_plot_props.pop("y", {})
+    new_y = {}
+    for filepath, val in old_y.items():
+        new_y[_normpath(fs.path.join(config_dir, filepath))] = val
+    new_plot_props["y"] = new_y
+    return new_plot_props
+
+
+def _resolve_definitions(
+    fs, targets, props, config_path, definitions, onerror=None
+):
+    config_dir = fs.path.dirname(config_path)
+    result: Dict[str, Dict] = {}
+    for plot_id, plot_props in definitions.items():
+        if plot_props is None:
+            plot_props = {}
+        if _id_is_path(plot_props):
+            data_path = _normpath(fs.path.join(config_dir, plot_id))
+            if _matches(targets, config_path, plot_id):
+                unpacked = unpack_if_dir(
+                    fs,
+                    data_path,
+                    props={**plot_props, **props},
+                    onerror=onerror,
+                )
+                dpath.util.merge(
+                    result,
+                    unpacked,
+                )
         else:
-            new_plot_def = deepcopy(plot_def)
-            old_y = new_plot_def.pop("y")
-            new_y = {}
-            for filepath, val in old_y.items():
-                new_y[_normpath(fs.path.join(config_dirname, filepath))] = val
-            new_plot_def["y"] = new_y
-            result[plot_id] = new_plot_def
-    return dict(result)
+            if _matches(targets, config_path, plot_id):
+                adjusted_props = _adjust_sources(fs, plot_props, config_dir)
+                dpath.util.merge(
+                    result, {"data": {plot_id: {**adjusted_props, **props}}}
+                )
+
+    return result
 
 
-def _collect_pipeline_files(repo, targets: List[str], props):
+def _collect_pipeline_files(repo, targets: List[str], props, onerror=None):
     from dvc.dvcfile import PipelineFile
 
     result: Dict[str, Dict] = {}
@@ -417,21 +443,19 @@ def _collect_pipeline_files(repo, targets: List[str], props):
     for dvcfile in dvcfiles:
         if isinstance(dvcfile, PipelineFile):
             dvcfile_path = _relpath(repo.dvcfs, dvcfile.path)
-            dvcfile_defs = _adjust_definitions_to_cwd(
-                repo.fs, dvcfile_path, dvcfile.load().get("plots", {})
+            dvcfile_defs = dvcfile.load().get("plots", {})
+            resolved = _resolve_definitions(
+                repo.dvcfs,
+                targets,
+                props,
+                dvcfile_path,
+                dvcfile_defs,
+                onerror=onerror,
+            )
+            dpath.util.merge(
+                result,
+                {dvcfile_path: resolved},
             )
-            for plot_id, plot_props in dvcfile_defs.items():
-                if plot_props is None:
-                    plot_props = {}
-                if _matches(targets, dvcfile_path, plot_id):
-                    dpath.util.merge(
-                        result,
-                        {
-                            dvcfile_path: {
-                                "data": {plot_id: {**plot_props, **props}}
-                            }
-                        },
-                    )
     return result
 
 
@@ -453,7 +477,10 @@ def _collect_definitions(
     fs = DvcFileSystem(repo=repo)
 
     if not config_files:
-        dpath.util.merge(result, _collect_pipeline_files(repo, targets, props))
+        dpath.util.merge(
+            result,
+            _collect_pipeline_files(repo, targets, props, onerror=onerror),
+        )
 
     if targets or (not targets and not config_files):
         dpath.util.merge(
@@ -463,12 +490,20 @@ def _collect_definitions(
 
     if config_files:
         for path in config_files:
-            definitions = parse(fs, path)
-            definitions = _adjust_definitions_to_cwd(
-                repo.fs, path, definitions
-            )
+            definitions = parse(fs, path).get("data", {})
             if definitions:
-                dpath.util.merge(result, {path: definitions})
+                resolved = _resolve_definitions(
+                    repo.dvcfs,
+                    targets,
+                    props,
+                    path,
+                    definitions,
+                    onerror=onerror,
+                )
+                dpath.util.merge(
+                    result,
+                    {path: resolved},
+                )
 
     for target in targets:
         if not result or fs.exists(target):

dvc/repo/plots/diff.py

@@ -12,6 +12,8 @@ def _revisions(repo, revs, experiment):
 
 
 def diff(repo, *args, revs=None, experiment=False, **kwargs):
+    if repo.scm.no_commits:
+        return {}
     return repo.plots.show(
         *args, revs=_revisions(repo, revs, experiment), **kwargs
     )

dvc/repo/remove.py

@@ -1,6 +1,13 @@
 import logging
 import typing
 
+from dvc.dvcfile import DVC_FILE_SUFFIX
+from dvc.stage.exceptions import (
+    StageFileDoesNotExistError,
+    StageFileIsNotDvcFileError,
+    StageNotFound,
+)
+
 from . import locked
 
 if typing.TYPE_CHECKING:
@@ -11,7 +18,16 @@ logger = logging.getLogger(__name__)
 
 @locked
 def remove(self: "Repo", target: str, outs: bool = False):
-    stages = self.stage.from_target(target)
+    try:
+        stages = self.stage.from_target(target)
+    except (StageNotFound, StageFileDoesNotExistError) as e:
+        # If the user specified a tracked file as a target instead of a stage,
+        # e.g. `data.csv` instead of `data.csv.dvc`,
+        # give a more helpful error message.
+        if self.fs.exists(target + DVC_FILE_SUFFIX):
+            raise StageFileIsNotDvcFileError(target) from e
+        else:
+            raise
 
     for stage in stages:
         stage.remove(remove_outs=outs, force=outs)

dvc/scm.py

@@ -160,7 +160,7 @@ def resolve_rev(scm: "Git", rev: str) -> str:
     except InternalRevError as exc:
         # `scm` will only resolve git branch and tag names,
         # if rev is not a sha it may be an abbreviated experiment name
-        if not rev.startswith("refs/"):
+        if not (rev == "HEAD" or rev.startswith("refs/")):
             from dvc.repo.experiments.utils import (
                 AmbiguousExpRefInfo,
                 resolve_name,

dvc/stage/__init__.py

@@ -554,9 +554,8 @@ class Stage(params.StageParams):
             self.remove_outs(ignore_remove=False, force=False)
 
         if (not self.frozen and self.is_import) or self.is_partial_import:
-            jobs = kwargs.get("jobs", None)
             self._sync_import(
-                dry, force, jobs, no_download, check_changed=self.frozen
+                dry, force, kwargs.get("jobs", None), no_download
             )
         elif not self.frozen and self.cmd:
             self._run_stage(dry, force, **kwargs)
@@ -580,10 +579,8 @@ class Stage(params.StageParams):
         return run_stage(self, dry, force, **kwargs)
 
     @rwlocked(read=["deps"], write=["outs"])
-    def _sync_import(
-        self, dry, force, jobs, no_download, check_changed: bool = False
-    ):
-        sync_import(self, dry, force, jobs, no_download, check_changed)
+    def _sync_import(self, dry, force, jobs, no_download):
+        sync_import(self, dry, force, jobs, no_download)
 
     @rwlocked(read=["outs"])
     def _check_missing_outputs(self):

dvc/stage/exceptions.py

@@ -31,7 +31,7 @@ class StageFileIsNotDvcFileError(DvcException):
 
         sname = fname + DVC_FILE_SUFFIX
         if is_dvc_file(sname):
-            msg += f" Do you mean '{sname}'?"
+            msg += f". Do you mean '{sname}'?"
 
         super().__init__(msg)
 

dvc/stage/imports.py

@@ -46,7 +46,6 @@ def sync_import(
     force=False,
     jobs=None,
     no_download=False,
-    check_changed=False,
 ):
     """Synchronize import's outs to the workspace."""
     logger.info("Importing '%s' -> '%s'", stage.deps[0], stage.outs[0])
@@ -56,13 +55,9 @@ def sync_import(
     if not force and stage.already_cached():
         stage.outs[0].checkout()
     else:
-        if check_changed:
-            old_hash_info = stage.deps[0].hash_info
         stage.save_deps()
-        if check_changed and not old_hash_info == stage.deps[0].hash_info:
-            from dvc.stage.exceptions import DataSourceChanged
-
-            raise DataSourceChanged(f"{stage} ({stage.deps[0]})")
-
         if not no_download:
-            stage.deps[0].download(stage.outs[0], jobs=jobs)
+            stage.deps[0].download(
+                stage.outs[0],
+                jobs=jobs,
+            )

pyproject.toml

@@ -30,11 +30,11 @@ line_length = 79
 
 [tool.pytest.ini_options]
 log_level = "debug"
-addopts = "-ra"
+addopts = "-ra -p no:hydra_pytest"
 markers = [
     "needs_internet: Might need network access for the tests",
-"vscode: Tests verifying contract between DVC and VSCode plugin",
-"studio: Tests verifying contract between DVC and Studio"
+    "vscode: Tests verifying contract between DVC and VSCode plugin",
+    "studio: Tests verifying contract between DVC and Studio",
 ]
 xfail_strict = true
 

scripts/pyinstaller/build.py

@@ -1,6 +1,6 @@
 import os
 import pathlib
-from subprocess import STDOUT, check_call
+from subprocess import STDOUT, check_call, check_output
 
 path = pathlib.Path(__file__).parent.absolute()
 hooks = path / "hooks"
@@ -21,10 +21,27 @@ check_call(
     stderr=STDOUT,
 )
 
-check_call(
+out = check_output(
     [
         path / "dist" / "dvc" / "dvc",
         "doctor",
     ],
     stderr=STDOUT,
-)
+).decode()
+
+remotes = [
+    "s3",
+    "oss",
+    "gdrive",
+    "gs",
+    "hdfs",
+    "http",
+    "webhdfs",
+    "azure",
+    "ssh",
+    "webdav",
+]
+
+print(out)
+for remote in remotes:
+    assert f"\t{remote}" in out, f"Missing support for {remote}"

scripts/pyinstaller/hooks/hook-dvc.py

@@ -16,5 +16,15 @@ datas += copy_metadata("webdav4")
 datas += copy_metadata("aiohttp")
 datas += copy_metadata("aiohttp_retry")
 
-# https://github.com/pypa/setuptools/issues/1963
-hiddenimports = ["pkg_resources.py2_warn"]
+hiddenimports = [
+    "dvc_azure",
+    "dvc_gdrive",
+    "dvc_gs",
+    "dvc_hdfs",
+    "dvc_oss",
+    "dvc_s3",
+    "dvc_webdav",
+    "dvc_webhdfs",
+    # https://github.com/pypa/setuptools/issues/1963
+    "pkg_resources.py2_warn",
+]

setup.cfg

@@ -62,14 +62,12 @@ install_requires =
     rich>=10.13.0
     pyparsing>=2.4.7
     typing-extensions>=3.7.4
-    fsspec[http]>=2021.10.1
-    aiohttp-retry>=2.4.5
     scmrepo==0.0.25
     dvc-render==0.0.9
     dvc-task==0.1.2
     dvclive>=0.10.0
-    dvc-data==0.1.22
-    dvc-http==0.0.2
+    dvc-data==0.2.0
+    dvc-http==2.19.1
     hydra-core>=1.1.0
 
 [options.extras_require]
@@ -88,25 +86,25 @@ dev =
     %(tests)s
     dvc-data[cli]
 # Extra dependencies for remote integrations
-azure = dvc-azure==0.0.1
-gdrive = dvc-gdrive==0.0.1
-gs = dvc-gs==0.0.1
-hdfs = dvc-hdfs==0.0.1
-oss = dvc-oss==0.0.1
-s3 = dvc-s3==0.0.2
-ssh = dvc-ssh==0.0.1
+azure = dvc-azure==2.19.0
+gdrive = dvc-gdrive==2.19.0
+gs = dvc-gs==2.19.1
+hdfs = dvc-hdfs==2.19.0; python_version < '3.11'
+oss = dvc-oss==2.19.0
+s3 = dvc-s3==2.19.0
+ssh = dvc-ssh==2.19.0
 # gssapi should not be included in all_remotes, because it doesn't have wheels
 # for Linux, so it will fail to compile if user doesn't have all the
 # requirements, including kerberos itself.
-ssh_gssapi = dvc-ssh[gssapi]==0.0.1
-webdav = dvc-webdav==0.0.1
-webhdfs = dvc-webhdfs==0.0.2
+ssh_gssapi = dvc-ssh[gssapi]==2.19.0
+webdav = dvc-webdav==2.19.0
+webhdfs = dvc-webhdfs==2.19.0
 # requests-kerberos requires krb5 & gssapi, which does not provide wheels Linux
-webhdfs_kerberos = dvc-webhdfs[kerberos]==0.0.2
+webhdfs_kerberos = dvc-webhdfs[kerberos]==2.19.0
 terraform = tpi[ssh]>=2.1.0
 tests =
     %(terraform)s
-    dvc-ssh==0.0.1
+    dvc-ssh==2.19.0
     # Test requirements
     pytest==7.1.2
     pytest-cov==3.0.0

tests/conftest.py

@@ -250,3 +250,15 @@ def run_copy_metrics(tmp_dir, run_copy):
         return stage
 
     return run
+
+
+@pytest.fixture(autouse=True)
+def mock_hydra_conf(mocker):
+    if sys.version_info < (3, 11):
+        return
+
+    # `hydra.conf` fails to import in 3.11, it raises ValueError due to changes
+    # in dataclasses. See https://github.com/python/cpython/pull/29867.
+    # NOTE: using sentinel here so that any imports from `hydra.conf`
+    # return a mock.
+    sys.modules["hydra.conf"] = mocker.sentinel

tests/func/api/test_params.py

@@ -1,3 +1,4 @@
+import os
 from textwrap import dedent
 
 import pytest
@@ -67,6 +68,9 @@ def test_params_show_targets(params_repo):
         "bar": 2,
         "foobar": 3,
     }
+    assert api.params_show("params.yaml", stages="stage-1") == {
+        "foo": 5,
+    }
 
 
 def test_params_show_deps(params_repo):
@@ -92,6 +96,27 @@ def test_params_show_stages(params_repo):
         api.params_show(stages="stage-0")
 
 
+def test_params_show_stage_addressing(tmp_dir, dvc):
+    for subdir in {"subdir1", "subdir2"}:
+        subdir = tmp_dir / subdir
+        subdir.mkdir()
+        with subdir.chdir():
+            subdir.gen("params.yaml", "foo: 1")
+
+            dvc.run(name="stage-0", cmd="echo stage-0", params=["foo"])
+
+    for s in {"subdir1", "subdir2"}:
+        dvcyaml = os.path.join(s, "dvc.yaml")
+        assert api.params_show(stages=f"{dvcyaml}:stage-0") == {"foo": 1}
+
+    with subdir.chdir():
+        nested = subdir / "nested"
+        nested.mkdir()
+        with nested.chdir():
+            dvcyaml = os.path.join("..", "dvc.yaml")
+            assert api.params_show(stages=f"{dvcyaml}:stage-0") == {"foo": 1}
+
+
 def test_params_show_revs(params_repo):
     assert api.params_show(rev="HEAD~1") == {
         "params.yaml:foo": 1,
@@ -168,3 +193,15 @@ def test_params_show_stage_without_params(tmp_dir, dvc):
 
     with pytest.raises(DvcException, match="No params found"):
         api.params_show(deps=True)
+
+
+def test_params_show_untracked_target(params_repo, tmp_dir):
+    tmp_dir.gen("params_foo.yaml", "foo: 1")
+
+    assert api.params_show("params_foo.yaml") == {"foo": 1}
+
+    with pytest.raises(DvcException, match="No params found"):
+        api.params_show("params_foo.yaml", stages="stage-0")
+
+    with pytest.raises(DvcException, match="No params found"):
+        api.params_show("params_foo.yaml", deps=True)

tests/func/experiments/test_diff.py

@@ -1,6 +1,17 @@
 from funcy import first
 
 
+def test_diff_no_commits(tmp_dir):
+    from scmrepo.git import Git
+
+    from dvc.repo import Repo
+
+    git = Git.init(tmp_dir.fs_path)
+    assert git.no_commits
+
+    assert Repo.init().experiments.diff() == {}
+
+
 def test_diff_empty(tmp_dir, scm, dvc, exp_stage):
     assert dvc.experiments.diff() == {"params": {}, "metrics": {}}
 

tests/func/experiments/test_queue.py

@@ -42,6 +42,7 @@ def failed_tasks(tmp_dir, dvc, scm, test_queue, failed_exp_stage):
     return name_list
 
 
+@pytest.mark.xfail(strict=False, reason="pytest-celery flaky")
 @pytest.mark.parametrize("follow", [True, False])
 def test_celery_logs(
     tmp_dir,
@@ -64,7 +65,7 @@ def test_celery_logs(
     assert "failed to reproduce 'failed-copy-file'" in captured.out
 
 
-@pytest.mark.flaky(max_runs=3, min_passes=1)
+@pytest.mark.xfail(strict=False, reason="pytest-celery flaky")
 def test_queue_remove_done(dvc, failed_tasks, success_tasks):
     assert len(dvc.experiments.celery_queue.failed_stash) == 3
     status = to_dict(dvc.experiments.celery_queue.status())

tests/func/experiments/test_show.py

@@ -35,6 +35,18 @@ def make_executor_info(**kwargs):
     return ExecutorInfo(**kwargs)
 
 
+@pytest.mark.vscode
+def test_show_no_commits(tmp_dir):
+    from scmrepo.git import Git
+
+    from dvc.repo import Repo
+
+    git = Git.init(tmp_dir.fs_path)
+    assert git.no_commits
+
+    assert Repo.init().experiments.show() == {}
+
+
 @pytest.mark.vscode
 def test_show_simple(tmp_dir, scm, dvc, exp_stage):
     assert dvc.experiments.show()["workspace"] == {

tests/func/metrics/test_show.py

@@ -21,6 +21,32 @@ def test_show_simple(tmp_dir, dvc, run_copy_metrics):
     }
 
 
+def test_show_simple_from_subdir(tmp_dir, dvc, run_copy_metrics):
+    subdir = tmp_dir / "subdir"
+    subdir.mkdir()
+    tmp_dir.gen("metrics_t.yaml", "1.1")
+    run_copy_metrics(
+        "metrics_t.yaml",
+        "subdir/metrics.yaml",
+        metrics=["subdir/metrics.yaml"],
+    )
+
+    expected_path = os.path.join("subdir", "metrics.yaml")
+    assert dvc.metrics.show() == {"": {"data": {expected_path: {"data": 1.1}}}}
+
+    expected_path = os.path.join("..", "subdir", "metrics.yaml")
+    with subdir.chdir():
+        assert dvc.metrics.show() == {
+            "": {"data": {expected_path: {"data": 1.1}}}
+        }
+    subdir2 = tmp_dir / "subdir2"
+    subdir2.mkdir()
+    with subdir2.chdir():
+        assert dvc.metrics.show() == {
+            "": {"data": {expected_path: {"data": 1.1}}}
+        }
+
+
 def test_show(tmp_dir, dvc, run_copy_metrics):
     tmp_dir.gen("metrics_t.yaml", "foo: 1.1")
     run_copy_metrics(

tests/func/plots/test_diff.py

@@ -1,3 +1,5 @@
+import pytest
+
 from tests.utils.plots import get_plot
 
 
@@ -46,3 +48,15 @@ def test_diff_dirty(tmp_dir, scm, dvc, run_copy_metrics):
     assert get_plot(
         diff_result, "workspace", "definitions", file="", endkey="data"
     ) == {"metric.json": props}
+
+
+@pytest.mark.vscode
+def test_no_commits(tmp_dir):
+    from scmrepo.git import Git
+
+    from dvc.repo import Repo
+
+    git = Git.init(tmp_dir.fs_path)
+    assert git.no_commits
+
+    assert Repo.init().plots.diff() == {}

tests/func/test_data_status.py

@@ -1,6 +1,9 @@
 from os.path import join
 
+import pytest
+
 from dvc.repo import Repo
+from dvc.repo.data import _transform_git_paths_to_dvc
 from dvc.testing.tmp_dir import make_subrepo
 from dvc.utils.fs import remove
 
@@ -14,24 +17,55 @@ EMPTY_STATUS = {
 }
 
 
-def test_file(M, tmp_dir, scm, dvc):
+@pytest.mark.parametrize("path", [None, ("sub", "repo")])
+def test_git_to_dvc_path_wdir_transformation(tmp_dir, scm, path):
+    struct = {"dir": {"foo": "foo", "bar": "bar"}, "file": "file", "dir2": {}}
+    tmp_dir.gen(struct)
+
+    subdir = tmp_dir.joinpath(*path) if path else tmp_dir
+    make_subrepo(subdir, scm)
+    dvc = subdir.dvc
+
+    with subdir.chdir():
+        subdir.gen(struct)
+        _, _, untracked = scm.status(untracked_files="all")
+        # make order independent of the platforms for easier test assertions
+        untracked = sorted(untracked, reverse=True)
+        assert _transform_git_paths_to_dvc(dvc, untracked) == [
+            "file",
+            join("dir", "foo"),
+            join("dir", "bar"),
+        ]
+        with (subdir / "dir").chdir():
+            assert _transform_git_paths_to_dvc(dvc, untracked) == [
+                join("..", "file"),
+                "foo",
+                "bar",
+            ]
+        with (subdir / "dir2").chdir():
+            assert _transform_git_paths_to_dvc(dvc, untracked) == [
+                join("..", "file"),
+                join("..", "dir", "foo"),
+                join("..", "dir", "bar"),
+            ]
+
+
+def test_file(M, tmp_dir, dvc, scm):
     tmp_dir.dvc_gen("foo", "foo", commit="add foo")
     tmp_dir.dvc_gen("foo", "foobar")
     remove(tmp_dir / "foo")
 
     expected = {
+        **EMPTY_STATUS,
         "committed": {"modified": ["foo"]},
         "uncommitted": {"deleted": ["foo"]},
-        "git": M.instance_of(dict),
-        "not_in_cache": [],
-        "unchanged": [],
-        "untracked": [],
+        "git": M.dict(),
     }
     assert dvc.data_status() == expected
     assert dvc.data_status(granular=True) == expected
 
 
-def test_directory(M, tmp_dir, scm, dvc):
+def test_directory(M, tmp_dir, dvc, scm):
     tmp_dir.dvc_gen({"dir": {"foo": "foo"}}, commit="add dir")
     tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar", "foobar": "foobar"}})
     remove(tmp_dir / "dir")
@@ -39,12 +73,10 @@ def test_directory(M, tmp_dir, scm, dvc):
     tmp_dir.gen("untracked", "untracked")
 
     assert dvc.data_status() == {
+        **EMPTY_STATUS,
         "committed": {"modified": [join("dir", "")]},
         "uncommitted": {"modified": [join("dir", "")]},
-        "git": M.instance_of(dict),
-        "not_in_cache": [],
-        "unchanged": [],
-        "untracked": [],
+        "git": M.dict(),
     }
 
     assert dvc.data_status(granular=True, untracked_files="all") == {
@@ -52,14 +84,15 @@ def test_directory(M, tmp_dir, scm, dvc):
             "added": M.unordered(
                 join("dir", "bar"),
                 join("dir", "foobar"),
-            )
+            ),
+            "modified": [join("dir", "")],
         },
         "uncommitted": {
             "added": [join("dir", "baz")],
-            "modified": [join("dir", "bar")],
+            "modified": M.unordered(join("dir", ""), join("dir", "bar")),
             "deleted": [join("dir", "foobar")],
         },
-        "git": M.instance_of(dict),
+        "git": M.dict(),
         "not_in_cache": [],
         "unchanged": [join("dir", "foo")],
         "untracked": ["untracked"],
@@ -81,52 +114,19 @@ def test_noscm_repo(dvc):
     assert dvc.data_status() == EMPTY_STATUS
 
 
-def test_unchanged(M, tmp_dir, scm, dvc):
+def test_unchanged(M, tmp_dir, dvc, scm):
     tmp_dir.dvc_gen({"dir": {"foo": "foo"}}, commit="add dir")
     tmp_dir.dvc_gen("bar", "bar", commit="add foo")
 
     assert dvc.data_status() == {
         **EMPTY_STATUS,
-        "git": M.instance_of(dict),
+        "git": M.dict(),
         "unchanged": M.unordered("bar", join("dir", "")),
     }
     assert dvc.data_status(granular=True) == {
         **EMPTY_STATUS,
-        "git": M.instance_of(dict),
-        "unchanged": M.unordered("bar", join("dir", "foo")),
-    }
-
-
-def test_not_in_cache(M, tmp_dir, scm, dvc):
-    # TODO: investigation required, might return wrong results
-    tmp_dir.dvc_gen({"dir": {"foo": "foo"}}, commit="add dir")
-    tmp_dir.dvc_gen("bar", "bar", commit="add foo")
-    remove(dvc.odb.local.path)
-
-    assert dvc.data_status() == {
-        **EMPTY_STATUS,
-        "not_in_cache": M.unordered("bar", join("dir", "")),
-        "git": M.instance_of(dict),
-        "unchanged": ["bar"],
-        "uncommitted": {"added": [join("dir", "")]},
-    }
-    assert dvc.data_status(granular=True) == {
-        **EMPTY_STATUS,
-        "not_in_cache": M.unordered("bar"),
-        "git": M.instance_of(dict),
-        "unchanged": [],
-        "committed": {"modified": ["bar"]},
-        "uncommitted": {"modified": ["bar"], "added": [join("dir", "foo")]},
-    }
-
-
-def test_withdirs(M, tmp_dir, scm, dvc):
-    tmp_dir.dvc_gen({"dir": {"foo": "foo"}}, commit="add dir")
-    tmp_dir.dvc_gen("bar", "bar", commit="add foo")
-    assert dvc.data_status(granular=True, with_dirs=True) == {
-        **EMPTY_STATUS,
-        "git": M.instance_of(dict),
-        "unchanged": M.unordered("bar", join("dir", "foo"), join("dir", "")),
+        "git": M.dict(),
+        "unchanged": M.unordered("bar", join("dir", ""), join("dir", "foo")),
     }
 
 
@@ -146,7 +146,8 @@ def test_skip_uncached_pipeline_outputs(tmp_dir, dvc, run_copy_metrics):
 
 def test_output_with_newly_added_stage(tmp_dir, dvc):
     dvc.stage.add(deps=["bar"], outs=["foo"], name="copy", cmd="cp foo bar")
-    assert dvc.data_status() == {**EMPTY_STATUS, "not_in_cache": ["foo"]}
+    assert dvc.data_status() == EMPTY_STATUS
+    assert dvc.data_status(granular=True) == EMPTY_STATUS
 
 
 def test_subdir(M, tmp_dir, scm):
@@ -161,7 +162,243 @@ def test_subdir(M, tmp_dir, scm):
         dvc = subrepo.dvc
         assert dvc.data_status(granular=True, untracked_files="all") == {
             **EMPTY_STATUS,
-            "git": M.instance_of(dict),
-            "unchanged": M.unordered("bar", join("dir", "foo")),
+            "git": M.dict(),
+            "unchanged": M.unordered(
+                "bar", join("dir", ""), join("dir", "foo")
+            ),
             "untracked": ["untracked"],
         }
+
+
+def test_untracked_newly_added_files(M, tmp_dir, dvc, scm):
+    tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}})
+    tmp_dir.gen("foobar", "foobar")
+
+    expected = {
+        **EMPTY_STATUS,
+        "untracked": M.unordered(
+            join("dir", "foo"), join("dir", "bar"), "foobar"
+        ),
+        "git": M.dict(),
+    }
+    assert dvc.data_status(untracked_files="all") == expected
+    assert dvc.data_status(granular=True, untracked_files="all") == expected
+
+
+def test_missing_cache_workspace_exists(M, tmp_dir, dvc, scm):
+    tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}})
+    tmp_dir.dvc_gen("foobar", "foobar")
+    remove(dvc.odb.repo.path)
+
+    assert dvc.data_status(untracked_files="all") == {
+        **EMPTY_STATUS,
+        "untracked": M.unordered("foobar.dvc", "dir.dvc", ".gitignore"),
+        "committed": {"added": M.unordered("foobar", join("dir", ""))},
+        "not_in_cache": M.unordered("foobar", join("dir", "")),
+        "git": M.dict(),
+    }
+
+    assert dvc.data_status(granular=True, untracked_files="all") == {
+        **EMPTY_STATUS,
+        "untracked": M.unordered("foobar.dvc", "dir.dvc", ".gitignore"),
+        "committed": {"added": M.unordered("foobar", join("dir", ""))},
+        "uncommitted": {
+            "unknown": M.unordered(join("dir", "foo"), join("dir", "bar"))
+        },
+        "not_in_cache": M.unordered(
+            "foobar",
+            join("dir", ""),
+        ),
+        "git": M.dict(),
+    }
+
+
+def test_missing_cache_missing_workspace(M, tmp_dir, dvc, scm):
+    tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}})
+    tmp_dir.dvc_gen("foobar", "foobar")
+    for path in [dvc.odb.repo.path, "dir", "foobar"]:
+        remove(path)
+
+    assert dvc.data_status(untracked_files="all") == {
+        **EMPTY_STATUS,
+        "untracked": M.unordered("foobar.dvc", "dir.dvc", ".gitignore"),
+        "uncommitted": {"deleted": M.unordered("foobar", join("dir", ""))},
+        "committed": {"added": M.unordered("foobar", join("dir", ""))},
+        "not_in_cache": M.unordered("foobar", join("dir", "")),
+        "git": M.dict(),
+    }
+
+    assert dvc.data_status(granular=True, untracked_files="all") == {
+        **EMPTY_STATUS,
+        "untracked": M.unordered("foobar.dvc", "dir.dvc", ".gitignore"),
+        "uncommitted": {"deleted": M.unordered("foobar", join("dir", ""))},
+        "committed": {"added": M.unordered("foobar", join("dir", ""))},
+        "not_in_cache": M.unordered("foobar", join("dir", "")),
+        "git": M.dict(),
+    }
+
+
+def test_git_committed_missing_cache_workspace_exists(M, tmp_dir, dvc, scm):
+    tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}}, commit="add dir")
+    tmp_dir.dvc_gen("foobar", "foobar", commit="add foobar")
+    remove(dvc.odb.local.path)
+
+    assert dvc.data_status(untracked_files="all") == {
+        **EMPTY_STATUS,
+        "not_in_cache": M.unordered("foobar", join("dir", "")),
+        "git": M.dict(),
+        "unchanged": M.unordered("foobar", join("dir", "")),
+    }
+    assert dvc.data_status(granular=True) == {
+        **EMPTY_STATUS,
+        "not_in_cache": M.unordered(
+            "foobar",
+            join("dir", ""),
+        ),
+        "uncommitted": {
+            "unknown": M.unordered(join("dir", "foo"), join("dir", "bar"))
+        },
+        "git": M.dict(),
+        "unchanged": M.unordered("foobar", join("dir", "")),
+    }
+
+
+def test_git_committed_missing_cache_missing_workspace(M, tmp_dir, dvc, scm):
+    tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}}, commit="add dir")
+    tmp_dir.dvc_gen("foobar", "foobar", commit="add foobar")
+    for path in [dvc.odb.repo.path, "dir", "foobar"]:
+        remove(path)
+
+    assert dvc.data_status(untracked_files="all") == {
+        **EMPTY_STATUS,
+        "uncommitted": {"deleted": M.unordered(join("dir", ""), "foobar")},
+        "not_in_cache": M.unordered(join("dir", ""), "foobar"),
+        "git": M.dict(),
+    }
+    assert dvc.data_status(granular=True, untracked_files="all") == {
+        **EMPTY_STATUS,
+        "committed": {},
+        "uncommitted": {"deleted": M.unordered(join("dir", ""), "foobar")},
+        "not_in_cache": M.unordered(join("dir", ""), "foobar"),
+        "git": M.dict(),
+    }
+
+
+def test_partial_missing_cache(M, tmp_dir, dvc, scm):
+    tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}})
+
+    # remove "foo" from cache
+    odb = dvc.odb.repo
+    odb.fs.rm(odb.oid_to_path("acbd18db4cc2f85cedef654fccc4a4d8"))
+
+    assert dvc.data_status() == {
+        **EMPTY_STATUS,
+        "committed": {"added": [join("dir", "")]},
+        "git": M.dict(),
+    }
+    assert dvc.data_status(granular=True) == {
+        **EMPTY_STATUS,
+        "committed": {
+            "added": M.unordered(
+                join("dir", ""), join("dir", "foo"), join("dir", "bar")
+            )
+        },
+        "not_in_cache": [join("dir", "foo")],
+        "git": M.dict(),
+    }
+
+
+def test_missing_dir_object_from_head(M, tmp_dir, dvc, scm):
+    (stage,) = tmp_dir.dvc_gen(
+        {"dir": {"foo": "foo", "bar": "bar"}}, commit="add dir"
+    )
+    remove("dir")
+    tmp_dir.dvc_gen({"dir": {"foobar": "foobar"}})
+    odb = dvc.odb.repo
+    odb.fs.rm(odb.oid_to_path(stage.outs[0].hash_info.value))
+
+    assert dvc.data_status() == {
+        **EMPTY_STATUS,
+        "committed": {"modified": [join("dir", "")]},
+        "git": M.dict(),
+    }
+    assert dvc.data_status(granular=True) == {
+        **EMPTY_STATUS,
+        "committed": {
+            "modified": [join("dir", "")],
+            "unknown": [join("dir", "foobar")],
+        },
+        "git": M.dict(),
+    }
+
+
+def test_missing_dir_object_from_index(M, tmp_dir, dvc, scm):
+    tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}}, commit="add dir")
+    remove("dir")
+    (stage,) = tmp_dir.dvc_gen({"dir": {"foobar": "foobar"}})
+    odb = dvc.odb.repo
+    odb.fs.rm(odb.oid_to_path(stage.outs[0].hash_info.value))
+
+    assert dvc.data_status() == {
+        **EMPTY_STATUS,
+        "committed": {"modified": [join("dir", "")]},
+        "not_in_cache": [join("dir", "")],
+        "git": M.dict(),
+    }
+    assert dvc.data_status(granular=True) == {
+        **EMPTY_STATUS,
+        "committed": {
+            "modified": [join("dir", "")],
+        },
+        "uncommitted": {"unknown": [join("dir", "foobar")]},
+        "not_in_cache": [join("dir", "")],
+        "git": M.dict(),
+    }
+
+
+def test_root_from_dir_to_file(M, tmp_dir, dvc, scm):
+    tmp_dir.dvc_gen({"data": {"foo": "foo", "bar": "bar"}})
+    remove("data")
+    tmp_dir.gen("data", "file")
+
+    assert dvc.data_status() == {
+        **EMPTY_STATUS,
+        "committed": {"added": [join("data", "")]},
+        "uncommitted": {"modified": ["data"]},
+        "git": M.dict(),
+    }
+    assert dvc.data_status(granular=True) == {
+        **EMPTY_STATUS,
+        "committed": {
+            "added": M.unordered(
+                join("data", ""), join("data", "foo"), join("data", "bar")
+            )
+        },
+        "uncommitted": {
+            "deleted": M.unordered(join("data", "foo"), join("data", "bar")),
+            "modified": ["data"],
+        },
+        "git": M.dict(),
+    }
+
+
+def test_root_from_file_to_dir(M, tmp_dir, dvc, scm):
+    tmp_dir.dvc_gen("data", "file")
+    remove("data")
+    tmp_dir.gen({"data": {"foo": "foo", "bar": "bar"}})
+
+    assert dvc.data_status() == {
+        **EMPTY_STATUS,
+        "committed": {"added": ["data"]},
+        "uncommitted": {"modified": [join("data", "")]},
+        "git": M.dict(),
+    }
+    assert dvc.data_status(granular=True) == {
+        **EMPTY_STATUS,
+        "committed": {"added": ["data"]},
+        "uncommitted": {
+            "modified": [join("data", "")],
+            "added": M.unordered(join("data", "foo"), join("data", "bar")),
+        },
+        "git": M.dict(),
+    }

tests/func/test_remove.py

@@ -4,7 +4,10 @@ import pytest
 
 from dvc.cli import main
 from dvc.fs import system
-from dvc.stage.exceptions import StageFileDoesNotExistError
+from dvc.stage.exceptions import (
+    StageFileDoesNotExistError,
+    StageFileIsNotDvcFileError,
+)
 from dvc.utils.fs import remove
 from dvc_objects.errors import ObjectDBError
 from tests.utils import get_gitignore_content
@@ -32,6 +35,18 @@ def test_remove(tmp_dir, scm, dvc, run_copy, remove_outs):
     assert not (tmp_dir / ".gitignore").exists()
 
 
+def test_remove_file_target(tmp_dir, dvc):
+    tmp_dir.dvc_gen("foo", "foo")
+
+    with pytest.raises(
+        StageFileIsNotDvcFileError,
+        match="'foo' is not a .dvc file. Do you mean 'foo.dvc'?",
+    ):
+        dvc.remove("foo")
+
+    dvc.remove("foo.dvc")
+
+
 def test_remove_non_existent_file(tmp_dir, dvc):
     with pytest.raises(StageFileDoesNotExistError):
         dvc.remove("non_existent_dvc_file.dvc")

tests/func/test_version.py

@@ -4,6 +4,7 @@ from dvc.cli import main
 from tests.unit.test_info import (
     DVC_VERSION_REGEX,
     PYTHON_VERSION_REGEX,
+    SUBPROJECTS,
     find_supported_remotes,
 )
 
@@ -14,8 +15,34 @@ def test_(tmp_dir, dvc, scm, capsys):
     out, _ = capsys.readouterr()
     assert re.search(rf"DVC version: {DVC_VERSION_REGEX}", out)
     assert re.search(f"Platform: {PYTHON_VERSION_REGEX} on .*", out)
+    for subproject in SUBPROJECTS:
+        assert re.search(rf"{subproject} = {DVC_VERSION_REGEX}", out)
+
     assert find_supported_remotes(out)
     assert re.search(r"Cache types: .*", out)
     assert re.search(r"Caches: local", out)
     assert re.search(r"Remotes: None", out)
     assert "Repo: dvc, git" in out
+
+
+def test_import_error(tmp_dir, dvc, scm, capsys, monkeypatch):
+    import importlib.metadata as importlib_metadata
+
+    original = importlib_metadata.version
+
+    def _import_error(name):
+        if name == "dvclive":
+            raise ImportError
+        return original(name)
+
+    monkeypatch.setattr(importlib_metadata, "version", _import_error)
+    assert main(["version"]) == 0
+
+    out, _ = capsys.readouterr()
+
+    for subproject in SUBPROJECTS:
+        match = re.search(rf"{subproject} = {DVC_VERSION_REGEX}", out)
+        if subproject != "dvclive":
+            assert match
+        else:
+            assert match is None

tests/func/utils/test_hydra.py

@@ -1,7 +1,6 @@
 import pytest
 
 from dvc.exceptions import InvalidArgumentError
-from dvc.utils.hydra import apply_overrides
 
 
 @pytest.mark.parametrize("suffix", ["yaml", "toml", "json"])
@@ -90,6 +89,8 @@ from dvc.utils.hydra import apply_overrides
     ],
 )
 def test_apply_overrides(tmp_dir, suffix, overrides, expected):
+    from dvc.utils.hydra import apply_overrides
+
     if suffix == "toml":
         if overrides in [
             ["foo=baz"],
@@ -114,6 +115,8 @@ def test_apply_overrides(tmp_dir, suffix, overrides, expected):
     [["foobar=2"], ["lorem=3,2"], ["+lorem=3"], ["foo[0]=bar"]],
 )
 def test_invalid_overrides(tmp_dir, overrides):
+    from dvc.utils.hydra import apply_overrides
+
     params_file = tmp_dir / "params.yaml"
     params_file.dump(
         {"foo": [{"bar": 1}, {"baz": 2}], "goo": {"bag": 3.0}, "lorem": False}

tests/unit/command/test_data_status.py

@@ -14,15 +14,21 @@ from tests.func.parsing.test_errors import escape_ansi
 def mocked_status():
     yield Status(
         not_in_cache=["notincache"],
-        committed={"added": ["dir/bar", "dir/foo"]},
+        committed={
+            "added": ["dir/bar", "dir/foo"],
+            "deleted": ["dir/baz"],
+            "modified": ["dir/foobar"],
+            "unknown": ["dir/unknown1"],
+        },
         uncommitted={
             "added": ["dir/baz"],
             "modified": ["dir/bar"],
             "deleted": ["dir/foobar"],
+            "unknown": ["dir2/unknown2"],
         },
         untracked=["untracked"],
         unchanged=["dir/foo"],
-        git={"is_dirty": True},
+        git={"is_dirty": True, "is_empty": False},
     )
 
 
@@ -38,7 +44,6 @@ def test_cli(dvc, mocker, mocked_status):
             "--json",
             "--unchanged",
             "--untracked-files",
-            "--with-dirs",
             "--granular",
         ]
     )
@@ -49,7 +54,6 @@ def test_cli(dvc, mocker, mocked_status):
     status.assert_called_once_with(
         untracked_files="all",
         granular=True,
-        with_dirs=True,
     )
 
 
@@ -106,24 +110,29 @@ def test_show_status(dvc, scm, mocker, capsys, mocked_status, args, is_dirty):
     out, err = capsys.readouterr()
     expected_out = """\
 Not in cache:
-  (use "dvc pull <file>..." to update your local storage)
+  (use "dvc fetch <file>..." to download files)
         notincache
 
 DVC committed changes:
   (git commit the corresponding dvc files to update the repo)
         added: dir/bar
         added: dir/foo
+        deleted: dir/baz
+        modified: dir/foobar
+        unknown: dir/unknown1
 
 DVC uncommitted changes:
   (use "dvc commit <file>..." to track changes)
+  (use "dvc checkout <file>..." to discard changes)
         added: dir/baz
         modified: dir/bar
         deleted: dir/foobar
+        unknown: dir2/unknown2
 """
     if "--untracked-files" in args:
         expected_out += """
 Untracked files:
-  (use "git add <file> ..." or dvc add <file>..." to commit to git or to dvc)
+  (use "git add <file> ..." or "dvc add <file>..." to commit to git or to dvc)
         untracked
 """
     if "--unchanged" in args:

tests/unit/command/test_plots.py

@@ -4,7 +4,6 @@ import posixpath
 from pathlib import Path
 
 import pytest
-from funcy import pluck_attr
 
 from dvc.cli import parse_args
 from dvc.commands.plots import CmdPlotsDiff, CmdPlotsShow, CmdPlotsTemplates
@@ -348,39 +347,38 @@ def test_plots_diff_json(dvc, mocker, capsys):
     render_mock.assert_not_called()
 
 
-@pytest.mark.parametrize("target", (("t1"), (None)))
-def test_plots_templates(tmp_dir, dvc, mocker, capsys, target):
-    assert not os.path.exists(dvc.plots.templates_dir)
-    mocker.patch(
-        "dvc.commands.plots.CmdPlotsTemplates.TEMPLATES_CHOICES",
-        ["t1", "t2"],
-    )
+@pytest.mark.parametrize(
+    "target,expected_out,expected_rtn",
+    (("t1", "\"{'t1'}\"", 0), (None, "t1\nt2", 0), ("t3", "", 1)),
+)
+def test_plots_templates(
+    dvc, mocker, capsys, target, expected_out, expected_rtn
+):
+    t1 = mocker.Mock()
+    t1.DEFAULT_NAME = "t1"
+    t1.DEFAULT_CONTENT = "{'t1'}"
 
-    arguments = ["plots", "templates", "--out", "output"]
+    t2 = mocker.Mock()
+    t2.DEFAULT_NAME = "t2"
+    t2.DEFAULT_CONTENT = "{'t2'}"
+
+    mocker.patch("dvc_render.vega_templates.TEMPLATES", [t1, t2])
+
+    arguments = ["plots", "templates"]
     if target:
         arguments += [target]
 
     cli_args = parse_args(arguments)
     assert cli_args.func == CmdPlotsTemplates
 
-    dump_mock = mocker.patch("dvc_render.vega_templates.dump_templates")
     cmd = cli_args.func(cli_args)
 
-    assert cmd.run() == 0
+    rtn = cmd.run()
+
     out, _ = capsys.readouterr()
 
-    dump_mock.assert_called_once_with(
-        output=os.path.abspath("output"), targets=[target] if target else None
-    )
-    assert "Templates have been written into 'output'." in out
-
-
-def test_plots_templates_choices(tmp_dir, dvc):
-    from dvc_render import TEMPLATES
-
-    assert CmdPlotsTemplates.TEMPLATES_CHOICES == list(
-        pluck_attr("DEFAULT_NAME", TEMPLATES)
-    )
+    assert out.strip() == expected_out
+    assert rtn == expected_rtn
 
 
 @pytest.mark.parametrize("split", (True, False))

tests/unit/fs/test_fs.py

@@ -1,5 +1,4 @@
 import pytest
-from dvc_hdfs import HDFSFileSystem
 from dvc_http import HTTPFileSystem, HTTPSFileSystem
 from dvc_s3 import S3FileSystem
 from dvc_ssh import SSHFileSystem
@@ -7,25 +6,31 @@ from dvc_ssh import SSHFileSystem
 from dvc.config import RemoteNotFoundError
 from dvc.fs import LocalFileSystem, get_fs_cls, get_fs_config
 
+url_cls_pairs = [
+    ("s3://bucket/path", S3FileSystem),
+    ("ssh://example.com:/dir/path", SSHFileSystem),
+    ("http://example.com/path/to/file", HTTPFileSystem),
+    ("https://example.com/path/to/file", HTTPSFileSystem),
+    ("path/to/file", LocalFileSystem),
+    ("path\\to\\file", LocalFileSystem),
+    ("file", LocalFileSystem),
+    ("./file", LocalFileSystem),
+    (".\\file", LocalFileSystem),
+    ("../file", LocalFileSystem),
+    ("..\\file", LocalFileSystem),
+    ("unknown://path", LocalFileSystem),
+]
 
-@pytest.mark.parametrize(
-    "url, cls",
-    [
-        ("s3://bucket/path", S3FileSystem),
-        ("ssh://example.com:/dir/path", SSHFileSystem),
-        ("hdfs://example.com/dir/path", HDFSFileSystem),
-        ("http://example.com/path/to/file", HTTPFileSystem),
-        ("https://example.com/path/to/file", HTTPSFileSystem),
-        ("path/to/file", LocalFileSystem),
-        ("path\\to\\file", LocalFileSystem),
-        ("file", LocalFileSystem),
-        ("./file", LocalFileSystem),
-        (".\\file", LocalFileSystem),
-        ("../file", LocalFileSystem),
-        ("..\\file", LocalFileSystem),
-        ("unknown://path", LocalFileSystem),
-    ],
-)
+
+try:
+    from dvc_hdfs import HDFSFileSystem
+
+    url_cls_pairs += [("hdfs://example.com/dir/path", HDFSFileSystem)]
+except ImportError:
+    pass
+
+
+@pytest.mark.parametrize("url, cls", url_cls_pairs)
 def test_get_fs_cls(url, cls):
     assert get_fs_cls({"url": url}) == cls
 

tests/unit/test_info.py

@@ -4,7 +4,7 @@ import shutil
 
 import pytest
 
-from dvc.info import get_dvc_info
+from dvc.info import SUBPROJECTS, get_dvc_info
 
 # Python's version is in the shape of:
 # <major>.<minor>.<patch>[{a|b|rc}N][.postN][.devN]
@@ -53,6 +53,9 @@ def test_info_in_repo(scm_init, tmp_dir):
 
     assert re.search(rf"DVC version: {DVC_VERSION_REGEX}", dvc_info)
     assert re.search(f"Platform: {PYTHON_VERSION_REGEX} on .*", dvc_info)
+    for subproject in SUBPROJECTS:
+        assert re.search(rf"{subproject} = {DVC_VERSION_REGEX}", dvc_info)
+
     assert find_supported_remotes(dvc_info)
     assert re.search(r"Cache types: .*", dvc_info)
 

tests/unit/test_scm.py

@@ -0,0 +1,9 @@
+import pytest
+
+from dvc.exceptions import DvcException
+from dvc.scm import resolve_rev
+
+
+def test_resolve_rev_empty_git_repo(scm):
+    with pytest.raises(DvcException):
+        resolve_rev(scm, "HEAD")