api: Fix params_show for untracked targets.

If an untracked target is provided alongside `stages` or `deps`, don't load the parameters from the target and raise no params found instead.

Closes #8161

dvc/commands/data.py

@@ -31,8 +31,7 @@ class CmdDataStatus(CmdBase):
         "unchanged": "DVC unchanged files",
     }
     HINTS = {
-        "not_in_cache": 'use "dvc pull <file>..." '
-        "to update your local storage",
+        "not_in_cache": 'use "dvc pull <file>..." to download files',
         "committed": "git commit the corresponding dvc files "
         "to update the repo",
         "uncommitted": 'use "dvc commit <file>..." to track changes',
@@ -52,6 +51,7 @@ class CmdDataStatus(CmdBase):
                     file: state
                     for state, files in stage_status.items()
                     for file in files
+                    if state != "unknown"
                 }
             if not items:
                 continue

dvc/repo/data.py

@@ -1,6 +1,15 @@
 import os
 from collections import defaultdict
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, TypedDict, cast
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    Iterable,
+    List,
+    Optional,
+    TypedDict,
+    cast,
+)
 
 from dvc.ui import ui
 
@@ -10,19 +19,16 @@ if TYPE_CHECKING:
     from dvc.output import Output
     from dvc.repo import Repo
     from dvc_data.hashfile.db import HashFileDB
+    from dvc_data.hashfile.hash_info import HashInfo
     from dvc_data.hashfile.obj import HashFile
 
 
-def _in_cache(obj: Optional["HashFile"], cache: "HashFileDB") -> bool:
+def _in_cache(obj: "HashInfo", cache: "HashFileDB") -> bool:
     from dvc_objects.errors import ObjectFormatError
 
-    if not obj:
-        return False
-    if not obj.hash_info.value:
-        return False
-
+    assert obj.value
     try:
-        cache.check(obj.hash_info.value)
+        cache.check(obj.value)
         return True
     except (FileNotFoundError, ObjectFormatError):
         return False
@@ -30,31 +36,30 @@ def _in_cache(obj: Optional["HashFile"], cache: "HashFileDB") -> bool:
 
 def _shallow_diff(
     root: str,
-    old_obj: Optional["HashFile"],
-    new_obj: Optional["HashFile"],
+    old: Optional["HashInfo"],
+    new: Optional["HashInfo"],
     cache: "HashFileDB",
 ) -> Dict[str, List[str]]:
-    # TODO: add support for shallow diff in dvc-data
-    # TODO: we may want to recursively do in_cache check
     d = {}
 
-    from dvc_data.objects.tree import Tree
+    old_root = new_root = root
+    if old and old.isdir:
+        old_root = os.path.sep.join([root, ""])
+    if new and new.isdir:
+        new_root = os.path.sep.join([root, ""])
 
-    if isinstance(new_obj, Tree):
-        root = os.path.sep.join([root, ""])
+    if old and not _in_cache(old, cache):
+        d["not_in_cache"] = [old_root]
 
-    if not _in_cache(old_obj, cache):
-        d["not_in_cache"] = [root]
-
-    if old_obj is None and new_obj is None:
+    if not old and not new:
         return d
-    if old_obj is None:
-        return {"added": [root], **d}
-    if new_obj is None:
-        return {"deleted": [root], **d}
-    if old_obj.hash_info != new_obj.hash_info:
-        return {"modified": [root], **d}
-    return {"unchanged": [root], **d}
+    if not new:
+        return {"deleted": [old_root], **d}
+    if not old:
+        return {"added": [new_root], **d}
+    if old != new:
+        return {"modified": [new_root], **d}
+    return {"unchanged": [new_root], **d}
 
 
 def _granular_diff(
@@ -66,55 +71,73 @@ def _granular_diff(
 ) -> Dict[str, List[str]]:
     from dvc_data.diff import ROOT
     from dvc_data.diff import diff as odiff
-    from dvc_data.objects.tree import Tree
 
-    drop_root = False
-    trees = isinstance(old_obj, Tree) or isinstance(new_obj, Tree)
-    if trees:
-        drop_root = not with_dirs
-
-    def path_join(root: str, *paths: str) -> str:
-        if not trees and paths == ROOT:
+    def path_join(root: str, *paths: str, isdir: bool = False) -> str:
+        if not isdir and paths == ROOT:
             return root
         return os.path.sep.join([root, *paths])
 
+    def type_changed(old, new):
+        if old.oid and new.oid:
+            return old.oid.isdir != new.oid.isdir
+        return False
+
     diff_data = odiff(old_obj, new_obj, cache)
 
     output: Dict[str, List[str]] = defaultdict(list)
     for state in ("added", "deleted", "modified", "unchanged"):
         items = getattr(diff_data, state)
-        output[state].extend(
-            path_join(root, *item.new.key)
-            for item in items  # pylint: disable=not-an-iterable
-            if not (drop_root and item.new.key == ROOT)
-        )
-        # TODO: PERF: diff is checking not_in_cache for each even if we only
-        # need it for the index.
-        # BUG: not_in_cache file also shows up as modified in staged and
-        # unstaged. We currently don't know if it is really modified.
-        output["not_in_cache"].extend(
-            path_join(root, *item.new.key)
-            for item in items  # pylint: disable=not-an-iterable
-            if not item.old.in_cache
-            and not (drop_root and item.new.key == ROOT)
-            and state != "added"
-        )
-    return output
+        for item in items:  # pylint: disable=not-an-iterable
+            old = item.old
+            new = item.new
+            entry = old if state == "deleted" else new
+            isdir = entry.oid.isdir if entry.oid else False
+            obj_type_changed = type_changed(old, new)
+            if (
+                not with_dirs
+                and isdir
+                and entry.key == ROOT
+                and not obj_type_changed
+            ):
+                continue
+
+            path = path_join(root, *entry.key, isdir=isdir)
+            output[state].append(path)
+            if not item.old.in_cache and state != "added":
+                output["not_in_cache"].append(path)
+    return dict(output)
+
+
+def _get_obj_items(root: str, obj: Optional["HashFile"]) -> List[str]:
+    if not obj:
+        return []
+
+    from dvc_data.objects.tree import Tree
+
+    sep = os.path.sep
+    if isinstance(obj, Tree):
+        return [sep.join([root, *key]) for key, _, _ in obj]
+    return [root]
 
 
 def _diff(
     root: str,
+    old_oid: Optional["HashInfo"],
     old_obj: Optional["HashFile"],
+    new_oid: Optional["HashInfo"],
     new_obj: Optional["HashFile"],
-    cache: "HashFileDB",
-    granular: bool = False,
+    odb: "HashFileDB",
     with_dirs: bool = False,
+    granular: bool = False,
 ) -> Dict[str, List[str]]:
-    if granular:
-        return _granular_diff(
-            root, old_obj, new_obj, cache, with_dirs=with_dirs
-        )
-    return _shallow_diff(root, old_obj, new_obj, cache)
+    if not granular:
+        return _shallow_diff(root, old_oid, new_oid, odb)
+    if (old_oid and not old_obj) or (new_oid and not new_obj):
+        # we don't have enough information to give full details
+        unknown = _get_obj_items(root, new_obj or old_obj)
+        shallow_diff = _shallow_diff(root, old_oid, new_oid, odb)
+        return {**shallow_diff, "unknown": unknown}
+    return _granular_diff(root, old_obj, new_obj, odb, with_dirs=with_dirs)
 
 
 class GitInfo(TypedDict, total=False):
@@ -175,8 +198,18 @@ def _diff_index_to_wtree(repo: "Repo", **kwargs: Any) -> Dict[str, List[str]]:
         cache = repo.odb.repo
         root = str(out)
         old = out.get_obj()
+
         with ui.status(f"Calculating diff for {root} between index/workspace"):
-            d = _diff(root, old, new, cache, **kwargs)
+            d = _diff(
+                root,
+                out.hash_info,
+                old,
+                new.hash_info if new else None,
+                new,
+                cache,
+                **kwargs,
+            )
+
         for state, items in d.items():
             if not items:
                 continue
@@ -198,14 +231,17 @@ def _diff_head_to_index(
 
             root = str(out)
             typ = "index" if rev == "workspace" else head
-            objs[root][typ] = out.get_obj()
+            objs[root][typ] = (out.get_obj(), out.hash_info)
 
     cache = repo.odb.repo
     for root, obj_d in objs.items():
-        old = obj_d.get(head, None)
-        new = obj_d.get("index", None)
+        old_obj, old_oid = obj_d.get(head, (None, None))
+        new_obj, new_oid = obj_d.get("index", (None, None))
         with ui.status(f"Calculating diff for {root} between head/index"):
-            d = _diff(root, old, new, cache, **kwargs)
+            d = _diff(
+                root, old_oid, old_obj, new_oid, new_obj, cache, **kwargs
+            )
+
         for state, items in d.items():
             if not items:
                 continue
@@ -216,22 +252,30 @@ def _diff_head_to_index(
 
 class Status(TypedDict):
     not_in_cache: List[str]
-    committed: Dict[str, Any]
-    uncommitted: Dict[str, Any]
+    committed: Dict[str, List[str]]
+    uncommitted: Dict[str, List[str]]
     untracked: List[str]
     unchanged: List[str]
     git: GitInfo
 
 
-def _transform_git_paths_to_dvc(repo: "Repo", files: List[str]):
+def _transform_git_paths_to_dvc(repo: "Repo", files: Iterable[str]):
     """Transform files rel. to Git root to DVC root, and drop outside files."""
     rel = repo.fs.path.relpath(repo.root_dir, repo.scm.root_dir).rstrip("/")
-    if rel in (os.curdir, ""):
-        return files
 
-    prefix = rel + os.sep
-    length = len(prefix)
-    return [file[length:] for file in files if file.startswith(prefix)]
+    # if we have repo root in a different location than scm's root,
+    # i.e. subdir repo, all git_paths need to be transformed rel. to the DVC
+    # repo root and anything outside need to be filtered out.
+    if rel not in (os.curdir, ""):
+        prefix = rel + os.sep
+        length = len(prefix)
+        files = (file[length:] for file in files if file.startswith(prefix))
+
+    start = repo.fs.path.relpath(repo.fs.path.getcwd(), repo.root_dir)
+    if start in (os.curdir, ""):
+        return list(files)
+    # we need to convert repo relative paths to curdir relative.
+    return [repo.fs.path.relpath(file, start) for file in files]
 
 
 def status(repo: "Repo", untracked_files: str = "no", **kwargs: Any) -> Status:

dvc/repo/metrics/show.py

@@ -85,12 +85,14 @@ def _read_metrics(repo, metrics, rev, onerror=None):
 
     res = {}
     for metric in metrics:
+        rel_metric_path = os.path.join(relpath, *fs.path.parts(metric))
         if not fs.isfile(metric):
-            continue
+            if fs.isfile(rel_metric_path):
+                metric = rel_metric_path
+            else:
+                continue
 
-        res[os.path.join(relpath, *fs.path.parts(metric))] = _read_metric(
-            metric, fs, rev, onerror=onerror
-        )
+        res[rel_metric_path] = _read_metric(metric, fs, rev, onerror=onerror)
 
     return res
 

dvc/repo/params/show.py

@@ -56,7 +56,9 @@ def _collect_configs(
             default_params
         ):
             fs_paths.append(default_params)
-
+    if targets and (deps or stages) and not params:
+        # A target has been provided but it is not used in the stages
+        fs_paths = []
     return params, fs_paths
 
 

scripts/pyinstaller/hooks/hook-fsspec.py

@@ -0,0 +1 @@
+hiddenimports = ["fsspec.implementations.memory"]

setup.cfg

@@ -68,8 +68,8 @@ install_requires =
     dvc-render==0.0.9
     dvc-task==0.1.2
     dvclive>=0.10.0
-    dvc-data==0.1.22
-    dvc-http==0.0.2
+    dvc-data==0.1.23
+    dvc-http==2.19.0
     hydra-core>=1.1.0
 
 [options.extras_require]
@@ -88,25 +88,25 @@ dev =
     %(tests)s
     dvc-data[cli]
 # Extra dependencies for remote integrations
-azure = dvc-azure==0.0.1
-gdrive = dvc-gdrive==0.0.1
-gs = dvc-gs==0.0.1
-hdfs = dvc-hdfs==0.0.1
-oss = dvc-oss==0.0.1
-s3 = dvc-s3==0.0.2
-ssh = dvc-ssh==0.0.1
+azure = dvc-azure==2.19.0
+gdrive = dvc-gdrive==2.19.0
+gs = dvc-gs==2.19.0
+hdfs = dvc-hdfs==2.19.0
+oss = dvc-oss==2.19.0
+s3 = dvc-s3==2.19.0
+ssh = dvc-ssh==2.19.0
 # gssapi should not be included in all_remotes, because it doesn't have wheels
 # for Linux, so it will fail to compile if user doesn't have all the
 # requirements, including kerberos itself.
-ssh_gssapi = dvc-ssh[gssapi]==0.0.1
-webdav = dvc-webdav==0.0.1
-webhdfs = dvc-webhdfs==0.0.2
+ssh_gssapi = dvc-ssh[gssapi]==2.19.0
+webdav = dvc-webdav==2.19.0
+webhdfs = dvc-webhdfs==2.19.0
 # requests-kerberos requires krb5 & gssapi, which does not provide wheels Linux
-webhdfs_kerberos = dvc-webhdfs[kerberos]==0.0.2
+webhdfs_kerberos = dvc-webhdfs[kerberos]==2.19.0
 terraform = tpi[ssh]>=2.1.0
 tests =
     %(terraform)s
-    dvc-ssh==0.0.1
+    dvc-ssh==2.19.0
     # Test requirements
     pytest==7.1.2
     pytest-cov==3.0.0

tests/func/api/test_params.py

@@ -67,6 +67,9 @@ def test_params_show_targets(params_repo):
         "bar": 2,
         "foobar": 3,
     }
+    assert api.params_show("params.yaml", stages="stage-1") == {
+        "foo": 5,
+    }
 
 
 def test_params_show_deps(params_repo):
@@ -168,3 +171,15 @@ def test_params_show_stage_without_params(tmp_dir, dvc):
 
     with pytest.raises(DvcException, match="No params found"):
         api.params_show(deps=True)
+
+
+def test_params_show_untracked_target(params_repo, tmp_dir):
+    tmp_dir.gen("params_foo.yaml", "foo: 1")
+
+    assert api.params_show("params_foo.yaml") == {"foo": 1}
+
+    with pytest.raises(DvcException, match="No params found"):
+        api.params_show("params_foo.yaml", stages="stage-0")
+
+    with pytest.raises(DvcException, match="No params found"):
+        api.params_show("params_foo.yaml", deps=True)

tests/func/experiments/test_queue.py

@@ -42,6 +42,7 @@ def failed_tasks(tmp_dir, dvc, scm, test_queue, failed_exp_stage):
     return name_list
 
 
+@pytest.mark.xfail(strict=False, reason="pytest-celery flaky")
 @pytest.mark.parametrize("follow", [True, False])
 def test_celery_logs(
     tmp_dir,
@@ -64,7 +65,7 @@ def test_celery_logs(
     assert "failed to reproduce 'failed-copy-file'" in captured.out
 
 
-@pytest.mark.flaky(max_runs=3, min_passes=1)
+@pytest.mark.xfail(strict=False, reason="pytest-celery flaky")
 def test_queue_remove_done(dvc, failed_tasks, success_tasks):
     assert len(dvc.experiments.celery_queue.failed_stash) == 3
     status = to_dict(dvc.experiments.celery_queue.status())

tests/func/metrics/test_show.py

@@ -21,6 +21,32 @@ def test_show_simple(tmp_dir, dvc, run_copy_metrics):
     }
 
 
+def test_show_simple_from_subdir(tmp_dir, dvc, run_copy_metrics):
+    subdir = tmp_dir / "subdir"
+    subdir.mkdir()
+    tmp_dir.gen("metrics_t.yaml", "1.1")
+    run_copy_metrics(
+        "metrics_t.yaml",
+        "subdir/metrics.yaml",
+        metrics=["subdir/metrics.yaml"],
+    )
+
+    expected_path = os.path.join("subdir", "metrics.yaml")
+    assert dvc.metrics.show() == {"": {"data": {expected_path: {"data": 1.1}}}}
+
+    expected_path = os.path.join("..", "subdir", "metrics.yaml")
+    with subdir.chdir():
+        assert dvc.metrics.show() == {
+            "": {"data": {expected_path: {"data": 1.1}}}
+        }
+    subdir2 = tmp_dir / "subdir2"
+    subdir2.mkdir()
+    with subdir2.chdir():
+        assert dvc.metrics.show() == {
+            "": {"data": {expected_path: {"data": 1.1}}}
+        }
+
+
 def test_show(tmp_dir, dvc, run_copy_metrics):
     tmp_dir.gen("metrics_t.yaml", "foo: 1.1")
     run_copy_metrics(

tests/func/test_data_status.py

@@ -1,6 +1,9 @@
 from os.path import join
 
+import pytest
+
 from dvc.repo import Repo
+from dvc.repo.data import _transform_git_paths_to_dvc
 from dvc.testing.tmp_dir import make_subrepo
 from dvc.utils.fs import remove
 
@@ -14,24 +17,54 @@ EMPTY_STATUS = {
 }
 
 
-def test_file(M, tmp_dir, scm, dvc):
+@pytest.mark.parametrize("path", [None, ("sub", "repo")])
+def test_git_to_dvc_path_wdir_transformation(tmp_dir, scm, path):
+    struct = {"dir": {"foo": "foo", "bar": "bar"}, "file": "file", "dir2": {}}
+    tmp_dir.gen(struct)
+
+    subdir = tmp_dir.joinpath(*path) if path else tmp_dir
+    make_subrepo(subdir, scm)
+    dvc = subdir.dvc
+
+    with subdir.chdir():
+        subdir.gen(struct)
+        _, _, untracked = scm.status(untracked_files="all")
+        untracked = sorted(untracked, reverse=True)
+        assert _transform_git_paths_to_dvc(dvc, untracked) == [
+            "file",
+            join("dir", "foo"),
+            join("dir", "bar"),
+        ]
+        with (subdir / "dir").chdir():
+            assert _transform_git_paths_to_dvc(dvc, untracked) == [
+                join("..", "file"),
+                "foo",
+                "bar",
+            ]
+        with (subdir / "dir2").chdir():
+            assert _transform_git_paths_to_dvc(dvc, untracked) == [
+                join("..", "file"),
+                join("..", "dir", "foo"),
+                join("..", "dir", "bar"),
+            ]
+
+
+def test_file(M, tmp_dir, dvc, scm):
     tmp_dir.dvc_gen("foo", "foo", commit="add foo")
     tmp_dir.dvc_gen("foo", "foobar")
     remove(tmp_dir / "foo")
 
     expected = {
+        **EMPTY_STATUS,
         "committed": {"modified": ["foo"]},
         "uncommitted": {"deleted": ["foo"]},
-        "git": M.instance_of(dict),
-        "not_in_cache": [],
-        "unchanged": [],
-        "untracked": [],
+        "git": M.dict(),
     }
     assert dvc.data_status() == expected
     assert dvc.data_status(granular=True) == expected
 
 
-def test_directory(M, tmp_dir, scm, dvc):
+def test_directory(M, tmp_dir, dvc, scm):
     tmp_dir.dvc_gen({"dir": {"foo": "foo"}}, commit="add dir")
     tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar", "foobar": "foobar"}})
     remove(tmp_dir / "dir")
@@ -39,12 +72,10 @@ def test_directory(M, tmp_dir, scm, dvc):
     tmp_dir.gen("untracked", "untracked")
 
     assert dvc.data_status() == {
+        **EMPTY_STATUS,
         "committed": {"modified": [join("dir", "")]},
         "uncommitted": {"modified": [join("dir", "")]},
-        "git": M.instance_of(dict),
-        "not_in_cache": [],
-        "unchanged": [],
-        "untracked": [],
+        "git": M.dict(),
     }
 
     assert dvc.data_status(granular=True, untracked_files="all") == {
@@ -59,7 +90,7 @@ def test_directory(M, tmp_dir, scm, dvc):
             "modified": [join("dir", "bar")],
             "deleted": [join("dir", "foobar")],
         },
-        "git": M.instance_of(dict),
+        "git": M.dict(),
         "not_in_cache": [],
         "unchanged": [join("dir", "foo")],
         "untracked": ["untracked"],
@@ -81,51 +112,28 @@ def test_noscm_repo(dvc):
     assert dvc.data_status() == EMPTY_STATUS
 
 
-def test_unchanged(M, tmp_dir, scm, dvc):
+def test_unchanged(M, tmp_dir, dvc, scm):
     tmp_dir.dvc_gen({"dir": {"foo": "foo"}}, commit="add dir")
     tmp_dir.dvc_gen("bar", "bar", commit="add foo")
 
     assert dvc.data_status() == {
         **EMPTY_STATUS,
-        "git": M.instance_of(dict),
+        "git": M.dict(),
         "unchanged": M.unordered("bar", join("dir", "")),
     }
     assert dvc.data_status(granular=True) == {
         **EMPTY_STATUS,
-        "git": M.instance_of(dict),
+        "git": M.dict(),
         "unchanged": M.unordered("bar", join("dir", "foo")),
     }
 
 
-def test_not_in_cache(M, tmp_dir, scm, dvc):
-    # TODO: investigation required, might return wrong results
-    tmp_dir.dvc_gen({"dir": {"foo": "foo"}}, commit="add dir")
-    tmp_dir.dvc_gen("bar", "bar", commit="add foo")
-    remove(dvc.odb.local.path)
-
-    assert dvc.data_status() == {
-        **EMPTY_STATUS,
-        "not_in_cache": M.unordered("bar", join("dir", "")),
-        "git": M.instance_of(dict),
-        "unchanged": ["bar"],
-        "uncommitted": {"added": [join("dir", "")]},
-    }
-    assert dvc.data_status(granular=True) == {
-        **EMPTY_STATUS,
-        "not_in_cache": M.unordered("bar"),
-        "git": M.instance_of(dict),
-        "unchanged": [],
-        "committed": {"modified": ["bar"]},
-        "uncommitted": {"modified": ["bar"], "added": [join("dir", "foo")]},
-    }
-
-
-def test_withdirs(M, tmp_dir, scm, dvc):
+def test_withdirs(M, tmp_dir, dvc, scm):
     tmp_dir.dvc_gen({"dir": {"foo": "foo"}}, commit="add dir")
     tmp_dir.dvc_gen("bar", "bar", commit="add foo")
     assert dvc.data_status(granular=True, with_dirs=True) == {
         **EMPTY_STATUS,
-        "git": M.instance_of(dict),
+        "git": M.dict(),
         "unchanged": M.unordered("bar", join("dir", "foo"), join("dir", "")),
     }
 
@@ -146,7 +154,8 @@ def test_skip_uncached_pipeline_outputs(tmp_dir, dvc, run_copy_metrics):
 
 def test_output_with_newly_added_stage(tmp_dir, dvc):
     dvc.stage.add(deps=["bar"], outs=["foo"], name="copy", cmd="cp foo bar")
-    assert dvc.data_status() == {**EMPTY_STATUS, "not_in_cache": ["foo"]}
+    assert dvc.data_status() == EMPTY_STATUS
+    assert dvc.data_status(granular=True) == EMPTY_STATUS
 
 
 def test_subdir(M, tmp_dir, scm):
@@ -161,7 +170,213 @@ def test_subdir(M, tmp_dir, scm):
         dvc = subrepo.dvc
         assert dvc.data_status(granular=True, untracked_files="all") == {
             **EMPTY_STATUS,
-            "git": M.instance_of(dict),
+            "git": M.dict(),
             "unchanged": M.unordered("bar", join("dir", "foo")),
             "untracked": ["untracked"],
         }
+
+
+def test_untracked_newly_added_files(M, tmp_dir, dvc, scm):
+    tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}})
+    tmp_dir.gen("foobar", "foobar")
+
+    expected = {
+        **EMPTY_STATUS,
+        "untracked": M.unordered(
+            join("dir", "foo"), join("dir", "bar"), "foobar"
+        ),
+        "git": M.dict(),
+    }
+    assert dvc.data_status(untracked_files="all") == expected
+    assert dvc.data_status(granular=True, untracked_files="all") == expected
+
+
+def test_missing_cache_workspace_exists(M, tmp_dir, dvc, scm):
+    tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}})
+    tmp_dir.dvc_gen("foobar", "foobar")
+    remove(dvc.odb.repo.path)
+
+    assert dvc.data_status(untracked_files="all") == {
+        **EMPTY_STATUS,
+        "untracked": M.unordered("foobar.dvc", "dir.dvc", ".gitignore"),
+        "committed": {"added": M.unordered("foobar", join("dir", ""))},
+        "not_in_cache": M.unordered("foobar", join("dir", "")),
+        "git": M.dict(),
+    }
+
+    assert dvc.data_status(granular=True, untracked_files="all") == {
+        **EMPTY_STATUS,
+        "untracked": M.unordered("foobar.dvc", "dir.dvc", ".gitignore"),
+        "committed": {"added": M.unordered("foobar", join("dir", ""))},
+        "uncommitted": {
+            "unknown": M.unordered(join("dir", "foo"), join("dir", "bar"))
+        },
+        "not_in_cache": M.unordered(
+            "foobar",
+            join("dir", ""),
+        ),
+        "git": M.dict(),
+    }
+
+
+def test_missing_cache_missing_workspace(M, tmp_dir, dvc, scm):
+    tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}})
+    tmp_dir.dvc_gen("foobar", "foobar")
+    for path in [dvc.odb.repo.path, "dir", "foobar"]:
+        remove(path)
+
+    assert dvc.data_status(untracked_files="all") == {
+        **EMPTY_STATUS,
+        "untracked": M.unordered("foobar.dvc", "dir.dvc", ".gitignore"),
+        "uncommitted": {"deleted": M.unordered("foobar", join("dir", ""))},
+        "committed": {"added": M.unordered("foobar", join("dir", ""))},
+        "not_in_cache": M.unordered("foobar", join("dir", "")),
+        "git": M.dict(),
+    }
+
+    assert dvc.data_status(granular=True, untracked_files="all") == {
+        **EMPTY_STATUS,
+        "untracked": M.unordered("foobar.dvc", "dir.dvc", ".gitignore"),
+        "uncommitted": {"deleted": M.unordered("foobar", join("dir", ""))},
+        "committed": {"added": M.unordered("foobar", join("dir", ""))},
+        "not_in_cache": M.unordered("foobar", join("dir", "")),
+        "git": M.dict(),
+    }
+
+
+def test_git_committed_missing_cache_workspace_exists(M, tmp_dir, dvc, scm):
+    tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}}, commit="add dir")
+    tmp_dir.dvc_gen("foobar", "foobar", commit="add foobar")
+    remove(dvc.odb.local.path)
+
+    assert dvc.data_status(untracked_files="all") == {
+        **EMPTY_STATUS,
+        "not_in_cache": M.unordered("foobar", join("dir", "")),
+        "git": M.dict(),
+        "unchanged": M.unordered("foobar", join("dir", "")),
+    }
+    assert dvc.data_status(granular=True) == {
+        **EMPTY_STATUS,
+        "not_in_cache": M.unordered(
+            "foobar",
+            join("dir", ""),
+        ),
+        "uncommitted": {
+            "unknown": M.unordered(join("dir", "foo"), join("dir", "bar"))
+        },
+        "git": M.dict(),
+        "unchanged": M.unordered("foobar", join("dir", "")),
+    }
+
+
+def test_git_committed_missing_cache_missing_workspace(M, tmp_dir, dvc, scm):
+    tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}}, commit="add dir")
+    tmp_dir.dvc_gen("foobar", "foobar", commit="add foobar")
+    for path in [dvc.odb.repo.path, "dir", "foobar"]:
+        remove(path)
+
+    assert dvc.data_status(untracked_files="all") == {
+        **EMPTY_STATUS,
+        "uncommitted": {"deleted": M.unordered(join("dir", ""), "foobar")},
+        "not_in_cache": M.unordered(join("dir", ""), "foobar"),
+        "git": M.dict(),
+    }
+    assert dvc.data_status(granular=True, untracked_files="all") == {
+        **EMPTY_STATUS,
+        "committed": {},
+        "uncommitted": {"deleted": M.unordered(join("dir", ""), "foobar")},
+        "not_in_cache": M.unordered(join("dir", ""), "foobar"),
+        "git": M.dict(),
+    }
+
+
+def test_partial_missing_cache(M, tmp_dir, dvc, scm):
+    tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}})
+
+    # remove "foo" from cache
+    odb = dvc.odb.repo
+    odb.fs.rm(odb.oid_to_path("acbd18db4cc2f85cedef654fccc4a4d8"))
+
+    assert dvc.data_status() == {
+        **EMPTY_STATUS,
+        "committed": {"added": [join("dir", "")]},
+        "git": M.dict(),
+    }
+    assert dvc.data_status(granular=True) == {
+        **EMPTY_STATUS,
+        "committed": {
+            "added": M.unordered(join("dir", "foo"), join("dir", "bar"))
+        },
+        "not_in_cache": [join("dir", "foo")],
+        "git": M.dict(),
+    }
+
+
+def test_missing_dir_object_from_head(M, tmp_dir, dvc, scm):
+    (stage,) = tmp_dir.dvc_gen(
+        {"dir": {"foo": "foo", "bar": "bar"}}, commit="add dir"
+    )
+    remove("dir")
+    tmp_dir.dvc_gen({"dir": {"foobar": "foobar"}})
+    odb = dvc.odb.repo
+    odb.fs.rm(odb.oid_to_path(stage.outs[0].hash_info.value))
+
+    assert dvc.data_status() == {
+        **EMPTY_STATUS,
+        "committed": {"modified": [join("dir", "")]},
+        "git": M.dict(),
+    }
+    assert dvc.data_status(granular=True) == {
+        **EMPTY_STATUS,
+        "committed": {
+            "modified": [join("dir", "")],
+            "unknown": [join("dir", "foobar")],
+        },
+        "git": M.dict(),
+    }
+
+
+def test_root_from_dir_to_file(M, tmp_dir, dvc, scm):
+    tmp_dir.dvc_gen({"data": {"foo": "foo", "bar": "bar"}})
+    remove("data")
+    tmp_dir.gen("data", "file")
+
+    assert dvc.data_status() == {
+        **EMPTY_STATUS,
+        "committed": {"added": [join("data", "")]},
+        "uncommitted": {"modified": [join("data")]},
+        "git": M.dict(),
+    }
+    assert dvc.data_status(granular=True) == {
+        **EMPTY_STATUS,
+        "committed": {
+            "added": M.unordered(join("data", "foo"), join("data", "bar"))
+        },
+        "uncommitted": {
+            "deleted": M.unordered(join("data", "foo"), join("data", "bar")),
+            "modified": ["data"],
+        },
+        "git": M.dict(),
+    }
+
+
+def test_root_from_file_to_dir(M, tmp_dir, dvc, scm):
+    tmp_dir.dvc_gen("data", "file")
+    remove("data")
+    tmp_dir.gen({"data": {"foo": "foo", "bar": "bar"}})
+
+    assert dvc.data_status() == {
+        **EMPTY_STATUS,
+        "committed": {"added": ["data"]},
+        "uncommitted": {"modified": [join("data", "")]},
+        "git": M.dict(),
+    }
+    assert dvc.data_status(granular=True) == {
+        **EMPTY_STATUS,
+        "committed": {"added": [join("data")]},
+        "uncommitted": {
+            "modified": [join("data", "")],
+            "added": M.unordered(join("data", "foo"), join("data", "bar")),
+        },
+        "git": M.dict(),
+    }

tests/unit/command/test_data_status.py

@@ -14,15 +14,21 @@ from tests.func.parsing.test_errors import escape_ansi
 def mocked_status():
     yield Status(
         not_in_cache=["notincache"],
-        committed={"added": ["dir/bar", "dir/foo"]},
+        committed={
+            "added": ["dir/bar", "dir/foo"],
+            "deleted": ["dir/baz"],
+            "modified": ["dir/foobar"],
+            "unknown": ["dir/unknown1"],
+        },
         uncommitted={
             "added": ["dir/baz"],
             "modified": ["dir/bar"],
             "deleted": ["dir/foobar"],
+            "unknown": ["dir2/unknown2"],
         },
         untracked=["untracked"],
         unchanged=["dir/foo"],
-        git={"is_dirty": True},
+        git={"is_dirty": True, "is_empty": False},
     )
 
 
@@ -106,13 +112,15 @@ def test_show_status(dvc, scm, mocker, capsys, mocked_status, args, is_dirty):
     out, err = capsys.readouterr()
     expected_out = """\
 Not in cache:
-  (use "dvc pull <file>..." to update your local storage)
+  (use "dvc pull <file>..." to download files)
         notincache
 
 DVC committed changes:
   (git commit the corresponding dvc files to update the repo)
         added: dir/bar
         added: dir/foo
+        deleted: dir/baz
+        modified: dir/foobar
 
 DVC uncommitted changes:
   (use "dvc commit <file>..." to track changes)