data status: remove --withdirs, show unknowns in CLI (#8189)

* data status: remove --withdirs, show unknowns in CLI

* data status: only consider unknown from newer object during diff

dvc/commands/data.py

@@ -51,7 +51,6 @@ class CmdDataStatus(CmdBase):
                     file: state
                     for state, files in stage_status.items()
                     for file in files
-                    if state != "unknown"
                 }
             if not items:
                 continue
@@ -102,7 +101,6 @@ class CmdDataStatus(CmdBase):
             status = self.repo.data_status(
                 granular=self.args.granular,
                 untracked_files=self.args.untracked_files,
-                with_dirs=self.args.with_dirs,
             )
 
         if not self.args.unchanged:
@@ -172,10 +170,4 @@ def add_parser(subparsers, parent_parser):
         nargs="?",
         help="Show untracked files.",
     )
-    data_status_parser.add_argument(
-        "--with-dirs",
-        action="store_true",
-        default=False,
-        help=argparse.SUPPRESS,
-    )
     data_status_parser.set_defaults(func=CmdDataStatus)

dvc/repo/data.py

@@ -67,7 +67,6 @@ def _granular_diff(
     old_obj: Optional["HashFile"],
     new_obj: Optional["HashFile"],
     cache: "HashFileDB",
-    with_dirs: bool = False,
 ) -> Dict[str, List[str]]:
     from dvc_data.diff import ROOT
     from dvc_data.diff import diff as odiff
@@ -77,29 +76,14 @@ def _granular_diff(
             return root
         return os.path.sep.join([root, *paths])
 
-    def type_changed(old, new):
-        if old.oid and new.oid:
-            return old.oid.isdir != new.oid.isdir
-        return False
-
     diff_data = odiff(old_obj, new_obj, cache)
 
     output: Dict[str, List[str]] = defaultdict(list)
     for state in ("added", "deleted", "modified", "unchanged"):
         items = getattr(diff_data, state)
         for item in items:  # pylint: disable=not-an-iterable
-            old = item.old
-            new = item.new
-            entry = old if state == "deleted" else new
+            entry = item.old if state == "deleted" else item.new
             isdir = entry.oid.isdir if entry.oid else False
-            obj_type_changed = type_changed(old, new)
-            if (
-                not with_dirs
-                and isdir
-                and entry.key == ROOT
-                and not obj_type_changed
-            ):
-                continue
 
             path = path_join(root, *entry.key, isdir=isdir)
             output[state].append(path)
@@ -127,17 +111,16 @@ def _diff(
     new_oid: Optional["HashInfo"],
     new_obj: Optional["HashFile"],
     odb: "HashFileDB",
-    with_dirs: bool = False,
     granular: bool = False,
 ) -> Dict[str, List[str]]:
     if not granular:
         return _shallow_diff(root, old_oid, new_oid, odb)
     if (old_oid and not old_obj) or (new_oid and not new_obj):
         # we don't have enough information to give full details
-        unknown = _get_obj_items(root, new_obj or old_obj)
+        unknown = _get_obj_items(root, new_obj)
         shallow_diff = _shallow_diff(root, old_oid, new_oid, odb)
         return {**shallow_diff, "unknown": unknown}
-    return _granular_diff(root, old_obj, new_obj, odb, with_dirs=with_dirs)
+    return _granular_diff(root, old_obj, new_obj, odb)
 
 
 class GitInfo(TypedDict, total=False):

tests/func/test_data_status.py

@@ -29,6 +29,7 @@ def test_git_to_dvc_path_wdir_transformation(tmp_dir, scm, path):
     with subdir.chdir():
         subdir.gen(struct)
         _, _, untracked = scm.status(untracked_files="all")
+        # make order independent of the platforms for easier test assertions
         untracked = sorted(untracked, reverse=True)
         assert _transform_git_paths_to_dvc(dvc, untracked) == [
             "file",
@@ -83,11 +84,12 @@ def test_directory(M, tmp_dir, dvc, scm):
             "added": M.unordered(
                 join("dir", "bar"),
                 join("dir", "foobar"),
-            )
+            ),
+            "modified": [join("dir", "")],
         },
         "uncommitted": {
             "added": [join("dir", "baz")],
-            "modified": [join("dir", "bar")],
+            "modified": M.unordered(join("dir", ""), join("dir", "bar")),
             "deleted": [join("dir", "foobar")],
         },
         "git": M.dict(),
@@ -124,17 +126,7 @@ def test_unchanged(M, tmp_dir, dvc, scm):
     assert dvc.data_status(granular=True) == {
         **EMPTY_STATUS,
         "git": M.dict(),
-        "unchanged": M.unordered("bar", join("dir", "foo")),
-    }
-
-
-def test_withdirs(M, tmp_dir, dvc, scm):
-    tmp_dir.dvc_gen({"dir": {"foo": "foo"}}, commit="add dir")
-    tmp_dir.dvc_gen("bar", "bar", commit="add foo")
-    assert dvc.data_status(granular=True, with_dirs=True) == {
-        **EMPTY_STATUS,
-        "git": M.dict(),
-        "unchanged": M.unordered("bar", join("dir", "foo"), join("dir", "")),
+        "unchanged": M.unordered("bar", join("dir", ""), join("dir", "foo")),
     }
 
 
@@ -171,7 +163,9 @@ def test_subdir(M, tmp_dir, scm):
         assert dvc.data_status(granular=True, untracked_files="all") == {
             **EMPTY_STATUS,
             "git": M.dict(),
-            "unchanged": M.unordered("bar", join("dir", "foo")),
+            "unchanged": M.unordered(
+                "bar", join("dir", ""), join("dir", "foo")
+            ),
             "untracked": ["untracked"],
         }
 
@@ -305,7 +299,9 @@ def test_partial_missing_cache(M, tmp_dir, dvc, scm):
     assert dvc.data_status(granular=True) == {
         **EMPTY_STATUS,
         "committed": {
-            "added": M.unordered(join("dir", "foo"), join("dir", "bar"))
+            "added": M.unordered(
+                join("dir", ""), join("dir", "foo"), join("dir", "bar")
+            )
         },
         "not_in_cache": [join("dir", "foo")],
         "git": M.dict(),
@@ -336,6 +332,30 @@ def test_missing_dir_object_from_head(M, tmp_dir, dvc, scm):
     }
 
 
+def test_missing_dir_object_from_index(M, tmp_dir, dvc, scm):
+    tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}}, commit="add dir")
+    remove("dir")
+    (stage,) = tmp_dir.dvc_gen({"dir": {"foobar": "foobar"}})
+    odb = dvc.odb.repo
+    odb.fs.rm(odb.oid_to_path(stage.outs[0].hash_info.value))
+
+    assert dvc.data_status() == {
+        **EMPTY_STATUS,
+        "committed": {"modified": [join("dir", "")]},
+        "not_in_cache": [join("dir", "")],
+        "git": M.dict(),
+    }
+    assert dvc.data_status(granular=True) == {
+        **EMPTY_STATUS,
+        "committed": {
+            "modified": [join("dir", "")],
+        },
+        "uncommitted": {"unknown": [join("dir", "foobar")]},
+        "not_in_cache": [join("dir", "")],
+        "git": M.dict(),
+    }
+
+
 def test_root_from_dir_to_file(M, tmp_dir, dvc, scm):
     tmp_dir.dvc_gen({"data": {"foo": "foo", "bar": "bar"}})
     remove("data")
@@ -344,13 +364,15 @@ def test_root_from_dir_to_file(M, tmp_dir, dvc, scm):
     assert dvc.data_status() == {
         **EMPTY_STATUS,
         "committed": {"added": [join("data", "")]},
-        "uncommitted": {"modified": [join("data")]},
+        "uncommitted": {"modified": ["data"]},
         "git": M.dict(),
     }
     assert dvc.data_status(granular=True) == {
         **EMPTY_STATUS,
         "committed": {
-            "added": M.unordered(join("data", "foo"), join("data", "bar"))
+            "added": M.unordered(
+                join("data", ""), join("data", "foo"), join("data", "bar")
+            )
         },
         "uncommitted": {
             "deleted": M.unordered(join("data", "foo"), join("data", "bar")),
@@ -373,7 +395,7 @@ def test_root_from_file_to_dir(M, tmp_dir, dvc, scm):
     }
     assert dvc.data_status(granular=True) == {
         **EMPTY_STATUS,
-        "committed": {"added": [join("data")]},
+        "committed": {"added": ["data"]},
         "uncommitted": {
             "modified": [join("data", "")],
             "added": M.unordered(join("data", "foo"), join("data", "bar")),

tests/unit/command/test_data_status.py

@@ -44,7 +44,6 @@ def test_cli(dvc, mocker, mocked_status):
             "--json",
             "--unchanged",
             "--untracked-files",
-            "--with-dirs",
             "--granular",
         ]
     )
@@ -55,7 +54,6 @@ def test_cli(dvc, mocker, mocked_status):
     status.assert_called_once_with(
         untracked_files="all",
         granular=True,
-        with_dirs=True,
     )
 
 
@@ -121,12 +119,14 @@ DVC committed changes:
         added: dir/foo
         deleted: dir/baz
         modified: dir/foobar
+        unknown: dir/unknown1
 
 DVC uncommitted changes:
   (use "dvc commit <file>..." to track changes)
         added: dir/baz
         modified: dir/bar
         deleted: dir/foobar
+        unknown: dir2/unknown2
 """
     if "--untracked-files" in args:
         expected_out += """