TabularData: Add ignore_empty flag to drop_duplicates.

Configures whether to consider missing values as relevant or not.

dvc/command/experiments/show.py

@@ -473,7 +473,7 @@ def show_experiments(
     )
 
     if kwargs.get("only_changed", False) or html:
-        td.drop_duplicates("cols")
+        td.drop_duplicates("cols", ignore_empty=False)
 
     html_args = {}
     if html:

dvc/compare.py

@@ -249,7 +249,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
         else:
             self.drop(*to_drop)
 
-    def drop_duplicates(self, axis: str = "rows"):
+    def drop_duplicates(self, axis: str = "rows", ignore_empty: bool = True):
         if axis not in ["rows", "cols"]:
             raise ValueError(
                 f"Invalid 'axis' value {axis}."
@@ -260,9 +260,9 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
             cols_to_drop: List[str] = []
             for n_col, col in enumerate(self.columns):
                 # Cast to str because Text is not hashable error
-                unique_vals = {
-                    str(x) for x in col if str(x) != self._fill_value
-                }
+                unique_vals = {str(x) for x in col}
+                if ignore_empty and self._fill_value in unique_vals:
+                    unique_vals -= {self._fill_value}
                 if len(unique_vals) == 1:
                     cols_to_drop.append(self.keys()[n_col])
             self.drop(*cols_to_drop)

tests/unit/test_tabular_data.py

@@ -221,7 +221,7 @@ def test_dropna(axis, how, data, expected):
 
 
 @pytest.mark.parametrize(
-    "axis,expected",
+    "axis,expected,ignore_empty",
     [
         (
             "rows",
@@ -230,11 +230,22 @@ def test_dropna(axis, how, data, expected):
                 ["foo", "foo", "-"],
                 ["foo", "bar", "foobar"],
             ],
+            True,
+        ),
+        ("cols", [["-"], ["foo"], ["foo"], ["bar"]], True),
+        (
+            "cols",
+            [
+                ["-", "-"],
+                ["foo", "-"],
+                ["foo", "-"],
+                ["bar", "foobar"],
+            ],
+            False,
         ),
-        ("cols", [["-"], ["foo"], ["foo"], ["bar"]]),
     ],
 )
-def test_drop_duplicates(axis, expected):
+def test_drop_duplicates(axis, expected, ignore_empty):
     td = TabularData(["col-1", "col-2", "col-3"], fill_value="-")
     td.extend(
         [["foo"], ["foo", "foo"], ["foo", "foo"], ["foo", "bar", "foobar"]]
@@ -247,11 +258,34 @@ def test_drop_duplicates(axis, expected):
         ["foo", "bar", "foobar"],
     ]
 
-    td.drop_duplicates(axis)
+    td.drop_duplicates(axis, ignore_empty=ignore_empty)
 
     assert list(td) == expected
 
 
+def test_drop_duplicates_ignore_empty():
+    td = TabularData(["col-1", "col-2", "col-3"], fill_value="-")
+    td.extend(
+        [["foo"], ["foo", "foo"], ["foo", "foo"], ["foo", "bar", "foobar"]]
+    )
+
+    assert list(td) == [
+        ["foo", "-", "-"],
+        ["foo", "foo", "-"],
+        ["foo", "foo", "-"],
+        ["foo", "bar", "foobar"],
+    ]
+
+    td.drop_duplicates("cols", ignore_empty=False)
+
+    assert list(td) == [
+        ["-", "-"],
+        ["foo", "-"],
+        ["foo", "-"],
+        ["bar", "foobar"],
+    ]
+
+
 def test_drop_duplicates_rich_text():
     from dvc.ui import ui