diff --git a/dvc/fs/dvc.py b/dvc/fs/dvc.py
index 4c15a7db7..440dd9544 100644
--- a/dvc/fs/dvc.py
+++ b/dvc/fs/dvc.py
@@ -320,7 +320,7 @@ class _DVCFileSystem(AbstractFileSystem):  # pylint:disable=abstract-method
             with suppress(FileNotFoundError, NotADirectoryError):
                 for info in dvc_fs.ls(dvc_path, detail=True):
                     dvc_infos[dvc_fs.path.name(info["name"])] = info
-            dvc_exists = bool(dvc_infos) or dvc_fs.exists(dvc_path)
+                dvc_exists = True
 
         fs_exists = False
         fs_infos = {}
@@ -333,10 +333,13 @@ class _DVCFileSystem(AbstractFileSystem):  # pylint:disable=abstract-method
                     fs, fs_path, detail=True, ignore_subrepos=ignore_subrepos
                 ):
                     fs_infos[fs.path.name(info["name"])] = info
+                fs_exists = True
             except (FileNotFoundError, NotADirectoryError):
                 pass
 
-            fs_exists = bool(fs_infos) or fs.exists(fs_path)
+        if not (dvc_exists or fs_exists):
+            # broken symlink or TreeError
+            raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), path)
 
         dvcfiles = kwargs.get("dvcfiles", False)
 
@@ -344,10 +347,6 @@ class _DVCFileSystem(AbstractFileSystem):  # pylint:disable=abstract-method
         paths = []
         names = set(dvc_infos.keys()) | set(fs_infos.keys())
 
-        if not names and (dvc_exists or fs_exists):
-            # broken symlink or TreeError
-            raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), path)
-
         for name in names:
             if not dvcfiles and _is_dvc_file(name):
                 continue
diff --git a/tests/unit/fs/test_dvc.py b/tests/unit/fs/test_dvc.py
index 030087efe..604ce0e3b 100644
--- a/tests/unit/fs/test_dvc.py
+++ b/tests/unit/fs/test_dvc.py
@@ -173,6 +173,23 @@ def test_ls_dirty(tmp_dir, dvc):
     assert set(fs.ls("data")) == {"data/foo", "data/bar"}
 
 
+def test_ls_file_not_found(tmp_dir, dvc):
+    tmp_dir.dvc_gen({"data": "data"})
+
+    fs = DVCFileSystem(repo=dvc)
+    with pytest.raises(FileNotFoundError):
+        fs.ls("missing")
+
+
+def test_ls_dir_empty(tmp_dir, dvc):
+    tmp_dir.dvc_gen({"data": "data"})
+    empty = tmp_dir / "empty"
+    empty.mkdir()
+
+    fs = DVCFileSystem(repo=dvc)
+    assert set(fs.ls("empty")) == set()
+
+
 @pytest.mark.parametrize(
     "dvcfiles,extra_expected",
     [