loader: don't forget to set meta.isdir

Leftover from #8273

Fixes #8300

.github/workflows/tests.yaml

@@ -43,6 +43,9 @@ jobs:
       matrix:
         os: [ubuntu-20.04, windows-latest, macos-latest]
         pyv: ["3.8", "3.9", "3.10"]
+        pytest-filter: 
+        - "import or plot or live or experiment"
+        - "not (import or plot or live or experiment)"
         include:
         - {os: ubuntu-latest, pyv: "3.11-dev"}
 
@@ -75,10 +78,7 @@ jobs:
         AIOHTTP_NO_EXTENSIONS: ${{ matrix.pyv == '3.11-dev' && '1' }}
     - name: run tests
       timeout-minutes: 40
-      run: >-
-        python -m tests -n=auto
-        --cov-report=xml --cov-report=term
-        ${{ env.extra_test_args }}
+      run: pytest -nauto -ra --durations 100 --cov=dvc --cov-report=xml --cov-report=term -k "${{ matrix.pytest-filter }}"
     - name: upload coverage report
       uses: codecov/codecov-action@v3
       with:

dvc/fs/dvc.py

@@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Any, Callable, Optional, Tuple, Type, Union
 from fsspec.spec import AbstractFileSystem
 from funcy import cached_property, wrap_prop, wrap_with
 
+from dvc.utils.fs import makedirs
 from dvc_objects.fs.base import FileSystem
 from dvc_objects.fs.callbacks import DEFAULT_CALLBACK
 from dvc_objects.fs.path import Path
@@ -296,6 +297,11 @@ class _DvcFileSystem(AbstractFileSystem):  # pylint:disable=abstract-method
         key = self._get_key_from_relative(rpath)
         fs_path = self._from_key(key)
         fs = self.repo.fs
+
+        if self.isdir(rpath):
+            makedirs(lpath, exist_ok=True)
+            return None
+
         try:
             fs.get_file(fs_path, lpath, callback=callback, **kwargs)
             return

dvc/output.py

@@ -345,7 +345,7 @@ class Output:
         # should be absolute and don't contain remote:// refs.
         self.stage = stage
         self.meta = meta
-        self._files = files
+        self.files = files
         self.use_cache = False if self.IS_DEPENDENCY else cache
         self.metric = False if self.IS_DEPENDENCY else metric
         self.plot = False if self.IS_DEPENDENCY else plot
@@ -420,7 +420,7 @@ class Output:
         self.hash_info = HashInfo.from_dict({})
         self.meta = Meta.from_dict({})
         self.obj = None
-        self._files = None
+        self.files = None
 
     @property
     def protocol(self):
@@ -620,7 +620,7 @@ class Output:
                 dry_run=True,
             )
             self.hash_info = obj.hash_info
-            self._files = None
+            self.files = None
             if not self.IS_DEPENDENCY:
                 logger.debug(
                     "Output '%s' doesn't use cache. Skipping saving.", self
@@ -637,7 +637,7 @@ class Output:
             ignore=self.dvcignore,
         )
         self.hash_info = self.obj.hash_info
-        self._files = None
+        self.files = None
 
     def set_exec(self):
         if self.isfile() and self.meta.isexec:
@@ -769,7 +769,7 @@ class Output:
             self.use_cache
             and self.is_in_repo
             and self.hash_info.isdir
-            and (kwargs.get("with_files") or self._files is not None)
+            and (kwargs.get("with_files") or self.files is not None)
         ):
             if self.obj:
                 obj = self.obj
@@ -816,8 +816,8 @@ class Output:
         if self.obj:
             obj = self.obj
         elif self.hash_info:
-            if self._files:
-                tree = Tree.from_list(self._files, hash_name=self.hash_name)
+            if self.files:
+                tree = Tree.from_list(self.files, hash_name=self.hash_name)
                 tree.digest()
                 obj = tree
             else:
@@ -944,7 +944,7 @@ class Output:
         )
 
         self.hash_info = obj.hash_info
-        self._files = None
+        self.files = None
         return obj
 
     def get_files_number(self, filter_info=None):
@@ -1152,7 +1152,7 @@ class Output:
         self.odb.add(merged.path, merged.fs, merged.oid)
 
         self.hash_info = merged.hash_info
-        self._files = None
+        self.files = None
         self.meta = Meta(
             size=du(self.odb, merged),
             nfiles=len(merged),

dvc/repo/index.py

@@ -194,7 +194,7 @@ class Index:
 
             data_index[key] = DataIndexEntry(
                 meta=out.meta,
-                obj=out.obj,
+                obj=out.get_obj() if out.files else out.obj,
                 hash_info=out.hash_info,
                 odb=out.odb,
                 cache=out.odb,

dvc/stage/loader.py

@@ -71,6 +71,8 @@ class StageLoader(Mapping):
             item.meta = Meta.from_dict(info)
             hash_value = getattr(item.meta, item.hash_name, None)
             item.hash_info = HashInfo(item.hash_name, hash_value)
+            if item.hash_info and item.hash_info.isdir:
+                item.meta.isdir = True
 
     @classmethod
     def load_stage(cls, dvcfile, name, stage_data, lock_data=None):

dvc/testing/test_api.py

@@ -1,4 +1,7 @@
+import pytest
+
 from dvc import api
+from dvc.api import DvcFileSystem
 from dvc.utils.fs import remove
 
 
@@ -18,3 +21,103 @@ class TestAPI:
 
         with api.open("foo") as fd:
             assert fd.read() == "foo-text"
+
+    @pytest.mark.parametrize(
+        "clear_cache", [True, False], ids=["cache", "no_cache"]
+    )
+    @pytest.mark.parametrize(
+        "fs_kwargs",
+        [
+            {},
+            {"url": "{path}"},
+            {"url": "{path}", "rev": "master"},
+            {"url": "file://{posixpath}"},
+            {"url": "file://{posixpath}", "rev": "master"},
+        ],
+        ids=["current", "local", "local_rev", "git", "git_rev"],
+    )
+    def test_filesystem(
+        self,
+        M,
+        tmp_dir,
+        make_tmp_dir,
+        scm,
+        dvc,
+        remote,
+        fs_kwargs,
+        clear_cache,
+    ):
+        tmp_dir.scm_gen({"scripts": {"script1": "script1"}}, commit="scripts")
+        tmp_dir.dvc_gen({"data": {"foo": "foo", "bar": "bar"}}, commit="data")
+        dvc.push()
+
+        if clear_cache:
+            remove(dvc.odb.repo.path)
+
+        if url := fs_kwargs.get("url"):
+            fs_kwargs["url"] = url.format(
+                path=tmp_dir, posixpath=tmp_dir.as_posix()
+            )
+
+        fs = DvcFileSystem(**fs_kwargs)
+
+        assert fs.ls("/", detail=False) == M.unordered(
+            "/.gitignore", "/scripts", "/data"
+        )
+        assert fs.ls("scripts", detail=False) == ["scripts/script1"]
+        assert fs.ls("data", detail=False) == M.unordered(
+            "data/foo", "data/bar"
+        )
+
+        data_info = M.dict(
+            name="/data",
+            type="directory",
+            dvc_info=M.dict(isdvc=True, isout=True),
+        )
+        scripts_info = M.dict(name="/scripts", type="directory", isexec=False)
+
+        assert sorted(fs.ls("/"), key=lambda i: i["name"]) == [
+            M.dict(name="/.gitignore", type="file", isexec=False),
+            data_info,
+            scripts_info,
+        ]
+
+        with pytest.raises(FileNotFoundError):
+            fs.info("/not-existing-path")
+
+        assert fs.info("/") == M.dict(name="/", isexec=False, type="directory")
+        assert fs.info("/data") == data_info
+        assert fs.info("/scripts") == scripts_info
+        assert fs.info("/data/foo") == M.dict(name="/data/foo", type="file")
+        assert fs.info("/scripts/script1") == M.dict(
+            name="/scripts/script1", type="file"
+        )
+
+        assert not fs.isdvc("/")
+        assert fs.isdvc("/data")
+        assert fs.isdvc("/data/foo")
+        assert not fs.isdvc("/scripts")
+        assert not fs.isdvc("/scripts/script1")
+
+        with pytest.raises((IsADirectoryError, PermissionError)):
+            fs.open("data")
+        with pytest.raises((IsADirectoryError, PermissionError)):
+            fs.open("scripts")
+        with fs.open("/data/foo") as fobj:
+            assert fobj.read() == b"foo"
+        with fs.open("/scripts/script1") as fobj:
+            assert fobj.read() == b"script1"
+
+        tmp = make_tmp_dir("temp-download")
+        fs.get_file("data/foo", tmp / "foo")
+        assert (tmp / "foo").read_text() == "foo"
+
+        fs.get_file("scripts/script1", tmp / "script1")
+        assert (tmp / "script1").read_text() == "script1"
+
+        fs.get("/", (tmp / "all").fs_path, recursive=True)
+        assert (tmp / "all").read_text() == {
+            ".gitignore": "/data\n",
+            "data": {"bar": "bar", "foo": "foo"},
+            "scripts": {"script1": "script1"},
+        }