dvc: detach gitfs from local paths

Our gitfs paths are currently tied to the local path in which the cloned
repository resides in, which results in a lot of hacky assumptions throughout
the code.

This PR detaches gitfs from local paths completely, making dvc work with a
proper fsspec gitfs implementation.

Part of #7543

dvc/config.py

@@ -89,6 +89,8 @@ class Config(dict):
         from dvc.fs.local import LocalFileSystem
 
         self.dvc_dir = dvc_dir
+        self.wfs = LocalFileSystem()
+        self.fs = fs or self.wfs
 
         if not dvc_dir:
             try:
@@ -98,10 +100,7 @@ class Config(dict):
             except NotDvcRepoError:
                 self.dvc_dir = None
         else:
-            self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
+            self.dvc_dir = self.fs.path.abspath(self.fs.path.realpath(dvc_dir))
-
-        self.wfs = LocalFileSystem(url=self.dvc_dir)
-        self.fs = fs or self.wfs
 
         self.load(validate=validate, config=config)
 
@@ -124,8 +123,8 @@ class Config(dict):
         }
 
         if self.dvc_dir is not None:
-            files["repo"] = os.path.join(self.dvc_dir, self.CONFIG)
+            files["repo"] = self.fs.path.join(self.dvc_dir, self.CONFIG)
-            files["local"] = os.path.join(self.dvc_dir, self.CONFIG_LOCAL)
+            files["local"] = self.fs.path.join(self.dvc_dir, self.CONFIG_LOCAL)
 
         return files
 

dvc/external_repo.py

@@ -50,19 +50,21 @@ def external_repo(
     config.update(cache_config)
 
     if for_write:
+        scm = None
         root_dir = path
         fs = None
     else:
-        root_dir = os.path.realpath(path)
+        scm = Git(path)
-        scm = Git(root_dir)
         fs = GitFileSystem(scm=scm, rev=rev)
+        root_dir = "/"
 
     repo_kwargs = dict(
         root_dir=root_dir,
         url=url,
         fs=fs,
         config=config,
-        repo_factory=erepo_factory(url, cache_config),
+        repo_factory=erepo_factory(url, root_dir, cache_config),
+        scm=scm,
         **kwargs,
     )
 
@@ -92,14 +94,16 @@ def external_repo(
             _remove(path)
 
 
-def erepo_factory(url, cache_config, *args, **kwargs):
+def erepo_factory(url, root_dir, cache_config):
-    def make_repo(path, **_kwargs):
+    from dvc.fs.local import localfs
+
+    def make_repo(path, fs=None, **_kwargs):
         _config = cache_config.copy()
         if os.path.isdir(url):
-            rel = os.path.relpath(path, _kwargs["fs"].fs_args["scm"].root_dir)
+            fs = fs or localfs
-            repo_path = os.path.join(url, rel)
+            repo_path = os.path.join(url, *fs.path.relparts(path, root_dir))
             _config.update(_get_remote_config(repo_path))
-        return Repo(path, config=_config, **_kwargs)
+        return Repo(path, fs=fs, config=_config, **_kwargs)
 
     return make_repo
 

dvc/fs/git.py

@@ -1,4 +1,3 @@
-import os
 import threading
 from typing import TYPE_CHECKING, Any
 
@@ -15,7 +14,6 @@ if TYPE_CHECKING:
 class GitFileSystem(FileSystem):  # pylint:disable=abstract-method
     """Proxies the repo file access methods to Git objects"""
 
-    sep = os.sep
     scheme = "local"
 
     def __init__(
@@ -40,12 +38,6 @@ class GitFileSystem(FileSystem):  # pylint:disable=abstract-method
             }
         )
 
-    @cached_property
-    def path(self):
-        from .path import Path
-
-        return Path(self.sep, getcwd=os.getcwd)
-
     @wrap_prop(threading.Lock())
     @cached_property
     def fs(self) -> "FsspecGitFileSystem":
@@ -53,6 +45,10 @@ class GitFileSystem(FileSystem):  # pylint:disable=abstract-method
 
         return FsspecGitFileSystem(**self.fs_args)
 
+    @cached_property
+    def path(self):
+        return self.fs.path
+
     @property
     def rev(self) -> str:
         return self.fs.rev

dvc/fs/local.py

@@ -182,7 +182,7 @@ class LocalFileSystem(FileSystem):
     def path(self):
         from .path import Path
 
-        return Path(self.sep, os.getcwd)
+        return Path(self.sep, getcwd=os.getcwd, realpath=os.path.realpath)
 
     def upload_fobj(self, fobj, to_info, **kwargs):
         self.makedirs(self.path.parent(to_info))

dvc/fs/path.py

@@ -3,11 +3,13 @@ import posixpath
 
 
 class Path:
-    def __init__(self, sep, getcwd=None):
+    def __init__(self, sep, getcwd=None, realpath=None):
         def _getcwd():
             return ""
 
         self.getcwd = getcwd or _getcwd
+        self.realpath = realpath or self.abspath
+
         if sep == posixpath.sep:
             self.flavour = posixpath
         elif sep == ntpath.sep:
@@ -15,6 +17,12 @@ class Path:
         else:
             raise ValueError(f"unsupported separator '{sep}'")
 
+    def chdir(self, path):
+        def _getcwd():
+            return path
+
+        self.getcwd = _getcwd
+
     def join(self, *parts):
         return self.flavour.join(*parts)
 
@@ -104,15 +112,15 @@ class Path:
         # pylint: disable=arguments-out-of-order
         return self.isin_or_eq(left, right) or self.isin(right, left)
 
-    def relpath(self, path, start):
+    def relpath(self, path, start=None):
-        if not start:
+        if start is None:
             start = "."
         return self.flavour.relpath(
             self.abspath(path), start=self.abspath(start)
         )
 
-    def relparts(self, path, base):
+    def relparts(self, path, start=None):
-        return self.parts(self.relpath(path, base))
+        return self.parts(self.relpath(path, start=start))
 
     def as_posix(self, path):
         return path.replace(self.flavour.sep, posixpath.sep)

dvc/fs/repo.py

@@ -106,7 +106,12 @@ class _RepoFileSystem(AbstractFileSystem):  # pylint:disable=abstract-method
             self.repo_factory = repo_factory
 
         def _getcwd():
-            return self.root_marker
+            relparts = ()
+            if repo.fs.path.isin(repo.fs.path.getcwd(), repo.root_dir):
+                relparts = repo.fs.path.relparts(
+                    repo.fs.path.getcwd(), repo.root_dir
+                )
+            return self.root_marker + self.sep.join(relparts)
 
         self.path = Path(self.sep, getcwd=_getcwd)
         self.repo = repo
@@ -190,7 +195,7 @@ class _RepoFileSystem(AbstractFileSystem):  # pylint:disable=abstract-method
             repo_kwargs["config"] = cache_config
         else:
             repo_kwargs["cache_dir"] = cache_dir
-            factory = erepo_factory(url, cache_config)
+            factory = erepo_factory(url, root, cache_config)
 
         with _open(
             url if url else root,
@@ -237,6 +242,7 @@ class _RepoFileSystem(AbstractFileSystem):  # pylint:disable=abstract-method
                 repo = self.repo_factory(
                     d,
                     fs=self.repo.fs,
+                    scm=self.repo.scm,
                     repo_factory=self.repo_factory,
                 )
                 self._dvcfss[key] = DvcFileSystem(repo=repo)
@@ -249,7 +255,7 @@ class _RepoFileSystem(AbstractFileSystem):  # pylint:disable=abstract-method
 
         from dvc.repo import Repo
 
-        repo_path = os.path.join(dir_path, Repo.DVC_DIR)
+        repo_path = self.repo.fs.path.join(dir_path, Repo.DVC_DIR)
         return self.repo.fs.isdir(repo_path)
 
     def _get_fs_pair(
@@ -417,7 +423,7 @@ class _RepoFileSystem(AbstractFileSystem):  # pylint:disable=abstract-method
         if not dvc_info and not fs_info:
             raise FileNotFoundError
 
-        info = _merge_info(dvc_fs.repo, fs_info, dvc_info)
+        info = _merge_info(repo, fs_info, dvc_info)
         info["name"] = path
         return info
 
@@ -445,6 +451,10 @@ class RepoFileSystem(FileSystem):
     def isdvc(self, path, **kwargs):
         return self.fs.isdvc(path, **kwargs)
 
+    @property
+    def path(self):  # pylint: disable=invalid-overridden-method
+        return self.fs.path
+
     @property
     def repo(self):
         return self.fs.repo

dvc/ignore.py

@@ -35,7 +35,6 @@ class DvcIgnorePatterns(DvcIgnore):
         self.sep = sep
         self.pattern_list = pattern_list
         self.dirname = dirname
-        self.prefix = self.dirname + self.sep
 
         self.regex_pattern_list = [
             GitWildMatchPattern.pattern_to_regex(pattern_info.patterns)
@@ -74,10 +73,13 @@ class DvcIgnorePatterns(DvcIgnore):
     def _get_normalize_path(self, dirname, basename):
         # NOTE: `relpath` is too slow, so we have to assume that both
         # `dirname` and `self.dirname` are relative or absolute together.
+
+        prefix = self.dirname.rstrip(self.sep) + self.sep
+
         if dirname == self.dirname:
             path = basename
-        elif dirname.startswith(self.prefix):
+        elif dirname.startswith(prefix):
-            rel = dirname[len(self.prefix) :]
+            rel = dirname[len(prefix) :]
             # NOTE: `os.path.join` is ~x5.5 slower
             path = f"{rel}{self.sep}{basename}"
         else:
@@ -210,6 +212,7 @@ class DvcIgnoreFilter:
             new_pattern = DvcIgnorePatterns.from_file(path, self.fs, name)
             if old_pattern:
                 plist, prefix = merge_patterns(
+                    self.fs.path.flavour,
                     old_pattern.pattern_list,
                     old_pattern.dirname,
                     new_pattern.pattern_list,
@@ -259,6 +262,7 @@ class DvcIgnoreFilter:
         old_pattern = ignore_trie.longest_prefix(key).value
         if old_pattern:
             plist, prefix = merge_patterns(
+                self.fs.path.flavour,
                 old_pattern.pattern_list,
                 old_pattern.dirname,
                 new_pattern.pattern_list,

dvc/output.py

@@ -295,12 +295,21 @@ class Output:
         if (
             self.fs.scheme == "local"
             and stage
+            and isinstance(stage.repo.fs, LocalFileSystem)
             and path_isin(path, stage.repo.root_dir)
         ):
             self.def_path = relpath(path, stage.wdir)
+            self.fs = stage.repo.fs
         else:
             self.def_path = path
 
+        if (
+            self.repo
+            and self.fs.scheme == "local"
+            and not self.fs.path.isabs(self.def_path)
+        ):
+            self.fs = self.repo.fs
+
         self._validate_output_path(path, stage)
         # This output (and dependency) objects have too many paths/urls
         # here is a list and comments:
@@ -342,7 +351,7 @@ class Output:
             if self.stage and not os.path.isabs(fs_path):
                 fs_path = fs.path.join(self.stage.wdir, fs_path)
 
-        abs_p = os.path.abspath(os.path.normpath(fs_path))
+        abs_p = fs.path.abspath(fs.path.normpath(fs_path))
         return abs_p
 
     def __repr__(self):
@@ -361,11 +370,11 @@ class Output:
         ):
             return str(self.def_path)
 
-        cur_dir = os.getcwd()
+        cur_dir = self.fs.path.getcwd()
-        if path_isin(cur_dir, self.repo.root_dir):
+        if self.fs.path.isin(cur_dir, self.repo.root_dir):
-            return relpath(self.fs_path, cur_dir)
+            return self.fs.path.relpath(self.fs_path, cur_dir)
 
-        return relpath(self.fs_path, self.repo.root_dir)
+        return self.fs.path.relpath(self.fs_path, self.repo.root_dir)
 
     @property
     def scheme(self):
@@ -379,11 +388,12 @@ class Output:
         if urlparse(self.def_path).scheme == "remote":
             return False
 
-        if os.path.isabs(self.def_path):
+        if self.fs.path.isabs(self.def_path):
             return False
 
-        return self.repo and path_isin(
+        return self.repo and self.fs.path.isin(
-            os.path.realpath(self.fs_path), self.repo.root_dir
+            self.fs.path.realpath(self.fs_path),
+            self.repo.root_dir,
         )
 
     @property

dvc/parsing/__init__.py

@@ -18,7 +18,6 @@ from funcy import cached_property, collecting, first, isa, join, reraise
 
 from dvc.exceptions import DvcException
 from dvc.parsing.interpolate import ParseError
-from dvc.utils import relpath
 
 from .context import (
     Context,
@@ -135,19 +134,14 @@ def make_definition(
 
 class DataResolver:
     def __init__(self, repo: "Repo", wdir: str, d: dict):
-        from dvc.fs import LocalFileSystem
-
         self.fs = fs = repo.fs
 
         if os.path.isabs(wdir):
-            start = (
+            wdir = fs.path.relpath(wdir)
-                os.curdir if isinstance(fs, LocalFileSystem) else repo.root_dir
-            )
-            wdir = relpath(wdir, start)
             wdir = "" if wdir == os.curdir else wdir
 
         self.wdir = wdir
-        self.relpath = os.path.normpath(fs.path.join(self.wdir, "dvc.yaml"))
+        self.relpath = fs.path.normpath(fs.path.join(self.wdir, "dvc.yaml"))
 
         vars_ = d.get(VARS_KWD, [])
         check_interpolations(vars_, VARS_KWD, self.relpath)

dvc/parsing/context.py

@@ -395,7 +395,7 @@ class Context(CtxDict):
 
     def merge_from(self, fs, item: str, wdir: str, overwrite=False):
         path, _, keys_str = item.partition(":")
-        path = os.path.normpath(fs.path.join(wdir, path))
+        path = fs.path.normpath(fs.path.join(wdir, path))
 
         select_keys = lfilter(bool, keys_str.split(",")) if keys_str else None
         if path in self.imports:

dvc/pathspec_math.py

@@ -2,7 +2,6 @@
 # Including changing base dir of path specification patterns and merging
 # of two path specification patterns with different base
 # All the operations follow the documents of `gitignore`
-import os
 from collections import namedtuple
 
 from pathspec.util import normalize_file
@@ -68,7 +67,7 @@ def _change_dirname(dirname, pattern_list, new_dirname):
     ]
 
 
-def merge_patterns(pattern_a, prefix_a, pattern_b, prefix_b):
+def merge_patterns(flavour, pattern_a, prefix_a, pattern_b, prefix_b):
     """
     Merge two path specification patterns.
 
@@ -81,7 +80,7 @@ def merge_patterns(pattern_a, prefix_a, pattern_b, prefix_b):
     if not pattern_b:
         return pattern_a, prefix_a
 
-    longest_common_dir = os.path.commonpath([prefix_a, prefix_b])
+    longest_common_dir = flavour.commonpath([prefix_a, prefix_b])
     new_pattern_a = _change_dirname(prefix_a, pattern_a, longest_common_dir)
     new_pattern_b = _change_dirname(prefix_b, pattern_b, longest_common_dir)
 

dvc/repo/__init__.py

@@ -18,6 +18,7 @@ if TYPE_CHECKING:
     from dvc.fs.base import FileSystem
     from dvc.objects.file import HashFile
     from dvc.repo.scm_context import SCMContext
+    from dvc.scm import Base
 
 logger = logging.getLogger(__name__)
 
@@ -85,29 +86,32 @@ class Repo:
         root_dir: str = None,
         fs: "FileSystem" = None,
         uninitialized: bool = False,
+        scm: "Base" = None,
     ):
-        from dvc.scm import SCM, Base, SCMError
+        from dvc.fs.local import localfs
-        from dvc.utils.fs import makedirs
+        from dvc.scm import SCM, SCMError
 
         dvc_dir = None
         tmp_dir = None
         try:
             root_dir = self.find_root(root_dir, fs)
-            dvc_dir = os.path.join(root_dir, self.DVC_DIR)
+            fs = fs or localfs
-            tmp_dir = os.path.join(dvc_dir, "tmp")
+            dvc_dir = fs.path.join(root_dir, self.DVC_DIR)
-            makedirs(tmp_dir, exist_ok=True)
+            tmp_dir = fs.path.join(dvc_dir, "tmp")
         except NotDvcRepoError:
             if not uninitialized:
                 raise
 
-            try:
+            if not scm:
-                scm = SCM(root_dir or os.curdir)
+                try:
-            except SCMError:
+                    scm = SCM(root_dir or os.curdir)
-                scm = SCM(os.curdir, no_scm=True)
+                except SCMError:
+                    scm = SCM(os.curdir, no_scm=True)
 
-            assert isinstance(scm, Base)
+            if not fs or not root_dir:
-            root_dir = scm.root_dir
+                root_dir = scm.root_dir
 
+        assert root_dir
         return root_dir, dvc_dir, tmp_dir
 
     def _get_database_dir(self, db_name):
@@ -148,6 +152,7 @@ class Repo:
         config=None,
         url=None,
         repo_factory=None,
+        scm=None,
     ):
         from dvc.config import Config
         from dvc.data.db import ODBManager
@@ -166,14 +171,18 @@ class Repo:
         self.url = url
         self._fs_conf = {"repo_factory": repo_factory}
         self._fs = fs or localfs
-        self._scm = None
+        self._scm = scm
 
         if rev and not fs:
-            self._scm = SCM(root_dir or os.curdir)
+            self._scm = scm = SCM(root_dir or os.curdir)
+            root_dir = "/"
             self._fs = GitFileSystem(scm=self._scm, rev=rev)
 
         self.root_dir, self.dvc_dir, self.tmp_dir = self._get_repo_dirs(
-            root_dir=root_dir, fs=self.fs, uninitialized=uninitialized
+            root_dir=root_dir,
+            fs=self.fs,
+            uninitialized=uninitialized,
+            scm=scm,
         )
 
         self.config = Config(self.dvc_dir, fs=self.fs, config=config)
@@ -189,7 +198,11 @@ class Repo:
             self.lock = LockNoop()
             self.state = StateNoop()
             self.odb = ODBManager(self)
+            self.tmp_dir = None
         else:
+            from dvc.utils.fs import makedirs
+
+            makedirs(self.tmp_dir, exist_ok=True)
             self.lock = make_lock(
                 os.path.join(self.tmp_dir, "lock"),
                 tmp_dir=self.tmp_dir,
@@ -312,9 +325,9 @@ class Repo:
     def find_root(cls, root=None, fs=None) -> str:
         from dvc.fs.local import LocalFileSystem, localfs
 
-        root = root or os.curdir
-        root_dir = os.path.realpath(root)
         fs = fs or localfs
+        root = root or os.curdir
+        root_dir = fs.path.realpath(root)
 
         if not fs.isdir(root_dir):
             raise NotDvcRepoError(f"directory '{root}' does not exist")
@@ -443,7 +456,7 @@ class Repo:
         # using `outs_graph` to ensure graph checks are run
         outs = outs or self.index.outs_graph
 
-        abs_path = os.path.abspath(path)
+        abs_path = self.fs.path.abspath(path)
         fs_path = abs_path
 
         def func(out):
@@ -467,7 +480,7 @@ class Repo:
         return matched
 
     def is_dvc_internal(self, path):
-        path_parts = os.path.normpath(path).split(os.path.sep)
+        path_parts = self.fs.path.normpath(path).split(self.fs.sep)
         return self.DVC_DIR in path_parts
 
     @cached_property

dvc/repo/brancher.py

@@ -31,7 +31,18 @@ def brancher(  # noqa: E302
 
     from dvc.fs.local import LocalFileSystem
 
+    repo_root_parts = ()
+    if self.fs.path.isin(self.root_dir, self.scm.root_dir):
+        repo_root_parts = self.fs.path.relparts(
+            self.root_dir, self.scm.root_dir
+        )
+
+    cwd_parts = ()
+    if self.fs.path.isin(self.fs.path.getcwd(), self.root_dir):
+        cwd_parts = self.fs.path.relparts(self.fs.path.getcwd(), self.root_dir)
+
     saved_fs = self.fs
+    saved_root = self.root_dir
 
     scm = self.scm
 
@@ -56,9 +67,16 @@ def brancher(  # noqa: E302
 
         for sha, names in found_revs.items():
             self.fs = GitFileSystem(scm=scm, rev=sha)
+            self.root_dir = self.fs.path.join("/", *repo_root_parts)
+
+            if cwd_parts:
+                cwd = self.fs.path.join("/", *cwd_parts)
+                self.fs.path.chdir(cwd)
+
             # ignore revs that don't contain repo root
             # (i.e. revs from before a subdir=True repo was init'ed)
             if self.fs.exists(self.root_dir):
                 yield sha if sha_only else ",".join(names)
     finally:
         self.fs = saved_fs
+        self.root_dir = saved_root

dvc/repo/collect.py

@@ -31,7 +31,6 @@ def _collect_paths(
     rev: str = None,
 ) -> StrPaths:
     from dvc.fs.repo import RepoFileSystem
-    from dvc.utils import relpath
 
     fs = RepoFileSystem(repo=repo)
     fs_paths = [fs.from_os_path(target) for target in targets]
@@ -41,7 +40,7 @@ def _collect_paths(
         if recursive and fs.isdir(fs_path):
             target_paths.extend(fs.find(fs_path))
 
-        rel = relpath(fs_path)
+        rel = fs.path.relpath(fs_path)
         if not fs.exists(fs_path):
             if rev == "workspace" or rev == "":
                 logger.warning("'%s' was not found in current workspace.", rel)

dvc/repo/diff.py

@@ -122,11 +122,11 @@ def _output_paths(repo, targets):
         return True
 
     def _to_path(output):
-        return (
+        relparts = output.fs.path.relparts(output.fs_path)
-            str(output)
+        base = os.path.join(*relparts)
-            if not output.is_dir_checksum
+        if output.is_dir_checksum:
-            else os.path.join(str(output), "")
+            return os.path.join(base, "")
-        )
+        return base
 
     for output in repo.index.outs:
         if _exists(output):
@@ -168,20 +168,14 @@ def _output_paths(repo, targets):
 
 
 def _dir_output_paths(fs, fs_path, obj, targets=None):
-    if fs.scheme == "local":
+    base = os.path.join(*fs.path.relparts(fs_path))
-        # NOTE: workaround for filesystems that are based on full local paths
-        # (e.g. gitfs, dvcfs, repofs). Proper solution is to use upcoming
-        # fsspec's prefixfs to use relpaths as fs_paths.
-        base = fs.path.relpath(fs_path, os.getcwd())
-    else:
-        base = fs_path
     for key, _, oid in obj:
         fname = fs.path.join(fs_path, *key)
         if targets is None or any(
             fs.path.isin_or_eq(fname, target) for target in targets
         ):
             # pylint: disable=no-member
-            yield fs.path.join(base, *key), oid.value
+            yield os.path.join(base, *key), oid.value
 
 
 def _filter_missing(repo_fs, paths):
@@ -206,7 +200,7 @@ def _targets_to_paths(repo_fs, targets):
 
     for target in targets:
         if repo_fs.exists(target):
-            paths.append(os.path.abspath(target))
+            paths.append(repo_fs.repo.fs.path.abspath(target))
         else:
             missing.append(target)
 

dvc/repo/params/show.py

@@ -10,12 +10,7 @@ from dvc.repo.collect import collect
 from dvc.scm import NoSCMError
 from dvc.stage import PipelineStage
 from dvc.ui import ui
-from dvc.utils import (
+from dvc.utils import error_handler, errored_revisions, onerror_collect
-    error_handler,
-    errored_revisions,
-    onerror_collect,
-    relpath,
-)
 from dvc.utils.serialize import LOADERS
 
 if TYPE_CHECKING:
@@ -42,7 +37,7 @@ def _collect_configs(
     )
     all_fs_paths = fs_paths + [p.fs_path for p in params]
     if not targets:
-        default_params = os.path.join(
+        default_params = repo.fs.path.join(
             repo.root_dir, ParamsDependency.DEFAULT_PARAMS_FILE
         )
         if default_params not in all_fs_paths and repo.fs.exists(
@@ -75,41 +70,33 @@ def _read_params(
                 onerror=onerror, flatten=False
             )
             if params_dict:
-                res[
+                name = os.sep.join(repo.fs.path.relparts(param.fs_path))
-                    repo.fs.path.relpath(param.fs_path, os.getcwd())
+                res[name] = params_dict
-                ] = params_dict
     else:
         fs_paths += [param.fs_path for param in params]
 
     for fs_path in fs_paths:
         from_path = _read_fs_path(repo.fs, fs_path, onerror=onerror)
         if from_path:
-            res[repo.fs.path.relpath(fs_path, os.getcwd())] = from_path
+            name = os.sep.join(repo.fs.path.relparts(fs_path))
+            res[name] = from_path
 
     return res
 
 
 def _collect_vars(repo, params) -> Dict:
-    from dvc.fs.git import GitFileSystem
-
     vars_params: Dict[str, Dict] = defaultdict(dict)
-    rel_to_root = relpath(repo.root_dir)
 
     for stage in repo.index.stages:
         if isinstance(stage, PipelineStage) and stage.tracked_vars:
             for file, vars_ in stage.tracked_vars.items():
-                if isinstance(repo.fs, GitFileSystem):
-                    # GitFileSystem uses relatively-absolute paths from the
-                    # root of the repo. We need to convert them to relative
-                    # paths based on the current working directory.
-                    file = os.path.normpath(os.path.join(rel_to_root, file))
-
                 # `params` file are shown regardless of `tracked` or not
                 # to reduce noise and duplication, they are skipped
                 if file in params:
                     continue
 
-                vars_params[file].update(vars_)
+                name = os.sep.join(repo.fs.path.parts(file))
+                vars_params[name].update(vars_)
     return vars_params
 
 

dvc/repo/plots/__init__.py

@@ -17,12 +17,7 @@ from typing import (
 from funcy import cached_property, first, project
 
 from dvc.exceptions import DvcException
-from dvc.utils import (
+from dvc.utils import error_handler, errored_revisions, onerror_collect
-    error_handler,
-    errored_revisions,
-    onerror_collect,
-    relpath,
-)
 from dvc.utils.serialize import LOADERS
 
 if TYPE_CHECKING:
@@ -110,6 +105,7 @@ class Plots:
         plots = _collect_plots(self.repo, targets, revision, recursive)
         res: Dict[str, Any] = {}
         for fs_path, rev_props in plots.items():
+            base = os.path.join(*fs.path.relparts(fs_path, fs.fs.root_marker))
             if fs.isdir(fs_path):
                 plot_files = []
                 unpacking_res = _unpack_dir_files(fs, fs_path, onerror=onerror)
@@ -118,12 +114,17 @@ class Plots:
                         "data"
                     ):
                         plot_files.append(
-                            (pi, relpath(pi, self.repo.root_dir))
+                            (
+                                pi,
+                                os.path.join(
+                                    base, *fs.path.relparts(pi, fs_path)
+                                ),
+                            )
                         )
                 else:
-                    res[relpath(fs_path, self.repo.root_dir)] = unpacking_res
+                    res[base] = unpacking_res
             else:
-                plot_files = [(fs_path, relpath(fs_path, self.repo.root_dir))]
+                plot_files = [(fs_path, base)]
 
             props = props or {}
 

dvc/repo/stage.py

@@ -1,6 +1,5 @@
 import fnmatch
 import logging
-import os
 import time
 import typing
 from contextlib import suppress
@@ -24,7 +23,7 @@ from dvc.exceptions import (
     OutputNotFoundError,
 )
 from dvc.repo import lock_repo
-from dvc.utils import parse_target, relpath
+from dvc.utils import as_posix, parse_target, relpath
 
 logger = logging.getLogger(__name__)
 
@@ -378,7 +377,7 @@ class StageLoad:
         if recursive and self.fs.isdir(target):
             from dvc.repo.graph import collect_inside_path
 
-            path = os.path.abspath(target)
+            path = self.fs.path.abspath(target)
             return collect_inside_path(path, graph or self.graph)
 
         stages = self.from_target(target, accept_group=accept_group, glob=glob)
@@ -416,6 +415,8 @@ class StageLoad:
         if not target:
             return [StageInfo(stage) for stage in self.repo.index]
 
+        target = as_posix(target)
+
         stages, file, _ = _collect_specific_target(
             self, target, with_deps, recursive, accept_group
         )
@@ -423,7 +424,7 @@ class StageLoad:
             if not (recursive and self.fs.isdir(target)):
                 try:
                     (out,) = self.repo.find_outs_by_path(target, strict=False)
-                    return [StageInfo(out.stage, os.path.abspath(target))]
+                    return [StageInfo(out.stage, self.fs.path.abspath(target))]
                 except OutputNotFoundError:
                     pass
 
@@ -467,7 +468,7 @@ class StageLoad:
         from dvc.fs.local import LocalFileSystem
 
         scm = self.repo.scm
-        sep = os.sep
+        sep = self.fs.sep
         outs: Set[str] = set()
 
         is_local_fs = isinstance(self.fs, LocalFileSystem)
@@ -493,7 +494,7 @@ class StageLoad:
         for root, dirs, files in walk_iter:
             dvcfile_filter = partial(is_dvcfile_and_not_ignored, root)
             for file in filter(dvcfile_filter, files):
-                file_path = os.path.join(root, file)
+                file_path = self.fs.path.join(root, file)
                 try:
                     new_stages = self.load_file(file_path)
                 except DvcException as exc:

dvc/stage/loader.py

@@ -77,7 +77,7 @@ class StageLoader(Mapping):
         assert stage_data and isinstance(stage_data, dict)
 
         path, wdir = resolve_paths(
-            dvcfile.path, stage_data.get(Stage.PARAM_WDIR)
+            dvcfile.repo.fs, dvcfile.path, stage_data.get(Stage.PARAM_WDIR)
         )
         stage = loads_from(PipelineStage, dvcfile.repo, path, wdir, stage_data)
         stage.name = name
@@ -184,7 +184,9 @@ class SingleStageLoader(Mapping):
 
     @classmethod
     def load_stage(cls, dvcfile, d, stage_text):
-        path, wdir = resolve_paths(dvcfile.path, d.get(Stage.PARAM_WDIR))
+        path, wdir = resolve_paths(
+            dvcfile.repo.fs, dvcfile.path, d.get(Stage.PARAM_WDIR)
+        )
         stage = loads_from(Stage, dvcfile.repo, path, wdir, d)
         stage._stage_text = stage_text  # noqa, pylint:disable=protected-access
         stage.deps = dependency.loadd_from(

dvc/stage/utils.py

@@ -210,10 +210,10 @@ def resolve_wdir(wdir, path):
     return pathlib.PurePath(rel_wdir).as_posix() if rel_wdir != "." else None
 
 
-def resolve_paths(path, wdir=None):
+def resolve_paths(fs, path, wdir=None):
-    path = os.path.abspath(path)
+    path = fs.path.abspath(path)
     wdir = wdir or os.curdir
-    wdir = os.path.abspath(os.path.join(os.path.dirname(path), wdir))
+    wdir = fs.path.abspath(fs.path.join(fs.path.dirname(path), wdir))
     return path, wdir
 
 

dvc/utils/__init__.py

@@ -332,7 +332,7 @@ def relpath(path, start=os.curdir):
     return os.path.relpath(path, start)
 
 
-def as_posix(path):
+def as_posix(path: str) -> str:
     import ntpath
     import posixpath
 

setup.cfg

@@ -72,7 +72,7 @@ install_requires =
     aiohttp-retry>=2.4.5
     diskcache>=5.2.1
     jaraco.windows>=5.7.0; python_version < '3.8' and sys_platform == 'win32'
-    scmrepo==0.0.19
+    scmrepo==0.0.22
     dvc-render==0.0.5
     dvclive>=0.7.3
 

tests/func/experiments/test_experiments.py

@@ -33,7 +33,7 @@ def test_new_simple(tmp_dir, scm, dvc, exp_stage, mocker, name, workspace):
 
     new_mock.assert_called_once()
     fs = scm.get_fs(exp)
-    with fs.open(tmp_dir / "metrics.yaml", mode="r", encoding="utf-8") as fobj:
+    with fs.open("metrics.yaml", mode="r", encoding="utf-8") as fobj:
         assert fobj.read().strip() == "foo: 2"
 
     if workspace:
@@ -75,7 +75,7 @@ def test_experiment_exists(tmp_dir, scm, dvc, exp_stage, mocker, workspace):
     exp = first(results)
 
     fs = scm.get_fs(exp)
-    with fs.open(tmp_dir / "metrics.yaml", mode="r", encoding="utf-8") as fobj:
+    with fs.open("metrics.yaml", mode="r", encoding="utf-8") as fobj:
         assert fobj.read().strip() == "foo: 3"
 
 
@@ -145,7 +145,7 @@ def test_modify_params(tmp_dir, scm, dvc, mocker, changes, expected):
 
     new_mock.assert_called_once()
     fs = scm.get_fs(exp)
-    with fs.open(tmp_dir / "metrics.yaml", mode="r") as fobj:
+    with fs.open("metrics.yaml", mode="r") as fobj:
         assert fobj.read().strip() == expected
 
 
@@ -263,9 +263,9 @@ def test_update_py_params(tmp_dir, scm, dvc):
     exp_a = first(results)
 
     fs = scm.get_fs(exp_a)
-    with fs.open(tmp_dir / "params.py", mode="r", encoding="utf-8") as fobj:
+    with fs.open("params.py", mode="r", encoding="utf-8") as fobj:
         assert fobj.read().strip() == "INT = 2"
-    with fs.open(tmp_dir / "metrics.py", mode="r", encoding="utf-8") as fobj:
+    with fs.open("metrics.py", mode="r", encoding="utf-8") as fobj:
         assert fobj.read().strip() == "INT = 2"
 
     tmp_dir.gen(
@@ -309,9 +309,9 @@ def test_update_py_params(tmp_dir, scm, dvc):
         return text.replace("\r\n", "\n")
 
     fs = scm.get_fs(exp_a)
-    with fs.open(tmp_dir / "params.py", mode="r", encoding="utf-8") as fobj:
+    with fs.open("params.py", mode="r", encoding="utf-8") as fobj:
         assert _dos2unix(fobj.read().strip()) == result
-    with fs.open(tmp_dir / "metrics.py", mode="r", encoding="utf-8") as fobj:
+    with fs.open("metrics.py", mode="r", encoding="utf-8") as fobj:
         assert _dos2unix(fobj.read().strip()) == result
 
     tmp_dir.gen("params.py", "INT = 1\n")
@@ -438,7 +438,7 @@ def test_untracked(tmp_dir, scm, dvc, caplog, workspace):
     assert fs.exists("dvc.yaml")
     assert fs.exists("dvc.lock")
     assert fs.exists("copy.py")
-    with fs.open(tmp_dir / "metrics.yaml", mode="r", encoding="utf-8") as fobj:
+    with fs.open("metrics.yaml", mode="r", encoding="utf-8") as fobj:
         assert fobj.read().strip() == "foo: 2"
 
 
@@ -515,8 +515,8 @@ def test_subdir(tmp_dir, scm, dvc, workspace):
 
     fs = scm.get_fs(exp)
     for fname in ["metrics.yaml", "dvc.lock"]:
-        assert fs.exists(subdir / fname)
+        assert fs.exists(f"dir/{fname}")
-    with fs.open(subdir / "metrics.yaml", mode="r", encoding="utf-8") as fobj:
+    with fs.open("dir/metrics.yaml", mode="r", encoding="utf-8") as fobj:
         assert fobj.read().strip() == "foo: 2"
 
     assert dvc.experiments.get_exact_name(exp) == ref_info.name
@@ -560,8 +560,8 @@ def test_subrepo(tmp_dir, scm, workspace):
 
     fs = scm.get_fs(exp)
     for fname in ["metrics.yaml", "dvc.lock"]:
-        assert fs.exists(subrepo / fname)
+        assert fs.exists(f"dir/repo/{fname}")
-    with fs.open(subrepo / "metrics.yaml", mode="r", encoding="utf-8") as fobj:
+    with fs.open("dir/repo/metrics.yaml", mode="r", encoding="utf-8") as fobj:
         assert fobj.read().strip() == "foo: 2"
 
     assert subrepo.dvc.experiments.get_exact_name(exp) == ref_info.name
@@ -582,9 +582,7 @@ def test_queue(tmp_dir, scm, dvc, exp_stage, mocker):
     metrics = set()
     for exp in results:
         fs = scm.get_fs(exp)
-        with fs.open(
+        with fs.open("metrics.yaml", mode="r", encoding="utf-8") as fobj:
-            tmp_dir / "metrics.yaml", mode="r", encoding="utf-8"
-        ) as fobj:
             metrics.add(fobj.read().strip())
     assert expected == metrics
 

tests/func/parsing/test_resolver.py

@@ -1,4 +1,3 @@
-import os
 from copy import deepcopy
 
 import pytest
@@ -71,13 +70,13 @@ def test_load_vars_from_file(tmp_dir, dvc):
 def test_load_vars_with_relpath(tmp_dir, scm, dvc):
     tmp_dir.scm_gen(DEFAULT_PARAMS_FILE, dumps_yaml(DATA), commit="add params")
 
-    subdir = tmp_dir / "subdir"
-    d = {"vars": [os.path.relpath(tmp_dir / DEFAULT_PARAMS_FILE, subdir)]}
-
     revisions = ["HEAD", "workspace"]
     for rev in dvc.brancher(revs=["HEAD"]):
         assert rev == revisions.pop()
-        resolver = DataResolver(dvc, subdir.fs_path, d)
+        d = {
+            "vars": [f"../{DEFAULT_PARAMS_FILE}"],
+        }
+        resolver = DataResolver(dvc, "subdir", d)
         assert resolver.context == deepcopy(DATA)
 
 

tests/func/test_fs.py

@@ -198,7 +198,7 @@ def test_walk_dont_ignore_subrepos(tmp_dir, scm, dvc):
     get_dirs = itemgetter(1)
 
     assert set(get_dirs(next(dvc_fs.walk(path)))) == {".dvc", "subdir", ".git"}
-    assert set(get_dirs(next(scm_fs.walk(path)))) == {".dvc", "subdir"}
+    assert set(get_dirs(next(scm_fs.walk("/")))) == {".dvc", "subdir"}
 
 
 def test_fs_size(dvc, cloud):

tests/func/test_ignore.py

@@ -111,6 +111,7 @@ def test_ignore_collecting_dvcignores(tmp_dir, dvc, dname):
     assert (
         DvcIgnorePatterns(
             *merge_patterns(
+                os.path,
                 _to_pattern_info_list([".hg/", ".git/", ".git", ".dvc/"]),
                 os.fspath(tmp_dir),
                 _to_pattern_info_list([os.path.basename(dname)]),
@@ -141,7 +142,8 @@ def test_ignore_on_branch(tmp_dir, scm, dvc):
     }
 
     dvc.fs = GitFileSystem(scm=scm, rev="branch")
-    assert dvc.dvcignore.is_ignored_file((tmp_dir / "foo").fs_path)
+    dvc.root_dir = "/"
+    assert dvc.dvcignore.is_ignored_file("/foo")
 
 
 def test_match_nested(tmp_dir, dvc):
@@ -177,26 +179,6 @@ def test_ignore_external(tmp_dir, scm, dvc, tmp_path_factory):
     )
 
 
-def test_ignore_subrepo(tmp_dir, scm, dvc):
-    tmp_dir.gen({".dvcignore": "foo", "subdir": {"foo": "foo"}})
-    scm.add([".dvcignore"])
-    scm.commit("init parent dvcignore")
-    dvc._reset()
-
-    subrepo_dir = tmp_dir / "subdir"
-
-    result = walk_files(dvc, dvc.fs, subrepo_dir)
-    assert set(result) == set()
-
-    with subrepo_dir.chdir():
-        subrepo = Repo.init(subdir=True)
-        scm.add(str(subrepo_dir / "foo"))
-        scm.commit("subrepo init")
-
-    for _ in subrepo.brancher(all_commits=True):
-        assert subrepo.fs.exists(subrepo_dir / "foo")
-
-
 def test_ignore_resurface_subrepo(tmp_dir, scm, dvc):
     tmp_dir.dvc_gen({"foo": "foo"}, commit="add foo")
     subrepo_dir = tmp_dir / "subdir"
@@ -358,16 +340,19 @@ def test_pattern_trie_fs(tmp_dir, dvc):
         os.fspath(tmp_dir),
     )
     first_pattern = merge_patterns(
+        os.path,
         *base_pattern,
         _to_pattern_info_list(["a", "b", "c"]),
         os.fspath(tmp_dir / "top" / "first"),
     )
     second_pattern = merge_patterns(
+        os.path,
         *first_pattern,
         _to_pattern_info_list(["d", "e", "f"]),
         os.fspath(tmp_dir / "top" / "first" / "middle" / "second"),
     )
     other_pattern = merge_patterns(
+        os.path,
         *base_pattern,
         _to_pattern_info_list(["1", "2", "3"]),
         os.fspath(tmp_dir / "other"),

tests/func/test_used_objs.py

@@ -3,14 +3,6 @@ import os
 
 import pytest
 
-from dvc.exceptions import NoOutputOrStageError
-from dvc.stage.exceptions import StageFileDoesNotExistError
-
-gitfs_xfail = pytest.mark.xfail(
-    raises=(NoOutputOrStageError, StageFileDoesNotExistError),
-    reason="gitfs works on absolute paths only",
-)
-
 
 @pytest.mark.parametrize(
     "stage_wdir, cwd, target",
@@ -19,23 +11,21 @@ gitfs_xfail = pytest.mark.xfail(
         (os.curdir, os.curdir, "train"),
         (os.curdir, os.curdir, "dvc.yaml:train"),
         (os.curdir, "sub", os.path.join(os.pardir, "foo")),
-        pytest.param(
+        (
             os.curdir,
             "sub",
             os.path.join(os.pardir, "dvc.yaml:train"),
-            marks=gitfs_xfail,
         ),
         ("sub", os.curdir, os.path.join("sub", "foo")),
         ("sub", os.curdir, os.path.join("sub", "dvc.yaml:train")),
         ("sub", "sub", "foo"),
-        pytest.param("sub", "sub", "train", marks=gitfs_xfail),
+        ("sub", "sub", "train"),
-        pytest.param("sub", "sub", "dvc.yaml:train", marks=gitfs_xfail),
+        ("sub", "sub", "dvc.yaml:train"),
         ("sub", "dir", os.path.join(os.pardir, "sub", "foo")),
-        pytest.param(
+        (
             "sub",
             "dir",
             os.path.join(os.pardir, "sub", "dvc.yaml:train"),
-            marks=gitfs_xfail,
         ),
     ],
 )

tests/unit/stage/test_utils.py

@@ -1,5 +1,6 @@
 import os
 
+from dvc.fs.local import localfs
 from dvc.stage.utils import resolve_paths
 
 
@@ -7,14 +8,16 @@ def test_resolve_paths():
     p = os.path.join("dir", "subdir")
     file_path = os.path.join(p, "dvc.yaml")
 
-    path, wdir = resolve_paths(path=file_path, wdir="dir")
+    path, wdir = resolve_paths(fs=localfs, path=file_path, wdir="dir")
     assert path == os.path.abspath(file_path)
     assert wdir == os.path.abspath(os.path.join(p, "dir"))
 
-    path, wdir = resolve_paths(path=file_path)
+    path, wdir = resolve_paths(fs=localfs, path=file_path)
     assert path == os.path.abspath(file_path)
     assert wdir == os.path.abspath(p)
 
-    path, wdir = resolve_paths(path=file_path, wdir="../../some-dir")
+    path, wdir = resolve_paths(
+        fs=localfs, path=file_path, wdir="../../some-dir"
+    )
     assert path == os.path.abspath(file_path)
     assert wdir == os.path.abspath("some-dir")

tests/unit/test_external_repo.py

@@ -31,12 +31,13 @@ def test_hook_is_called(tmp_dir, erepo_dir, mocker):
         list(repo.repo_fs.walk("", ignore_subrepos=False))  # drain
         assert spy.call_count == len(subrepos)
 
-        paths = [os.path.join(repo.root_dir, path) for path in subrepo_paths]
+        paths = ["/" + path.replace("\\", "/") for path in subrepo_paths]
         spy.assert_has_calls(
             [
                 call(
                     path,
                     fs=repo.fs,
+                    scm=repo.scm,
                     repo_factory=repo.repo_fs.fs.repo_factory,
                 )
                 for path in paths

tests/unit/test_ignore.py

@@ -11,6 +11,7 @@ def mock_dvcignore(dvcignore_path, patterns):
 
     fs = MagicMock()
     fs.path = localfs.path
+    fs.sep = localfs.sep
     with patch.object(fs, "open", mock_open(read_data="\n".join(patterns))):
         ignore_patterns = DvcIgnorePatterns.from_file(
             dvcignore_path, fs, "mocked"