fs: path: use ntpath/posixpath

This is less error-prone than implementing these methods ourselves.

.github/workflows/packages.yaml

@@ -13,10 +13,10 @@ jobs:
     - uses: actions/[email protected]
       with:
           fetch-depth: 0
-    - name: Set up Python 3.8
+    - name: Set up Python 3.9
       uses: actions/[email protected]
       with:
-        python-version: 3.8
+        python-version: 3.9
     - name: Set up Ruby 2.4
       uses: actions/[email protected]
       with:
@@ -44,10 +44,10 @@ jobs:
     - uses: actions/[email protected]
       with:
         fetch-depth: 0
-    - name: Set up Python 3.8
+    - name: Set up Python 3.9
       uses: actions/[email protected]
       with:
-        python-version: 3.8
+        python-version: 3.9
     - name: Set up Ruby 2.4
       uses: actions/[email protected]
       with:
@@ -75,10 +75,10 @@ jobs:
     - uses: actions/[email protected]
       with:
         fetch-depth: 0
-    - name: Set up Python 3.8
+    - name: Set up Python 3.9
       uses: actions/[email protected]
       with:
-        python-version: 3.8
+        python-version: 3.9
     - name: Set up Ruby 2.4
       uses: actions/[email protected]
       with:
@@ -109,10 +109,10 @@ jobs:
     - uses: actions/[email protected]
       with:
         fetch-depth: 0
-    - name: Set up Python 3.8
+    - name: Set up Python 3.9
       uses: actions/[email protected]
       with:
-        python-version: 3.8
+        python-version: 3.9
     - name: Install deps
       run: |
         pip install .[all]

.github/workflows/tests.yaml

@@ -29,12 +29,8 @@ jobs:
         python-version: 3.8
     - name: Install requirements
       run: |
-        pip install wheel
-        pip install ".[all,tests,terraform]" pre-commit
-    - name: install temporary dependencies
-      run: |
-        pip install git+https://github.com/isidentical/gdrivefs@service-account
-        pip install git+https://github.com/isidentical/pydata-google-auth@service-account
+        pip install --upgrade pip setuptools wheel
+        pip install ".[dev]" pre-commit
     - name: Check README
       run: python setup.py checkdocs
     - uses: pre-commit/[email protected]
@@ -46,10 +42,6 @@ jobs:
       matrix:
         os: [ubuntu-20.04, windows-latest, macos-latest]
         pyv: ["3.7", "3.8", "3.9", "3.10"]
-        exclude:
-          # no wheels for pygit2 yet
-          - os: windows-latest
-            pyv: "3.10"
     steps:
     - uses: actions/[email protected]
       with:
@@ -64,7 +56,7 @@ jobs:
         echo "::set-output name=dir::$(pip cache dir)"
     - name: set pip cache
       id: pip-cache
-      uses: actions/[email protected].6
+      uses: actions/[email protected].7
       with:
         path: ${{ steps.pip-cache-dir.outputs.dir }}
         key: ${{ runner.os }}-pip-${{ hashFiles('setup.cfg') }}
@@ -74,42 +66,17 @@ jobs:
       if: steps.cache.pip-cache-dir.cache-hit != 'true'
       run: |
         pip install --upgrade pip setuptools wheel
-        pip install -e ".[all,tests,terraform]"
-    - name: install temporary dependencies
-      run: |
-        pip install git+https://github.com/isidentical/gdrivefs@service-account
-        pip install git+https://github.com/isidentical/pydata-google-auth@service-account
+        pip install -e ".[dev]"
     - name: setup git
       run: |
         git config --global user.email "[email protected]"
         git config --global user.name "DVC Tester"
-    - name: setup gs creds
-      shell: bash
-      env:
-        GS_CREDS_JSON: ${{ secrets.GS_CREDS_JSON }}
-      if: env.GS_CREDS_JSON != ''
-      run: |
-        mkdir -p scripts/ci
-        echo "$GS_CREDS_JSON" > scripts/ci/gcp-creds.json
-    - name: disable gdrive tests
-      if: >-
-        github.event_name != 'schedule' &&
-        (matrix.pyv != '3.8' || matrix.os != 'ubuntu-18.04')
-      run: |
-        echo "extra_test_args=--disable-gdrive" >> $GITHUB_ENV
     - name: run tests
       timeout-minutes: 40
-      env:
-        GDRIVE_CREDENTIALS_DATA: ${{ secrets.GDRIVE_CREDENTIALS_DATA }}
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-        OSS_ACCESS_KEY_ID: ${{ secrets.OSS_ACCESS_KEY_ID}}
-        OSS_ACCESS_KEY_SECRET: ${{ secrets.OSS_ACCESS_KEY_SECRET}}
-        OSS_ENDPOINT: ${{ secrets.OSS_ENDPOINT}}
       run: >-
         python -m tests -n=auto
         --cov-report=xml --cov-report=term
-        --enable-ssh ${{ env.extra_test_args }}
+        ${{ env.extra_test_args }}
     - name: upload coverage report
       uses: codecov/[email protected]
       with:

.pre-commit-config.yaml

@@ -6,7 +6,7 @@ repos:
       - id: black
         language_version: python3
     repo: https://github.com/ambv/black
-    rev: 21.10b0
+    rev: 21.12b0
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.0.1
     hooks:
@@ -16,14 +16,14 @@ repos:
       - id: codespell
         args: 
           - --ignore-words-list
-          - ba,datas,fo,uptodate
+          - ba,datas,fo,uptodate,cachable
     repo: https://github.com/codespell-project/codespell
     rev: v2.1.0
   - hooks:
       - id: isort
         language_version: python3
     repo: https://github.com/timothycrosley/isort
-    rev: 5.9.3
+    rev: 5.10.1
   - hooks:
       - id: flake8
         language_version: python3

dvc/analytics.py

@@ -82,10 +82,12 @@ def send(path):
 
 
 def _scm_in_use():
+    from scmrepo.noscm import NoSCM
+
     from dvc.exceptions import NotDvcRepoError
     from dvc.repo import Repo
-    from dvc.scm import SCM, NoSCM
-    from dvc.scm.base import SCMError
+
+    from .scm import SCM, SCMError
 
     try:
         scm = SCM(root_dir=Repo.find_root())

dvc/api.py

@@ -4,7 +4,6 @@ from contextlib import _GeneratorContextManager as GCM
 from funcy import reraise
 
 from dvc.exceptions import OutputNotFoundError, PathMissingError
-from dvc.path_info import PathInfo
 from dvc.repo import Repo
 
 
@@ -20,15 +19,15 @@ def get_url(path, repo=None, rev=None, remote=None):
     directory in the remote storage.
     """
     with Repo.open(repo, rev=rev, subrepos=True, uninitialized=True) as _repo:
-        path_info = PathInfo(_repo.root_dir) / path
+        fs_path = _repo.fs.path.join(_repo.root_dir, path)
         with reraise(FileNotFoundError, PathMissingError(path, repo)):
-            metadata = _repo.repo_fs.metadata(path_info)
+            metadata = _repo.repo_fs.metadata(fs_path)
 
         if not metadata.is_dvc:
             raise OutputNotFoundError(path, repo)
 
         cloud = metadata.repo.cloud
-        md5 = metadata.repo.dvcfs.info(path_info)["md5"]
+        md5 = metadata.repo.dvcfs.info(fs_path)["md5"]
         return cloud.get_url_for(remote, checksum=md5)
 
 

dvc/command/dag.py

@@ -38,12 +38,14 @@ def _collect_targets(repo, target, outs):
     targets = []
 
     outs_trie = repo.index.outs_trie
-    for stage, info in pairs:
-        if not info:
+    for stage, path in pairs:
+        if not path:
             targets.extend([str(out) for out in stage.outs])
             continue
 
-        for out in outs_trie.itervalues(prefix=info.parts):  # noqa: B301
+        for out in outs_trie.itervalues(
+            prefix=repo.fs.path.parts(path)
+        ):  # noqa: B301
             targets.extend(str(out))
 
     return targets

dvc/command/experiments.py

@@ -142,7 +142,7 @@ def _collect_rows(
     fill_value=FILL_VALUE,
     iso=False,
 ):
-    from dvc.scm.git import Git
+    from scmrepo.git import Git
 
     if sort_by:
         sort_path, sort_name, sort_type = _sort_column(
@@ -471,6 +471,9 @@ def show_experiments(
         }
     )
 
+    if kwargs.get("only_changed", False):
+        td.drop_duplicates("cols")
+
     td.render(
         pager=pager,
         borders=True,
@@ -537,6 +540,7 @@ class CmdExperimentsShow(CmdBase):
                 pager=not self.args.no_pager,
                 csv=self.args.csv,
                 markdown=self.args.markdown,
+                only_changed=self.args.only_changed,
             )
         return 0
 
@@ -583,7 +587,6 @@ class CmdExperimentsDiff(CmdBase):
                     title=title,
                     markdown=self.args.markdown,
                     no_path=self.args.no_path,
-                    old=self.args.old,
                     on_empty_diff="diff not supported",
                     precision=precision if key == "metrics" else None,
                     a_rev=self.args.a_rev,
@@ -790,6 +793,7 @@ class CmdExperimentsRemove(CmdBase):
 
 
 class CmdExperimentsInit(CmdBase):
+    DEFAULT_NAME = "train"
     CODE = "src"
     DATA = "data"
     MODELS = "models"
@@ -806,15 +810,12 @@ class CmdExperimentsInit(CmdBase):
         "plots": PLOTS,
         "live": DVCLIVE,
     }
-    EXP_LINK = (
-        "https://dvc.org/doc"
-        "/user-guide/experiment-management/running-experiments"
-    )
+    EXP_LINK = "https://s.dvc.org/g/exp/run"
 
     def run(self):
         from dvc.command.stage import parse_cmd
 
-        cmd = parse_cmd(self.args.cmd)
+        cmd = parse_cmd(self.args.command)
         if not self.args.interactive and not cmd:
             raise InvalidArgumentError("command is not specified")
 
@@ -848,12 +849,10 @@ class CmdExperimentsInit(CmdBase):
             force=self.args.force,
         )
 
-        name = self.args.name or self.args.type
-
         text = ui.rich_text.assemble(
             "\n" if self.args.interactive else "",
             "Created ",
-            (name, "bright_blue"),
+            (self.args.name, "bright_blue"),
             " stage in ",
             ("dvc.yaml", "green"),
             ".",
@@ -1033,6 +1032,15 @@ def add_parser(subparsers, parent_parser):
         ),
         metavar="<n>",
     )
+    experiments_show_parser.add_argument(
+        "--only-changed",
+        action="store_true",
+        default=False,
+        help=(
+            "Only show metrics/params with values varying "
+            "across the selected experiments."
+        ),
+    )
     experiments_show_parser.set_defaults(func=CmdExperimentsShow)
 
     EXPERIMENTS_APPLY_HELP = (
@@ -1101,12 +1109,6 @@ def add_parser(subparsers, parent_parser):
         dest="markdown",
         help="Show tabulated output in the Markdown format (GFM).",
     )
-    experiments_diff_parser.add_argument(
-        "--old",
-        action="store_true",
-        default=False,
-        help="Show old ('a_rev') metric/param value.",
-    )
     experiments_diff_parser.add_argument(
         "--no-path",
         action="store_true",
@@ -1414,9 +1416,10 @@ def add_parser(subparsers, parent_parser):
         parents=[parent_parser],
         description=append_doc_link(EXPERIMENTS_INIT_HELP, "exp/init"),
         formatter_class=argparse.RawDescriptionHelpFormatter,
+        help=EXPERIMENTS_INIT_HELP,
     )
     experiments_init_parser.add_argument(
-        "cmd",
+        "command",
         nargs=argparse.REMAINDER,
         help="Command to execute.",
         metavar="command",
@@ -1446,7 +1449,10 @@ def add_parser(subparsers, parent_parser):
         help="Only use the path values explicitly provided",
     )
     experiments_init_parser.add_argument(
-        "--name", "-n", help="Name of the stage to create"
+        "--name",
+        "-n",
+        help="Name of the stage to create (default: %(default)s)",
+        default=CmdExperimentsInit.DEFAULT_NAME,
     )
     experiments_init_parser.add_argument(
         "--code",
@@ -1487,7 +1493,7 @@ def add_parser(subparsers, parent_parser):
     )
     experiments_init_parser.add_argument(
         "--type",
-        choices=["default", "live"],
+        choices=["default", "dl"],
         default="default",
         help="Select type of stage to create (default: %(default)s)",
     )

dvc/command/machine.py

@@ -2,8 +2,10 @@ import argparse
 
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig
+from dvc.compare import TabularData
 from dvc.config import ConfigError
 from dvc.exceptions import DvcException
+from dvc.types import Dict, List
 from dvc.ui import ui
 from dvc.utils import format_link
 
@@ -71,18 +73,56 @@ class CmdMachineRemove(CmdMachineConfig):
 
 
 class CmdMachineList(CmdMachineConfig):
-    def run(self):
+    TABLE_COLUMNS = [
+        "name",
+        "cloud",
+        "region",
+        "image",
+        "spot",
+        "spot_price",
+        "instance_hdd_size",
+        "instance_type",
+        "ssh_private",
+        "startup_script",
+    ]
+
+    PRIVATE_COLUMNS = ["ssh_private", "startup_script"]
+
+    def _hide_private(self, conf):
+        for machine in conf:
+            for column in self.PRIVATE_COLUMNS:
+                if column in conf[machine]:
+                    conf[machine][column] = "***"
+
+    def _show_origin(self):
         levels = [self.args.level] if self.args.level else self.config.LEVELS
         for level in levels:
             conf = self.config.read(level)["machine"]
             if self.args.name:
                 conf = conf.get(self.args.name, {})
-            prefix = self._config_file_prefix(
-                self.args.show_origin, self.config, level
-            )
+            self._hide_private(conf)
+            prefix = self._config_file_prefix(True, self.config, level)
             configs = list(self._format_config(conf, prefix))
             if configs:
                 ui.write("\n".join(configs))
+
+    def _show_table(self):
+        td = TabularData(self.TABLE_COLUMNS, fill_value="-")
+        conf = self.config.read()["machine"]
+        if self.args.name:
+            conf = {self.args.name: conf.get(self.args.name, {})}
+        self._hide_private(conf)
+        for machine, machine_config in conf.items():
+            machine_config["name"] = machine
+            td.row_from_dict(machine_config)
+        td.dropna("cols", "all")
+        td.render()
+
+    def run(self):
+        if self.args.show_origin:
+            self._show_origin()
+        else:
+            self._show_table()
         return 0
 
 
@@ -193,8 +233,8 @@ class CmdMachineCreate(CmdBase):
 
 
 class CmdMachineStatus(CmdBase):
+    INSTANCE_FIELD = ["name", "instance", "status"]
     SHOWN_FIELD = [
-        "name",
         "cloud",
         "instance_ip",
         "instance_type",
@@ -202,16 +242,46 @@ class CmdMachineStatus(CmdBase):
         "instance_gpu",
     ]
 
+    def _add_row(
+        self,
+        name: str,
+        all_status: List[Dict],
+        td: TabularData,
+    ):
+
+        if not all_status:
+            row = [name, None, "offline"]
+            td.append(row)
+        for i, status in enumerate(all_status, start=1):
+            row = [name, f"num_{i}", "running" if status else "offline"]
+            for field in self.SHOWN_FIELD:
+                value = str(status.get(field, ""))
+                row.append(value)
+            td.append(row)
+
     def run(self):
         if self.repo.machine is None:
             raise MachineDisabledError
 
-        all_status = self.repo.machine.status(self.args.name)
-        for i, status in enumerate(all_status, start=1):
-            ui.write(f"instance_num_{i}:")
-            for field in self.SHOWN_FIELD:
-                value = status.get(field, None)
-                ui.write(f"\t{field:20}: {value}")
+        td = TabularData(
+            self.INSTANCE_FIELD + self.SHOWN_FIELD, fill_value="-"
+        )
+
+        if self.args.name:
+            all_status = list(self.repo.machine.status(self.args.name))
+            self._add_row(self.args.name, all_status, td)
+        else:
+            name_set = set()
+            for level in self.repo.config.LEVELS:
+                conf = self.repo.config.read(level)["machine"]
+                name_set.update(conf.keys())
+            name_list = list(name_set)
+            for name in sorted(name_list):
+                all_status = list(self.repo.machine.status(name))
+                self._add_row(name, all_status, td)
+
+        td.dropna("cols", "all")
+        td.render()
         return 0
 
 
@@ -380,7 +450,7 @@ def add_parser(subparsers, parent_parser):
     machine_CREATE_HELP = "Create and start a machine instance."
     machine_create_parser = machine_subparsers.add_parser(
         "create",
-        parents=[parent_config_parser, parent_parser],
+        parents=[parent_parser],
         description=append_doc_link(machine_CREATE_HELP, "machine/create"),
         help=machine_CREATE_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
@@ -390,23 +460,25 @@ def add_parser(subparsers, parent_parser):
     )
     machine_create_parser.set_defaults(func=CmdMachineCreate)
 
-    machine_STATUS_HELP = "List the status of a running machine."
+    machine_STATUS_HELP = (
+        "List the status of running instances for one/all machines."
+    )
     machine_status_parser = machine_subparsers.add_parser(
         "status",
-        parents=[parent_config_parser, parent_parser],
+        parents=[parent_parser],
         description=append_doc_link(machine_STATUS_HELP, "machine/status"),
         help=machine_STATUS_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     machine_status_parser.add_argument(
-        "name", help="Name of the running machine."
+        "name", nargs="?", help="(optional) Name of the machine."
     )
     machine_status_parser.set_defaults(func=CmdMachineStatus)
 
     machine_DESTROY_HELP = "Destroy an machine instance."
     machine_destroy_parser = machine_subparsers.add_parser(
         "destroy",
-        parents=[parent_config_parser, parent_parser],
+        parents=[parent_parser],
         description=append_doc_link(machine_DESTROY_HELP, "machine/destroy"),
         help=machine_DESTROY_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
@@ -419,7 +491,7 @@ def add_parser(subparsers, parent_parser):
     machine_SSH_HELP = "Connect to a machine via SSH."
     machine_ssh_parser = machine_subparsers.add_parser(
         "ssh",
-        parents=[parent_config_parser, parent_parser],
+        parents=[parent_parser],
         description=append_doc_link(machine_SSH_HELP, "machine/ssh"),
         help=machine_SSH_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,

dvc/command/plots.py

@@ -62,23 +62,11 @@ class CmdPlots(CmdBase):
                 html_template_path=self.args.html_template,
             )
 
-            assert index_path.is_absolute()
-            url = index_path.as_uri()
-            ui.write(url)
+            ui.write(index_path.as_uri())
 
             if self.args.open:
-                import webbrowser
-                from platform import uname
+                return ui.open_browser(index_path)
 
-                if "Microsoft" in uname().release:
-                    url = Path(rel) / "index.html"
-
-                opened = webbrowser.open(url)
-                if not opened:
-                    ui.error_write(
-                        "Failed to open. Please try opening it manually."
-                    )
-                    return 1
             return 0
 
         except DvcException:

dvc/command/run.py

@@ -23,7 +23,7 @@ class CmdRun(CmdBase):
                 self.args.outs_persist_no_cache,
                 self.args.checkpoints,
                 self.args.params,
-                self.args.cmd,
+                self.args.command,
             ]
         ):  # pragma: no cover
             logger.error(
@@ -36,7 +36,7 @@ class CmdRun(CmdBase):
         kwargs = vars(self.args)
         kwargs.update(
             {
-                "cmd": parse_cmd(self.args.cmd),
+                "cmd": parse_cmd(self.args.command),
                 "fname": kwargs.pop("file"),
                 "no_exec": (self.args.no_exec or bool(self.args.checkpoints)),
                 "run_cache": not kwargs.pop("no_run_cache"),

dvc/command/stage.py

@@ -129,7 +129,7 @@ class CmdStageAdd(CmdBase):
         kwargs = vars(self.args)
         kwargs.update(
             {
-                "cmd": parse_cmd(kwargs.pop("cmd")),
+                "cmd": parse_cmd(kwargs.pop("command")),
                 "params": parse_params(self.args.params),
             }
         )
@@ -281,7 +281,7 @@ def _add_common_args(parser):
         ),
     )
     parser.add_argument(
-        "cmd",
+        "command",
         nargs=argparse.REMAINDER,
         help="Command to execute.",
         metavar="command",

dvc/compare.py

@@ -116,6 +116,8 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
             del col[item]
 
     def __len__(self) -> int:
+        if not self._columns:
+            return 0
         return len(self.columns[0])
 
     @property
@@ -182,22 +184,38 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
             {k: self._columns[k][i] for k in keys} for i in range(len(self))
         ]
 
-    def dropna(self, axis: str = "rows"):
+    def dropna(self, axis: str = "rows", how="any"):
         if axis not in ["rows", "cols"]:
             raise ValueError(
                 f"Invalid 'axis' value {axis}."
                 "Choose one of ['rows', 'cols']"
             )
+        if how not in ["any", "all"]:
+            raise ValueError(
+                f"Invalid 'how' value {how}." "Choose one of ['any', 'all']"
+            )
+
+        match_line: Set = set()
+        match = True
+        if how == "all":
+            match = False
 
-        to_drop: Set = set()
         for n_row, row in enumerate(self):
             for n_col, col in enumerate(row):
-                if col == self._fill_value:
+                if (col == self._fill_value) is match:
                     if axis == "rows":
-                        to_drop.add(n_row)
+                        match_line.add(n_row)
                         break
                     else:
-                        to_drop.add(self.keys()[n_col])
+                        match_line.add(self.keys()[n_col])
+
+        to_drop = match_line
+        if how == "all":
+            if axis == "rows":
+                to_drop = set(range(len(self)))
+            else:
+                to_drop = set(self.keys())
+            to_drop -= match_line
 
         if axis == "rows":
             for name in self.keys():
@@ -211,6 +229,41 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
         else:
             self.drop(*to_drop)
 
+    def drop_duplicates(self, axis: str = "rows"):
+        if axis not in ["rows", "cols"]:
+            raise ValueError(
+                f"Invalid 'axis' value {axis}."
+                "Choose one of ['rows', 'cols']"
+            )
+
+        if axis == "cols":
+            cols_to_drop: List[str] = []
+            for n_col, col in enumerate(self.columns):
+                # Cast to str because Text is not hashable error
+                unique_vals = {str(x) for x in col if x != self._fill_value}
+                if len(unique_vals) == 1:
+                    cols_to_drop.append(self.keys()[n_col])
+            self.drop(*cols_to_drop)
+
+        elif axis == "rows":
+            unique_rows = []
+            rows_to_drop: List[int] = []
+            for n_row, row in enumerate(self):
+                tuple_row = tuple(row)
+                if tuple_row in unique_rows:
+                    rows_to_drop.append(n_row)
+                else:
+                    unique_rows.append(tuple_row)
+
+            for name in self.keys():
+                self._columns[name] = Column(
+                    [
+                        x
+                        for n, x in enumerate(self._columns[name])
+                        if n not in rows_to_drop
+                    ]
+                )
+
 
 def _normalize_float(val: float, precision: int):
     return f"{val:.{precision}g}"

dvc/config.py

@@ -8,7 +8,6 @@ from functools import partial
 from funcy import cached_property, compact, memoize, re_find
 
 from dvc.exceptions import DvcException, NotDvcRepoError
-from dvc.path_info import PathInfo
 
 logger = logging.getLogger(__name__)
 
@@ -229,6 +228,7 @@ class Config(dict):
 
     @staticmethod
     def _to_relpath(conf_dir, path):
+        from dvc.fs.local import localfs
         from dvc.utils import relpath
 
         from .config_schema import RelPath
@@ -238,8 +238,9 @@ class Config(dict):
 
         if isinstance(path, RelPath) or not os.path.isabs(path):
             path = relpath(path, conf_dir)
+            return localfs.path.as_posix(path)
 
-        return PathInfo(path).as_posix()
+        return path
 
     @staticmethod
     def _save_paths(conf, filename):

dvc/config_schema.py

@@ -241,7 +241,6 @@ SCHEMA = {
                 Lower, Choices("us-west", "us-east", "eu-west", "eu-north")
             ),
             "image": str,
-            "name": str,
             "spot": Bool,
             "spot_price": Coerce(float),
             "instance_hdd_size": Coerce(int),

dvc/data_cloud.py

@@ -58,9 +58,9 @@ class DataCloud:
         from dvc.fs import get_cloud_fs
         from dvc.objects.db import get_odb
 
-        cls, config, path_info = get_cloud_fs(self.repo, name=name)
+        cls, config, fs_path = get_cloud_fs(self.repo, name=name)
         config["tmp_dir"] = self.repo.index_db_dir
-        return get_odb(cls(**config), path_info, **config)
+        return get_odb(cls(**config), fs_path, **config)
 
     def push(
         self,
@@ -156,5 +156,6 @@ class DataCloud:
         )
 
     def get_url_for(self, remote, checksum):
-        remote_odb = self.get_remote_odb(remote)
-        return str(remote_odb.hash_to_path_info(checksum))
+        odb = self.get_remote_odb(remote)
+        path = odb.hash_to_path(checksum)
+        return odb.fs.unstrip_protocol(path)

dvc/dependency/param.py

@@ -92,10 +92,10 @@ class ParamsDependency(Dependency):
         if not self.exists:
             return {}
 
-        suffix = self.path_info.suffix.lower()
+        suffix = self.repo.fs.path.suffix(self.fs_path).lower()
         loader = LOADERS[suffix]
         try:
-            return loader(self.path_info, fs=self.repo.fs)
+            return loader(self.fs_path, fs=self.repo.fs)
         except ParseError as exc:
             raise BadParamFileError(
                 f"Unable to read parameters from '{self}'"

dvc/dependency/repo.py

@@ -5,8 +5,6 @@ from typing import TYPE_CHECKING, Dict, Optional, Set, Tuple
 
 from voluptuous import Required
 
-from dvc.path_info import PathInfo
-
 from .base import Dependency
 
 if TYPE_CHECKING:
@@ -34,7 +32,7 @@ class RepoDependency(Dependency):
         self._staged_objs: Dict[str, "HashFile"] = {}
         super().__init__(stage, *args, **kwargs)
 
-    def _parse_path(self, fs, path_info):
+    def _parse_path(self, fs, fs_path):
         return None
 
     @property
@@ -70,7 +68,7 @@ class RepoDependency(Dependency):
 
         obj = self.get_obj()
         checkout(
-            to.path_info,
+            to.fs_path,
             to.fs,
             obj,
             self.repo.odb.local,
@@ -114,11 +112,11 @@ class RepoDependency(Dependency):
             if locked and self.def_repo.get(self.PARAM_REV_LOCK) is None:
                 self.def_repo[self.PARAM_REV_LOCK] = rev
 
-            path_info = PathInfo(repo.root_dir) / str(self.def_path)
+            path = os.path.abspath(os.path.join(repo.root_dir, self.def_path))
             if not obj_only:
                 try:
                     for odb, obj_ids in repo.used_objs(
-                        [os.fspath(path_info)],
+                        [path],
                         force=True,
                         jobs=kwargs.get("jobs"),
                         recursive=True,
@@ -134,7 +132,7 @@ class RepoDependency(Dependency):
             try:
                 staging, _, staged_obj = stage(
                     local_odb,
-                    path_info,
+                    path,
                     repo.repo_fs,
                     local_odb.fs.PARAM_CHECKSUM,
                 )

dvc/dvcfile.py

@@ -60,7 +60,7 @@ def is_lock_file(path):
 
 def is_git_ignored(repo, path):
     from dvc.fs.local import LocalFileSystem
-    from dvc.scm.base import NoSCMError
+    from dvc.scm import NoSCMError
 
     try:
         return isinstance(repo.fs, LocalFileSystem) and repo.scm.is_ignored(
@@ -199,7 +199,7 @@ class SingleStageFile(FileMixin):
             check_dvcfile_path(self.repo, self.path)
         logger.debug(f"Saving information to '{relpath(self.path)}'.")
         dump_yaml(self.path, serialize.to_single_stage_file(stage))
-        self.repo.scm.track_file(self.relpath)
+        self.repo.scm_context.track_file(self.relpath)
 
     def remove_stage(self, stage):  # pylint: disable=unused-argument
         self.remove()
@@ -268,7 +268,7 @@ class PipelineFile(FileMixin):
             else:
                 data["stages"].update(stage_data)
 
-        self.repo.scm.track_file(self.relpath)
+        self.repo.scm_context.track_file(self.relpath)
 
     @property
     def stage(self):
@@ -388,7 +388,7 @@ class Lockfile(FileMixin):
             data["stages"].update(stage_data)
 
         if modified:
-            self.repo.scm.track_file(self.relpath)
+            self.repo.scm_context.track_file(self.relpath)
 
     def remove_stage(self, stage):
         if not self.exists():

dvc/exceptions.py

@@ -216,18 +216,6 @@ class DvcIgnoreInCollectedDirError(DvcException):
         )
 
 
-class GitHookAlreadyExistsError(DvcException):
-    def __init__(self, hook_name):
-        from dvc.utils import format_link
-
-        super().__init__(
-            "Hook '{}' already exists. Please refer to {} for more "
-            "info.".format(
-                hook_name, format_link("https://man.dvc.org/install")
-            )
-        )
-
-
 class FileTransferError(DvcException):
     _METHOD = "transfer"
 
@@ -305,7 +293,7 @@ class PathMissingError(DvcException):
 
 
 class RemoteCacheRequiredError(DvcException):
-    def __init__(self, path_info):
+    def __init__(self, scheme, fs_path):
         from dvc.utils import format_link
 
         super().__init__(
@@ -314,8 +302,8 @@ class RemoteCacheRequiredError(DvcException):
                 "existing cache on '{}' remote. See {} for information on how "
                 "to set up remote cache."
             ).format(
-                path_info,
-                path_info.scheme,
+                fs_path,
+                scheme,
                 format_link("https://man.dvc.org/config#cache"),
             )
         )
@@ -350,12 +338,12 @@ class CacheLinkError(DvcException):
         )
     )
 
-    def __init__(self, path_infos):
+    def __init__(self, fs_paths):
         msg = "No possible cache link types for '{}'. {}".format(
-            ", ".join([str(path) for path in path_infos]), self.SUPPORT_LINK
+            ", ".join(fs_paths), self.SUPPORT_LINK
         )
         super().__init__(msg)
-        self.path_infos = path_infos
+        self.fs_paths = fs_paths
 
 
 class CircularImportError(DvcException):

dvc/external_repo.py

@@ -16,17 +16,20 @@ from dvc.exceptions import (
     PathMissingError,
 )
 from dvc.repo import Repo
+from dvc.scm import CloneError, map_scm_exception
 from dvc.utils import relpath
 
 logger = logging.getLogger(__name__)
 
 
 @contextmanager
+@map_scm_exception()
 def external_repo(
     url, rev=None, for_write=False, cache_dir=None, cache_types=None, **kwargs
 ):
+    from scmrepo.git import Git
+
     from dvc.config import NoRemoteError
-    from dvc.scm.git import Git
 
     logger.debug("Creating external repo %s@%s", url, rev)
     path = _cached_clone(url, rev, for_write=for_write)
@@ -173,7 +176,7 @@ def _clone_default_branch(url, rev, for_write=False):
 
     The cloned is reactualized with git pull unless rev is a known sha.
     """
-    from dvc.scm.git import Git
+    from scmrepo.git import Git
 
     clone_path, shallow = CLONES.get(url, (None, False))
 
@@ -198,14 +201,15 @@ def _clone_default_branch(url, rev, for_write=False):
                     logger.debug("erepo: git pull '%s'", url)
                     git.pull()
         else:
+            from dvc.scm import clone
+
             logger.debug("erepo: git clone '%s' to a temporary dir", url)
             clone_path = tempfile.mkdtemp("dvc-clone")
             if not for_write and rev and not Git.is_sha(rev):
                 # If rev is a tag or branch name try shallow clone first
-                from dvc.scm.base import CloneError
 
                 try:
-                    git = Git.clone(url, clone_path, shallow_branch=rev)
+                    git = clone(url, clone_path, shallow_branch=rev)
                     shallow = True
                     logger.debug(
                         "erepo: using shallow clone for branch '%s'", rev
@@ -213,7 +217,7 @@ def _clone_default_branch(url, rev, for_write=False):
                 except CloneError:
                     pass
             if not git:
-                git = Git.clone(url, clone_path)
+                git = clone(url, clone_path)
                 shallow = False
             CLONES[url] = (clone_path, shallow)
     finally:
@@ -237,7 +241,7 @@ def _unshallow(git):
 
 
 def _git_checkout(repo_path, rev):
-    from dvc.scm.git import Git
+    from scmrepo.git import Git
 
     logger.debug("erepo: git checkout %s@%s", repo_path, rev)
     git = Git(repo_path)

dvc/fs/__init__.py

@@ -1,4 +1,3 @@
-import posixpath
 from urllib.parse import urlparse
 
 from ..scheme import Schemes
@@ -39,7 +38,7 @@ def get_fs_cls(remote_conf, scheme=None):
     return FS_MAP.get(scheme, LocalFileSystem)
 
 
-def get_fs_config(config, **kwargs):
+def get_fs_config(repo, config, **kwargs):
     name = kwargs.get("name")
     if name:
         try:
@@ -50,10 +49,10 @@ def get_fs_config(config, **kwargs):
             raise RemoteNotFoundError(f"remote '{name}' doesn't exist")
     else:
         remote_conf = kwargs
-    return _resolve_remote_refs(config, remote_conf)
+    return _resolve_remote_refs(repo, config, remote_conf)
 
 
-def _resolve_remote_refs(config, remote_conf):
+def _resolve_remote_refs(repo, config, remote_conf):
     # Support for cross referenced remotes.
     # This will merge the settings, shadowing base ref with remote_conf.
     # For example, having:
@@ -79,8 +78,10 @@ def _resolve_remote_refs(config, remote_conf):
     if parsed.scheme != "remote":
         return remote_conf
 
-    base = get_fs_config(config, name=parsed.netloc)
-    url = posixpath.join(base["url"], parsed.path.lstrip("/"))
+    base = get_fs_config(repo, config, name=parsed.netloc)
+    cls, _, _ = get_cloud_fs(repo, **base)
+    relpath = parsed.path.lstrip("/").replace("/", cls.sep)
+    url = cls.sep.join((base["url"], relpath))
     return {**base, **remote_conf, "url": url}
 
 
@@ -91,7 +92,7 @@ def get_cloud_fs(repo, **kwargs):
     repo_config = repo.config if repo else {}
     core_config = repo_config.get("core", {})
 
-    remote_conf = get_fs_config(repo_config, **kwargs)
+    remote_conf = get_fs_config(repo, repo_config, **kwargs)
     try:
         remote_conf = SCHEMA["remote"][str](remote_conf)
     except Invalid as exc:
@@ -113,10 +114,8 @@ def get_cloud_fs(repo, **kwargs):
         remote_conf["gdrive_credentials_tmp_dir"] = repo.tmp_dir
 
     url = remote_conf.pop("url")
-    path_info = cls.PATH_CLS(
-        cls._strip_protocol(url)  # pylint:disable=protected-access
-    )
+    fs_path = cls._strip_protocol(url)  # pylint:disable=protected-access
 
     extras = cls._get_kwargs_from_urls(url)  # pylint:disable=protected-access
     conf = {**extras, **remote_conf}  # remote config takes priority
-    return cls, conf, path_info
+    return cls, conf, fs_path

dvc/fs/_metadata.py

@@ -2,7 +2,6 @@ from dataclasses import dataclass, field
 from typing import TYPE_CHECKING, List
 
 from dvc.output import Output
-from dvc.path_info import PathInfo
 
 if TYPE_CHECKING:
     from dvc.repo import Repo
@@ -16,7 +15,7 @@ class Metadata:
     """
 
     # required field
-    path_info: PathInfo
+    fs_path: str
     repo: "Repo"
 
     # computed fields
@@ -47,11 +46,13 @@ class Metadata:
         else:
             out = self.outs[0]
             # or, the path itself could be an output
-            self.is_output = self.path_info == out.path_info
+            self.is_output = self.fs_path == out.fs_path
 
             # or, a directory must have an output somewhere deep inside it
             if not self.is_output:
-                self.contains_outputs = out.path_info.isin(self.path_info)
+                self.contains_outputs = self.repo.fs.path.isin(
+                    out.fs_path, self.fs_path
+                )
 
         # or, the path could be a part of an output, i.e. inside of an output
         self.part_of_output = not self.is_output and not self.contains_outputs
@@ -69,4 +70,4 @@ class Metadata:
         return not self.isdir
 
     def __str__(self):
-        return f"Metadata: {self.path_info}"
+        return f"Metadata: {self.fs_path}"

dvc/fs/azure.py

@@ -7,7 +7,6 @@ from fsspec.utils import infer_storage_options
 from funcy import cached_property, memoize, wrap_prop
 
 from dvc.exceptions import DvcException
-from dvc.path_info import CloudURLInfo
 from dvc.scheme import Schemes
 from dvc.utils import format_link
 
@@ -41,7 +40,6 @@ def _az_config():
 # pylint:disable=abstract-method
 class AzureFileSystem(CallbackMixin, ObjectFSWrapper):
     scheme = Schemes.AZURE
-    PATH_CLS = CloudURLInfo
     PARAM_CHECKSUM = "etag"
     DETAIL_FIELDS = frozenset(("etag", "size"))
     REQUIRES = {
@@ -52,12 +50,14 @@ class AzureFileSystem(CallbackMixin, ObjectFSWrapper):
 
     @classmethod
     def _strip_protocol(cls, path: str):
-        bucket = infer_storage_options(path).get("host")
-        if bucket:
-            return path
+        opts = infer_storage_options(path)
+        if opts.get("host"):
+            return "{host}{path}".format(**opts)
 
-        bucket = _az_config().get("storage", "container_name", None)
-        return f"azure://{bucket}"
+        return _az_config().get("storage", "container_name", None)
+
+    def unstrip_protocol(self, path: str) -> str:
+        return "azure://" + path.lstrip("/")
 
     @staticmethod
     def _get_kwargs_from_urls(urlpath):

dvc/fs/base.py

@@ -4,12 +4,12 @@ import os
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from functools import partialmethod
 from multiprocessing import cpu_count
-from typing import Any, ClassVar, Dict, FrozenSet, Optional
+from typing import ClassVar, Dict, FrozenSet, Optional
 
+from funcy import cached_property
 from tqdm.utils import CallbackIOWrapper
 
 from dvc.exceptions import DvcException
-from dvc.path_info import URLInfo
 from dvc.progress import DEFAULT_CALLBACK, FsspecCallback
 from dvc.ui import ui
 from dvc.utils import tmp_fname
@@ -36,12 +36,11 @@ class RemoteMissingDepsError(DvcException):
     pass
 
 
-class BaseFileSystem:
+class FileSystem:
     sep = "/"
 
     scheme = "base"
     REQUIRES: ClassVar[Dict[str, str]] = {}
-    PATH_CLS = URLInfo  # type: Any
     _JOBS = 4 * cpu_count()
 
     CHECKSUM_DIR_SUFFIX = ".dir"
@@ -69,10 +68,19 @@ class BaseFileSystem:
     def config(self):
         return self._config
 
+    @cached_property
+    def path(self):
+        from .path import Path
+
+        return Path(self.sep)
+
     @classmethod
     def _strip_protocol(cls, path: str):
         return path
 
+    def unstrip_protocol(self, path):
+        return path
+
     @staticmethod
     def _get_kwargs_from_urls(urlpath):  # pylint:disable=unused-argument
         return {}
@@ -129,36 +137,36 @@ class BaseFileSystem:
             f"dependencies: {missing}. {hint}"
         )
 
-    def checksum(self, path_info) -> str:
+    def checksum(self, path) -> str:
         raise NotImplementedError
 
-    def open(self, path_info, mode: str = "r", encoding: str = None, **kwargs):
+    def open(self, path, mode: str = "r", encoding: str = None, **kwargs):
         raise RemoteActionNotImplemented("open", self.scheme)
 
-    def exists(self, path_info) -> bool:
+    def exists(self, path) -> bool:
         raise NotImplementedError
 
     # pylint: disable=unused-argument
 
-    def isdir(self, path_info):
+    def isdir(self, path):
         """Optional: Overwrite only if the remote has a way to distinguish
         between a directory and a file.
         """
         return False
 
-    def isfile(self, path_info):
+    def isfile(self, path):
         """Optional: Overwrite only if the remote has a way to distinguish
         between a directory and a file.
         """
         return True
 
-    def isexec(self, path_info):
+    def isexec(self, path):
         """Optional: Overwrite only if the remote has a way to distinguish
         between executable and non-executable file.
         """
         return False
 
-    def iscopy(self, path_info):
+    def iscopy(self, path):
         """Check if this file is an independent copy."""
         return False  # We can't be sure by default
 
@@ -166,28 +174,28 @@ class BaseFileSystem:
         """Return a generator with (root, dirs, files)."""
         raise NotImplementedError
 
-    def walk_files(self, path_info, **kwargs):
-        """Return a generator with `PathInfo`s to all the files.
+    def find(self, path, prefix=None):
+        """Return a generator with `str`s to all the files.
 
         Optional kwargs:
-            prefix (bool): If true `path_info` will be treated as a prefix
+            prefix (bool): If true `path` will be treated as a prefix
                 rather than directory path.
         """
         raise NotImplementedError
 
-    def is_empty(self, path_info):
+    def is_empty(self, path):
         return False
 
-    def info(self, path_info):
+    def info(self, path):
         raise NotImplementedError
 
-    def getsize(self, path_info):
-        return self.info(path_info).get("size")
+    def getsize(self, path):
+        return self.info(path).get("size")
 
-    def remove(self, path_info):
+    def remove(self, path):
         raise RemoteActionNotImplemented("remove", self.scheme)
 
-    def makedirs(self, path_info, **kwargs):
+    def makedirs(self, path, **kwargs):
         """Optional: Implement only if the remote needs to create
         directories before copying/linking/moving data
         """
@@ -231,11 +239,10 @@ class BaseFileSystem:
         if not hasattr(self, method):
             raise RemoteActionNotImplemented(method, self.scheme)
 
-        if to_info.scheme != self.scheme:
-            raise NotImplementedError
-
         if not is_file_obj:
-            desc = desc or from_info.name
+            from .local import localfs
+
+            desc = desc or localfs.path.name(from_info)
 
         stack = contextlib.ExitStack()
         if not callback:
@@ -261,9 +268,6 @@ class BaseFileSystem:
                 # pylint: disable=no-member
                 return self.upload_fobj(wrapped, to_info, size=total)
 
-            if from_info.scheme != "local":
-                raise NotImplementedError
-
             logger.debug("Uploading '%s' to '%s'", from_info, to_info)
             # pylint: disable=no-member
             return self.put_file(
@@ -281,18 +285,14 @@ class BaseFileSystem:
         _only_file=False,
         **kwargs,
     ):
+        from .local import localfs
+
         if not hasattr(self, "get_file"):
             raise RemoteActionNotImplemented("get_file", self.scheme)
 
-        if from_info.scheme != self.scheme:
-            raise NotImplementedError
-
-        if to_info.scheme != "local":
-            raise NotImplementedError
-
         download_dir = not _only_file and self.isdir(from_info)
 
-        desc = name or to_info.name
+        desc = name or localfs.path.name(to_info)
         stack = contextlib.ExitStack()
         if not callback:
             pbar_kwargs = {"unit": "Files"} if download_dir else {}
@@ -323,13 +323,16 @@ class BaseFileSystem:
         jobs=None,
         **kwargs,
     ):
-        from_infos = list(self.walk_files(from_info, **kwargs))
+        from .local import localfs
+
+        from_infos = list(self.find(from_info, **kwargs))
         if not from_infos:
             makedirs(to_info, exist_ok=True)
             return None
 
         to_infos = (
-            to_info / info.relative_to(from_info) for info in from_infos
+            localfs.path.join(to_info, *self.path.relparts(info, from_info))
+            for info in from_infos
         )
         callback.set_size(len(from_infos))
 
@@ -362,7 +365,9 @@ class BaseFileSystem:
         to_info,
         callback=DEFAULT_CALLBACK,
     ):
-        makedirs(to_info.parent, exist_ok=True)
+        from .local import localfs
+
+        makedirs(localfs.path.parent(to_info), exist_ok=True)
         tmp_file = tmp_fname(to_info)
 
         logger.debug("Downloading '%s' to '%s'", from_info, to_info)

dvc/fs/dvc.py

@@ -3,21 +3,20 @@ import os
 import typing
 
 from dvc.exceptions import OutputNotFoundError
-from dvc.path_info import PathInfo
 from dvc.utils import relpath
 
 from ..progress import DEFAULT_CALLBACK
 from ._metadata import Metadata
-from .base import BaseFileSystem
+from .base import FileSystem
 
 if typing.TYPE_CHECKING:
     from dvc.output import Output
-
+    from dvc.types import AnyPath
 
 logger = logging.getLogger(__name__)
 
 
-class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
+class DvcFileSystem(FileSystem):  # pylint:disable=abstract-method
     """DVC repo fs.
 
     Args:
@@ -49,12 +48,9 @@ class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
 
         return outs
 
-    def _get_granular_hash(
-        self, path_info: PathInfo, out: "Output", remote=None
-    ):
-        assert isinstance(path_info, PathInfo)
+    def _get_granular_hash(self, path: "AnyPath", out: "Output", remote=None):
         # NOTE: use string paths here for performance reasons
-        key = tuple(relpath(path_info, out.path_info).split(os.sep))
+        key = tuple(relpath(path, out.fs_path).split(os.sep))
         out.get_dir_cache(remote=remote)
         if out.obj is None:
             raise FileNotFoundError
@@ -63,14 +59,14 @@ class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             return oid
         raise FileNotFoundError
 
-    def _get_fs_path(self, path: PathInfo, remote=None):
+    def _get_fs_path(self, path: "AnyPath", remote=None):
         try:
             outs = self._find_outs(path, strict=False)
         except OutputNotFoundError as exc:
             raise FileNotFoundError from exc
 
         if len(outs) != 1 or (
-            outs[0].is_dir_checksum and path == outs[0].path_info
+            outs[0].is_dir_checksum and path == outs[0].fs_path
         ):
             raise IsADirectoryError
 
@@ -90,18 +86,20 @@ class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
                 checksum = self._get_granular_hash(path, out).value
             else:
                 checksum = out.hash_info.value
-            remote_info = remote_odb.hash_to_path_info(checksum)
-            return remote_odb.fs, remote_info
+            remote_fs_path = remote_odb.hash_to_path(checksum)
+            return remote_odb.fs, remote_fs_path
 
         if out.is_dir_checksum:
             checksum = self._get_granular_hash(path, out).value
-            cache_path = out.odb.hash_to_path_info(checksum).url
+            cache_path = out.odb.fs.unstrip_protocol(
+                out.odb.hash_to_path(checksum)
+            )
         else:
             cache_path = out.cache_path
         return out.odb.fs, cache_path
 
     def open(  # type: ignore
-        self, path: PathInfo, mode="r", encoding=None, **kwargs
+        self, path: str, mode="r", encoding=None, **kwargs
     ):  # pylint: disable=arguments-renamed
         fs, fspath = self._get_fs_path(path, **kwargs)
         return fs.open(fspath, mode=mode, encoding=encoding)
@@ -120,18 +118,18 @@ class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         except FileNotFoundError:
             return False
 
-    def check_isdir(self, path_info, outs):
+    def check_isdir(self, path, outs):
         if len(outs) != 1:
             return True
 
         out = outs[0]
         if not out.is_dir_checksum:
-            return out.path_info != path_info
-        if out.path_info == path_info:
+            return out.fs_path != path
+        if out.fs_path == path:
             return True
 
         try:
-            self._get_granular_hash(path_info, out)
+            self._get_granular_hash(path, out)
             return False
         except FileNotFoundError:
             return True
@@ -153,7 +151,7 @@ class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
     def _add_dir(self, trie, out, **kwargs):
         self._fetch_dir(out, **kwargs)
 
-        base = out.path_info.parts
+        base = out.fs.path.parts(out.fs_path)
         for key, _, _ in out.obj:  # noqa: B301
             trie[base + key] = None
 
@@ -161,14 +159,15 @@ class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         dirs = set()
         files = []
 
-        out = trie.get(root.parts)
+        root_parts = self.path.parts(root)
+        out = trie.get(root_parts)
         if out and out.is_dir_checksum:
             self._add_dir(trie, out, **kwargs)
 
-        root_len = len(root.parts)
+        root_len = len(root_parts)
         try:
-            for key, out in trie.iteritems(prefix=root.parts):  # noqa: B301
-                if key == root.parts:
+            for key, out in trie.iteritems(prefix=root_parts):  # noqa: B301
+                if key == root_parts:
                     continue
 
                 name = key[root_len]
@@ -182,16 +181,16 @@ class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
 
         assert topdown
         dirs = list(dirs)
-        yield root.fspath, dirs, files
+        yield root, dirs, files
 
         for dname in dirs:
-            yield from self._walk(root / dname, trie)
+            yield from self._walk(self.path.join(root, dname), trie)
 
     def walk(self, top, topdown=True, onerror=None, **kwargs):
         from pygtrie import Trie
 
         assert topdown
-        root = PathInfo(os.path.abspath(top))
+        root = os.path.abspath(top)
         try:
             meta = self.metadata(root)
         except FileNotFoundError:
@@ -206,18 +205,18 @@ class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
 
         trie = Trie()
         for out in meta.outs:
-            trie[out.path_info.parts] = out
+            trie[out.fs.path.parts(out.fs_path)] = out
 
-            if out.is_dir_checksum and root.isin_or_eq(out.path_info):
+            if out.is_dir_checksum and self.path.isin_or_eq(root, out.fs_path):
                 self._add_dir(trie, out, **kwargs)
 
         yield from self._walk(root, trie, topdown=topdown, **kwargs)
 
-    def walk_files(self, path_info, **kwargs):
-        for root, _, files in self.walk(path_info):
+    def find(self, path, prefix=None):
+        for root, _, files in self.walk(path):
             for fname in files:
                 # NOTE: os.path.join is ~5.5 times slower
-                yield PathInfo(f"{root}{os.sep}{fname}")
+                yield f"{root}{os.sep}{fname}"
 
     def isdvc(self, path, recursive=False, strict=True):
         try:
@@ -228,23 +227,23 @@ class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         recurse = recursive or not strict
         return meta.output_exists if recurse else meta.is_output
 
-    def isexec(self, path_info):  # pylint: disable=unused-argument
+    def isexec(self, path):  # pylint: disable=unused-argument
         return False
 
-    def metadata(self, path_info):
-        path_info = PathInfo(os.path.abspath(path_info))
+    def metadata(self, fs_path):
+        abspath = os.path.abspath(fs_path)
 
         try:
-            outs = self._find_outs(path_info, strict=False, recursive=True)
+            outs = self._find_outs(abspath, strict=False, recursive=True)
         except OutputNotFoundError as exc:
             raise FileNotFoundError from exc
 
-        meta = Metadata(path_info=path_info, outs=outs, repo=self.repo)
-        meta.isdir = meta.isdir or self.check_isdir(meta.path_info, meta.outs)
+        meta = Metadata(fs_path=abspath, outs=outs, repo=self.repo)
+        meta.isdir = meta.isdir or self.check_isdir(meta.fs_path, meta.outs)
         return meta
 
-    def info(self, path_info):
-        meta = self.metadata(path_info)
+    def info(self, path):
+        meta = self.metadata(path)
         ret = {"type": "directory" if meta.isdir else "file"}
         if meta.is_output and len(meta.outs) == 1 and meta.outs[0].hash_info:
             out = meta.outs[0]
@@ -252,7 +251,7 @@ class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             ret[out.hash_info.name] = out.hash_info.value
         elif meta.part_of_output:
             (out,) = meta.outs
-            key = path_info.relative_to(out.path_info).parts
+            key = self.path.parts(self.path.relpath(path, out.fs_path))
             (obj_meta, oid) = out.obj.trie.get(key) or (None, None)
             if oid:
                 ret["size"] = obj_meta.size if obj_meta else 0
@@ -268,8 +267,8 @@ class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             path, to_file, callback=callback, **kwargs
         )
 
-    def checksum(self, path_info):
-        info = self.info(path_info)
+    def checksum(self, path):
+        info = self.info(path)
         md5 = info.get("md5")
         if md5:
             return md5

dvc/fs/fsspec_wrapper.py

@@ -1,31 +1,18 @@
 import os
 import shutil
-from functools import lru_cache
-from typing import (
-    IO,
-    TYPE_CHECKING,
-    Any,
-    Dict,
-    Iterator,
-    Optional,
-    Union,
-    overload,
-)
+from typing import IO, TYPE_CHECKING, Any, Dict, Iterator, Optional, overload
 
 from funcy import cached_property
 from tqdm.utils import CallbackIOWrapper
 
-from dvc.path_info import URLInfo
 from dvc.progress import DEFAULT_CALLBACK
 
-from .base import BaseFileSystem
+from .base import FileSystem
 
-FSPath = URLInfo
-AnyFSPath = Union[FSPath, str]
+FSPath = str
+AnyFSPath = str
 
 if TYPE_CHECKING:
-    from typing import Iterable
-
     from typing_extensions import Literal
 
 
@@ -35,7 +22,7 @@ Entry = Dict[str, Any]
 
 
 # pylint: disable=no-member
-class FSSpecWrapper(BaseFileSystem):
+class FSSpecWrapper(FileSystem):
     TRAVERSE_PREFIX_LEN = 2
 
     def __init__(self, **kwargs):
@@ -56,38 +43,6 @@ class FSSpecWrapper(BaseFileSystem):
     def fs(self):
         raise NotImplementedError
 
-    def _with_bucket(self, path: AnyFSPath) -> str:
-        return str(path)
-
-    def _strip_bucket(self, entry: str) -> str:
-        return entry
-
-    @overload
-    def _strip_buckets(
-        self, entries: "Iterable[Entry]", detail: "Literal[True]"
-    ) -> "Iterator[Entry]":
-        ...
-
-    @overload
-    def _strip_buckets(
-        self, entries: "Iterable[str]", detail: "Literal[False]" = False
-    ) -> "Iterator[str]":
-        ...
-
-    def _strip_buckets(self, entries, detail=False):
-        for entry in entries:
-            if detail:
-                entry = self._entry_hook(entry.copy())
-                entry["name"] = self._strip_bucket(entry["name"])
-            else:
-                entry = self._strip_bucket(entry)
-            yield entry
-
-    def _entry_hook(self, entry: "Entry") -> "Entry":
-        """Simple hook method to be overridden when wanted to process
-        entries within info() and ls(detail=True) calls"""
-        return entry
-
     def _prepare_credentials(
         self, **config: Dict[str, Any]  # pylint: disable=unused-argument
     ) -> Dict[str, Any]:
@@ -95,134 +50,95 @@ class FSSpecWrapper(BaseFileSystem):
         host filesystem"""
         return {}
 
-    def _isdir(self, path_info: FSPath) -> bool:
-        return self.fs.isdir(self._with_bucket(path_info))
+    def _isdir(self, path: AnyFSPath) -> bool:
+        return self.fs.isdir(path)
 
-    def isdir(self, path_info: FSPath) -> bool:
+    def isdir(self, path: AnyFSPath) -> bool:
         try:
-            return self._isdir(path_info)
+            return self._isdir(path)
         except FileNotFoundError:
             return False
 
-    def isfile(self, path_info: FSPath) -> bool:
+    def isfile(self, path: AnyFSPath) -> bool:
         try:
-            return not self._isdir(path_info)
+            return not self._isdir(path)
         except FileNotFoundError:
             return False
 
-    def is_empty(self, path_info: FSPath) -> bool:
-        entry = self.info(path_info)
+    def is_empty(self, path: AnyFSPath) -> bool:
+        entry = self.info(path)
         if entry["type"] == "directory":
-            return not self.fs.ls(self._with_bucket(path_info))
+            return not self.fs.ls(path)
         return entry["size"] == 0
 
     def open(
         self,
-        path_info: FSPath,
+        path: AnyFSPath,
         mode: str = "r",
         encoding: Optional[str] = None,
         **kwargs,
     ) -> "IO":  # pylint: disable=arguments-differ
-        return self.fs.open(self._with_bucket(path_info), mode=mode)
+        return self.fs.open(path, mode=mode)
 
-    def checksum(self, path_info: FSPath) -> str:
-        return self.fs.checksum(self._with_bucket(path_info))
+    def checksum(self, path: AnyFSPath) -> str:
+        return self.fs.checksum(path)
 
-    def copy(self, from_info: FSPath, to_info: FSPath) -> None:
-        self.makedirs(to_info.parent)
-        self.fs.copy(self._with_bucket(from_info), self._with_bucket(to_info))
+    def copy(self, from_info: AnyFSPath, to_info: AnyFSPath) -> None:
+        self.makedirs(self.path.parent(to_info))
+        self.fs.copy(from_info, to_info)
 
-    def exists(self, path_info: FSPath) -> bool:
-        return self.fs.exists(self._with_bucket(path_info))
+    def exists(self, path: AnyFSPath) -> bool:
+        return self.fs.exists(path)
 
     @overload
     def ls(
-        self, path_info: FSPath, detail: "Literal[True]"
+        self, path: AnyFSPath, detail: "Literal[True]"
     ) -> "Iterator[Entry]":
         ...
 
     @overload
-    def ls(self, path_info: FSPath, detail: "Literal[False]") -> Iterator[str]:
+    def ls(self, path: AnyFSPath, detail: "Literal[False]") -> Iterator[str]:
         ...
 
-    def ls(self, path_info, detail=False):
-        path = self._with_bucket(path_info)
-        files = self.fs.ls(path, detail=detail)
-        yield from self._strip_buckets(files, detail=detail)
+    def ls(self, path, detail=False):
+        yield from self.fs.ls(path, detail=detail)
 
-    @overload
-    def find(
-        self,
-        path_info: FSPath,
-        detail: "Literal[True]",
-        prefix: Optional[bool],
-    ) -> "Iterator[Entry]":
-        ...
+    def find(self, path, prefix=None):
+        yield from self.fs.find(path)
 
-    @overload
-    def find(
-        self,
-        path_info: FSPath,
-        detail: "Literal[False]",
-        prefix: Optional[bool],
-    ) -> Iterator[str]:
-        ...
+    def move(self, from_info: AnyFSPath, to_info: AnyFSPath) -> None:
+        self.fs.move(from_info, to_info)
 
-    # pylint: disable=unused-argument
-    def find(self, path_info, detail=False, prefix=None):
-        path = self._with_bucket(path_info)
-        files = self.fs.find(path, detail=detail)
-        if detail:
-            files = files.values()
+    def remove(self, path: AnyFSPath) -> None:
+        self.fs.rm_file(path)
 
-        yield from self._strip_buckets(files, detail=detail)
+    def info(self, path: AnyFSPath) -> "Entry":
+        return self.fs.info(path)
 
-    def walk_files(self, path_info: FSPath, **kwargs) -> Iterator[FSPath]:
-        for file in self.find(path_info, **kwargs):
-            yield path_info.replace(path=file)
-
-    def move(self, from_info: FSPath, to_info: FSPath) -> None:
-        self.fs.move(self._with_bucket(from_info), self._with_bucket(to_info))
-
-    def remove(self, path_info: FSPath) -> None:
-        self.fs.rm_file(self._with_bucket(path_info))
-
-    def info(self, path_info: FSPath) -> "Entry":
-        info = self.fs.info(self._with_bucket(path_info)).copy()
-        info = self._entry_hook(info)
-        info["name"] = self._strip_bucket(info["name"])
-        return info
-
-    def makedirs(self, path_info: FSPath, **kwargs) -> None:
-        self.fs.makedirs(
-            self._with_bucket(path_info), exist_ok=kwargs.pop("exist_ok", True)
-        )
+    def makedirs(self, path: AnyFSPath, **kwargs) -> None:
+        self.fs.makedirs(path, exist_ok=kwargs.pop("exist_ok", True))
 
     def put_file(
         self,
-        from_file: FSPath,
-        to_info: FSPath,
+        from_file: AnyFSPath,
+        to_info: AnyFSPath,
         callback: Any = DEFAULT_CALLBACK,
         **kwargs,
     ) -> None:
-        self.fs.put_file(
-            from_file, self._with_bucket(to_info), callback=callback, **kwargs
-        )
-        self.fs.invalidate_cache(self._with_bucket(to_info.parent))
+        self.fs.put_file(from_file, to_info, callback=callback, **kwargs)
+        self.fs.invalidate_cache(self.path.parent(to_info))
 
     def get_file(
         self,
-        from_info: FSPath,
-        to_info: FSPath,
+        from_info: AnyFSPath,
+        to_info: AnyFSPath,
         callback: Any = DEFAULT_CALLBACK,
         **kwargs,
     ) -> None:
-        self.fs.get_file(
-            self._with_bucket(from_info), to_info, callback=callback, **kwargs
-        )
+        self.fs.get_file(from_info, to_info, callback=callback, **kwargs)
 
-    def upload_fobj(self, fobj: IO, to_info: FSPath, **kwargs) -> None:
-        self.makedirs(to_info.parent)
+    def upload_fobj(self, fobj: IO, to_info: AnyFSPath, **kwargs) -> None:
+        self.makedirs(self.path.parent(to_info))
         with self.open(to_info, "wb") as fdest:
             shutil.copyfileobj(
                 fobj, fdest, length=getattr(fdest, "blocksize", None)
@@ -233,22 +149,7 @@ class FSSpecWrapper(BaseFileSystem):
 class ObjectFSWrapper(FSSpecWrapper):
     TRAVERSE_PREFIX_LEN = 3
 
-    @lru_cache(512)
-    def _with_bucket(self, path: AnyFSPath) -> str:
-        if isinstance(path, self.PATH_CLS):
-            return f"{path.bucket}/{path.path}"
-        return str(path)
-
-    def _strip_bucket(self, entry: str) -> str:
-        try:
-            bucket, path = entry.split("/", 1)
-        except ValueError:
-            # If there is no path attached, only returns
-            # the bucket (top-level).
-            bucket, path = entry, ""
-        return path or bucket
-
-    def makedirs(self, path_info: FSPath, **kwargs) -> None:
+    def makedirs(self, path: AnyFSPath, **kwargs) -> None:
         # For object storages make this method a no-op. The original
         # fs.makedirs() method will only check if the bucket exists
         # and create if it doesn't though we don't want to support
@@ -256,44 +157,37 @@ class ObjectFSWrapper(FSSpecWrapper):
         # simply ignore all mkdir()/makedirs() calls.
         return None
 
-    def _isdir(self, path_info: FSPath) -> bool:
+    def _isdir(self, path: AnyFSPath) -> bool:
         # Directory in object storages are interpreted differently
         # among different fsspec providers, so this logic is a temporary
         # measure for us to adapt as of now. It checks whether it is a
         # directory (as in a prefix with contents) or whether it is an empty
         # file where it's name ends with a forward slash
 
-        entry = self.info(path_info)
+        entry = self.info(path)
         return entry["type"] == "directory" or (
             entry["size"] == 0
             and entry["type"] == "file"
             and entry["name"].endswith("/")
         )
 
-    def find(self, path_info, detail=False, prefix=None):
+    def find(self, path, prefix=None):
         if prefix:
-            path = self._with_bucket(path_info.parent)
-            files = self.fs.find(
-                path, detail=detail, prefix=path_info.parts[-1]
-            )
+            with_prefix = self.path.parent(path)
+            files = self.fs.find(with_prefix, prefix=self.path.parts(path)[-1])
         else:
-            path = self._with_bucket(path_info)
-            files = self.fs.find(path, detail=detail)
-
-        if detail:
-            files = list(files.values())
+            with_prefix = path
+            files = self.fs.find(path)
 
         # When calling find() on a file, it returns the same file in a list.
         # For object-based storages, the same behavior applies to empty
         # directories since they are represented as files. This condition
         # checks whether we should yield an empty list (if it is an empty
         # directory) or just yield the file itself.
-        if len(files) == 1 and files[0] == path and self.isdir(path_info):
+        if len(files) == 1 and files[0] == with_prefix and self.isdir(path):
             return None
 
-        yield from self._strip_buckets(  # pylint: disable=not-an-iterable
-            files, detail=detail
-        )
+        yield from files
 
 
 # pylint: disable=arguments-differ
@@ -310,9 +204,6 @@ class NoDirectoriesMixin:
     def walk(self, *args, **kwargs):
         raise NotImplementedError
 
-    def walk_files(self, *args, **kwargs):
-        raise NotImplementedError
-
     def ls(self, *args, **kwargs):
         raise NotImplementedError
 
@@ -329,13 +220,13 @@ class CallbackMixin:
     ):
         """Add compatibility support for Callback."""
         # pylint: disable=protected-access
-        self.makedirs(to_info.parent)
+        self.makedirs(self.path.parent(to_info))
         size = os.path.getsize(from_file)
         with open(from_file, "rb") as fobj:
             callback.set_size(size)
             wrapped = CallbackIOWrapper(callback.relative_update, fobj)
             self.upload_fobj(wrapped, to_info)
-            self.fs.invalidate_cache(self._with_bucket(to_info.parent))
+            self.fs.invalidate_cache(self.path.parent(to_info))
 
     def get_file(
         self,

dvc/fs/gdrive.py

@@ -1,14 +1,10 @@
 import logging
 import os
-import posixpath
-import re
 import threading
-from urllib.parse import urlparse
 
 from funcy import cached_property, wrap_prop
 
 from dvc.exceptions import DvcException
-from dvc.path_info import CloudURLInfo
 from dvc.scheme import Schemes
 from dvc.utils import format_link, tmp_fname
 
@@ -34,24 +30,8 @@ class GDriveAuthError(DvcException):
         super().__init__(message)
 
 
-class GDriveURLInfo(CloudURLInfo):
-    def __init__(self, url):
-        super().__init__(url)
-
-        # GDrive URL host part is case sensitive,
-        # we are restoring it here.
-        p = urlparse(url)
-        self.host = p.netloc
-        assert self.netloc == self.host
-
-        # Normalize path. Important since we have a cache (path to ID)
-        # and don't want to deal with different variations of path in it.
-        self._spath = re.sub("/{2,}", "/", self._spath.rstrip("/"))
-
-
 class GDriveFileSystem(FSSpecWrapper):  # pylint:disable=abstract-method
     scheme = Schemes.GDRIVE
-    PATH_CLS = GDriveURLInfo
     PARAM_CHECKSUM = "checksum"
     REQUIRES = {"pydrive2": "pydrive2"}
     # Always prefer traverse for GDrive since API usage quotas are a concern.
@@ -65,11 +45,14 @@ class GDriveFileSystem(FSSpecWrapper):  # pylint:disable=abstract-method
     DEFAULT_GDRIVE_CLIENT_SECRET = "a1Fz59uTpVNeG_VGuSKDLJXv"
 
     def __init__(self, **config):
+        from fsspec.utils import infer_storage_options
+
         super().__init__(**config)
 
-        self.path_info = self.PATH_CLS(config["url"])
+        self.url = config["url"]
+        opts = infer_storage_options(self.url)
 
-        if not self.path_info.bucket:
+        if not opts["host"]:
             raise DvcException(
                 "Empty GDrive URL '{}'. Learn more at {}".format(
                     config["url"],
@@ -77,8 +60,8 @@ class GDriveFileSystem(FSSpecWrapper):  # pylint:disable=abstract-method
                 )
             )
 
-        self._bucket = self.path_info.bucket
-        self._path = self.path_info.path
+        self._bucket = opts["host"]
+        self._path = opts["path"].lstrip("/")
         self._trash_only = config.get("gdrive_trash_only")
         self._use_service_account = config.get("gdrive_use_service_account")
         self._service_account_user_email = config.get(
@@ -106,6 +89,17 @@ class GDriveFileSystem(FSSpecWrapper):  # pylint:disable=abstract-method
             )
         )
 
+    @classmethod
+    def _strip_protocol(cls, path):
+        from fsspec.utils import infer_storage_options
+
+        opts = infer_storage_options(path)
+
+        return "{host}{path}".format(**opts)
+
+    def unstrip_protocol(self, path):
+        return f"gdrive://{path}"
+
     @staticmethod
     def _get_kwargs_from_urls(urlpath):
         return {"url": urlpath}
@@ -255,27 +249,11 @@ class GDriveFileSystem(FSSpecWrapper):  # pylint:disable=abstract-method
                 os.remove(temporary_save_path)
 
         return _GDriveFileSystem(
-            self._with_bucket(self.path_info),
+            f"{self._bucket}/{self._path}",
             gauth,
             trash_only=self._trash_only,
         )
 
-    def _with_bucket(self, path):
-        if isinstance(path, self.PATH_CLS):
-            return posixpath.join(path.bucket, path.path)
-
-        return super()._with_bucket(path)
-
-    def _strip_bucket(self, entry):
-        try:
-            bucket, path = entry.split("/", 1)
-        except ValueError:
-            # If there is no path attached, only returns
-            # the bucket (top-level).
-            bucket, path = entry, None
-        return path or bucket
-
     def upload_fobj(self, fobj, to_info, **kwargs):
-        rpath = self._with_bucket(to_info)
-        self.makedirs(os.path.dirname(rpath))
-        return self.fs.upload_fobj(fobj, rpath, **kwargs)
+        self.makedirs(os.path.dirname(to_info))
+        return self.fs.upload_fobj(fobj, to_info, **kwargs)

dvc/fs/git.py

@@ -1,136 +1,72 @@
-import errno
 import os
+import threading
+from typing import TYPE_CHECKING, Any, Callable
 
-from tqdm.utils import CallbackIOWrapper
+from funcy import cached_property, wrap_prop
 
-from dvc.utils import is_exec, relpath
+from .fsspec_wrapper import AnyFSPath, FSSpecWrapper
 
-from ..progress import DEFAULT_CALLBACK
-from .base import BaseFileSystem
+if TYPE_CHECKING:
+    from scmrepo.fs import GitFileSystem as FsspecGitFileSystem
+    from scmrepo.git import Git
+    from scmrepo.git.objects import GitTrie
 
 
-class GitFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
+class GitFileSystem(FSSpecWrapper):  # pylint:disable=abstract-method
     """Proxies the repo file access methods to Git objects"""
 
     sep = os.sep
-
     scheme = "local"
 
-    def __init__(self, root_dir, trie):
+    def __init__(
+        self,
+        path: str = None,
+        rev: str = None,
+        scm: "Git" = None,
+        trie: "GitTrie" = None,
+        **kwargs: Any,
+    ) -> None:
+        from dvc.scm import resolve_rev
+
         super().__init__()
-        self._root = root_dir
-        self.trie = trie
+        self.fs_args.update(
+            {
+                "path": path,
+                "rev": rev,
+                "scm": scm,
+                "trie": trie,
+                "rev_resolver": resolve_rev,
+                **kwargs,
+            }
+        )
+
+    @wrap_prop(threading.Lock())
+    @cached_property
+    def fs(self) -> "FsspecGitFileSystem":
+        from scmrepo.fs import GitFileSystem as FsspecGitFileSystem
+
+        return FsspecGitFileSystem(**self.fs_args)
 
     @property
-    def rev(self):
-        return self.trie.rev
+    def rev(self) -> str:
+        return self.fs.rev
 
-    def _get_key(self, path):
-        if isinstance(path, str):
-            if not os.path.isabs(path):
-                relparts = path.split(os.sep)
-            else:
-                relparts = relpath(path, self._root).split(os.sep)
-        else:
-            relparts = path.relative_to(self._root).parts
-        if relparts == ["."]:
-            return ()
-        return tuple(relparts)
+    def isexec(self, path: AnyFSPath) -> bool:
+        from dvc.utils import is_exec
 
-    def open(
-        self, path, mode="r", encoding=None, **kwargs
-    ):  # pylint: disable=arguments-renamed
-        # NOTE: this is incorrect, we need to use locale to determine default
-        # encoding.
-        encoding = encoding or "utf-8"
-
-        key = self._get_key(path)
         try:
-            return self.trie.open(key, mode=mode, encoding=encoding)
-        except KeyError as exc:
-            msg = os.strerror(errno.ENOENT) + f"in branch '{self.rev}'"
-            raise FileNotFoundError(errno.ENOENT, msg, path) from exc
-        except IsADirectoryError as exc:
-            raise IsADirectoryError(
-                errno.EISDIR, os.strerror(errno.EISDIR), path
-            ) from exc
-
-    def exists(self, path_info) -> bool:
-        key = self._get_key(path_info)
-        return self.trie.exists(key)
-
-    def isdir(self, path_info) -> bool:
-        key = self._get_key(path_info)
-        return self.trie.isdir(key)
-
-    def isfile(self, path_info) -> bool:
-        key = self._get_key(path_info)
-        return self.trie.isfile(key)
-
-    def walk(self, top, topdown=True, onerror=None, **kwargs):
-        """Directory tree generator.
-
-        See `os.walk` for the docs. Differences:
-        - no support for symlinks
-        """
-        if not self.isdir(top):
-            if onerror:
-                if self.exists(top):
-                    exc = NotADirectoryError(
-                        errno.ENOTDIR, os.strerror(errno.ENOTDIR), top
-                    )
-                else:
-                    exc = FileNotFoundError(
-                        errno.ENOENT, os.strerror(errno.ENOENT), top
-                    )
-                onerror(exc)
-            return []
-
-        key = self._get_key(top)
-        for prefix, dirs, files in self.trie.walk(key, topdown=topdown):
-            if prefix:
-                root = os.path.join(self._root, os.sep.join(prefix))
-            else:
-                root = self._root
-            yield root, dirs, files
-
-    def isexec(self, path_info):
-        if not self.exists(path_info):
+            return is_exec(self.info(path)["mode"])
+        except FileNotFoundError:
             return False
 
-        mode = self.info(path_info)["mode"]
-        return is_exec(mode)
+    def isfile(self, path: AnyFSPath) -> bool:
+        return self.fs.isfile(path)
 
-    def info(self, path_info):
-        key = self._get_key(path_info)
-        try:
-            return self.trie.info(key)
-        except KeyError:
-            raise FileNotFoundError(
-                errno.ENOENT, os.strerror(errno.ENOENT), path_info
-            )
-
-    def checksum(self, path_info):
-        return self.info(path_info)["sha"]
-
-    def walk_files(self, path_info, **kwargs):
-        for root, _, files in self.walk(path_info, **kwargs):
-            for file in files:
-                # NOTE: os.path.join is ~5.5 times slower
-                yield f"{root}{os.sep}{file}"
-
-    def get_file(
-        self, from_info, to_file, callback=DEFAULT_CALLBACK, **kwargs
+    def walk(
+        self,
+        top: AnyFSPath,
+        topdown: bool = True,
+        onerror: Callable[[OSError], None] = None,
+        **kwargs: Any,
     ):
-        import shutil
-
-        total = self.getsize(from_info)
-        if total:
-            callback.set_size(total)
-
-        with self.open(from_info, "rb", **kwargs) as from_fobj:
-            with open(to_file, "wb+") as to_fobj:
-                wrapped = CallbackIOWrapper(
-                    callback.relative_update, from_fobj
-                )
-                shutil.copyfileobj(wrapped, to_fobj)
+        return self.fs.walk(top, topdown=topdown, onerror=onerror, **kwargs)

dvc/fs/gs.py

@@ -1,9 +1,7 @@
-import base64
 import threading
 
 from funcy import cached_property, wrap_prop
 
-from dvc.path_info import CloudURLInfo
 from dvc.scheme import Schemes
 
 # pylint:disable=abstract-method
@@ -12,7 +10,6 @@ from .fsspec_wrapper import CallbackMixin, ObjectFSWrapper
 
 class GSFileSystem(CallbackMixin, ObjectFSWrapper):
     scheme = Schemes.GS
-    PATH_CLS = CloudURLInfo
     REQUIRES = {"gcsfs": "gcsfs"}
     PARAM_CHECKSUM = "etag"
     DETAIL_FIELDS = frozenset(("etag", "size"))
@@ -23,11 +20,6 @@ class GSFileSystem(CallbackMixin, ObjectFSWrapper):
         login_info["token"] = config.get("credentialpath")
         return login_info
 
-    def _entry_hook(self, entry):
-        if "etag" in entry:
-            entry["etag"] = base64.b64decode(entry["etag"]).hex()
-        return entry
-
     @wrap_prop(threading.Lock())
     @cached_property
     def fs(self):

dvc/fs/hdfs.py

@@ -19,10 +19,17 @@ class HDFSFileSystem(CallbackMixin, FSSpecWrapper):
     REQUIRES = {"fsspec": "fsspec", "pyarrow": "pyarrow"}
     PARAM_CHECKSUM = "checksum"
 
-    def _with_bucket(self, path):
-        if isinstance(path, self.PATH_CLS):
-            return path.path
-        return super()._with_bucket(path)
+    @classmethod
+    def _strip_protocol(cls, path: str) -> str:
+        from fsspec.utils import infer_storage_options
+
+        return infer_storage_options(path)["path"].lstrip("/")
+
+    def unstrip_protocol(self, path: str) -> str:
+        host = self.fs_args["host"]
+        port = self.fs_args["port"]
+        path = path.lstrip("/")
+        return f"hdfs://{host}:{port}/{path}"
 
     @staticmethod
     def _get_kwargs_from_urls(urlpath):
@@ -41,17 +48,19 @@ class HDFSFileSystem(CallbackMixin, FSSpecWrapper):
 
         return HadoopFileSystem(**self.fs_args)
 
-    def checksum(self, path_info):
-        return self._checksum(path_info)
+    def checksum(self, path):
+        return self._checksum(path)
 
-    def _checksum(self, path_info, **kwargs):
+    def _checksum(self, path, **kwargs):
         # PyArrow doesn't natively support retrieving the
         # checksum, so we have to use hadoop fs
 
+        url = self.unstrip_protocol(path)
+
         result = self._run_command(
-            f"checksum {path_info.url}",
+            f"checksum {url}",
             env=fix_env(os.environ),
-            user=path_info.user,
+            user=self.fs_args["user"],
         )
         if result is None:
             return None

dvc/fs/http.py

@@ -1,12 +1,13 @@
 import threading
+from typing import Any
 
 from funcy import cached_property, memoize, wrap_with
 
 from dvc import prompt
-from dvc.path_info import HTTPURLInfo
+from dvc.progress import DEFAULT_CALLBACK
 from dvc.scheme import Schemes
 
-from .fsspec_wrapper import FSSpecWrapper, NoDirectoriesMixin
+from .fsspec_wrapper import AnyFSPath, FSSpecWrapper, NoDirectoriesMixin
 
 
 @wrap_with(threading.Lock())
@@ -34,7 +35,6 @@ def make_context(ssl_verify):
 # pylint: disable=abstract-method
 class HTTPFileSystem(NoDirectoriesMixin, FSSpecWrapper):
     scheme = Schemes.HTTP
-    PATH_CLS = HTTPURLInfo
     PARAM_CHECKSUM = "checksum"
     REQUIRES = {"aiohttp": "aiohttp", "aiohttp-retry": "aiohttp_retry"}
     CAN_TRAVERSE = False
@@ -127,6 +127,15 @@ class HTTPFileSystem(NoDirectoriesMixin, FSSpecWrapper):
 
         return _HTTPFileSystem(**self.fs_args)
 
-    def _entry_hook(self, entry):
-        entry["checksum"] = entry.get("ETag") or entry.get("Content-MD5")
-        return entry
+    def unstrip_protocol(self, path: str) -> str:
+        return path
+
+    def put_file(
+        self,
+        from_file: AnyFSPath,
+        to_info: AnyFSPath,
+        callback: Any = DEFAULT_CALLBACK,
+        **kwargs,
+    ) -> None:
+        kwargs.setdefault("method", self.upload_method)
+        super().put_file(from_file, to_info, callback=callback, **kwargs)

dvc/fs/local.py

@@ -1,23 +1,21 @@
 import logging
 import os
 
-from dvc.path_info import PathInfo
 from dvc.scheme import Schemes
 from dvc.system import System
 from dvc.utils import is_exec, tmp_fname
 from dvc.utils.fs import copy_fobj_to_file, copyfile, makedirs, move, remove
 
 from ..progress import DEFAULT_CALLBACK
-from .base import BaseFileSystem
+from .base import FileSystem
 
 logger = logging.getLogger(__name__)
 
 
-class LocalFileSystem(BaseFileSystem):
+class LocalFileSystem(FileSystem):
     sep = os.sep
 
     scheme = Schemes.LOCAL
-    PATH_CLS = PathInfo
     PARAM_CHECKSUM = "md5"
     PARAM_PATH = "path"
     TRAVERSE_PREFIX_LEN = 2
@@ -29,32 +27,29 @@ class LocalFileSystem(BaseFileSystem):
         self.fs = LocalFS()
 
     @staticmethod
-    def open(path_info, mode="r", encoding=None, **kwargs):
-        return open(path_info, mode=mode, encoding=encoding)
+    def open(path, mode="r", encoding=None, **kwargs):
+        return open(path, mode=mode, encoding=encoding)
 
-    def exists(self, path_info) -> bool:
-        assert isinstance(path_info, str) or path_info.scheme == "local"
+    def exists(self, path) -> bool:
         # TODO: replace this with os.path.exists once the problem is fixed on
         # the fsspec https://github.com/intake/filesystem_spec/issues/742
-        return os.path.lexists(path_info)
+        return os.path.lexists(path)
 
-    def checksum(self, path_info) -> str:
+    def checksum(self, path) -> str:
         from fsspec.utils import tokenize
 
-        st = os.stat(path_info)
+        st = os.stat(path)
 
         return str(int(tokenize([st.st_ino, st.st_mtime, st.st_size]), 16))
 
-    def isfile(self, path_info) -> bool:
-        return os.path.isfile(path_info)
+    def isfile(self, path) -> bool:
+        return os.path.isfile(path)
 
-    def isdir(self, path_info) -> bool:
-        return os.path.isdir(path_info)
+    def isdir(self, path) -> bool:
+        return os.path.isdir(path)
 
-    def iscopy(self, path_info):
-        return not (
-            System.is_symlink(path_info) or System.is_hardlink(path_info)
-        )
+    def iscopy(self, path):
+        return not (System.is_symlink(path) or System.is_hardlink(path))
 
     def walk(self, top, topdown=True, onerror=None, **kwargs):
         """Directory fs generator.
@@ -67,45 +62,37 @@ class LocalFileSystem(BaseFileSystem):
         ):
             yield os.path.normpath(root), dirs, files
 
-    def walk_files(self, path_info, **kwargs):
-        for root, _, files in self.walk(path_info):
+    def find(self, path, prefix=None):
+        for root, _, files in self.walk(path):
             for file in files:
                 # NOTE: os.path.join is ~5.5 times slower
-                yield PathInfo(f"{root}{os.sep}{file}")
+                yield f"{root}{os.sep}{file}"
 
-    def is_empty(self, path_info):
-        if self.isfile(path_info) and os.path.getsize(path_info) == 0:
+    def is_empty(self, path):
+        if self.isfile(path) and os.path.getsize(path) == 0:
             return True
 
-        if self.isdir(path_info) and len(os.listdir(path_info)) == 0:
+        if self.isdir(path) and len(os.listdir(path)) == 0:
             return True
 
         return False
 
-    def remove(self, path_info):
-        if isinstance(path_info, PathInfo):
-            if path_info.scheme != "local":
-                raise NotImplementedError
-        remove(path_info)
+    def remove(self, path):
+        remove(path)
 
-    def makedirs(self, path_info, **kwargs):
-        makedirs(path_info, exist_ok=kwargs.pop("exist_ok", True))
+    def makedirs(self, path, **kwargs):
+        makedirs(path, exist_ok=kwargs.pop("exist_ok", True))
 
-    def isexec(self, path_info):
-        mode = self.info(path_info)["mode"]
+    def isexec(self, path):
+        mode = self.info(path)["mode"]
         return is_exec(mode)
 
     def move(self, from_info, to_info):
-        if (
-            isinstance(from_info, PathInfo) and from_info.scheme != "local"
-        ) or (isinstance(to_info, PathInfo) and to_info.scheme != "local"):
-            raise NotImplementedError
-
-        self.makedirs(to_info.parent)
+        self.makedirs(self.path.parent(to_info))
         move(from_info, to_info)
 
     def copy(self, from_info, to_info):
-        tmp_info = to_info.parent / tmp_fname("")
+        tmp_info = self.path.join(self.path.parent(to_info), tmp_fname(""))
         try:
             copyfile(from_info, tmp_info)
             os.rename(tmp_info, to_info)
@@ -114,8 +101,8 @@ class LocalFileSystem(BaseFileSystem):
             raise
 
     def upload_fobj(self, fobj, to_info, **kwargs):
-        self.makedirs(to_info.parent)
-        tmp_info = to_info.parent / tmp_fname("")
+        self.makedirs(self.path.parent(to_info))
+        tmp_info = self.path.join(self.path.parent(to_info), tmp_fname(""))
         try:
             copy_fobj_to_file(fobj, tmp_info)
             os.rename(tmp_info, to_info)
@@ -128,8 +115,8 @@ class LocalFileSystem(BaseFileSystem):
         System.symlink(from_info, to_info)
 
     @staticmethod
-    def is_symlink(path_info):
-        return System.is_symlink(path_info)
+    def is_symlink(path):
+        return System.is_symlink(path)
 
     def hardlink(self, from_info, to_info):
         # If there are a lot of empty files (which happens a lot in datasets),
@@ -154,20 +141,21 @@ class LocalFileSystem(BaseFileSystem):
         System.hardlink(from_info, to_info)
 
     @staticmethod
-    def is_hardlink(path_info):
-        return System.is_hardlink(path_info)
+    def is_hardlink(path):
+        return System.is_hardlink(path)
 
     def reflink(self, from_info, to_info):
         System.reflink(from_info, to_info)
 
-    def info(self, path_info):
-        return self.fs.info(path_info)
+    def info(self, path):
+        return self.fs.info(path)
 
     def put_file(
         self, from_file, to_info, callback=DEFAULT_CALLBACK, **kwargs
     ):
-        makedirs(to_info.parent, exist_ok=True)
-        tmp_file = tmp_fname(to_info)
+        parent = self.path.parent(to_info)
+        makedirs(parent, exist_ok=True)
+        tmp_file = self.path.join(parent, tmp_fname())
         copyfile(from_file, tmp_file, callback=callback)
         os.replace(tmp_file, to_info)
 
@@ -175,3 +163,6 @@ class LocalFileSystem(BaseFileSystem):
         self, from_info, to_file, callback=DEFAULT_CALLBACK, **kwargs
     ):
         copyfile(from_info, to_file, callback=callback)
+
+
+localfs = LocalFileSystem()

dvc/fs/memory.py

@@ -2,7 +2,6 @@ import threading
 
 from funcy import cached_property, wrap_prop
 
-from dvc.path_info import CloudURLInfo
 from dvc.scheme import Schemes
 
 from .fsspec_wrapper import FSSpecWrapper
@@ -11,7 +10,6 @@ from .fsspec_wrapper import FSSpecWrapper
 class MemoryFileSystem(FSSpecWrapper):  # pylint:disable=abstract-method
     scheme = Schemes.MEMORY
     PARAM_CHECKSUM = "md5"
-    PATH_CLS = CloudURLInfo
     TRAVERSE_PREFIX_LEN = 2
     DEFAULT_BLOCKSIZE = 4096
 

dvc/fs/oss.py

@@ -4,7 +4,6 @@ import threading
 
 from funcy import cached_property, wrap_prop
 
-from dvc.path_info import CloudURLInfo
 from dvc.scheme import Schemes
 
 from .fsspec_wrapper import ObjectFSWrapper
@@ -15,7 +14,6 @@ logger = logging.getLogger(__name__)
 # pylint:disable=abstract-method
 class OSSFileSystem(ObjectFSWrapper):
     scheme = Schemes.OSS
-    PATH_CLS = CloudURLInfo
     REQUIRES = {"ossfs": "ossfs"}
     PARAM_CHECKSUM = "etag"
     COPY_POLL_SECONDS = 5

dvc/fs/path.py

@@ -0,0 +1,94 @@
+import ntpath
+import posixpath
+
+
+class Path:
+    def __init__(self, sep):
+        if sep == posixpath.sep:
+            self.flavour = posixpath
+        elif sep == ntpath.sep:
+            self.flavour = ntpath
+        else:
+            raise ValueError(f"unsupported separator '{sep}'")
+
+    def join(self, *parts):
+        return self.flavour.join(*parts)
+
+    def parts(self, path):
+        drive, path = self.flavour.splitdrive(path)
+
+        ret = []
+        while True:
+            path, part = self.flavour.split(path)
+
+            if part:
+                ret.append(part)
+                continue
+
+            if path:
+                ret.append(path)
+
+            break
+
+        ret.reverse()
+
+        if drive:
+            ret = [drive] + ret
+
+        return tuple(ret)
+
+    def parent(self, path):
+        return self.flavour.dirname(path)
+
+    def parents(self, path):
+        parts = self.parts(path)
+        return tuple(
+            self.join(*parts[:length])
+            for length in range(len(parts) - 1, 0, -1)
+        )
+
+    def name(self, path):
+        return self.parts(path)[-1]
+
+    def suffix(self, path):
+        name = self.name(path)
+        _, dot, suffix = name.partition(".")
+        return dot + suffix
+
+    def with_name(self, path, name):
+        parts = list(self.parts(path))
+        parts[-1] = name
+        return self.join(*parts)
+
+    def with_suffix(self, path, suffix):
+        parts = list(self.parts(path))
+        real_path, _, _ = parts[-1].partition(".")
+        parts[-1] = real_path + suffix
+        return self.join(*parts)
+
+    def isin(self, left, right):
+        left_parts = self.parts(left)
+        right_parts = self.parts(right)
+        left_len = len(left_parts)
+        right_len = len(right_parts)
+        return left_len > right_len and left_parts[:right_len] == right_parts
+
+    def isin_or_eq(self, left, right):
+        return left == right or self.isin(left, right)
+
+    def overlaps(self, left, right):
+        # pylint: disable=arguments-out-of-order
+        return self.isin_or_eq(left, right) or self.isin(right, left)
+
+    def relpath(self, path, base):
+        assert len(path) > len(base)
+        assert path.startswith(base)
+        normpath = path.rstrip(self.flavour.sep)
+        normbase = base.rstrip(self.flavour.sep)
+        return normpath[len(normbase) + 1 :]
+
+    def relparts(self, path, base):
+        return self.parts(self.relpath(path, base))
+
+    def as_posix(self, path):
+        return path.replace(self.flavour.sep, posixpath.sep)

dvc/fs/repo.py

@@ -7,10 +7,8 @@ from typing import TYPE_CHECKING, Callable, Optional, Tuple, Type, Union
 
 from funcy import lfilter, wrap_with
 
-from dvc.path_info import PathInfo
-
 from ..progress import DEFAULT_CALLBACK
-from .base import BaseFileSystem
+from .base import FileSystem
 from .dvc import DvcFileSystem
 
 if TYPE_CHECKING:
@@ -21,7 +19,7 @@ logger = logging.getLogger(__name__)
 RepoFactory = Union[Callable[[str], "Repo"], Type["Repo"]]
 
 
-class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
+class RepoFileSystem(FileSystem):  # pylint:disable=abstract-method
     """DVC + git-tracked files fs.
 
     Args:
@@ -164,7 +162,7 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         if not prefix:
             return None
 
-        parents = (parent.fspath for parent in PathInfo(path).parents)
+        parents = (parent for parent in self.path.parents(path))
         dirs = [path] + list(takewhile(lambda p: p != prefix, parents))
         dirs.reverse()
         self._update(dirs, starting_repo=repo)
@@ -195,9 +193,7 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         repo_path = os.path.join(dir_path, Repo.DVC_DIR)
         return self._main_repo.fs.isdir(repo_path)
 
-    def _get_fs_pair(
-        self, path
-    ) -> Tuple[BaseFileSystem, Optional[DvcFileSystem]]:
+    def _get_fs_pair(self, path) -> Tuple[FileSystem, Optional[DvcFileSystem]]:
         """
         Returns a pair of fss based on repo the path falls in, using prefix.
         """
@@ -217,34 +213,33 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             encoding = None
 
         fs, dvc_fs = self._get_fs_pair(path)
-        path_info = PathInfo(path)
         try:
-            return fs.open(path_info, mode=mode, encoding=encoding)
+            return fs.open(path, mode=mode, encoding=encoding)
         except FileNotFoundError:
             if not dvc_fs:
                 raise
 
-        return dvc_fs.open(path_info, mode=mode, encoding=encoding, **kwargs)
+        return dvc_fs.open(path, mode=mode, encoding=encoding, **kwargs)
 
-    def exists(self, path_info) -> bool:
-        fs, dvc_fs = self._get_fs_pair(path_info)
+    def exists(self, path) -> bool:
+        fs, dvc_fs = self._get_fs_pair(path)
 
         if not dvc_fs:
-            return fs.exists(path_info)
+            return fs.exists(path)
 
-        if dvc_fs.repo.dvcignore.is_ignored(fs, path_info):
+        if dvc_fs.repo.dvcignore.is_ignored(fs, path):
             return False
 
-        if fs.exists(path_info):
+        if fs.exists(path):
             return True
 
         try:
-            meta = dvc_fs.metadata(path_info)
+            meta = dvc_fs.metadata(path)
         except FileNotFoundError:
             return False
 
         for out in meta.outs:
-            if fs.exists(out.path_info):
+            if fs.exists(out.fs_path):
                 return False
 
         return True
@@ -271,7 +266,7 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             return False
 
         for out in meta.outs:
-            if fs.exists(out.path_info):
+            if fs.exists(out.fs_path):
                 return False
 
         return meta.isdir
@@ -303,15 +298,15 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
 
         (out,) = meta.outs
         assert len(meta.outs) == 1
-        if fs.exists(out.path_info):
+        if fs.exists(out.fs_path):
             return False
         return meta.isfile
 
-    def isexec(self, path_info):
-        fs, dvc_fs = self._get_fs_pair(path_info)
-        if dvc_fs and dvc_fs.exists(path_info):
-            return dvc_fs.isexec(path_info)
-        return fs.isexec(path_info)
+    def isexec(self, path):
+        fs, dvc_fs = self._get_fs_pair(path)
+        if dvc_fs and dvc_fs.exists(path):
+            return dvc_fs.isexec(path)
+        return fs.isexec(path)
 
     def _dvc_walk(self, walk):
         try:
@@ -444,10 +439,10 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
 
         yield from self._walk(repo_walk, dvc_walk, dvcfiles=dvcfiles)
 
-    def walk_files(self, path_info, **kwargs):
-        for root, _, files in self.walk(path_info, **kwargs):
+    def find(self, path, prefix=None):
+        for root, _, files in self.walk(path):
             for fname in files:
-                yield PathInfo(root) / fname
+                yield self.path.join(root, fname)
 
     def get_file(
         self, from_info, to_file, callback=DEFAULT_CALLBACK, **kwargs
@@ -467,18 +462,17 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         )
 
     def metadata(self, path):
-        abspath = os.path.abspath(path)
-        path_info = PathInfo(abspath)
-        fs, dvc_fs = self._get_fs_pair(path_info)
+        fs_path = os.path.abspath(path)
+        fs, dvc_fs = self._get_fs_pair(fs_path)
 
         dvc_meta = None
         if dvc_fs:
             with suppress(FileNotFoundError):
-                dvc_meta = dvc_fs.metadata(path_info)
+                dvc_meta = dvc_fs.metadata(fs_path)
 
         info_result = None
         with suppress(FileNotFoundError):
-            info_result = fs.info(path_info)
+            info_result = fs.info(fs_path)
 
         if not info_result and not dvc_meta:
             raise FileNotFoundError
@@ -486,8 +480,8 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         from ._metadata import Metadata
 
         meta = dvc_meta or Metadata(
-            path_info=path_info,
-            repo=self._get_repo(abspath) or self._main_repo,
+            fs_path=fs_path,
+            repo=self._get_repo(fs_path) or self._main_repo,
         )
 
         isdir = bool(info_result) and info_result["type"] == "directory"
@@ -499,18 +493,18 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             meta.is_exec = bool(info_result) and is_exec(info_result["mode"])
         return meta
 
-    def info(self, path_info):
-        fs, dvc_fs = self._get_fs_pair(path_info)
+    def info(self, path):
+        fs, dvc_fs = self._get_fs_pair(path)
 
         try:
-            return fs.info(path_info)
+            return fs.info(path)
         except FileNotFoundError:
-            return dvc_fs.info(path_info)
+            return dvc_fs.info(path)
 
-    def checksum(self, path_info):
-        fs, dvc_fs = self._get_fs_pair(path_info)
+    def checksum(self, path):
+        fs, dvc_fs = self._get_fs_pair(path)
 
         try:
-            return fs.checksum(path_info)
+            return fs.checksum(path)
         except FileNotFoundError:
-            return dvc_fs.checksum(path_info)
+            return dvc_fs.checksum(path)

dvc/fs/s3.py

@@ -5,7 +5,6 @@ from collections import defaultdict
 
 from funcy import cached_property, wrap_prop
 
-from dvc.path_info import CloudURLInfo
 from dvc.progress import DEFAULT_CALLBACK
 from dvc.scheme import Schemes
 
@@ -17,7 +16,6 @@ _AWS_CONFIG_PATH = os.path.join(os.path.expanduser("~"), ".aws", "config")
 # pylint:disable=abstract-method
 class BaseS3FileSystem(ObjectFSWrapper):
     scheme = Schemes.S3
-    PATH_CLS = CloudURLInfo
     REQUIRES = {"s3fs": "s3fs", "boto3": "boto3"}
     PARAM_CHECKSUM = "etag"
     DETAIL_FIELDS = frozenset(("etag", "size"))
@@ -148,12 +146,6 @@ class BaseS3FileSystem(ObjectFSWrapper):
             }
         )
 
-    def _entry_hook(self, entry):
-        entry = entry.copy()
-        if "ETag" in entry:
-            entry["etag"] = entry["ETag"].strip('"')
-        return entry
-
     @wrap_prop(threading.Lock())
     @cached_property
     def fs(self):
@@ -161,6 +153,15 @@ class BaseS3FileSystem(ObjectFSWrapper):
 
         return _S3FileSystem(**self.fs_args)
 
+    @classmethod
+    def _strip_protocol(cls, path: str) -> str:
+        from fsspec.utils import infer_storage_options
+
+        return infer_storage_options(path)["path"]
+
+    def unstrip_protocol(self, path):
+        return "s3://" + path.lstrip("/")
+
 
 def _translate_exceptions(func):
     @functools.wraps(func)
@@ -204,9 +205,10 @@ class S3FileSystem(BaseS3FileSystem):  # pylint:disable=abstract-method
             verify=client_kwargs.get("verify"),
         )
 
-    def _get_obj(self, path_info):
-        bucket = self.s3.Bucket(path_info.bucket)
-        return bucket.Object(path_info.path)
+    def _get_obj(self, path):
+        bucket_name, key, _ = self.fs.split_path(path)
+        bucket = self.s3.Bucket(bucket_name)
+        return bucket.Object(key)
 
     @_translate_exceptions
     def put_file(
@@ -220,7 +222,7 @@ class S3FileSystem(BaseS3FileSystem):  # pylint:disable=abstract-method
             ExtraArgs=self.fs_args.get("s3_additional_kwargs"),
             Config=self._transfer_config,
         )
-        self.fs.invalidate_cache(self._with_bucket(to_info.parent))
+        self.fs.invalidate_cache(self.path.parent(to_info))
 
     @_translate_exceptions
     def get_file(

dvc/fs/ssh.py

@@ -34,10 +34,17 @@ class SSHFileSystem(FSSpecWrapper):
     DEFAULT_PORT = 22
     PARAM_CHECKSUM = "md5"
 
-    def _with_bucket(self, path):
-        if isinstance(path, self.PATH_CLS):
-            return path.path
-        return super()._with_bucket(path)
+    @classmethod
+    def _strip_protocol(cls, path: str) -> str:
+        from fsspec.utils import infer_storage_options
+
+        return infer_storage_options(path)["path"]
+
+    def unstrip_protocol(self, path: str) -> str:
+        host = self.fs_args["host"]
+        port = self.fs_args["port"]
+        path = path.lstrip("/")
+        return f"ssh://{host}:{port}/{path}"
 
     def _prepare_credentials(self, **config):
         self.CAN_TRAVERSE = True

dvc/fs/utils.py

@@ -1,73 +1,99 @@
 import errno
 import logging
 import os
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, List, Optional
 
 from .base import RemoteActionNotImplemented
 from .local import LocalFileSystem
 
 if TYPE_CHECKING:
-    from dvc.types import AnyPath, DvcPath
-
-    from .base import BaseFileSystem
+    from .base import FileSystem
+    from .types import AnyPath
 
 logger = logging.getLogger(__name__)
 
 
 def _link(
-    from_fs: "BaseFileSystem",
-    from_info: "AnyPath",
-    to_fs: "BaseFileSystem",
-    to_info: "DvcPath",
-    hardlink: bool = False,
+    link: "AnyPath",
+    from_fs: "FileSystem",
+    from_path: "AnyPath",
+    to_fs: "FileSystem",
+    to_path: "AnyPath",
 ) -> None:
     if not isinstance(from_fs, type(to_fs)):
         raise OSError(errno.EXDEV, "can't link across filesystems")
 
-    links = ["reflink"] + ["hardlink"] if hardlink else []
+    try:
+        func = getattr(to_fs, link)
+        func(from_path, to_path)
+    except (OSError, AttributeError, RemoteActionNotImplemented) as exc:
+        # NOTE: there are so many potential error codes that a link call can
+        # throw, that is safer to just catch them all and try another link.
+        raise OSError(
+            errno.ENOTSUP, f"'{link}' is not supported by {type(from_fs)}"
+        ) from exc
+
+
+def _copy(
+    from_fs: "FileSystem",
+    from_path: "AnyPath",
+    to_fs: "FileSystem",
+    to_path: "AnyPath",
+) -> None:
+    if isinstance(from_fs, LocalFileSystem):
+        return to_fs.upload(from_path, to_path)
+
+    if isinstance(to_fs, LocalFileSystem):
+        return from_fs.download_file(from_path, to_path)
+
+    with from_fs.open(from_path, mode="rb") as fobj:
+        size = from_fs.getsize(from_path)
+        return to_fs.upload(fobj, to_path, total=size)
+
+
+def _try_links(
+    links: List["AnyPath"],
+    from_fs: "FileSystem",
+    from_path: "AnyPath",
+    to_fs: "FileSystem",
+    to_path: "AnyPath",
+) -> None:
+    error = None
     while links:
-        link = links.pop(0)
-        try:
-            func = getattr(to_fs, link)
-        except AttributeError:
-            continue
+        link = links[0]
+
+        if link == "copy":
+            return _copy(from_fs, from_path, to_fs, to_path)
 
         try:
-            return func(from_info, to_info)
-        except RemoteActionNotImplemented:
-            continue
+            return _link(link, from_fs, from_path, to_fs, to_path)
         except OSError as exc:
-            if exc.errno not in [errno.EXDEV, errno.ENOTSUP]:
+            if exc.errno not in [errno.ENOTSUP, errno.EXDEV]:
                 raise
+            error = exc
 
-    raise OSError(errno.ENOTSUP, "reflink and hardlink are not supported")
+        del links[0]
+
+    raise OSError(
+        errno.ENOTSUP, "no more link types left to try out"
+    ) from error
 
 
 def transfer(
-    from_fs: "BaseFileSystem",
-    from_info: "AnyPath",
-    to_fs: "BaseFileSystem",
-    to_info: "DvcPath",
+    from_fs: "FileSystem",
+    from_path: "AnyPath",
+    to_fs: "FileSystem",
+    to_path: "AnyPath",
     hardlink: bool = False,
+    links: Optional[List["AnyPath"]] = None,
 ) -> None:
     try:
-        try:
-            return _link(from_fs, from_info, to_fs, to_info, hardlink=hardlink)
-        except OSError as exc:
-            if exc.errno not in [errno.EXDEV, errno.ENOTSUP]:
-                raise
-
-        if isinstance(from_fs, LocalFileSystem):
-            if not isinstance(from_info, from_fs.PATH_CLS):
-                from_info = from_fs.PATH_CLS(from_info)
-            return to_fs.upload(from_info, to_info)
-
-        if isinstance(to_fs, LocalFileSystem):
-            return from_fs.download_file(from_info, to_info)
-
-        with from_fs.open(from_info, mode="rb") as fobj:
-            size = from_fs.getsize(from_info)
-            return to_fs.upload(fobj, to_info, total=size)
+        assert not (hardlink and links)
+        if hardlink:
+            links = links or ["reflink", "hardlink", "copy"]
+        else:
+            links = links or ["reflink", "copy"]
+        _try_links(links, from_fs, from_path, to_fs, to_path)
     except OSError as exc:
         # If the target file already exists, we are going to simply
         # ignore the exception (#4992).
@@ -81,7 +107,62 @@ def transfer(
             and exc.__context__
             and isinstance(exc.__context__, FileExistsError)
         ):
-            logger.debug("'%s' file already exists, skipping", to_info)
+            logger.debug("'%s' file already exists, skipping", to_path)
             return None
 
         raise
+
+
+def _test_link(
+    link: "AnyPath",
+    from_fs: "FileSystem",
+    from_file: "AnyPath",
+    to_fs: "FileSystem",
+    to_file: "AnyPath",
+) -> bool:
+    try:
+        _try_links([link], from_fs, from_file, to_fs, to_file)
+    except OSError:
+        logger.debug("", exc_info=True)
+        return False
+
+    try:
+        _is_link_func = getattr(to_fs, f"is_{link}")
+        return _is_link_func(to_file)
+    except AttributeError:
+        pass
+
+    return True
+
+
+def test_links(
+    links: List["AnyPath"],
+    from_fs: "FileSystem",
+    from_path: "AnyPath",
+    to_fs: "FileSystem",
+    to_path: "AnyPath",
+) -> List["AnyPath"]:
+    from dvc.utils import tmp_fname
+
+    from_file = from_fs.path.join(from_path, tmp_fname())
+    to_file = to_fs.path.join(
+        to_fs.path.parent(to_path),
+        tmp_fname(),
+    )
+
+    from_fs.makedirs(from_fs.path.parent(from_file))
+    with from_fs.open(from_file, "wb") as fobj:
+        fobj.write(b"test")
+
+    ret = []
+    try:
+        for link in links:
+            try:
+                if _test_link(link, from_fs, from_file, to_fs, to_file):
+                    ret.append(link)
+            finally:
+                to_fs.remove(to_file)
+    finally:
+        from_fs.remove(from_file)
+
+    return ret

dvc/fs/webdav.py

@@ -1,10 +1,8 @@
 import logging
 import threading
-from functools import lru_cache
 
 from funcy import cached_property, wrap_prop
 
-from dvc.path_info import WebDAVURLInfo
 from dvc.scheme import Schemes
 
 from .fsspec_wrapper import FSSpecWrapper
@@ -15,7 +13,6 @@ logger = logging.getLogger(__name__)
 
 class WebDAVFileSystem(FSSpecWrapper):  # pylint:disable=abstract-method
     scheme = Schemes.WEBDAV
-    PATH_CLS = WebDAVURLInfo
     CAN_TRAVERSE = True
     TRAVERSE_PREFIX_LEN = 2
     REQUIRES = {"webdav4": "webdav4"}
@@ -37,14 +34,35 @@ class WebDAVFileSystem(FSSpecWrapper):  # pylint:disable=abstract-method
             }
         )
 
+    @classmethod
+    def _strip_protocol(cls, path: str) -> str:
+        from fsspec.utils import infer_storage_options
+
+        return infer_storage_options(path)["path"].lstrip("/")
+
+    def unstrip_protocol(self, path: str) -> str:
+        return self.fs_args["base_url"] + path
+
     @staticmethod
     def _get_kwargs_from_urls(urlpath):
-        path_info = WebDAVURLInfo(urlpath)
+        from urllib.parse import urlparse, urlunparse
+
+        parsed = urlparse(urlpath)
+        scheme = parsed.scheme.replace("webdav", "http")
         return {
-            "prefix": path_info.path.rstrip("/"),
-            "host": path_info.replace(path="").url,
-            "url": path_info.url.rstrip("/"),
-            "user": path_info.user,
+            "prefix": parsed.path.rstrip("/"),
+            "host": urlunparse((scheme, parsed.netloc, "", None, None, None)),
+            "url": urlunparse(
+                (
+                    scheme,
+                    parsed.netloc,
+                    parsed.path.rstrip("/"),
+                    None,
+                    None,
+                    None,
+                )
+            ),
+            "user": parsed.username,
         }
 
     def _prepare_credentials(self, **config):
@@ -71,20 +89,11 @@ class WebDAVFileSystem(FSSpecWrapper):  # pylint:disable=abstract-method
         return WebdavFileSystem(**self.fs_args)
 
     def upload_fobj(self, fobj, to_info, **kwargs):
-        rpath = self.translate_path_info(to_info)
         size = kwargs.get("size")
         # using upload_fileobj to directly upload fileobj rather than buffering
         # and using overwrite=True to avoid check for an extra exists call,
         # as caller should ensure that the file does not exist beforehand.
-        return self.fs.upload_fileobj(fobj, rpath, overwrite=True, size=size)
-
-    @lru_cache(512)
-    def translate_path_info(self, path):
-        if isinstance(path, self.PATH_CLS):
-            return path.path[len(self.prefix) :].lstrip("/")
-        return path
-
-    _with_bucket = translate_path_info  # type: ignore
+        return self.fs.upload_fileobj(fobj, to_info, overwrite=True, size=size)
 
 
 class WebDAVSFileSystem(WebDAVFileSystem):  # pylint:disable=abstract-method

dvc/fs/webhdfs.py

@@ -2,7 +2,6 @@ import threading
 
 from funcy import cached_property, wrap_prop
 
-from dvc.path_info import CloudURLInfo
 from dvc.scheme import Schemes
 
 # pylint:disable=abstract-method
@@ -11,14 +10,20 @@ from .fsspec_wrapper import CallbackMixin, FSSpecWrapper
 
 class WebHDFSFileSystem(CallbackMixin, FSSpecWrapper):
     scheme = Schemes.WEBHDFS
-    PATH_CLS = CloudURLInfo
     REQUIRES = {"fsspec": "fsspec"}
     PARAM_CHECKSUM = "checksum"
 
-    def _with_bucket(self, path):
-        if isinstance(path, self.PATH_CLS):
-            return f"/{path.path.rstrip('/')}"
-        return path
+    @classmethod
+    def _strip_protocol(cls, path: str) -> str:
+        from fsspec.utils import infer_storage_options
+
+        return infer_storage_options(path)["path"]
+
+    def unstrip_protocol(self, path: str) -> str:
+        host = self.fs_args["host"]
+        port = self.fs_args["port"]
+        path = path.lstrip("/")
+        return f"webhdfs://{host}:{port}/{path}"
 
     @staticmethod
     def _get_kwargs_from_urls(urlpath):
@@ -46,7 +51,6 @@ class WebHDFSFileSystem(CallbackMixin, FSSpecWrapper):
         fs.session.verify = self._ssl_verify
         return fs
 
-    def checksum(self, path_info):
-        path = self._with_bucket(path_info)
+    def checksum(self, path):
         ukey = self.fs.ukey(path)
         return ukey["bytes"]

dvc/ignore.py

@@ -7,8 +7,8 @@ from itertools import groupby, takewhile
 from pathspec.patterns import GitWildMatchPattern
 from pathspec.util import normalize_file
 
-from dvc.fs.base import BaseFileSystem
-from dvc.path_info import PathInfo
+from dvc.fs.base import FileSystem
+from dvc.fs.local import localfs
 from dvc.pathspec_math import PatternInfo, merge_patterns
 from dvc.scheme import Schemes
 from dvc.types import AnyPath, List, Optional
@@ -267,25 +267,25 @@ class DvcIgnoreFilter:
             dirs, files = ignore_pattern(abs_root, dirs, files)
         return dirs, files
 
-    def walk(self, fs: BaseFileSystem, path_info: AnyPath, **kwargs):
+    def walk(self, fs: FileSystem, path: AnyPath, **kwargs):
         ignore_subrepos = kwargs.pop("ignore_subrepos", True)
         if fs.scheme == Schemes.LOCAL:
-            for root, dirs, files in fs.walk(path_info, **kwargs):
+            for root, dirs, files in fs.walk(path, **kwargs):
                 dirs[:], files[:] = self(
                     root, dirs, files, ignore_subrepos=ignore_subrepos
                 )
                 yield root, dirs, files
         else:
-            yield from fs.walk(path_info, **kwargs)
+            yield from fs.walk(path, **kwargs)
 
-    def walk_files(self, fs: BaseFileSystem, path_info: AnyPath, **kwargs):
+    def find(self, fs: FileSystem, path: AnyPath, **kwargs):
         if fs.scheme == Schemes.LOCAL:
-            for root, _, files in self.walk(fs, path_info, **kwargs):
+            for root, _, files in self.walk(fs, path, **kwargs):
                 for file in files:
                     # NOTE: os.path.join is ~5.5 times slower
-                    yield PathInfo(f"{root}{os.sep}{file}")
+                    yield f"{root}{os.sep}{file}"
         else:
-            yield from fs.walk_files(path_info)
+            yield from fs.find(path)
 
     def _get_trie_pattern(
         self, dirname, dnames: Optional["List"] = None, ignore_subrepos=True
@@ -307,7 +307,7 @@ class DvcIgnoreFilter:
         dirs = list(
             takewhile(
                 lambda path: path != prefix,
-                (parent.fspath for parent in PathInfo(dirname).parents),
+                (parent for parent in localfs.path.parents(dirname)),
             )
         )
         dirs.reverse()
@@ -343,8 +343,6 @@ class DvcIgnoreFilter:
         return self._is_ignored(path, False)
 
     def _outside_repo(self, path):
-        path = PathInfo(path)
-
         # paths outside of the repo should be ignored
         path = relpath(path, self.root_dir)
         if path.startswith("..") or (
@@ -373,7 +371,7 @@ class DvcIgnoreFilter:
         return _no_match(target)
 
     def is_ignored(
-        self, fs: BaseFileSystem, path: str, ignore_subrepos: bool = True
+        self, fs: FileSystem, path: str, ignore_subrepos: bool = True
     ) -> bool:
         # NOTE: can't use self.check_ignore(path).match for now, see
         # https://github.com/iterative/dvc/issues/4555

dvc/info.py

@@ -1,18 +1,16 @@
-import errno
 import itertools
 import os
 import pathlib
 import platform
-import uuid
 
 import psutil
 
 from dvc import __version__
 from dvc.exceptions import NotDvcRepoError
 from dvc.fs import FS_MAP, get_fs_cls, get_fs_config
+from dvc.fs.utils import test_links
 from dvc.repo import Repo
-from dvc.scm.base import SCMError
-from dvc.system import System
+from dvc.scm import SCMError
 from dvc.utils import error_link
 from dvc.utils.pkg import PKG
 
@@ -75,7 +73,7 @@ def _get_caches(cache):
 
 def _get_remotes(config):
     schemes = (
-        get_fs_cls(get_fs_config(config, name=remote)).scheme
+        get_fs_cls(get_fs_config(None, config, name=remote)).scheme
         for remote in config["remote"]
     )
 
@@ -83,37 +81,17 @@ def _get_remotes(config):
 
 
 def _get_linktype_support_info(repo):
+    odb = repo.odb.local
 
-    links = {
-        "reflink": (System.reflink, None),
-        "hardlink": (System.hardlink, System.is_hardlink),
-        "symlink": (System.symlink, System.is_symlink),
-    }
+    links = test_links(
+        ["reflink", "hardlink", "symlink"],
+        odb.fs,
+        odb.fs_path,
+        repo.fs,
+        repo.root_dir,
+    )
 
-    fname = "." + str(uuid.uuid4())
-    src = os.path.join(repo.odb.local.cache_dir, fname)
-    open(src, "w", encoding="utf-8").close()
-    dst = os.path.join(repo.root_dir, fname)
-
-    cache = []
-
-    for name, (link, is_link) in links.items():
-        try:
-            link(src, dst)
-            status = "supported"
-            if is_link and not is_link(dst):
-                status = "broken"
-            os.unlink(dst)
-        except OSError as exc:
-            if exc.errno != errno.ENOTSUP:
-                raise
-            status = "not supported"
-
-        if status == "supported":
-            cache.append(name)
-    os.remove(src)
-
-    return ", ".join(cache)
+    return ", ".join(links)
 
 
 def _get_supported_remotes():

dvc/objects/checkout.py

@@ -1,16 +1,9 @@
-import errno
 import logging
 from itertools import chain
 
-from shortuuid import uuid
-
 from dvc import prompt
-from dvc.exceptions import (
-    CacheLinkError,
-    CheckoutError,
-    ConfirmRemoveError,
-    DvcException,
-)
+from dvc.exceptions import CacheLinkError, CheckoutError, ConfirmRemoveError
+from dvc.fs.utils import test_links, transfer
 from dvc.ignore import DvcIgnoreFilter
 from dvc.objects.db.slow_link_detection import (  # type: ignore[attr-defined]
     slow_link_guard,
@@ -23,106 +16,29 @@ from dvc.types import Optional
 logger = logging.getLogger(__name__)
 
 
-def _remove(path_info, fs, in_cache, force=False):
-    if not fs.exists(path_info):
+def _remove(fs_path, fs, in_cache, force=False):
+    if not fs.exists(fs_path):
         return
 
     if force:
-        fs.remove(path_info)
+        fs.remove(fs_path)
         return
 
     if not in_cache:
         msg = (
-            f"file/directory '{path_info}' is going to be removed. "
+            f"file/directory '{fs_path}' is going to be removed. "
             "Are you sure you want to proceed?"
         )
 
         if not prompt.confirm(msg):
-            raise ConfirmRemoveError(str(path_info))
+            raise ConfirmRemoveError(fs_path)
 
-    fs.remove(path_info)
+    fs.remove(fs_path)
 
 
-def _verify_link(cache, path_info, link_type):
-    if link_type == "hardlink" and cache.fs.getsize(path_info) == 0:
-        return
-
-    if cache.cache_type_confirmed:
-        return
-
-    is_link = getattr(cache.fs, f"is_{link_type}", None)
-    if is_link and not is_link(path_info):
-        cache.fs.remove(path_info)
-        raise DvcException(f"failed to verify {link_type}")
-
-    cache.cache_type_confirmed = True
-
-
-def _do_link(cache, from_info, to_info, link_method):
-    if cache.fs.exists(to_info):
-        cache.fs.remove(to_info)  # broken symlink
-
-    link_method(from_info, to_info)
-
-    logger.debug(
-        "Created '%s': %s -> %s", cache.cache_types[0], from_info, to_info
-    )
-
-
-@slow_link_guard
-def _try_links(cache, from_info, to_info, link_types):
-    while link_types:
-        link_method = getattr(cache.fs, link_types[0])
-        try:
-            _do_link(cache, from_info, to_info, link_method)
-            _verify_link(cache, to_info, link_types[0])
-            return
-
-        except OSError as exc:
-            if exc.errno not in [errno.EXDEV, errno.ENOTSUP]:
-                raise
-            logger.debug(
-                "Cache type '%s' is not supported: %s", link_types[0], exc
-            )
-            del link_types[0]
-
-    raise CacheLinkError([to_info])
-
-
-def _link(cache, from_info, to_info):
-    cache.makedirs(to_info.parent)
-    try:
-        _try_links(cache, from_info, to_info, cache.cache_types)
-    except FileNotFoundError as exc:
-        raise CheckoutError([str(to_info)]) from exc
-
-
-def _confirm_cache_type(cache, path_info):
-    """Checks whether cache uses copies."""
-    if cache.cache_type_confirmed:
-        return
-
-    if set(cache.cache_types) <= {"copy"}:
-        return
-
-    workspace_file = path_info.with_name("." + uuid())
-    test_cache_file = cache.path_info / ".cache_type_test_file"
-    if not cache.fs.exists(test_cache_file):
-        cache.makedirs(test_cache_file.parent)
-        with cache.fs.open(test_cache_file, "wb") as fobj:
-            fobj.write(bytes(1))
-    try:
-        _link(cache, test_cache_file, workspace_file)
-    finally:
-        cache.fs.remove(workspace_file)
-        cache.fs.remove(test_cache_file)
-
-    cache.cache_type_confirmed = True
-
-
-def _relink(cache, cache_info, fs, path_info, in_cache, force):
-    _remove(path_info, fs, in_cache, force=force)
-    _link(cache, cache_info, path_info)
+def _relink(link, cache, cache_info, fs, path, in_cache, force):
+    _remove(path, fs, in_cache, force=force)
+    link(cache, cache_info, fs, path)
     # NOTE: Depending on a file system (e.g. on NTFS), `_remove` might reset
     # read-only permissions in order to delete a hardlink to protected object,
     # which will also reset it for the object itself, making it unprotected,
@@ -131,7 +47,8 @@ def _relink(cache, cache_info, fs, path_info, in_cache, force):
 
 
 def _checkout_file(
-    path_info,
+    link,
+    fs_path,
     fs,
     change,
     cache,
@@ -143,47 +60,47 @@ def _checkout_file(
     """The file is changed we need to checkout a new copy"""
     modified = False
 
-    _confirm_cache_type(cache, path_info)
-
-    cache_info = cache.hash_to_path_info(change.new.oid.value)
+    cache_fs_path = cache.hash_to_path(change.new.oid.value)
     if change.old.oid:
         if relink:
-            if fs.iscopy(path_info) and cache.cache_types[0] == "copy":
-                cache.unprotect(path_info)
+            if fs.iscopy(fs_path) and cache.cache_types[0] == "copy":
+                cache.unprotect(fs_path)
             else:
                 _relink(
+                    link,
                     cache,
-                    cache_info,
+                    cache_fs_path,
                     fs,
-                    path_info,
+                    fs_path,
                     change.old.in_cache,
                     force=force,
                 )
         else:
             modified = True
             _relink(
+                link,
                 cache,
-                cache_info,
+                cache_fs_path,
                 fs,
-                path_info,
+                fs_path,
                 change.old.in_cache,
                 force=force,
             )
     else:
-        _link(cache, cache_info, path_info)
+        link(cache, cache_fs_path, fs, fs_path)
         modified = True
 
     if state:
-        state.save(path_info, fs, change.new.oid)
+        state.save(fs_path, fs, change.new.oid)
 
     if progress_callback:
-        progress_callback(str(path_info))
+        progress_callback(fs_path)
 
     return modified
 
 
 def _diff(
-    path_info,
+    fs_path,
     fs,
     obj,
     cache,
@@ -194,7 +111,7 @@ def _diff(
     try:
         _, _, old = stage(
             cache,
-            path_info,
+            fs_path,
             fs,
             obj.hash_info.name if obj else cache.fs.PARAM_CHECKSUM,
             dry_run=True,
@@ -211,9 +128,27 @@ def _diff(
     return diff
 
 
+class Link:
+    def __init__(self, links):
+        self._links = links
+
+    @slow_link_guard
+    def __call__(self, cache, from_path, to_fs, to_path):
+        if to_fs.exists(to_path):
+            to_fs.remove(to_path)  # broken symlink
+
+        cache.makedirs(cache.fs.path.parent(to_path))
+        try:
+            transfer(cache.fs, from_path, to_fs, to_path, links=self._links)
+        except FileNotFoundError as exc:
+            raise CheckoutError([to_path]) from exc
+        except OSError as exc:
+            raise CacheLinkError([to_path]) from exc
+
+
 def _checkout(
     diff,
-    path_info,
+    fs_path,
     fs,
     cache,
     force=False,
@@ -224,20 +159,25 @@ def _checkout(
     if not diff:
         return
 
+    links = test_links(cache.cache_types, cache.fs, cache.fs_path, fs, fs_path)
+    if not links:
+        raise CacheLinkError([fs_path])
+    link = Link(links)
+
     for change in diff.deleted:
         entry_path = (
-            path_info.joinpath(*change.old.key)
+            fs.path.join(fs_path, *change.old.key)
             if change.old.key != ROOT
-            else path_info
+            else fs_path
         )
         _remove(entry_path, fs, change.old.in_cache, force=force)
 
     failed = []
     for change in chain(diff.added, diff.modified):
         entry_path = (
-            path_info.joinpath(*change.new.key)
+            fs.path.join(fs_path, *change.new.key)
             if change.new.key != ROOT
-            else path_info
+            else fs_path
         )
         if change.new.oid.isdir:
             fs.makedirs(entry_path)
@@ -245,6 +185,7 @@ def _checkout(
 
         try:
             _checkout_file(
+                link,
                 entry_path,
                 fs,
                 change,
@@ -262,7 +203,7 @@ def _checkout(
 
 
 def checkout(
-    path_info,
+    fs_path,
     fs,
     obj,
     cache,
@@ -273,12 +214,11 @@ def checkout(
     dvcignore: Optional[DvcIgnoreFilter] = None,
     state=None,
 ):
-
-    if path_info.scheme not in ["local", cache.fs.scheme]:
-        raise NotImplementedError
+    # if scheme(fs_path) not in ["local", cache.fs.scheme]:
+    #    raise NotImplementedError
 
     diff = _diff(
-        path_info,
+        fs_path,
         fs,
         obj,
         cache,
@@ -291,16 +231,16 @@ def checkout(
         if not quiet:
             logger.warning(
                 "No file hash info found for '%s'. It won't be created.",
-                path_info,
+                fs_path,
             )
-        failed.append(str(path_info))
+        failed.append(fs_path)
     elif not diff:
-        logger.trace("Data '%s' didn't change.", path_info)  # type: ignore
+        logger.trace("Data '%s' didn't change.", fs_path)  # type: ignore
 
     try:
         _checkout(
             diff,
-            path_info,
+            fs_path,
             fs,
             cache,
             force=force,
@@ -312,13 +252,13 @@ def checkout(
         failed.extend(exc.target_infos)
 
     if diff and state:
-        state.save_link(path_info, fs)
+        state.save_link(fs_path, fs)
         if not failed:
-            state.save(path_info, fs, obj.hash_info)
+            state.save(fs_path, fs, obj.hash_info)
 
     if failed or not diff:
         if progress_callback and obj:
-            progress_callback(str(path_info), len(obj))
+            progress_callback(fs_path, len(obj))
         if failed:
             raise CheckoutError(failed)
         return

dvc/objects/db/__init__.py

@@ -6,22 +6,22 @@ if TYPE_CHECKING:
     from .index import ObjectDBIndexBase
 
 
-def get_odb(fs, path_info, **config):
+def get_odb(fs, fs_path, **config):
     from .base import ObjectDB
     from .gdrive import GDriveObjectDB
     from .local import LocalObjectDB
     from .oss import OSSObjectDB
 
     if fs.scheme == Schemes.LOCAL:
-        return LocalObjectDB(fs, path_info, **config)
+        return LocalObjectDB(fs, fs_path, **config)
 
     if fs.scheme == Schemes.GDRIVE:
-        return GDriveObjectDB(fs, path_info, **config)
+        return GDriveObjectDB(fs, fs_path, **config)
 
     if fs.scheme == Schemes.OSS:
-        return OSSObjectDB(fs, path_info, **config)
+        return OSSObjectDB(fs, fs_path, **config)
 
-    return ObjectDB(fs, path_info, **config)
+    return ObjectDB(fs, fs_path, **config)
 
 
 def _get_odb(repo, settings):
@@ -30,9 +30,9 @@ def _get_odb(repo, settings):
     if not settings:
         return None
 
-    cls, config, path_info = get_cloud_fs(repo, **settings)
+    cls, config, fs_path = get_cloud_fs(repo, **settings)
     config["tmp_dir"] = repo.tmp_dir
-    return get_odb(cls(**config), path_info, state=repo.state, **config)
+    return get_odb(cls(**config), fs_path, state=repo.state, **config)
 
 
 def get_index(odb) -> "ObjectDBIndexBase":
@@ -43,7 +43,9 @@ def get_index(odb) -> "ObjectDBIndexBase":
     cls = ObjectDBIndex if odb.tmp_dir else ObjectDBIndexNoop
     return cls(
         odb.tmp_dir,
-        hashlib.sha256(odb.path_info.url.encode("utf-8")).hexdigest(),
+        hashlib.sha256(
+            odb.fs.unstrip_protocol(odb.fs_path).encode("utf-8")
+        ).hexdigest(),
         odb.fs.CHECKSUM_DIR_SUFFIX,
     )
 

dvc/objects/db/base.py

@@ -10,9 +10,9 @@ from dvc.objects.file import HashFile
 from dvc.progress import Tqdm
 
 if TYPE_CHECKING:
-    from dvc.fs.base import BaseFileSystem
+    from dvc.fs.base import FileSystem
     from dvc.hash_info import HashInfo
-    from dvc.types import AnyPath, DvcPath
+    from dvc.types import AnyPath
 
 logger = logging.getLogger(__name__)
 
@@ -23,15 +23,14 @@ class ObjectDB:
     DEFAULT_CACHE_TYPES = ["copy"]
     CACHE_MODE: Optional[int] = None
 
-    def __init__(self, fs: "BaseFileSystem", path_info: "AnyPath", **config):
+    def __init__(self, fs: "FileSystem", path: str, **config):
         from dvc.state import StateNoop
 
         self.fs = fs
-        self.path_info = path_info
+        self.fs_path = path
         self.state = config.get("state", StateNoop())
         self.verify = config.get("verify", self.DEFAULT_VERIFY)
         self.cache_types = config.get("type") or copy(self.DEFAULT_CACHE_TYPES)
-        self.cache_type_confirmed = False
         self.slow_link_warning = config.get("slow_link_warning", True)
         self.tmp_dir = config.get("tmp_dir")
         self.read_only = config.get("read_only", False)
@@ -50,26 +49,25 @@ class ObjectDB:
     def __eq__(self, other):
         return (
             self.fs == other.fs
-            and self.path_info == other.path_info
+            and self.fs_path == other.fs_path
             and self.read_only == other.read_only
         )
 
     def __hash__(self):
-        return hash((self.fs.scheme, self.path_info))
+        return hash((self.fs.scheme, self.fs_path))
 
     def exists(self, hash_info: "HashInfo"):
-        return self.fs.exists(self.hash_to_path_info(hash_info.value))
+        return self.fs.exists(self.hash_to_path(hash_info.value))
 
     def move(self, from_info, to_info):
         self.fs.move(from_info, to_info)
 
-    def makedirs(self, path_info):
-        self.fs.makedirs(path_info)
+    def makedirs(self, fs_path):
+        self.fs.makedirs(fs_path)
 
     def get(self, hash_info: "HashInfo"):
         """get raw object"""
         return HashFile(
-            # Prefer string path over PathInfo when possible due to performance
             self.hash_to_path(hash_info.value),
             self.fs,
             hash_info,
@@ -77,15 +75,15 @@ class ObjectDB:
 
     def _add_file(
         self,
-        from_fs: "BaseFileSystem",
+        from_fs: "FileSystem",
         from_info: "AnyPath",
-        to_info: "DvcPath",
+        to_info: "AnyPath",
         _hash_info: "HashInfo",
         hardlink: bool = False,
     ):
         from dvc import fs
 
-        self.makedirs(to_info.parent)
+        self.makedirs(self.fs.path.parent(to_info))
         return fs.utils.transfer(
             from_fs,
             from_info,
@@ -96,8 +94,8 @@ class ObjectDB:
 
     def add(
         self,
-        path_info: "AnyPath",
-        fs: "BaseFileSystem",
+        fs_path: "AnyPath",
+        fs: "FileSystem",
         hash_info: "HashInfo",
         hardlink: bool = False,
         verify: Optional[bool] = None,
@@ -113,35 +111,32 @@ class ObjectDB:
         except (ObjectFormatError, FileNotFoundError):
             pass
 
-        cache_info = self.hash_to_path_info(hash_info.value)
-        self._add_file(fs, path_info, cache_info, hash_info, hardlink=hardlink)
+        cache_fs_path = self.hash_to_path(hash_info.value)
+        self._add_file(
+            fs, fs_path, cache_fs_path, hash_info, hardlink=hardlink
+        )
 
         try:
             if verify:
                 self.check(hash_info, check_hash=True)
-            self.protect(cache_info)
-            self.state.save(cache_info, self.fs, hash_info)
+            self.protect(cache_fs_path)
+            self.state.save(cache_fs_path, self.fs, hash_info)
         except (ObjectFormatError, FileNotFoundError):
             pass
 
-    def hash_to_path_info(self, hash_) -> "DvcPath":
-        return self.path_info / hash_[0:2] / hash_[2:]
-
-    # Override to return path as a string instead of PathInfo for clouds
-    # which support string paths (see local)
     def hash_to_path(self, hash_):
-        return self.hash_to_path_info(hash_)
+        return self.fs.path.join(self.fs_path, hash_[0:2], hash_[2:])
 
-    def protect(self, path_info):  # pylint: disable=unused-argument
+    def protect(self, fs_path):  # pylint: disable=unused-argument
         pass
 
-    def is_protected(self, path_info):  # pylint: disable=unused-argument
+    def is_protected(self, fs_path):  # pylint: disable=unused-argument
         return False
 
-    def unprotect(self, path_info):  # pylint: disable=unused-argument
+    def unprotect(self, fs_path):  # pylint: disable=unused-argument
         pass
 
-    def set_exec(self, path_info):  # pylint: disable=unused-argument
+    def set_exec(self, fs_path):  # pylint: disable=unused-argument
         pass
 
     def check(
@@ -164,52 +159,54 @@ class ObjectDB:
         """
 
         obj = self.get(hash_info)
-        if self.is_protected(obj.path_info):
+        if self.is_protected(obj.fs_path):
             logger.trace(  # type: ignore[attr-defined]
                 "Assuming '%s' is unchanged since it is read-only",
-                obj.path_info,
+                obj.fs_path,
             )
             return
 
         try:
             obj.check(self, check_hash=check_hash)
         except ObjectFormatError:
-            logger.warning("corrupted cache file '%s'.", obj.path_info)
+            logger.warning("corrupted cache file '%s'.", obj.fs_path)
             with suppress(FileNotFoundError):
-                self.fs.remove(obj.path_info)
+                self.fs.remove(obj.fs_path)
             raise
 
         if check_hash:
             # making cache file read-only so we don't need to check it
             # next time
-            self.protect(obj.path_info)
+            self.protect(obj.fs_path)
 
     def _list_paths(self, prefix=None, progress_callback=None):
         if prefix:
             if len(prefix) > 2:
-                path_info = self.path_info / prefix[:2] / prefix[2:]
+                fs_path = self.fs.path.join(
+                    self.fs_path, prefix[:2], prefix[2:]
+                )
             else:
-                path_info = self.path_info / prefix[:2]
+                fs_path = self.fs.path.join(self.fs_path, prefix[:2])
             prefix = True
         else:
-            path_info = self.path_info
+            fs_path = self.fs_path
             prefix = False
         if progress_callback:
-            for file_info in self.fs.walk_files(path_info, prefix=prefix):
+            for file_info in self.fs.find(fs_path, prefix=prefix):
                 progress_callback()
-                yield file_info.path
+                yield file_info
         else:
-            yield from self.fs.walk_files(path_info, prefix=prefix)
+            yield from self.fs.find(fs_path, prefix=prefix)
 
     def _path_to_hash(self, path):
-        parts = self.fs.PATH_CLS(path).parts[-2:]
+        parts = self.fs.path.parts(path)[-2:]
 
         if not (len(parts) == 2 and parts[0] and len(parts[0]) == 2):
             raise ValueError(f"Bad cache file path '{path}'")
 
         return "".join(parts)
 
-    def list_hashes(self, prefix=None, progress_callback=None):
+    def _list_hashes(self, prefix=None, progress_callback=None):
         """Iterate over hashes in this fs.
 
         If `prefix` is specified, only hashes which begin with `prefix`
@@ -225,12 +222,12 @@ class ObjectDB:
 
     def _hashes_with_limit(self, limit, prefix=None, progress_callback=None):
         count = 0
-        for hash_ in self.list_hashes(prefix, progress_callback):
+        for hash_ in self._list_hashes(prefix, progress_callback):
             yield hash_
             count += 1
             if count > limit:
                 logger.debug(
-                    "`list_hashes()` returned max '{}' hashes, "
+                    "`_list_hashes()` returned max '{}' hashes, "
                     "skipping remaining results".format(limit)
                 )
                 return
@@ -269,7 +266,7 @@ class ObjectDB:
                     max_hashes / total_prefixes, prefix, update
                 )
             else:
-                hashes = self.list_hashes(prefix, update)
+                hashes = self._list_hashes(prefix, update)
 
             remote_hashes = set(hashes)
             if remote_hashes:
@@ -279,7 +276,7 @@ class ObjectDB:
             logger.debug(f"Estimated remote size: {remote_size} files")
         return remote_size, remote_hashes
 
-    def list_hashes_traverse(
+    def _list_hashes_traverse(
         self, remote_size, remote_hashes, jobs=None, name=None
     ):
         """Iterate over all hashes found in this fs.
@@ -325,7 +322,7 @@ class ObjectDB:
 
             def list_with_update(prefix):
                 return list(
-                    self.list_hashes(
+                    self._list_hashes(
                         prefix=prefix, progress_callback=pbar.update
                     )
                 )
@@ -349,10 +346,10 @@ class ObjectDB:
         )
 
         if not self.fs.CAN_TRAVERSE:
-            return self.list_hashes()
+            return self._list_hashes()
 
         remote_size, remote_hashes = self._estimate_remote_size(name=name)
-        return self.list_hashes_traverse(
+        return self._list_hashes_traverse(
             remote_size, remote_hashes, jobs, name
         )
 
@@ -378,18 +375,18 @@ class ObjectDB:
         removed = False
         # hashes must be sorted to ensure we always remove .dir files first
         for hash_ in sorted(
-            self.all(jobs, str(self.path_info)),
+            self.all(jobs, self.fs_path),
             key=self.fs.is_dir_hash,
             reverse=True,
         ):
             if hash_ in used_hashes:
                 continue
-            path_info = self.hash_to_path_info(hash_)
+            fs_path = self.hash_to_path(hash_)
             if self.fs.is_dir_hash(hash_):
                 # backward compatibility
                 # pylint: disable=protected-access
                 self._remove_unpacked_dir(hash_)
-            self.fs.remove(path_info)
+            self.fs.remove(fs_path)
             removed = True
 
         return removed
@@ -406,16 +403,16 @@ class ObjectDB:
             unit="file",
         ) as pbar:
 
-            def exists_with_progress(path_info):
-                ret = self.fs.exists(path_info)
-                pbar.update_msg(str(path_info))
+            def exists_with_progress(fs_path):
+                ret = self.fs.exists(fs_path)
+                pbar.update_msg(fs_path)
                 return ret
 
             with ThreadPoolExecutor(
                 max_workers=jobs or self.fs.jobs
             ) as executor:
-                path_infos = map(self.hash_to_path_info, hashes)
-                in_remote = executor.map(exists_with_progress, path_infos)
+                fs_paths = map(self.hash_to_path, hashes)
+                in_remote = executor.map(exists_with_progress, fs_paths)
                 ret = list(itertools.compress(hashes, in_remote))
                 return ret
 
@@ -458,7 +455,7 @@ class ObjectDB:
 
         # During the tests, for ensuring that the traverse behavior
         # is working we turn on this option. It will ensure the
-        # list_hashes_traverse() is called.
+        # _list_hashes_traverse() is called.
         always_traverse = getattr(self.fs, "_ALWAYS_TRAVERSE", False)
 
         hashes = set(hashes)
@@ -495,6 +492,6 @@ class ObjectDB:
 
         logger.debug(f"Querying '{len(hashes)}' hashes via traverse")
         remote_hashes = set(
-            self.list_hashes_traverse(remote_size, remote_hashes, jobs, name)
+            self._list_hashes_traverse(remote_size, remote_hashes, jobs, name)
         )
         return list(hashes & set(remote_hashes))

dvc/objects/db/local.py

@@ -7,7 +7,6 @@ from shortuuid import uuid
 
 from dvc.hash_info import HashInfo
 from dvc.objects.errors import ObjectFormatError
-from dvc.path_info import PathInfo
 from dvc.progress import Tqdm
 from dvc.utils import relpath
 from dvc.utils.fs import copyfile, remove, umask, walk_files
@@ -22,9 +21,9 @@ class LocalObjectDB(ObjectDB):
     CACHE_MODE = 0o444
     UNPACKED_DIR_SUFFIX = ".unpacked"
 
-    def __init__(self, fs, path_info, **config):
-        super().__init__(fs, path_info, **config)
-        self.cache_dir = path_info.fspath
+    def __init__(self, fs, fs_path, **config):
+        super().__init__(fs, fs_path, **config)
+        self.cache_dir = fs_path
 
         shared = config.get("shared")
         if shared:
@@ -36,11 +35,11 @@ class LocalObjectDB(ObjectDB):
 
     @property
     def cache_dir(self):
-        return self.path_info.fspath if self.path_info else None
+        return self.fs_path if self.fs_path else None
 
     @cache_dir.setter
     def cache_dir(self, value):
-        self.path_info = PathInfo(value) if value else None
+        self.fs_path = value
 
     @cached_property
     def cache_path(self):
@@ -50,10 +49,10 @@ class LocalObjectDB(ObjectDB):
         super().move(from_info, to_info)
         os.chmod(to_info, self._file_mode)
 
-    def makedirs(self, path_info):
+    def makedirs(self, fs_path):
         from dvc.utils.fs import makedirs
 
-        makedirs(path_info, exist_ok=True, mode=self._dir_mode)
+        makedirs(fs_path, exist_ok=True, mode=self._dir_mode)
 
     def hash_to_path(self, hash_):
         # NOTE: `self.cache_path` is already normalized so we can simply use
@@ -81,26 +80,30 @@ class LocalObjectDB(ObjectDB):
         return ret
 
     def _list_paths(self, prefix=None, progress_callback=None):
-        assert self.path_info is not None
+        assert self.fs_path is not None
         if prefix:
-            path_info = self.path_info / prefix[:2]
-            if not self.fs.exists(path_info):
+            fs_path = self.fs.path.join(self.fs_path, prefix[:2])
+            if not self.fs.exists(fs_path):
                 return
         else:
-            path_info = self.path_info
+            fs_path = self.fs_path
+
         # NOTE: use utils.fs walk_files since fs.walk_files will not follow
         # symlinks
         if progress_callback:
-            for path in walk_files(path_info):
+            for path in walk_files(fs_path):
                 progress_callback()
                 yield path
         else:
-            yield from walk_files(path_info)
+            yield from walk_files(fs_path)
 
     def _remove_unpacked_dir(self, hash_):
-        info = self.hash_to_path_info(hash_)
-        path_info = info.with_name(info.name + self.UNPACKED_DIR_SUFFIX)
-        self.fs.remove(path_info)
+        hash_fs_path = self.hash_to_path(hash_)
+        fs_path = self.fs.path.with_name(
+            hash_fs_path,
+            self.fs.path.name(hash_fs_path) + self.UNPACKED_DIR_SUFFIX,
+        )
+        self.fs.remove(fs_path)
 
     def _unprotect_file(self, path):
         if self.fs.is_symlink(path) or self.fs.is_hardlink(path):
@@ -126,47 +129,47 @@ class LocalObjectDB(ObjectDB):
         os.chmod(path, self._file_mode)
 
     def _unprotect_dir(self, path):
-        for fname in self.fs.walk_files(path):
+        for fname in self.fs.find(path):
             self._unprotect_file(fname)
 
-    def unprotect(self, path_info):
-        if not os.path.exists(path_info):
+    def unprotect(self, fs_path):
+        if not os.path.exists(fs_path):
             from dvc.exceptions import DvcException
 
             raise DvcException(
-                f"can't unprotect non-existing data '{path_info}'"
+                f"can't unprotect non-existing data '{fs_path}'"
             )
 
-        if os.path.isdir(path_info):
-            self._unprotect_dir(path_info)
+        if os.path.isdir(fs_path):
+            self._unprotect_dir(fs_path)
         else:
-            self._unprotect_file(path_info)
+            self._unprotect_file(fs_path)
 
-    def protect(self, path_info):
+    def protect(self, fs_path):
         try:
-            os.chmod(path_info, self.CACHE_MODE)
+            os.chmod(fs_path, self.CACHE_MODE)
         except OSError:
             # NOTE: not being able to protect cache file is not fatal, it
             # might happen on funky filesystems (e.g. Samba, see #5255),
             # read-only filesystems or in a shared cache scenario.
-            logger.trace("failed to protect '%s'", path_info, exc_info=True)
+            logger.trace("failed to protect '%s'", fs_path, exc_info=True)
 
-    def is_protected(self, path_info):
+    def is_protected(self, fs_path):
         try:
-            mode = os.stat(path_info).st_mode
+            mode = os.stat(fs_path).st_mode
         except FileNotFoundError:
             return False
 
         return stat.S_IMODE(mode) == self.CACHE_MODE
 
-    def set_exec(self, path_info):
-        mode = os.stat(path_info).st_mode | stat.S_IEXEC
+    def set_exec(self, fs_path):
+        mode = os.stat(fs_path).st_mode | stat.S_IEXEC
         try:
-            os.chmod(path_info, mode)
+            os.chmod(fs_path, mode)
         except OSError:
             logger.trace(
                 "failed to chmod '%s' '%s'",
                 oct(mode),
-                path_info,
+                fs_path,
                 exc_info=True,
             )

dvc/objects/db/reference.py

@@ -10,9 +10,9 @@ from ..reference import ReferenceHashFile
 from .base import ObjectDB
 
 if TYPE_CHECKING:
-    from dvc.fs.base import BaseFileSystem
+    from dvc.fs.base import FileSystem
     from dvc.hash_info import HashInfo
-    from dvc.types import AnyPath, DvcPath
+    from dvc.types import AnyPath
 
 logger = logging.getLogger(__name__)
 
@@ -26,7 +26,7 @@ class ReferenceObjectDB(ObjectDB):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self._fs_cache: Dict[tuple, "BaseFileSystem"] = {}
+        self._fs_cache: Dict[tuple, "FileSystem"] = {}
         self._obj_cache: Dict["HashInfo", "ReferenceHashFile"] = {}
 
     def get(self, hash_info: "HashInfo"):
@@ -36,9 +36,9 @@ class ReferenceObjectDB(ObjectDB):
             return self._obj_cache[hash_info]
         except KeyError:
             pass
-        path_info = self.hash_to_path(hash_info.value)
+        fs_path = self.hash_to_path(hash_info.value)
         try:
-            with self.fs.open(path_info, "rb") as fobj:
+            with self.fs.open(fs_path, "rb") as fobj:
                 ref_file = ReferenceHashFile.from_bytes(
                     fobj.read(), fs_cache=self._fs_cache
                 )
@@ -47,20 +47,20 @@ class ReferenceObjectDB(ObjectDB):
         try:
             ref_file.check(self, check_hash=False)
         except ObjectFormatError:
-            self.fs.remove(path_info)
+            self.fs.remove(fs_path)
             raise
         self._obj_cache[hash_info] = ref_file
         return ref_file
 
     def _add_file(
         self,
-        from_fs: "BaseFileSystem",
+        from_fs: "FileSystem",
         from_info: "AnyPath",
-        to_info: "DvcPath",
+        to_info: "AnyPath",
         hash_info: "HashInfo",
         hardlink: bool = False,
     ):
-        self.makedirs(to_info.parent)
+        self.makedirs(self.fs.path.parent(to_info))
         if hash_info.isdir:
             return super()._add_file(
                 from_fs,

dvc/objects/db/slow_link_detection.py

@@ -25,15 +25,15 @@ this.message = (
 
 def slow_link_guard(f):
     @wraps(f)
-    def wrapper(odb, *args, **kwargs):
+    def wrapper(self, odb, *args, **kwargs):
         if this.already_displayed:
-            return f(odb, *args, **kwargs)
+            return f(self, odb, *args, **kwargs)
 
         if not odb.slow_link_warning or odb.cache_types:
-            return f(odb, *args, **kwargs)
+            return f(self, odb, *args, **kwargs)
 
         start = time.time()
-        result = f(odb, *args, **kwargs)
+        result = f(self, odb, *args, **kwargs)
         delta = time.time() - start
 
         if delta >= this.timeout_seconds:

dvc/objects/file.py

@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Optional
 from .errors import ObjectFormatError
 
 if TYPE_CHECKING:
-    from dvc.fs.base import BaseFileSystem
+    from dvc.fs.base import FileSystem
     from dvc.hash_info import HashInfo
     from dvc.types import AnyPath
 
@@ -18,12 +18,12 @@ logger = logging.getLogger(__name__)
 class HashFile:
     def __init__(
         self,
-        path_info: Optional["AnyPath"],
-        fs: Optional["BaseFileSystem"],
+        fs_path: Optional["AnyPath"],
+        fs: Optional["FileSystem"],
         hash_info: "HashInfo",
         name: Optional[str] = None,
     ):
-        self.path_info = path_info
+        self.fs_path = fs_path
         self.fs = fs
         self.hash_info = hash_info
         self.name = name
@@ -41,7 +41,7 @@ class HashFile:
         if not isinstance(other, HashFile):
             return False
         return (
-            self.path_info == other.path_info
+            self.fs_path == other.fs_path
             and self.fs == other.fs
             and self.hash_info == other.hash_info
         )
@@ -50,7 +50,7 @@ class HashFile:
         return hash(
             (
                 self.hash_info,
-                self.path_info,
+                self.fs_path,
                 self.fs.scheme if self.fs else None,
             )
         )
@@ -58,9 +58,9 @@ class HashFile:
     def check(self, odb: "ObjectDB", check_hash: bool = True):
         if not check_hash:
             assert self.fs
-            if not self.fs.exists(self.path_info):
+            if not self.fs.exists(self.fs_path):
                 raise FileNotFoundError(
-                    errno.ENOENT, os.strerror(errno.ENOENT), self.path_info
+                    errno.ENOENT, os.strerror(errno.ENOENT), self.fs_path
                 )
             else:
                 return None
@@ -71,12 +71,12 @@ class HashFile:
         from .stage import get_file_hash
 
         _, actual = get_file_hash(
-            self.path_info, self.fs, self.hash_info.name, odb.state
+            self.fs_path, self.fs, self.hash_info.name, odb.state
         )
 
         logger.trace(
             "cache '%s' expected '%s' actual '%s'",
-            self.path_info,
+            self.fs_path,
             self.hash_info,
             actual,
         )

dvc/objects/reference.py

@@ -8,7 +8,7 @@ from .errors import ObjectFormatError
 from .file import HashFile
 
 if TYPE_CHECKING:
-    from dvc.fs.base import BaseFileSystem
+    from dvc.fs.base import FileSystem
     from dvc.hash_info import HashInfo
     from dvc.types import AnyPath
 
@@ -25,23 +25,23 @@ class ReferenceHashFile(HashFile):
 
     def __init__(
         self,
-        path_info: "AnyPath",
-        fs: "BaseFileSystem",
+        fs_path: "AnyPath",
+        fs: "FileSystem",
         hash_info: "HashInfo",
         checksum: Optional[str] = None,
         **kwargs,
     ):
-        super().__init__(path_info, fs, hash_info, **kwargs)
-        self.checksum = checksum or fs.checksum(path_info)
+        super().__init__(fs_path, fs, hash_info, **kwargs)
+        self.checksum = checksum or fs.checksum(fs_path)
 
     def __str__(self):
-        return f"ref object {self.hash_info} -> {self.path_info}"
+        return f"ref object {self.hash_info} -> {self.fs_path}"
 
     def check(self, odb: "ObjectDB", check_hash: bool = True):
         assert self.fs
-        if not self.fs.exists(self.path_info):
+        if not self.fs.exists(self.fs_path):
             raise FileNotFoundError(
-                errno.ENOENT, os.strerror(errno.ENOENT), self.path_info
+                errno.ENOENT, os.strerror(errno.ENOENT), self.fs_path
             )
         if self.checksum != self._get_checksum():
             raise ObjectFormatError(f"{self} is changed")
@@ -50,23 +50,22 @@ class ReferenceHashFile(HashFile):
 
     def _get_checksum(self) -> str:
         assert self.fs
-        return self.fs.checksum(self.path_info)
+        return self.fs.checksum(self.fs_path)
 
     def to_bytes(self):
         from dvc.fs.repo import RepoFileSystem
-        from dvc.path_info import PathInfo
 
         # NOTE: dumping reference FS's this way is insecure, as the
         # fully parsed remote FS config will include credentials
         #
         # ReferenceHashFiles should currently only be serialized in
         # memory and not to disk
-        path_info = self.path_info
+        fs_path = self.fs_path
         if isinstance(self.fs, RepoFileSystem):
-            path_info = PathInfo(path_info).relative_to(self.fs.root_dir)
+            fs_path = self.fs.path.relpath(fs_path, self.fs.root_dir)
 
         dict_ = {
-            self.PARAM_PATH: path_info,
+            self.PARAM_PATH: fs_path,
             self.PARAM_HASH: self.hash_info,
             self.PARAM_CHECKSUM: self.checksum,
             self.PARAM_FS_CONFIG: self.config_tuple(self.fs),
@@ -87,7 +86,7 @@ class ReferenceHashFile(HashFile):
             raise ObjectFormatError("ReferenceHashFile is corrupted") from exc
 
         try:
-            path_info = dict_[cls.PARAM_PATH]
+            fs_path = dict_[cls.PARAM_PATH]
             hash_info = dict_[cls.PARAM_HASH]
         except KeyError as exc:
             raise ObjectFormatError("ReferenceHashFile is corrupted") from exc
@@ -98,19 +97,19 @@ class ReferenceHashFile(HashFile):
             config = dict(config_pairs)
             if RepoFileSystem.PARAM_REPO_URL in config:
                 fs = RepoFileSystem(**config)
-                path_info = fs.root_dir / path_info
+                fs_path = fs.path.join(fs.root_dir, fs_path)
             else:
-                fs_cls = get_fs_cls(config, scheme=path_info.scheme)
+                fs_cls = get_fs_cls(config, scheme=scheme)
                 fs = fs_cls(**config)
         return ReferenceHashFile(
-            path_info,
+            fs_path,
             fs,
             hash_info,
             checksum=dict_.get(cls.PARAM_CHECKSUM),
         )
 
     @staticmethod
-    def config_tuple(fs: "BaseFileSystem"):
+    def config_tuple(fs: "FileSystem"):
         return (
             fs.scheme,
             tuple(

dvc/objects/stage.py

@@ -1,10 +1,9 @@
 import hashlib
 import logging
 import os
-import pathlib
 from concurrent.futures import ThreadPoolExecutor
 from functools import partial
-from typing import TYPE_CHECKING, Dict, Tuple
+from typing import TYPE_CHECKING, Dict, Optional, Tuple
 
 from dvc.exceptions import DvcIgnoreInCollectedDirError
 from dvc.hash_info import HashInfo
@@ -17,8 +16,8 @@ from .file import HashFile
 from .meta import Meta
 
 if TYPE_CHECKING:
-    from dvc.fs.base import BaseFileSystem
-    from dvc.types import AnyPath, DvcPath
+    from dvc.fs.base import FileSystem
+    from dvc.types import AnyPath
 
     from .db.base import ObjectDB
 
@@ -28,31 +27,49 @@ logger = logging.getLogger(__name__)
 _STAGING_MEMFS_PATH = "dvc-staging"
 
 
-def _upload_file(path_info, fs, odb, upload_odb):
+def _upload_file(from_fs_path, fs, odb, upload_odb):
     from dvc.utils import tmp_fname
     from dvc.utils.stream import HashedStreamReader
 
-    tmp_info = upload_odb.path_info / tmp_fname()
-    with fs.open(path_info, mode="rb", chunk_size=fs.CHUNK_SIZE) as stream:
+    fs_path = upload_odb.fs.path
+    tmp_info = fs_path.join(upload_odb.fs_path, tmp_fname())
+    with fs.open(from_fs_path, mode="rb", chunk_size=fs.CHUNK_SIZE) as stream:
         stream = HashedStreamReader(stream)
-        size = fs.getsize(path_info)
-        upload_odb.fs.upload(stream, tmp_info, desc=path_info.name, total=size)
+        size = fs.getsize(from_fs_path)
+        upload_odb.fs.upload(
+            stream, tmp_info, desc=fs_path.name(from_fs_path), total=size
+        )
 
     odb.add(tmp_info, upload_odb.fs, stream.hash_info)
     meta = Meta(size=size)
-    return path_info, meta, odb.get(stream.hash_info)
+    return from_fs_path, meta, odb.get(stream.hash_info)
 
 
-def _get_file_hash(path_info, fs, name):
-    info = fs.info(path_info)
+def _adapt_info(info, scheme):
+    if scheme == "s3" and "ETag" in info:
+        info["etag"] = info["ETag"].strip('"')
+    elif scheme == "gs" and "etag" in info:
+        import base64
+
+        info["etag"] = base64.b64decode(info["etag"]).hex()
+    elif scheme.startswith("http") and (
+        "ETag" in info or "Content-MD5" in info
+    ):
+        info["checksum"] = info.get("ETag") or info.get("Content-MD5")
+    return info
+
+
+def _get_file_hash(fs_path, fs, name):
+    info = _adapt_info(fs.info(fs_path), fs.scheme)
+
     if name in info:
         assert not info[name].endswith(".dir")
         hash_value = info[name]
     elif hasattr(fs, name):
         func = getattr(fs, name)
-        hash_value = func(path_info)
+        hash_value = func(fs_path)
     elif name == "md5":
-        hash_value = file_md5(path_info, fs)
+        hash_value = file_md5(fs_path, fs)
     else:
         raise NotImplementedError
 
@@ -61,41 +78,41 @@ def _get_file_hash(path_info, fs, name):
     return meta, hash_info
 
 
-def get_file_hash(path_info, fs, name, state=None):
+def get_file_hash(fs_path, fs, name, state=None):
     if state:
         meta, hash_info = state.get(  # pylint: disable=assignment-from-none
-            path_info, fs
+            fs_path, fs
         )
         if hash_info:
             return meta, hash_info
 
-    meta, hash_info = _get_file_hash(path_info, fs, name)
+    meta, hash_info = _get_file_hash(fs_path, fs, name)
 
     if state:
         assert ".dir" not in hash_info.value
-        state.save(path_info, fs, hash_info)
+        state.save(fs_path, fs, hash_info)
 
     return meta, hash_info
 
 
-def _stage_file(path_info, fs, name, odb=None, upload_odb=None, dry_run=False):
+def _stage_file(fs_path, fs, name, odb=None, upload_odb=None, dry_run=False):
     state = odb.state if odb else None
-    meta, hash_info = get_file_hash(path_info, fs, name, state=state)
+    meta, hash_info = get_file_hash(fs_path, fs, name, state=state)
     if upload_odb and not dry_run:
         assert odb and name == "md5"
-        return _upload_file(path_info, fs, odb, upload_odb)
+        return _upload_file(fs_path, fs, odb, upload_odb)
 
     if dry_run:
-        obj = HashFile(path_info, fs, hash_info)
+        obj = HashFile(fs_path, fs, hash_info)
     else:
-        odb.add(path_info, fs, hash_info, hardlink=False)
+        odb.add(fs_path, fs, hash_info, hardlink=False)
         obj = odb.get(hash_info)
 
-    return path_info, meta, obj
+    return fs_path, meta, obj
 
 
 def _build_objects(
-    path_info,
+    fs_path,
     fs,
     name,
     dvcignore=None,
@@ -104,9 +121,9 @@ def _build_objects(
     **kwargs,
 ):
     if dvcignore:
-        walk_iterator = dvcignore.walk_files(fs, path_info)
+        walk_iterator = dvcignore.find(fs, fs_path)
     else:
-        walk_iterator = fs.walk_files(path_info)
+        walk_iterator = fs.find(fs_path)
     with Tqdm(
         unit="md5",
         desc="Computing file/dir hashes (only done once)",
@@ -126,18 +143,18 @@ def _build_objects(
             yield from executor.map(worker, walk_iterator)
 
 
-def _iter_objects(path_info, fs, name, **kwargs):
-    yield from _build_objects(path_info, fs, name, **kwargs)
+def _iter_objects(fs_path, fs, name, **kwargs):
+    yield from _build_objects(fs_path, fs, name, **kwargs)
 
 
-def _build_tree(path_info, fs, name, **kwargs):
+def _build_tree(fs_path, fs, name, **kwargs):
     from .tree import Tree
 
     tree_meta = Meta(size=0, nfiles=0)
     tree = Tree(None, None, None)
-    for file_info, meta, obj in _iter_objects(path_info, fs, name, **kwargs):
-        if DvcIgnore.DVCIGNORE_FILE == file_info.name:
-            raise DvcIgnoreInCollectedDirError(file_info.parent)
+    for file_fs_path, meta, obj in _iter_objects(fs_path, fs, name, **kwargs):
+        if DvcIgnore.DVCIGNORE_FILE == fs.path.name(file_fs_path):
+            raise DvcIgnoreInCollectedDirError(fs.path.parent(file_fs_path))
 
         # NOTE: this is lossy transformation:
         #   "hey\there" -> "hey/there"
@@ -147,7 +164,10 @@ def _build_tree(path_info, fs, name, **kwargs):
         #
         # Yes, this is a BUG, as long as we permit "/" in
         # filenames on Windows and "\" on Unix
-        tree.add(file_info.relative_to(path_info).parts, meta, obj.hash_info)
+
+        key = fs.path.relparts(file_fs_path, fs_path)
+        assert key
+        tree.add(key, meta, obj.hash_info)
 
         tree_meta.size += meta.size
         tree_meta.nfiles += 1
@@ -155,7 +175,7 @@ def _build_tree(path_info, fs, name, **kwargs):
     return tree_meta, tree
 
 
-def _stage_tree(path_info, fs, fs_info, name, odb=None, **kwargs):
+def _stage_tree(fs_path, fs, fs_info, name, odb=None, **kwargs):
     from .tree import Tree
 
     value = fs_info.get(name)
@@ -167,41 +187,46 @@ def _stage_tree(path_info, fs, fs_info, name, odb=None, **kwargs):
         except FileNotFoundError:
             pass
 
-    meta, tree = _build_tree(path_info, fs, name, odb=odb, **kwargs)
+    meta, tree = _build_tree(fs_path, fs, name, odb=odb, **kwargs)
     state = odb.state if odb and odb.state else None
     hash_info = None
     if state:
         _, hash_info = state.get(  # pylint: disable=assignment-from-none
-            path_info, fs
+            fs_path, fs
         )
     tree.digest(hash_info=hash_info)
-    odb.add(tree.path_info, tree.fs, tree.hash_info, hardlink=False)
+    odb.add(tree.fs_path, tree.fs, tree.hash_info, hardlink=False)
     raw = odb.get(tree.hash_info)
     # cleanup unneeded memfs tmpfile and return tree based on the
     # ODB fs/path
     if odb.fs != tree.fs:
-        tree.fs.remove(tree.path_info)
+        tree.fs.remove(tree.fs_path)
     tree.fs = raw.fs
-    tree.path_info = raw.path_info
+    tree.fs_path = raw.fs_path
     return meta, tree
 
 
 _url_cache: Dict[str, str] = {}
 
 
-def _make_staging_url(path_info: "AnyPath"):
-    from dvc.path_info import CloudURLInfo
+def _make_staging_url(
+    fs: "FileSystem", odb: "ObjectDB", fs_path: Optional[str]
+):
     from dvc.scheme import Schemes
 
-    url = CloudURLInfo(f"{Schemes.MEMORY}://{_STAGING_MEMFS_PATH}")
-    if path_info:
-        if isinstance(path_info, (str, pathlib.PurePath)):
-            path = os.path.abspath(path_info)
-        else:
-            path = str(path_info)
-        if path not in _url_cache:
-            _url_cache[path] = hashlib.sha256(path.encode("utf-8")).hexdigest()
-        url /= _url_cache[path]
+    url = f"{Schemes.MEMORY}://{_STAGING_MEMFS_PATH}"
+
+    if fs_path is not None:
+        if odb.fs.scheme == Schemes.LOCAL:
+            fs_path = os.path.abspath(fs_path)
+
+        if fs_path not in _url_cache:
+            _url_cache[fs_path] = hashlib.sha256(
+                fs_path.encode("utf-8")
+            ).hexdigest()
+
+        url = fs.path.join(url, _url_cache[fs_path])
+
     return url
 
 
@@ -214,18 +239,18 @@ def _get_staging(odb: "ObjectDB") -> "ObjectDB":
     from dvc.fs.memory import MemoryFileSystem
 
     fs = MemoryFileSystem()
-    path_info = _make_staging_url(odb.path_info)
+    fs_path = _make_staging_url(fs, odb, odb.fs_path)
     state = odb.state
-    return ReferenceObjectDB(fs, path_info, state=state)
+    return ReferenceObjectDB(fs, fs_path, state=state)
 
 
-def _load_from_state(odb, staging, path_info, fs, name):
+def _load_from_state(odb, staging, fs_path, fs, name):
     from . import check, load
     from .errors import ObjectFormatError
     from .tree import Tree
 
     state = odb.state
-    meta, hash_info = state.get(path_info, fs)
+    meta, hash_info = state.get(fs_path, fs)
     if hash_info:
         for odb_ in (odb, staging):
             if odb_.exists(hash_info):
@@ -248,10 +273,10 @@ def _stage_external_tree_info(odb, tree, name):
     # that would be universal for all clouds.
     assert odb and name != "md5"
 
-    odb.add(tree.path_info, tree.fs, tree.hash_info)
+    odb.add(tree.fs_path, tree.fs, tree.hash_info)
     raw = odb.get(tree.hash_info)
-    _, hash_info = get_file_hash(raw.path_info, raw.fs, name, state=odb.state)
-    tree.path_info = raw.path_info
+    _, hash_info = get_file_hash(raw.fs_path, raw.fs, name, state=odb.state)
+    tree.fs_path = raw.fs_path
     tree.fs = raw.fs
     tree.hash_info.name = hash_info.name
     tree.hash_info.value = hash_info.value
@@ -262,8 +287,8 @@ def _stage_external_tree_info(odb, tree, name):
 
 def stage(
     odb: "ObjectDB",
-    path_info: "DvcPath",
-    fs: "BaseFileSystem",
+    fs_path: "AnyPath",
+    fs: "FileSystem",
     name: str,
     upload: bool = False,
     dry_run: bool = False,
@@ -282,19 +307,20 @@ def stage(
     ODB filesystem, and staged objects will reference the uploaded path rather
     than the original source path.
     """
-    assert path_info and path_info.scheme == fs.scheme
+    assert fs_path
+    # assert scheme(fs_path) == fs.scheme
 
-    details = fs.info(path_info)
+    details = fs.info(fs_path)
     staging = _get_staging(odb)
     if odb:
         try:
-            return _load_from_state(odb, staging, path_info, fs, name)
+            return _load_from_state(odb, staging, fs_path, fs, name)
         except FileNotFoundError:
             pass
 
     if details["type"] == "directory":
         meta, obj = _stage_tree(
-            path_info,
+            fs_path,
             fs,
             details,
             name,
@@ -308,7 +334,7 @@ def stage(
             obj = _stage_external_tree_info(odb, obj, name)
     else:
         _, meta, obj = _stage_file(
-            path_info,
+            fs_path,
             fs,
             name,
             odb=staging,
@@ -317,6 +343,6 @@ def stage(
         )
 
     if odb and odb.state and obj.hash_info:
-        odb.state.save(path_info, fs, obj.hash_info)
+        odb.state.save(fs_path, fs, obj.hash_info)
 
     return staging, meta, obj

dvc/objects/status.py

@@ -86,7 +86,7 @@ def status(
         exists: objs that exist in odb
         missing: objs that do not exist in ODB
     """
-    logger.debug("Preparing to collect status from '%s'", odb.path_info)
+    logger.debug("Preparing to collect status from '%s'", odb.fs_path)
     if not name:
         name = odb.fs.PARAM_CHECKSUM
 
@@ -116,7 +116,7 @@ def status(
     hashes: Set[str] = set(hash_infos.keys())
     exists: Set[str] = set()
 
-    logger.debug("Collecting status from '%s'", odb.path_info)
+    logger.debug("Collecting status from '%s'", odb.fs_path)
     if index and hashes:
         if dir_objs:
             exists = hashes.intersection(
@@ -128,9 +128,7 @@ def status(
             hashes.difference_update(exists)
 
     if hashes:
-        exists.update(
-            odb.hashes_exist(hashes, name=str(odb.path_info), **kwargs)
-        )
+        exists.update(odb.hashes_exist(hashes, name=odb.fs_path, **kwargs))
     return StatusResult(
         {hash_infos[hash_] for hash_ in exists},
         {hash_infos[hash_] for hash_ in (hashes - exists)},

dvc/objects/transfer.py

@@ -20,9 +20,9 @@ logger = logging.getLogger(__name__)
 
 def _log_exceptions(func):
     @wraps(func)
-    def wrapper(path_info, *args, **kwargs):
+    def wrapper(fs_path, *args, **kwargs):
         try:
-            func(path_info, *args, **kwargs)
+            func(fs_path, *args, **kwargs)
             return 0
         except Exception as exc:  # pylint: disable=broad-except
             # NOTE: this means we ran out of file descriptors and there is no
@@ -31,7 +31,7 @@ def _log_exceptions(func):
             if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
                 raise
 
-            logger.exception("failed to transfer '%s'", path_info)
+            logger.exception("failed to transfer '%s'", fs_path)
             return 1
 
     return wrapper
@@ -87,8 +87,8 @@ def _do_transfer(
             )
             logger.error(
                 "failed to upload '%s' to '%s'",
-                src.get(dir_obj.hash_info).path_info,
-                dest.get(dir_obj.hash_info).path_info,
+                src.get(dir_obj.hash_info).fs_path,
+                dest.get(dir_obj.hash_info).fs_path,
             )
             total_fails += dir_fails + 1
         elif entry_ids.intersection(missing_ids):
@@ -144,8 +144,8 @@ def transfer(
 
     logger.debug(
         "Preparing to transfer data from '%s' to '%s'",
-        src.path_info,
-        dest.path_info,
+        src.fs_path,
+        dest.fs_path,
     )
     if src == dest:
         return 0
@@ -157,7 +157,7 @@ def transfer(
     def func(hash_info: "HashInfo") -> None:
         obj = src.get(hash_info)
         return dest.add(
-            obj.path_info,
+            obj.fs_path,
             obj.fs,
             obj.hash_info,
             verify=verify,

dvc/objects/tree.py

@@ -36,19 +36,18 @@ class Tree(HashFile):
 
     def digest(self, hash_info: Optional["HashInfo"] = None):
         from dvc.fs.memory import MemoryFileSystem
-        from dvc.path_info import CloudURLInfo
         from dvc.utils import tmp_fname
 
         memfs = MemoryFileSystem()
-        path_info = CloudURLInfo("memory://{}".format(tmp_fname("")))
-        with memfs.open(path_info, "wb") as fobj:
+        fs_path = "memory://{}".format(tmp_fname(""))
+        with memfs.open(fs_path, "wb") as fobj:
             fobj.write(self.as_bytes())
         self.fs = memfs
-        self.path_info = path_info
+        self.fs_path = fs_path
         if hash_info:
             self.hash_info = hash_info
         else:
-            _, self.hash_info = get_file_hash(path_info, memfs, "md5")
+            _, self.hash_info = get_file_hash(fs_path, memfs, "md5")
             assert self.hash_info.value
             self.hash_info.value += ".dir"
 
@@ -101,7 +100,7 @@ class Tree(HashFile):
         obj = odb.get(hash_info)
 
         try:
-            with obj.fs.open(obj.path_info, "r") as fobj:
+            with obj.fs.open(obj.fs_path, "r") as fobj:
                 raw = json.load(fobj)
         except ValueError as exc:
             raise ObjectFormatError(f"{obj} is corrupted") from exc
@@ -109,12 +108,12 @@ class Tree(HashFile):
         if not isinstance(raw, list):
             logger.error(
                 "dir cache file format error '%s' [skipping the file]",
-                obj.path_info,
+                obj.fs_path,
             )
             raise ObjectFormatError(f"{obj} is corrupted")
 
         tree = cls.from_list(raw)
-        tree.path_info = obj.path_info
+        tree.fs_path = obj.fs_path
         tree.fs = obj.fs
         tree.hash_info = hash_info
 
@@ -125,11 +124,11 @@ class Tree(HashFile):
         inside prefix.
 
         The returned tree will contain the original tree's hash_info and
-        path_info.
+        fs_path.
 
         Returns an empty tree if no object exists at the specified prefix.
         """
-        tree = Tree(self.path_info, self.fs, self.hash_info)
+        tree = Tree(self.fs_path, self.fs, self.hash_info)
         try:
             for key, (meta, oid) in self.trie.items(prefix):
                 tree.add(key, meta, oid)
@@ -160,8 +159,7 @@ class Tree(HashFile):
 def du(odb, tree):
     try:
         return sum(
-            odb.fs.getsize(odb.hash_to_path_info(oid.value))
-            for _, _, oid in tree
+            odb.fs.getsize(odb.hash_to_path(oid.value)) for _, _, oid in tree
         )
     except FileNotFoundError:
         return None

dvc/output.py

@@ -292,7 +292,7 @@ class Output:
     ):
         self.repo = stage.repo if stage else None
 
-        fs_cls, fs_config, path_info = get_cloud_fs(self.repo, url=path)
+        fs_cls, fs_config, fs_path = get_cloud_fs(self.repo, url=path)
         self.fs = fs_cls(**fs_config)
 
         if (
@@ -309,7 +309,6 @@ class Output:
         # here is a list and comments:
         #
         #   .def_path - path from definition in DVC file
-        #   .path_info - PathInfo/URLInfo structured resolved path
         #   .fspath - local only, resolved
         #   .__str__ - for presentation purposes, def_path/relpath
         #
@@ -326,32 +325,32 @@ class Output:
         self.live = live
         self.desc = desc
 
-        self.path_info = self._parse_path(self.fs, path_info)
+        self.fs_path = self._parse_path(self.fs, fs_path)
         if self.use_cache and self.odb is None:
-            raise RemoteCacheRequiredError(self.path_info)
+            raise RemoteCacheRequiredError(self.fs.scheme, self.fs_path)
 
         self.obj = None
         self.isexec = False if self.IS_DEPENDENCY else isexec
 
         self.remote = remote
 
-    def _parse_path(self, fs, path_info):
+    def _parse_path(self, fs, fs_path):
         if fs.scheme != "local":
-            return path_info
+            return fs_path
 
         parsed = urlparse(self.def_path)
         if parsed.scheme != "remote":
             # NOTE: we can path either from command line or .dvc file,
             # so we should expect both posix and windows style paths.
-            # PathInfo accepts both, i.e. / works everywhere, \ only on win.
+            # paths accepts both, i.e. / works everywhere, \ only on win.
             #
             # FIXME: if we have Windows path containing / or posix one with \
             # then we have #2059 bug and can't really handle that.
-            if self.stage and not path_info.is_absolute():
-                path_info = self.stage.wdir / path_info
+            if self.stage and not os.path.isabs(fs_path):
+                fs_path = fs.path.join(self.stage.wdir, fs_path)
 
-        abs_p = os.path.abspath(os.path.normpath(path_info))
-        return fs.PATH_CLS(abs_p)
+        abs_p = os.path.abspath(os.path.normpath(fs_path))
+        return abs_p
 
     def __repr__(self):
         return "{class_name}: '{def_path}'".format(
@@ -371,9 +370,9 @@ class Output:
 
         cur_dir = os.getcwd()
         if path_isin(cur_dir, self.repo.root_dir):
-            return relpath(self.path_info, cur_dir)
+            return relpath(self.fs_path, cur_dir)
 
-        return relpath(self.path_info, self.repo.root_dir)
+        return relpath(self.fs_path, self.repo.root_dir)
 
     @property
     def scheme(self):
@@ -391,7 +390,7 @@ class Output:
             return False
 
         return self.repo and path_isin(
-            os.path.realpath(self.path_info), self.repo.root_dir
+            os.path.realpath(self.fs_path), self.repo.root_dir
         )
 
     @property
@@ -407,7 +406,9 @@ class Output:
 
     @property
     def cache_path(self):
-        return self.odb.hash_to_path_info(self.hash_info.value).url
+        return self.odb.fs.unstrip_protocol(
+            self.odb.hash_to_path(self.hash_info.value)
+        )
 
     def get_hash(self):
         if self.use_cache:
@@ -418,7 +419,7 @@ class Output:
             name = self.fs.PARAM_CHECKSUM
         _, _, obj = ostage(
             odb,
-            self.path_info,
+            self.fs_path,
             self.fs,
             name,
             dvcignore=self.dvcignore,
@@ -438,10 +439,10 @@ class Output:
 
     @property
     def exists(self):
-        if self._is_path_dvcignore(self.path_info):
+        if self._is_path_dvcignore(self.fs_path):
             return False
 
-        return self.fs.exists(self.path_info)
+        return self.fs.exists(self.fs_path)
 
     def changed_checksum(self):
         return self.hash_info != self.get_hash()
@@ -491,17 +492,17 @@ class Output:
 
     @property
     def is_empty(self):
-        return self.fs.is_empty(self.path_info)
+        return self.fs.is_empty(self.fs_path)
 
     def isdir(self):
-        if self._is_path_dvcignore(self.path_info):
+        if self._is_path_dvcignore(self.fs_path):
             return False
-        return self.fs.isdir(self.path_info)
+        return self.fs.isdir(self.fs_path)
 
     def isfile(self):
-        if self._is_path_dvcignore(self.path_info):
+        if self._is_path_dvcignore(self.fs_path):
             return False
-        return self.fs.isfile(self.path_info)
+        return self.fs.isfile(self.fs_path)
 
     # pylint: disable=no-member
 
@@ -512,13 +513,13 @@ class Output:
         if self.repo.scm.is_tracked(self.fspath):
             raise OutputAlreadyTrackedError(self)
 
-        self.repo.scm.ignore(self.fspath)
+        self.repo.scm_context.ignore(self.fspath)
 
     def ignore_remove(self):
         if not self.use_scm_ignore:
             return
 
-        self.repo.scm.ignore_remove(self.fspath)
+        self.repo.scm_context.ignore_remove(self.fspath)
 
     # pylint: enable=no-member
 
@@ -540,7 +541,7 @@ class Output:
         if not self.use_cache:
             _, self.meta, obj = ostage(
                 self.repo.odb.local,
-                self.path_info,
+                self.fs_path,
                 self.fs,
                 self.fs.PARAM_CHECKSUM,
                 dvcignore=self.dvcignore,
@@ -557,17 +558,17 @@ class Output:
 
         _, self.meta, self.obj = ostage(
             self.odb,
-            self.path_info,
+            self.fs_path,
             self.fs,
             self.odb.fs.PARAM_CHECKSUM,
             dvcignore=self.dvcignore,
         )
         self.hash_info = self.obj.hash_info
-        self.isexec = self.isfile() and self.fs.isexec(self.path_info)
+        self.isexec = self.isfile() and self.fs.isexec(self.fs_path)
 
     def set_exec(self):
         if self.isfile() and self.isexec:
-            self.odb.set_exec(self.path_info)
+            self.odb.set_exec(self.fs_path)
 
     def commit(self, filter_info=None):
         if not self.exists:
@@ -579,14 +580,14 @@ class Output:
             granular = (
                 self.is_dir_checksum
                 and filter_info
-                and filter_info != self.path_info
+                and filter_info != self.fs_path
             )
             if granular:
                 obj = self._commit_granular_dir(filter_info)
             else:
                 staging, _, obj = ostage(
                     self.odb,
-                    filter_info or self.path_info,
+                    filter_info or self.fs_path,
                     self.fs,
                     self.odb.fs.PARAM_CHECKSUM,
                     dvcignore=self.dvcignore,
@@ -599,7 +600,7 @@ class Output:
                     hardlink=True,
                 )
             checkout(
-                filter_info or self.path_info,
+                filter_info or self.fs_path,
                 self.fs,
                 obj,
                 self.odb,
@@ -610,10 +611,12 @@ class Output:
             self.set_exec()
 
     def _commit_granular_dir(self, filter_info):
-        prefix = filter_info.relative_to(self.path_info).parts
+        prefix = self.fs.path.parts(
+            self.fs.path.relpath(filter_info, self.fs_path)
+        )
         staging, _, save_obj = ostage(
             self.odb,
-            self.path_info,
+            self.fs_path,
             self.fs,
             self.odb.fs.PARAM_CHECKSUM,
             dvcignore=self.dvcignore,
@@ -633,7 +636,9 @@ class Output:
         ret = {**self.hash_info.to_dict(), **self.meta.to_dict()}
 
         if self.is_in_repo:
-            path = self.path_info.relpath(self.stage.wdir).as_posix()
+            path = self.fs.path.as_posix(
+                relpath(self.fs_path, self.stage.wdir)
+            )
         else:
             path = self.def_path
 
@@ -684,22 +689,21 @@ class Output:
         if not self.metric or self.plot:
             return
 
-        path = os.fspath(self.path_info)
-        if not os.path.exists(path):
+        if not os.path.exists(self.fs_path):
             return
 
         name = "metrics" if self.metric else "plot"
-        if os.path.isdir(path):
+        if os.path.isdir(self.fs_path):
             msg = "directory '%s' cannot be used as %s."
-            logger.debug(msg, str(self.path_info), name)
+            logger.debug(msg, str(self), name)
             return
 
-        if not istextfile(path, self.fs):
+        if not istextfile(self.fs_path, self.fs):
             msg = "binary file '{}' cannot be used as {}."
-            raise DvcException(msg.format(self.path_info, name))
+            raise DvcException(msg.format(self.fs_path, name))
 
     def download(self, to, jobs=None):
-        self.fs.download(self.path_info, to.path_info, jobs=jobs)
+        self.fs.download(self.fs_path, to.fs_path, jobs=jobs)
 
     def get_obj(self, filter_info=None, **kwargs):
         if self.obj:
@@ -712,8 +716,9 @@ class Output:
         else:
             return None
 
-        if filter_info and filter_info != self.path_info:
-            prefix = filter_info.relative_to(self.path_info).parts
+        fs_path = self.fs.path
+        if filter_info and filter_info != self.fs_path:
+            prefix = fs_path.relparts(filter_info, self.fs_path)
             obj = obj.get(self.odb, prefix)
 
         return obj
@@ -731,12 +736,12 @@ class Output:
         if not self.use_cache:
             if progress_callback:
                 progress_callback(
-                    str(self.path_info), self.get_files_number(filter_info)
+                    self.fs_path, self.get_files_number(filter_info)
                 )
             return None
 
         obj = self.get_obj(filter_info=filter_info)
-        if not obj and (filter_info and filter_info != self.path_info):
+        if not obj and (filter_info and filter_info != self.fs_path):
             # backward compatibility
             return None
 
@@ -749,7 +754,7 @@ class Output:
 
         try:
             modified = checkout(
-                filter_info or self.path_info,
+                filter_info or self.fs_path,
                 self.fs,
                 obj,
                 self.odb,
@@ -767,7 +772,7 @@ class Output:
         return added, False if added else modified
 
     def remove(self, ignore_remove=False):
-        self.fs.remove(self.path_info)
+        self.fs.remove(self.fs_path)
         if self.scheme != Schemes.LOCAL:
             return
 
@@ -777,16 +782,16 @@ class Output:
     def move(self, out):
         # pylint: disable=no-member
         if self.scheme == "local" and self.use_scm_ignore:
-            self.repo.scm.ignore_remove(self.fspath)
+            self.repo.scm_context.ignore_remove(self.fspath)
 
-        self.fs.move(self.path_info, out.path_info)
+        self.fs.move(self.fs_path, out.fs_path)
         self.def_path = out.def_path
-        self.path_info = out.path_info
+        self.fs_path = out.fs_path
         self.save()
         self.commit()
 
         if self.scheme == "local" and self.use_scm_ignore:
-            self.repo.scm.ignore(self.fspath)
+            self.repo.scm_context.ignore(self.fspath)
 
     def transfer(
         self, source, odb=None, jobs=None, update=False, no_progress_bar=False
@@ -836,7 +841,7 @@ class Output:
         if not self.hash_info.isdir:
             return 1
 
-        if not filter_info or filter_info == self.path_info:
+        if not filter_info or filter_info == self.fs_path:
             return self.meta.nfiles or 0
 
         obj = self.get_obj(filter_info=filter_info)
@@ -844,7 +849,7 @@ class Output:
 
     def unprotect(self):
         if self.exists:
-            self.odb.unprotect(self.path_info)
+            self.odb.unprotect(self.fs_path)
 
     def get_dir_cache(self, **kwargs):
         if not self.is_dir_checksum:
@@ -886,7 +891,7 @@ class Output:
                 "Cache for files inside will be lost. "
                 "Would you like to continue? Use '-f' to force."
             )
-            if not force and not prompt.confirm(msg.format(self.path_info)):
+            if not force and not prompt.confirm(msg.format(self.fs_path)):
                 raise CollectCacheError(
                     "unable to fully collect used cache"
                     " without cache for directory '{}'".format(self)
@@ -894,8 +899,10 @@ class Output:
             return None
 
         obj = self.get_obj()
-        if filter_info and filter_info != self.path_info:
-            prefix = filter_info.relative_to(self.path_info).parts
+        if filter_info and filter_info != self.fs_path:
+            prefix = self.fs.path.parts(
+                self.fs.path.relpath(filter_info, self.fs_path)
+            )
             obj = obj.filter(prefix)
         return obj
 
@@ -1021,7 +1028,7 @@ class Output:
         merged = merge(
             self.odb, ancestor_info, self.hash_info, other.hash_info
         )
-        self.odb.add(merged.path_info, merged.fs, merged.hash_info)
+        self.odb.add(merged.fs_path, merged.fs, merged.hash_info)
 
         self.hash_info = merged.hash_info
         self.meta = Meta(
@@ -1031,7 +1038,7 @@ class Output:
 
     @property
     def fspath(self):
-        return self.path_info.fspath
+        return self.fs_path
 
     @property
     def is_decorated(self) -> bool:

dvc/parsing/__init__.py

@@ -17,7 +17,6 @@ from funcy import cached_property, collecting, first, isa, join, reraise
 
 from dvc.exceptions import DvcException
 from dvc.parsing.interpolate import ParseError
-from dvc.path_info import PathInfo
 from dvc.utils import relpath
 
 from .context import (
@@ -134,10 +133,10 @@ def make_definition(
 
 
 class DataResolver:
-    def __init__(self, repo: "Repo", wdir: PathInfo, d: dict):
+    def __init__(self, repo: "Repo", wdir: str, d: dict):
         self.fs = fs = repo.fs
         self.wdir = wdir
-        self.relpath = relpath(self.wdir / "dvc.yaml")
+        self.relpath = relpath(fs.path.join(self.wdir, "dvc.yaml"))
 
         vars_ = d.get(VARS_KWD, [])
         check_interpolations(vars_, VARS_KWD, self.relpath)
@@ -232,7 +231,7 @@ class EntryDefinition:
 
     def _resolve_wdir(
         self, context: Context, name: str, wdir: str = None
-    ) -> PathInfo:
+    ) -> str:
         if not wdir:
             return self.wdir
 
@@ -240,7 +239,7 @@ class EntryDefinition:
             wdir = to_str(context.resolve_str(wdir))
         except (ContextError, ParseError) as exc:
             format_and_raise(exc, f"'{self.where}.{name}.wdir'", self.relpath)
-        return self.wdir / wdir
+        return self.resolver.fs.path.join(self.wdir, wdir)
 
     def resolve(self, **kwargs):
         try:

dvc/parsing/context.py

@@ -19,7 +19,6 @@ from dvc.parsing.interpolate import (
     recurse,
     str_interpolate,
 )
-from dvc.path_info import PathInfo
 from dvc.utils import relpath
 
 logger = logging.getLogger(__name__)
@@ -356,7 +355,7 @@ class Context(CtxDict):
 
     @classmethod
     def load_from(
-        cls, fs, path: PathInfo, select_keys: List[str] = None
+        cls, fs, path: str, select_keys: List[str] = None
     ) -> "Context":
         from dvc.utils.serialize import LOADERS
 
@@ -396,18 +395,17 @@ class Context(CtxDict):
             raise ReservedKeyError(matches)
         return super().merge_update(other, overwrite=overwrite)
 
-    def merge_from(self, fs, item: str, wdir: PathInfo, overwrite=False):
+    def merge_from(self, fs, item: str, wdir: str, overwrite=False):
         path, _, keys_str = item.partition(":")
         select_keys = lfilter(bool, keys_str.split(",")) if keys_str else None
 
-        abspath = os.path.abspath(wdir / path)
-        path_info = PathInfo(abspath)
+        abspath = os.path.abspath(fs.path.join(wdir, path))
         if abspath in self.imports:
             if not select_keys and self.imports[abspath] is None:
                 return  # allow specifying complete filepath multiple times
             self.check_loaded(abspath, item, select_keys)
 
-        ctx = Context.load_from(fs, path_info, select_keys)
+        ctx = Context.load_from(fs, abspath, select_keys)
 
         try:
             self.merge_update(ctx, overwrite=overwrite)
@@ -439,12 +437,12 @@ class Context(CtxDict):
         self,
         fs,
         vars_: List,
-        wdir: PathInfo,
+        wdir: str,
         stage_name: str = None,
         default: str = None,
     ):
         if default:
-            to_import = wdir / default
+            to_import = fs.path.join(wdir, default)
             if fs.exists(to_import):
                 self.merge_from(fs, default, wdir)
             else:

dvc/proc/exceptions.py

@@ -0,0 +1,25 @@
+from dvc.exceptions import DvcException
+
+
+class ProcessNotTerminatedError(DvcException):
+    def __init__(self, name):
+        super().__init__(f"Managed process '{name}' has not been terminated.")
+
+
+class ProcessNotFoundError(DvcException):
+    def __init__(self, name):
+        super().__init__(f"Managed process '{name}' does not exist.")
+
+
+class TimeoutExpired(DvcException):
+    def __init__(self, cmd, timeout):
+        super().__init__(
+            f"'{cmd}' did not complete before timeout '{timeout}'"
+        )
+        self.cmd = cmd
+        self.timeout = timeout
+
+
+class UnsupportedSignalError(DvcException):
+    def __init__(self, sig):
+        super().__init__(f"Unsupported signal: {sig}")

dvc/proc/manager.py

@@ -0,0 +1,154 @@
+"""Serverless process manager."""
+
+import json
+import logging
+import os
+import signal
+import sys
+from typing import Generator, List, Optional, Tuple, Union
+
+from funcy.flow import reraise
+from shortuuid import uuid
+
+from .exceptions import ProcessNotTerminatedError, UnsupportedSignalError
+from .process import ManagedProcess, ProcessInfo
+
+logger = logging.getLogger(__name__)
+
+
+class ProcessManager:
+    """Manager for controlling background ManagedProcess(es).
+
+    Spawned process entries are kept in the manager directory until they
+    are explicitly removed (with remove() or cleanup()) so that return
+    value and log information can be accessed after a process has completed.
+    """
+
+    def __init__(self, wdir: Optional[str] = None):
+        self.wdir = wdir or "."
+
+    def __iter__(self) -> Generator[str, None, None]:
+        if not os.path.exists(self.wdir):
+            return
+        for name in os.listdir(self.wdir):
+            yield name
+
+    def __getitem__(self, key: str) -> "ProcessInfo":
+        info_path = os.path.join(self.wdir, key, f"{key}.json")
+        try:
+            with open(info_path, encoding="utf-8") as fobj:
+                return ProcessInfo.from_dict(json.load(fobj))
+        except FileNotFoundError:
+            raise KeyError
+
+    @reraise(FileNotFoundError, KeyError)
+    def __setitem__(self, key: str, value: "ProcessInfo"):
+        info_path = os.path.join(self.wdir, key, f"{key}.json")
+        with open(info_path, "w", encoding="utf-8") as fobj:
+            return json.dump(value.asdict(), fobj)
+
+    def __delitem__(self, key: str) -> None:
+        from dvc.utils.fs import remove
+
+        path = os.path.join(self.wdir, key)
+        if os.path.exists(path):
+            remove(path)
+
+    def get(self, key: str, default=None):
+        try:
+            return self[key]
+        except KeyError:
+            return default
+
+    def processes(self) -> Generator[Tuple[str, "ProcessInfo"], None, None]:
+        for name in self:
+            try:
+                yield name, self[name]
+            except KeyError:
+                continue
+
+    def spawn(self, args: Union[str, List[str]], name: Optional[str] = None):
+        """Run the given command in the background."""
+        name = name or uuid()
+        pid = ManagedProcess.spawn(
+            args,
+            wdir=os.path.join(self.wdir, name),
+            name=name,
+        )
+        logger.debug(
+            "Spawned managed process '%s' (PID: '%d')",
+            name,
+            pid,
+        )
+
+    def send_signal(self, name: str, sig: int):
+        """Send `signal` to the specified named process."""
+        process_info = self[name]
+        if sys.platform == "win32":
+            if sig not in (
+                signal.SIGTERM,
+                signal.CTRL_C_EVENT,
+                signal.CTRL_BREAK_EVENT,
+            ):
+                raise UnsupportedSignalError(sig)
+
+        def handle_closed_process():
+            logging.warning(
+                f"Process {name} had already aborted unexpectedly."
+            )
+            process_info.returncode = -1
+            self[name] = process_info
+
+        if process_info.returncode is None:
+            try:
+                os.kill(process_info.pid, sig)
+            except ProcessLookupError:
+                handle_closed_process()
+                raise
+            except OSError as exc:
+                if sys.platform == "win32":
+                    if exc.winerror == 87:
+                        handle_closed_process()
+                        raise ProcessLookupError from exc
+                raise
+
+    def terminate(self, name: str):
+        """Terminate the specified named process."""
+        self.send_signal(name, signal.SIGTERM)
+
+    def kill(self, name: str):
+        """Kill the specified named process."""
+        if sys.platform == "win32":
+            self.send_signal(name, signal.SIGTERM)
+        else:
+            self.send_signal(name, signal.SIGKILL)
+
+    def remove(self, name: str, force: bool = False):
+        """Remove the specified named process from this manager.
+
+        If the specified process is still running, it will be forcefully killed
+        if `force` is True`, otherwise an exception will be raised.
+
+        Raises:
+            ProcessNotTerminatedError if the specified process is still
+            running and was not forcefully killed.
+        """
+        try:
+            process_info = self[name]
+        except KeyError:
+            return
+        if process_info.returncode is None and not force:
+            raise ProcessNotTerminatedError(name)
+        try:
+            self.kill(name)
+        except ProcessLookupError:
+            pass
+        del self[name]
+
+    def cleanup(self, force: bool = False):
+        """Remove stale (terminated) processes from this manager."""
+        for name in self:
+            try:
+                self.remove(name, force)
+            except ProcessNotTerminatedError:
+                continue

dvc/proc/process.py

@@ -0,0 +1,180 @@
+import json
+import logging
+import os
+import shlex
+import subprocess
+from contextlib import AbstractContextManager
+from dataclasses import asdict, dataclass
+from typing import List, Optional, TextIO, Union
+
+from funcy import cached_property
+from shortuuid import uuid
+
+from dvc.utils.fs import makedirs
+
+from .exceptions import TimeoutExpired
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ProcessInfo:
+    pid: int
+    stdin: Optional[str]
+    stdout: Optional[str]
+    stderr: Optional[str]
+    returncode: Optional[int]
+
+    @classmethod
+    def from_dict(cls, d):
+        return cls(**d)
+
+    def asdict(self):
+        return asdict(self)
+
+
+class ManagedProcess(AbstractContextManager):
+    """Run the specified command with redirected output.
+
+    stdout and stderr will both be redirected to <name>.out.
+    Interactive processes (requiring stdin input) are currently unsupported.
+
+    Parameters:
+        args: Command to be run.
+        wdir: If specified, redirected output files will be placed in `wdir`.
+        name: Name to use for this process, if not specified a UUID will be
+            generated instead.
+    """
+
+    def __init__(
+        self,
+        args: Union[str, List[str]],
+        wdir: Optional[str] = None,
+        name: Optional[str] = None,
+    ):
+        self.args: List[str] = (
+            shlex.split(args, posix=os.name == "posix")
+            if isinstance(args, str)
+            else list(args)
+        )
+        self.wdir = wdir
+        self.name = name or uuid()
+        self.returncode: Optional[int] = None
+        self._stdout: Optional[TextIO] = None
+        self._stderr: Optional[TextIO] = None
+
+        self._run()
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.wait()
+
+    def _close_fds(self):
+        if self._stdout:
+            self._stdout.close()
+            self._stdout = None
+        if self._stderr:
+            self._stderr.close()
+            self._stderr = None
+
+    def _make_path(self, path: str) -> str:
+        return os.path.join(self.wdir, path) if self.wdir else path
+
+    @cached_property
+    def stdout_path(self) -> str:
+        return self._make_path(f"{self.name}.out")
+
+    @cached_property
+    def info_path(self) -> str:
+        return self._make_path(f"{self.name}.json")
+
+    @cached_property
+    def pidfile_path(self) -> str:
+        return self._make_path(f"{self.name}.pid")
+
+    @property
+    def info(self) -> "ProcessInfo":
+        return ProcessInfo(
+            pid=self.pid,
+            stdin=None,
+            stdout=self.stdout_path,
+            stderr=None,
+            returncode=self.returncode,
+        )
+
+    def _make_wdir(self):
+        if self.wdir:
+            makedirs(self.wdir, exist_ok=True)
+
+    def _dump(self):
+        self._make_wdir()
+        with open(self.info_path, "w", encoding="utf-8") as fobj:
+            json.dump(self.info.asdict(), fobj)
+        with open(self.pidfile_path, "w", encoding="utf-8") as fobj:
+            fobj.write(str(self.pid))
+
+    def _run(self):
+        self._make_wdir()
+        logger.debug(
+            "Appending output to '%s'",
+            self.stdout_path,
+        )
+        self._stdout = open(self.stdout_path, "ab")
+        try:
+            self._proc = subprocess.Popen(
+                self.args,
+                stdin=subprocess.DEVNULL,
+                stdout=self._stdout,
+                stderr=subprocess.STDOUT,
+                close_fds=True,
+                shell=False,
+            )
+            self.pid: int = self._proc.pid
+            self._dump()
+        except Exception:
+            if self._proc is not None:
+                self._proc.kill()
+            self._close_fds()
+            raise
+
+    def wait(self, timeout: Optional[int] = None) -> Optional[int]:
+        """Block until a process started with `run` has completed.
+
+        Raises:
+            TimeoutExpired if `timeout` was set and the process
+            did not terminate after `timeout` seconds.
+        """
+        if self.returncode is not None:
+            return self.returncode
+        try:
+            self._proc.wait(timeout=timeout)
+        except subprocess.TimeoutExpired as exc:
+            raise TimeoutExpired(exc.cmd, exc.timeout) from exc
+        self.returncode = self._proc.returncode
+        self._close_fds()
+        self._dump()
+        return self.returncode
+
+    @classmethod
+    def spawn(cls, *args, **kwargs) -> Optional[int]:
+        """Spawn a ManagedProcess command in the background.
+
+        Returns: The spawned process PID.
+        """
+        import multiprocessing as mp
+
+        proc = mp.Process(
+            target=cls._spawn,
+            args=args,
+            kwargs=kwargs,
+            daemon=True,
+        )
+        proc.start()
+        # Do not terminate the child daemon when the main process exits
+        # pylint: disable=protected-access
+        mp.process._children.discard(proc)  # type: ignore[attr-defined]
+        return proc.pid
+
+    @classmethod
+    def _spawn(cls, *args, **kwargs):
+        with cls(*args, **kwargs):
+            pass

dvc/render/__init__.py

@@ -1,33 +1,4 @@
-import logging
-from typing import TYPE_CHECKING, Dict
+from .image import ImageRenderer
+from .vega import VegaRenderer
 
-if TYPE_CHECKING:
-    from dvc.types import StrPath
-
-logger = logging.getLogger(__name__)
-
-
-class Renderer:
-    def __init__(self, data: Dict):
-        self.data = data
-
-        from dvc.render.utils import get_files
-
-        files = get_files(self.data)
-
-        # we assume comparison of same file between revisions for now
-        assert len(files) == 1
-        self.filename = files.pop()
-
-    def _convert(self, path: "StrPath"):
-        raise NotImplementedError
-
-    @property
-    def DIV(self):
-        raise NotImplementedError
-
-    def generate_html(self, path: "StrPath"):
-        """this method might edit content of path"""
-        partial = self._convert(path)
-        div_id = f"plot_{self.filename.replace('.', '_').replace('/', '_')}"
-        return self.DIV.format(id=div_id, partial=partial)
+RENDERERS = [ImageRenderer, VegaRenderer]

dvc/render/base.py

@@ -0,0 +1,61 @@
+import abc
+from typing import TYPE_CHECKING, Dict
+
+from dvc.exceptions import DvcException
+
+if TYPE_CHECKING:
+    from dvc.types import StrPath
+
+
+REVISION_FIELD = "rev"
+INDEX_FIELD = "step"
+
+
+class BadTemplateError(DvcException):
+    pass
+
+
+class Renderer(abc.ABC):
+    def __init__(self, data: Dict, templates=None):
+        self.data = data
+        self.templates = templates
+
+        from dvc.render.utils import get_files
+
+        files = get_files(self.data)
+
+        # we assume comparison of same file between revisions for now
+        assert len(files) == 1
+        self.filename = files.pop()
+
+    def _convert(self, path: "StrPath"):
+        raise NotImplementedError
+
+    @property
+    @abc.abstractmethod
+    def DIV(self):
+        raise NotImplementedError
+
+    @property
+    @abc.abstractmethod
+    def SCRIPTS(self):
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def as_json(self):
+        raise NotImplementedError
+
+    @staticmethod
+    def _remove_special_chars(string: str):
+        return string.translate(
+            {ord(c): "_" for c in r"!@#$%^&*()[]{};,<>?\/:.|`~=_+"}
+        )
+
+    def generate_html(self, path: "StrPath"):
+        """this method might edit content of path"""
+        partial = self._convert(path)
+
+        div_id = self._remove_special_chars(self.filename)
+        div_id = f"plot_{div_id}"
+
+        return self.DIV.format(id=div_id, partial=partial)

dvc/render/html.py

@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Dict, List, Optional
 from dvc.exceptions import DvcException
 
 if TYPE_CHECKING:
-    from dvc.render import Renderer
+    from dvc.render.base import Renderer
     from dvc.types import StrPath
 
 PAGE_HTML = """<!DOCTYPE html>
@@ -13,9 +13,7 @@ PAGE_HTML = """<!DOCTYPE html>
 <head>
     {refresh_tag}
     <title>DVC Plot</title>
-    <script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
-    <script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
-    <script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
+    {scripts}
 </head>
 <body>
     {plot_divs}
@@ -29,8 +27,9 @@ class MissingPlaceholderError(DvcException):
 
 
 class HTML:
-    PLACEHOLDER = "plot_divs"
-    PLACEHOLDER_FORMAT_STR = f"{{{PLACEHOLDER}}}"
+    SCRIPTS_PLACEHOLDER = "scripts"
+    PLOTS_PLACEHOLDER = "plot_divs"
+    PLOTS_PLACEHOLDER_FORMAT_STR = f"{{{PLOTS_PLACEHOLDER}}}"
     REFRESH_PLACEHOLDER = "refresh_tag"
     REFRESH_TAG = '<meta http-equiv="refresh" content="{}">'
 
@@ -40,11 +39,12 @@ class HTML:
         refresh_seconds: Optional[int] = None,
     ):
         template = template or PAGE_HTML
-        if self.PLACEHOLDER_FORMAT_STR not in template:
-            raise MissingPlaceholderError(self.PLACEHOLDER_FORMAT_STR)
+        if self.PLOTS_PLACEHOLDER_FORMAT_STR not in template:
+            raise MissingPlaceholderError(self.PLOTS_PLACEHOLDER_FORMAT_STR)
 
         self.template = template
         self.elements: List[str] = []
+        self.scripts: str = ""
         self.refresh_tag = ""
         if refresh_seconds is not None:
             self.refresh_tag = self.REFRESH_TAG.format(refresh_seconds)
@@ -65,13 +65,19 @@ class HTML:
         self.elements.append(tabulate.tabulate(rows, header, tablefmt="html"))
         return self
 
+    def with_scripts(self, scripts: str) -> "HTML":
+        if scripts not in self.scripts:
+            self.scripts += f"\n{scripts}"
+        return self
+
     def with_element(self, html: str) -> "HTML":
         self.elements.append(html)
         return self
 
     def embed(self) -> str:
         kwargs = {
-            self.PLACEHOLDER: "\n".join(self.elements),
+            self.SCRIPTS_PLACEHOLDER: self.scripts,
+            self.PLOTS_PLACEHOLDER: "\n".join(self.elements),
             self.REFRESH_PLACEHOLDER: self.refresh_tag,
         }
         return self.template.format(**kwargs)
@@ -98,6 +104,7 @@ def write(
         document.with_element("<br>")
 
     for renderer in renderers:
+        document.with_scripts(renderer.SCRIPTS)
         document.with_element(renderer.generate_html(path))
 
     index = Path(os.path.join(path, "index.html"))

dvc/render/image.py

@@ -1,7 +1,7 @@
 import os
 from typing import TYPE_CHECKING
 
-from dvc.render import Renderer
+from dvc.render.base import Renderer
 from dvc.render.utils import get_files
 from dvc.utils import relpath
 
@@ -13,10 +13,13 @@ class ImageRenderer(Renderer):
     DIV = """
         <div
             id="{id}"
-            style="border: 1px solid;">
+            style="border:1px solid black;text-align:center;
+            white-space: nowrap;overflow-y:hidden;">
             {partial}
         </div>"""
 
+    SCRIPTS = ""
+
     def _write_image(
         self,
         page_dir_path: "StrPath",
@@ -33,7 +36,9 @@ class ImageRenderer(Renderer):
         with open(img_path, "wb") as fd:
             fd.write(image_data)
         return """
-        <div>
+        <div
+            style="border:1px dotted black;margin:2px;display: inline-block;
+            overflow:hidden;margin-left:8px;">
             <p>{title}</p>
             <img src="{src}">
         </div>""".format(
@@ -56,6 +61,9 @@ class ImageRenderer(Renderer):
             return "\n".join(div_content)
         return ""
 
+    def as_json(self):
+        raise NotImplementedError
+
     @staticmethod
     def matches(data):
         files = get_files(data)

dvc/render/utils.py

@@ -26,20 +26,18 @@ def find_vega(repo, plots_data, target):
     from dvc.render.vega import VegaRenderer
 
     if found and VegaRenderer.matches(found):
-        return VegaRenderer(found, repo.plots.templates).get_vega()
+        return VegaRenderer(found, repo.plots.templates).as_json()
     return ""
 
 
 def match_renderers(plots_data, templates):
-    from dvc.render.image import ImageRenderer
-    from dvc.render.vega import VegaRenderer
+    from dvc.render import RENDERERS
 
     renderers = []
     for g in group_by_filename(plots_data):
-        if VegaRenderer.matches(g):
-            renderers.append(VegaRenderer(g, templates))
-        if ImageRenderer.matches(g):
-            renderers.append(ImageRenderer(g))
+        for renderer_class in RENDERERS:
+            if renderer_class.matches(g):
+                renderers.append(renderer_class(g, templates))
     return renderers
 
 

dvc/render/vega.py

@@ -5,24 +5,17 @@ from typing import TYPE_CHECKING, Dict, List, Optional, Union
 from funcy import first
 
 from dvc.exceptions import DvcException
-from dvc.render import Renderer
+from dvc.render.base import (
+    INDEX_FIELD,
+    REVISION_FIELD,
+    BadTemplateError,
+    Renderer,
+)
 from dvc.render.utils import get_files
 
 if TYPE_CHECKING:
-    from dvc.repo.plots.template import PlotTemplates
     from dvc.types import StrPath
 
-REVISION_FIELD = "rev"
-INDEX_FIELD = "step"
-
-
-class PlotMetricTypeError(DvcException):
-    def __init__(self, file):
-        super().__init__(
-            "'{}' - file type error\n"
-            "Only JSON, YAML, CSV and TSV formats are supported.".format(file)
-        )
-
 
 class PlotDataStructureError(DvcException):
     def __init__(self):
@@ -108,9 +101,11 @@ class VegaRenderer(Renderer):
     </div>
     """
 
-    def __init__(self, data: Dict, templates: "PlotTemplates"):
-        super().__init__(data)
-        self.templates = templates
+    SCRIPTS = """
+    <script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
+    <script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
+    <script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
+    """
 
     def _squash_props(self) -> Dict:
         resolved: Dict[str, str] = {}
@@ -144,7 +139,36 @@ class VegaRenderer(Renderer):
                     datapoints.extend(tmp)
         return datapoints
 
-    def get_vega(self) -> Optional[str]:
+    def fill_template(self, template, datapoints, props=None):
+        props = props or {}
+
+        content = deepcopy(template.content)
+        if template.anchor_str("data") not in template.content:
+            anchor = template.anchor("data")
+            raise BadTemplateError(
+                f"Template '{template.name}' is not using '{anchor}' anchor"
+            )
+
+        if props.get("x"):
+            template.check_field_exists(datapoints, props.get("x"))
+        if props.get("y"):
+            template.check_field_exists(datapoints, props.get("y"))
+
+        content = template.fill_anchor(content, "data", datapoints)
+
+        props.setdefault("title", "")
+        props.setdefault("x_label", props.get("x"))
+        props.setdefault("y_label", props.get("y"))
+
+        names = ["title", "x", "y", "x_label", "y_label"]
+        for name in names:
+            value = props.get(name)
+            if value is not None:
+                content = template.fill_anchor(content, name, value)
+
+        return content
+
+    def as_json(self) -> Optional[str]:
         props = self._squash_props()
 
         template = self.templates.load(props.get("template", None))
@@ -162,11 +186,14 @@ class VegaRenderer(Renderer):
                 props["y"] = first(
                     f for f in reversed(fields) if f not in skip
                 )
-            return template.render(datapoints, props=props)
+            filled_template = self.fill_template(template, datapoints, props)
+
+            return filled_template
+
         return None
 
     def _convert(self, path: "StrPath"):
-        return self.get_vega()
+        return self.as_json()
 
     @staticmethod
     def matches(data):

dvc/repo/__init__.py

@@ -11,16 +11,15 @@ from dvc.exceptions import FileMissingError
 from dvc.exceptions import IsADirectoryError as DvcIsADirectoryError
 from dvc.exceptions import NotDvcRepoError, OutputNotFoundError
 from dvc.ignore import DvcIgnoreFilter
-from dvc.path_info import PathInfo
 from dvc.utils import env2bool
 from dvc.utils.fs import path_isin
 
 if TYPE_CHECKING:
-    from dvc.fs.base import BaseFileSystem
+    from dvc.fs.base import FileSystem
     from dvc.objects.file import HashFile
+    from dvc.repo.scm_context import SCMContext
     from dvc.scm import Base
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -91,15 +90,18 @@ class Repo:
     ):
         assert bool(scm) == bool(rev)
 
-        from dvc.scm import SCM
-        from dvc.scm.base import SCMError
-        from dvc.scm.git import Git
+        from dvc.fs.git import GitFileSystem
+        from dvc.scm import SCM, Base, Git, SCMError
         from dvc.utils.fs import makedirs
 
         dvc_dir = None
         tmp_dir = None
         try:
-            fs = scm.get_fs(rev) if isinstance(scm, Git) and rev else None
+            fs = (
+                GitFileSystem(scm=scm, rev=rev)
+                if isinstance(scm, Git) and rev
+                else None
+            )
             root_dir = self.find_root(root_dir, fs)
             dvc_dir = os.path.join(root_dir, self.DVC_DIR)
             tmp_dir = os.path.join(dvc_dir, "tmp")
@@ -113,8 +115,6 @@ class Repo:
             except SCMError:
                 scm = SCM(os.curdir, no_scm=True)
 
-            from dvc.scm import Base
-
             assert isinstance(scm, Base)
             root_dir = scm.root_dir
 
@@ -161,6 +161,7 @@ class Repo:
     ):
         from dvc.config import Config
         from dvc.data_cloud import DataCloud
+        from dvc.fs.git import GitFileSystem
         from dvc.fs.local import LocalFileSystem
         from dvc.lock import LockNoop, make_lock
         from dvc.objects.db import ODBManager
@@ -169,7 +170,7 @@ class Repo:
         from dvc.repo.params import Params
         from dvc.repo.plots import Plots
         from dvc.repo.stage import StageLoad
-        from dvc.scm import SCM
+        from dvc.scm import SCM, Git
         from dvc.stage.cache import StageCache
         from dvc.state import State, StateNoop
 
@@ -184,7 +185,8 @@ class Repo:
         )
 
         if scm:
-            self._fs = scm.get_fs(rev)
+            assert isinstance(scm, Git)
+            self._fs = GitFileSystem(scm=scm, rev=rev)
         else:
             self._fs = LocalFileSystem(url=self.root_dir)
 
@@ -254,8 +256,7 @@ class Repo:
 
     @cached_property
     def scm(self):
-        from dvc.scm import SCM
-        from dvc.scm.base import SCMError
+        from dvc.scm import SCM, SCMError
 
         if self._scm:
             return self._scm
@@ -270,6 +271,12 @@ class Repo:
                 return SCM(self.root_dir, no_scm=True)
             raise
 
+    @cached_property
+    def scm_context(self) -> "SCMContext":
+        from dvc.repo.scm_context import SCMContext
+
+        return SCMContext(self.scm, self.config)
+
     @cached_property
     def dvcignore(self) -> DvcIgnoreFilter:
 
@@ -280,7 +287,10 @@ class Repo:
 
         assert self.scm
         if isinstance(self.fs, LocalFileSystem):
-            return self.scm.get_rev()
+            from dvc.scm import map_scm_exception
+
+            with map_scm_exception():
+                return self.scm.get_rev()
         return self.fs.rev
 
     @cached_property
@@ -301,11 +311,11 @@ class Repo:
         return None
 
     @property
-    def fs(self) -> "BaseFileSystem":
+    def fs(self) -> "FileSystem":
         return self._fs
 
     @fs.setter
-    def fs(self, fs: "BaseFileSystem"):
+    def fs(self, fs: "FileSystem"):
         self._fs = fs
         # Our graph cache is no longer valid, as it was based on the previous
         # fs.
@@ -354,7 +364,7 @@ class Repo:
         )
 
     def unprotect(self, target):
-        return self.odb.local.unprotect(PathInfo(target))
+        return self.odb.local.unprotect(target)
 
     def _ignore(self):
         flist = [self.config.files["local"], self.tmp_dir]
@@ -362,7 +372,8 @@ class Repo:
         if path_isin(self.odb.local.cache_dir, self.root_dir):
             flist += [self.odb.local.cache_dir]
 
-        self.scm.ignore_list(flist)
+        for file in flist:
+            self.scm_context.ignore(file)
 
     def brancher(self, *args, **kwargs):
         from dvc.repo.brancher import brancher
@@ -446,14 +457,18 @@ class Repo:
         outs = outs or self.index.outs_graph
 
         abs_path = os.path.abspath(path)
-        path_info = PathInfo(abs_path)
-        match = path_info.__eq__ if strict else path_info.isin_or_eq
+        fs_path = abs_path
 
         def func(out):
-            if out.scheme == "local" and match(out.path_info):
+            def eq(one, two):
+                return one == two
+
+            match = eq if strict else out.fs.path.isin_or_eq
+
+            if out.scheme == "local" and match(fs_path, out.fs_path):
                 return True
 
-            if recursive and out.path_info.isin(path_info):
+            if recursive and out.fs.path.isin(out.fs_path, fs_path):
                 return True
 
             return False
@@ -490,7 +505,7 @@ class Repo:
         from dvc.fs.repo import RepoFileSystem
 
         fs = RepoFileSystem(self, subrepos=True)
-        path = PathInfo(self.root_dir) / path
+        path = self.fs.path.join(self.root_dir, path)
         try:
             with fs.open(
                 path, mode=mode, encoding=encoding, remote=remote

dvc/repo/add.py

@@ -87,8 +87,8 @@ def translate_graph_error(stages: Stages) -> Iterator[None]:
         yield
     except OverlappingOutputPathsError as exc:
         msg = OVERLAPPING_OUTPUT_FMT.format(
-            out=exc.overlapping_out.path_info,
-            parent=exc.parent.path_info,
+            out=exc.overlapping_out.fs_path,
+            parent=exc.parent.fs_path,
             parent_stage=exc.parent.stage.addressing,
         )
         raise OverlappingOutputPathsError(exc.parent, exc.overlapping_out, msg)
@@ -228,7 +228,7 @@ def _find_all_targets(
     from dvc.dvcfile import is_dvc_file
 
     if os.path.isdir(target) and recursive:
-        files = map(os.fspath, repo.dvcignore.walk_files(repo.fs, target))
+        files = repo.dvcignore.find(repo.fs, target)
         yield from (
             path
             for path in files

dvc/repo/brancher.py

@@ -1,3 +1,5 @@
+from functools import partial
+
 from funcy import group_by
 
 
@@ -58,8 +60,12 @@ def brancher(  # noqa: E302
 
     try:
         if revs:
-            for sha, names in group_by(scm.resolve_rev, revs).items():
-                self.fs = scm.get_fs(sha)
+            from dvc.fs.git import GitFileSystem
+            from dvc.scm import resolve_rev
+
+            rev_resolver = partial(resolve_rev, scm)
+            for sha, names in group_by(rev_resolver, revs).items():
+                self.fs = GitFileSystem(scm=scm, rev=sha)
                 # ignore revs that don't contain repo root
                 # (i.e. revs from before a subdir=True repo was init'ed)
                 if self.fs.exists(self.root_dir):

dvc/repo/collect.py

@@ -2,8 +2,7 @@ import logging
 import os
 from typing import TYPE_CHECKING, Callable, Iterable, List, Tuple
 
-from dvc.path_info import PathInfo
-from dvc.types import DvcPath
+from dvc.types import AnyPath
 
 if TYPE_CHECKING:
     from dvc.output import Output
@@ -14,7 +13,8 @@ logger = logging.getLogger(__name__)
 
 FilterFn = Callable[["Output"], bool]
 Outputs = List["Output"]
-DvcPaths = List[DvcPath]
+AnyPaths = List[AnyPath]
+StrPaths = List[str]
 
 
 def _collect_outs(
@@ -32,39 +32,40 @@ def _collect_paths(
     rev: str = None,
 ):
     from dvc.fs.repo import RepoFileSystem
+    from dvc.utils import relpath
 
-    path_infos = [PathInfo(os.path.abspath(target)) for target in targets]
+    fs_paths = [os.path.abspath(target) for target in targets]
     fs = RepoFileSystem(repo)
 
-    target_infos = []
-    for path_info in path_infos:
+    target_paths = []
+    for fs_path in fs_paths:
 
-        if recursive and fs.isdir(path_info):
-            target_infos.extend(repo.dvcignore.walk_files(fs, path_info))
+        if recursive and fs.isdir(fs_path):
+            target_paths.extend(repo.dvcignore.find(fs, fs_path))
 
-        if not fs.exists(path_info):
+        if not fs.exists(fs_path):
+            rel = relpath(fs_path)
             if rev == "workspace" or rev == "":
-                logger.warning(
-                    "'%s' was not found in current workspace.", path_info
-                )
+                logger.warning("'%s' was not found in current workspace.", rel)
             else:
-                logger.warning("'%s' was not found at: '%s'.", path_info, rev)
-        target_infos.append(path_info)
-    return target_infos
+                logger.warning("'%s' was not found at: '%s'.", rel, rev)
+        target_paths.append(fs_path)
+    return target_paths
 
 
 def _filter_duplicates(
-    outs: Outputs, path_infos: DvcPaths
-) -> Tuple[Outputs, DvcPaths]:
+    outs: Outputs, fs_paths: StrPaths
+) -> Tuple[Outputs, StrPaths]:
     res_outs: Outputs = []
-    res_infos = path_infos
+    fs_res_paths = fs_paths
 
     for out in outs:
-        if out.path_info in path_infos:
+        if out.fs_path in fs_paths:
             res_outs.append(out)
-            res_infos.remove(out.path_info)
+            # MUTATING THE SAME LIST!!
+            fs_res_paths.remove(out.fs_path)
 
-    return res_outs, res_infos
+    return res_outs, fs_res_paths
 
 
 def collect(
@@ -74,15 +75,15 @@ def collect(
     output_filter: FilterFn = None,
     rev: str = None,
     recursive: bool = False,
-) -> Tuple[Outputs, DvcPaths]:
+) -> Tuple[Outputs, StrPaths]:
     assert targets or output_filter
 
     outs: Outputs = _collect_outs(repo, output_filter=output_filter, deps=deps)
 
     if not targets:
-        path_infos: DvcPaths = []
-        return outs, path_infos
+        fs_paths: StrPaths = []
+        return outs, fs_paths
 
-    target_infos = _collect_paths(repo, targets, recursive=recursive, rev=rev)
+    target_paths = _collect_paths(repo, targets, recursive=recursive, rev=rev)
 
-    return _filter_duplicates(outs, target_infos)
+    return _filter_duplicates(outs, target_paths)

dvc/repo/diff.py

@@ -37,14 +37,14 @@ def diff(self, a_rev="HEAD", b_rev=None, targets=None):
             # workspace paths/checksums if b_rev was None
             continue
 
-        targets_path_infos = None
+        targets_paths = None
         if targets is not None:
-            # convert targets to path_infos, and capture any missing targets
-            targets_path_infos, missing_targets[rev] = _targets_to_path_infos(
+            # convert targets to paths, and capture any missing targets
+            targets_paths, missing_targets[rev] = _targets_to_paths(
                 repo_fs, targets
             )
 
-        results[rev] = _paths_checksums(self, targets_path_infos)
+        results[rev] = _paths_checksums(self, targets_paths)
 
     if targets is not None:
         # check for overlapping missing targets between a_rev and b_rev
@@ -133,13 +133,14 @@ def _output_paths(repo, targets):
     for output in repo.index.outs:
         if _exists(output):
             yield_output = targets is None or any(
-                output.path_info.isin_or_eq(target) for target in targets
+                output.fs.path.isin_or_eq(output.fs_path, target)
+                for target in targets
             )
 
             if on_working_fs:
                 _, _, obj = ostage(
                     repo.odb.local,
-                    output.path_info,
+                    output.fs_path,
                     repo.odb.local.fs,
                     "md5",
                     dry_run=True,
@@ -158,19 +159,24 @@ def _output_paths(repo, targets):
 
             if output.is_dir_checksum and (
                 yield_output
-                or any(target.isin(output.path_info) for target in targets)
+                or any(
+                    output.fs.path.isin(target, output.fs_path)
+                    for target in targets
+                )
             ):
-                yield from _dir_output_paths(output.path_info, obj, targets)
+                yield from _dir_output_paths(
+                    output.fs, output.fs_path, obj, targets
+                )
 
 
-def _dir_output_paths(path_info, obj, targets=None):
+def _dir_output_paths(fs, fs_path, obj, targets=None):
     for key, _, oid in obj:
-        fname = path_info.joinpath(*key)
+        fname = fs.path.join(fs_path, *key)
         if targets is None or any(
-            fname.isin_or_eq(target) for target in targets
+            fs.path.isin_or_eq(fname, target) for target in targets
         ):
             # pylint: disable=no-member
-            yield str(fname), oid.value
+            yield fs.path.join(fs.path.name(fs_path), *key), oid.value
 
 
 def _filter_missing(repo_fs, paths):
@@ -185,17 +191,17 @@ def _filter_missing(repo_fs, paths):
             pass
 
 
-def _targets_to_path_infos(repo_fs, targets):
-    path_infos = []
+def _targets_to_paths(repo_fs, targets):
+    paths = []
     missing = []
 
     for target in targets:
         if repo_fs.exists(target):
-            path_infos.append(repo_fs.metadata(target).path_info)
+            paths.append(repo_fs.metadata(target).fs_path)
         else:
             missing.append(target)
 
-    return path_infos, missing
+    return paths, missing
 
 
 def _calculate_renamed(new, old, added, deleted):

dvc/repo/experiments/__init__.py

@@ -13,8 +13,6 @@ from funcy import cached_property, first
 from dvc.dependency.param import MissingParamsError
 from dvc.env import DVCLIVE_RESUME
 from dvc.exceptions import DvcException
-from dvc.path_info import PathInfo
-from dvc.stage.monitor import CheckpointKilledError
 from dvc.utils import relpath
 
 from .base import (
@@ -43,7 +41,9 @@ def scm_locked(f):
     # different sequences of git operations at once
     @wraps(f)
     def wrapper(exp, *args, **kwargs):
-        with exp.scm_lock:
+        from dvc.scm import map_scm_exception
+
+        with map_scm_exception(), exp.scm_lock:
             return f(exp, *args, **kwargs)
 
     return wrapper
@@ -90,7 +90,7 @@ class Experiments:
 
     def __init__(self, repo):
         from dvc.lock import make_lock
-        from dvc.scm.base import NoSCMError
+        from dvc.scm import NoSCMError
 
         if repo.config["core"].get("no_scm", False):
             raise NoSCMError
@@ -99,6 +99,7 @@ class Experiments:
         self.scm_lock = make_lock(
             os.path.join(self.repo.tmp_dir, "exp_scm_lock"),
             tmp_dir=self.repo.tmp_dir,
+            hardlink_lock=repo.config["core"].get("hardlink_lock", False),
         )
 
     @property
@@ -117,7 +118,7 @@ class Experiments:
 
     @cached_property
     def stash(self):
-        from dvc.scm.git import Stash
+        from scmrepo.git import Stash
 
         return Stash(self.scm, EXPS_STASH)
 
@@ -163,7 +164,7 @@ class Experiments:
                 the human-readable name in the experiment branch ref. Has no
                 effect of branch is specified.
         """
-        with self.scm.detach_head() as orig_head:
+        with self.scm.detach_head(client="dvc") as orig_head:
             stash_head = orig_head
             if baseline_rev is None:
                 baseline_rev = orig_head
@@ -368,13 +369,12 @@ class Experiments:
 
         logger.debug("Using experiment params '%s'", params)
 
-        for params_fname in params:
-            path = PathInfo(params_fname)
-            suffix = path.suffix.lower()
+        for path in params:
+            suffix = self.repo.fs.path.suffix(path).lower()
             modify_data = MODIFIERS[suffix]
             with modify_data(path, fs=self.repo.fs) as data:
                 try:
-                    merge_params(data, params[params_fname], allow_new=False)
+                    merge_params(data, params[path], allow_new=False)
                 except NewParamsFound as e:
                     msg = self._format_new_params_msg(e.new_params, path)
                     raise MissingParamsError(msg)
@@ -410,7 +410,9 @@ class Experiments:
                 self.scm.reset()
 
         if checkpoint_resume:
-            resume_rev = self.scm.resolve_rev(checkpoint_resume)
+            from dvc.scm import resolve_rev
+
+            resume_rev = resolve_rev(self.scm, checkpoint_resume)
             try:
                 self.check_baseline(resume_rev)
                 checkpoint_resume = resume_rev
@@ -693,6 +695,8 @@ class Experiments:
             dict mapping stash revs to the successfully executed experiments
             for each stash rev.
         """
+        from dvc.stage.monitor import CheckpointKilledError
+
         result: Dict[str, Dict[str, str]] = defaultdict(dict)
 
         manager = Manager()
@@ -795,6 +799,7 @@ class Experiments:
     @unlocked_repo
     def _workspace_repro(self) -> Mapping[str, str]:
         """Run the most recently stashed experiment in the workspace."""
+        from dvc.stage.monitor import CheckpointKilledError
         from dvc.utils.fs import makedirs
 
         from .executor.base import BaseExecutor
@@ -807,7 +812,7 @@ class Experiments:
         # `checkout --force` here will not lose any data (popping stash commit
         # will result in conflict between workspace params and stashed CLI
         # params, but we always want the stashed version).
-        with self.scm.detach_head(entry.rev, force=True):
+        with self.scm.detach_head(entry.rev, force=True, client="dvc"):
             rev = self.stash.pop()
             self.scm.set_ref(EXEC_BASELINE, entry.baseline_rev)
             if entry.branch:
@@ -880,7 +885,9 @@ class Experiments:
         return self._get_baseline(rev)
 
     def _get_baseline(self, rev):
-        rev = self.scm.resolve_rev(rev)
+        from dvc.scm import resolve_rev
+
+        rev = resolve_rev(self.scm, rev)
 
         if rev in self.stash_revs:
             entry = self.stash_revs.get(rev)
@@ -929,7 +936,7 @@ class Experiments:
         from .executor.base import BaseExecutor, ExecutorInfo
 
         result = {}
-        for pidfile in self.repo.fs.walk_files(
+        for pidfile in self.repo.fs.find(
             os.path.join(self.repo.tmp_dir, self.EXEC_PID_DIR)
         ):
             rev, _ = os.path.splitext(os.path.basename(pidfile))

dvc/repo/experiments/apply.py

@@ -3,7 +3,6 @@ import os
 
 from dvc.repo import locked
 from dvc.repo.scm_context import scm_context
-from dvc.scm.base import RevError
 from dvc.utils.fs import remove
 
 from .base import (
@@ -20,13 +19,15 @@ logger = logging.getLogger(__name__)
 @locked
 @scm_context
 def apply(repo, rev, force=True, **kwargs):
+    from scmrepo.exceptions import MergeConflictError
+
     from dvc.repo.checkout import checkout as dvc_checkout
-    from dvc.scm.base import MergeConflictError, SCMError
+    from dvc.scm import RevError, SCMError, resolve_rev
 
     exps = repo.experiments
 
     try:
-        exp_rev = repo.scm.resolve_rev(rev)
+        exp_rev = resolve_rev(repo.scm, rev)
         exps.check_baseline(exp_rev)
     except (RevError, BaselineMismatchError) as exc:
         raise InvalidExpRevError(rev) from exc
@@ -45,7 +46,12 @@ def apply(repo, rev, force=True, **kwargs):
     else:
         workspace = None
 
-    repo.scm.merge(exp_rev, commit=False)
+    from scmrepo.exceptions import SCMError as _SCMError
+
+    try:
+        repo.scm.merge(exp_rev, commit=False)
+    except _SCMError as exc:
+        raise SCMError(str(exc))
 
     if workspace:
         try:
@@ -60,7 +66,7 @@ def apply(repo, rev, force=True, **kwargs):
                 repo.scm.reset(hard=True)
                 repo.scm.stash.pop()
                 raise ApplyConflictError(rev) from exc
-        except SCMError as exc:
+        except _SCMError as exc:
             raise ApplyConflictError(rev) from exc
         repo.scm.stash.drop()
     repo.scm.reset()

dvc/repo/experiments/branch.py

@@ -3,7 +3,7 @@ import logging
 from dvc.exceptions import InvalidArgumentError
 from dvc.repo import locked
 from dvc.repo.scm_context import scm_context
-from dvc.scm.base import RevError
+from dvc.scm import RevError
 
 from .base import InvalidExpRevError
 from .utils import exp_refs_by_rev
@@ -14,8 +14,10 @@ logger = logging.getLogger(__name__)
 @locked
 @scm_context
 def branch(repo, exp_rev, branch_name, *args, **kwargs):
+    from dvc.scm import resolve_rev
+
     try:
-        rev = repo.scm.resolve_rev(exp_rev)
+        rev = resolve_rev(repo.scm, exp_rev)
     except RevError:
         raise InvalidArgumentError(exp_rev)
     ref_info = None

dvc/repo/experiments/diff.py

@@ -9,13 +9,14 @@ logger = logging.getLogger(__name__)
 
 def diff(repo, *args, a_rev=None, b_rev=None, param_deps=False, **kwargs):
     from dvc.repo.experiments.show import _collect_experiment_commit
+    from dvc.scm import resolve_rev
 
     if repo.scm.no_commits:
         return {}
 
     if a_rev:
         a_rev = fix_exp_head(repo.scm, a_rev)
-        rev = repo.scm.resolve_rev(a_rev)
+        rev = resolve_rev(repo.scm, a_rev)
         old = _collect_experiment_commit(repo, rev, param_deps=param_deps)
     else:
         old = _collect_experiment_commit(
@@ -24,7 +25,7 @@ def diff(repo, *args, a_rev=None, b_rev=None, param_deps=False, **kwargs):
 
     if b_rev:
         b_rev = fix_exp_head(repo.scm, b_rev)
-        rev = repo.scm.resolve_rev(b_rev)
+        rev = resolve_rev(repo.scm, b_rev)
         new = _collect_experiment_commit(repo, rev, param_deps=param_deps)
     else:
         new = _collect_experiment_commit(

dvc/repo/experiments/executor/base.py

@@ -16,7 +16,6 @@ from typing import (
 
 from dvc.env import DVC_EXP_AUTO_PUSH, DVC_EXP_GIT_REMOTE
 from dvc.exceptions import DvcException
-from dvc.path_info import PathInfo
 from dvc.repo import Repo
 from dvc.repo.experiments.base import (
     EXEC_BASELINE,
@@ -39,7 +38,7 @@ from dvc.utils.fs import remove
 if TYPE_CHECKING:
     from multiprocessing import Queue
 
-    from dvc.scm.git import Git
+    from scmrepo.git import Git
 
 logger = logging.getLogger(__name__)
 
@@ -101,7 +100,7 @@ class BaseExecutor(ABC):
         self,
         src: "Git",
         dvc_dir: str,
-        root_dir: Optional[Union[str, PathInfo]] = None,
+        root_dir: Optional[str] = None,
         branch: Optional[str] = None,
         name: Optional[str] = None,
         **kwargs,
@@ -184,9 +183,11 @@ class BaseExecutor(ABC):
             on_diverged: Callback in the form on_diverged(ref, is_checkpoint)
                 to be called when an experiment ref has diverged.
         """
+        from ..utils import iter_remote_refs
+
         refs = []
         has_checkpoint = False
-        for ref in dest_scm.iter_remote_refs(url, base=EXPS_NAMESPACE):
+        for ref in iter_remote_refs(dest_scm, url, base=EXPS_NAMESPACE):
             if ref == EXEC_CHECKPOINT:
                 has_checkpoint = True
             elif not ref.startswith(EXEC_NAMESPACE) and ref != EXPS_STASH:
@@ -223,6 +224,9 @@ class BaseExecutor(ABC):
 
     @classmethod
     def _validate_remotes(cls, dvc: "Repo", git_remote: Optional[str]):
+        from scmrepo.exceptions import InvalidRemote
+
+        from dvc.scm import InvalidRemoteSCMRepo
 
         if git_remote == dvc.root_dir:
             logger.warning(
@@ -231,7 +235,10 @@ class BaseExecutor(ABC):
                 "will automatically be pushed to the default DVC remote "
                 "(if any) on each experiment commit."
             )
-        dvc.scm.validate_git_remote(git_remote)
+        try:
+            dvc.scm.validate_git_remote(git_remote)
+        except InvalidRemote as exc:
+            raise InvalidRemoteSCMRepo(str(exc))
         dvc.cloud.get_remote_odb()
 
     @classmethod
@@ -393,7 +400,7 @@ class BaseExecutor(ABC):
 
         dvc = Repo(dvc_dir)
         if cls.QUIET:
-            dvc.scm.quiet = cls.QUIET
+            dvc.scm_context.quiet = cls.QUIET
         if dvc_dir is not None:
             old_cwd: Optional[str] = os.getcwd()
             if rel_cwd:

dvc/repo/experiments/executor/local.py

@@ -17,8 +17,9 @@ from dvc.scm import SCM
 from .base import BaseExecutor
 
 if TYPE_CHECKING:
+    from scmrepo.git import Git
+
     from dvc.repo import Repo
-    from dvc.scm.git import Git
 
 logger = logging.getLogger(__name__)
 
@@ -85,12 +86,14 @@ class TempDirExecutor(BaseLocalExecutor):
     def _init_git(self, scm: "Git", branch: Optional[str] = None, **kwargs):
         from dulwich.repo import Repo as DulwichRepo
 
+        from ..utils import push_refspec
+
         DulwichRepo.init(os.fspath(self.root_dir))
 
         refspec = f"{EXEC_NAMESPACE}/"
-        scm.push_refspec(self.git_url, refspec, refspec, **kwargs)
+        push_refspec(scm, self.git_url, refspec, refspec, **kwargs)
         if branch:
-            scm.push_refspec(self.git_url, branch, branch, **kwargs)
+            push_refspec(scm, self.git_url, branch, branch, **kwargs)
             self.scm.set_ref(EXEC_BRANCH, branch, symbolic=True)
         elif self.scm.get_ref(EXEC_BRANCH):
             self.scm.remove_ref(EXEC_BRANCH)

dvc/repo/experiments/executor/ssh.py

@@ -17,9 +17,10 @@ from dvc.repo.experiments.base import (
 from .base import BaseExecutor
 
 if TYPE_CHECKING:
+    from scmrepo.git import Git
+
     from dvc.machine import MachineManager
     from dvc.repo import Repo
-    from dvc.scm.git import Git
 
 logger = logging.getLogger(__name__)
 
@@ -104,6 +105,8 @@ class SSHExecutor(BaseExecutor):
         return kwargs
 
     def _init_git(self, scm: "Git", branch: Optional[str] = None, **kwargs):
+        from ..utils import push_refspec
+
         with self.sshfs() as fs:
             fs.makedirs(self.root_dir)
             self._ssh_cmd(fs, "git init .")
@@ -120,9 +123,9 @@ class SSHExecutor(BaseExecutor):
             # (see https://github.com/iterative/dvc/issues/6508)
             kwargs.update(self._git_client_args(fs))
             refspec = f"{EXEC_NAMESPACE}/"
-            scm.push_refspec(self.git_url, refspec, refspec, **kwargs)
+            push_refspec(scm, self.git_url, refspec, refspec, **kwargs)
             if branch:
-                scm.push_refspec(self.git_url, branch, branch, **kwargs)
+                push_refspec(scm, self.git_url, branch, branch, **kwargs)
                 self._ssh_cmd(fs, f"git symbolic-ref {EXEC_BRANCH} {branch}")
             else:
                 self._ssh_cmd(
@@ -148,12 +151,12 @@ class SSHExecutor(BaseExecutor):
         from dvc.repo import Repo
         from dvc.repo.push import push
 
-        cache_url = posixpath.join(
-            self.abs_url,
-            self._dvc_dir,
+        cache_path = posixpath.join(
+            self._repo_abspath,
             Repo.DVC_DIR,
             ODBManager.CACHE_DIR,
         )
+
         with self.sshfs() as fs:
-            odb = get_odb(fs, fs.PATH_CLS(cache_url), **fs.config)
+            odb = get_odb(fs, cache_path, **fs.config)
             push(dvc, revs=[rev], run_cache=run_cache, odb=odb)

dvc/repo/experiments/init.py

@@ -1,10 +1,22 @@
 import logging
 import os
 from contextlib import contextmanager
-from typing import TYPE_CHECKING, Dict, Iterable, cast
+from functools import partial
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Dict,
+    Iterable,
+    List,
+    Optional,
+    TextIO,
+    Tuple,
+    Union,
+    cast,
+)
 
-from funcy import compact, lremove, post_processing
-from rich.prompt import Confirm, Prompt
+from funcy import compact, lremove
 from rich.rule import Rule
 from rich.syntax import Syntax
 
@@ -17,7 +29,7 @@ from dvc.utils.serialize import dumps_yaml
 if TYPE_CHECKING:
     from dvc.repo import Repo
     from dvc.dvcfile import DVCFile
-    from dvc.stage import Stage
+    from rich.tree import Tree
 
 from dvc.ui import ui
 
@@ -33,93 +45,27 @@ PROMPTS = {
 }
 
 
-class RichInputMixin:
-    """Prevents exc message from printing in the same line on Ctrl + D/C."""
+def _prompts(
+    keys: Iterable[str],
+    defaults: Dict[str, str] = None,
+    validator: Callable[[str, str], Union[str, Tuple[str, str]]] = None,
+    allow_omission: bool = True,
+    stream: Optional[TextIO] = None,
+) -> Dict[str, OptStr]:
+    from dvc.ui.prompt import Prompt
 
-    @classmethod
-    def get_input(cls, *args, **kwargs) -> str:
-        try:
-            return super().get_input(*args, **kwargs)  # type: ignore[misc]
-        except (KeyboardInterrupt, EOFError):
-            ui.error_write()
-            raise
-
-
-class ConfirmationPrompt(RichInputMixin, Confirm):
-    def make_prompt(self, default):
-        prompt = self.prompt.copy()
-        prompt.end = ""
-
-        prompt.append(" [")
-        yes, no = self.choices
-        for idx, (val, choice) in enumerate(((True, yes), (False, no))):
-            if idx:
-                prompt.append("/")
-            if val == default:
-                prompt.append(choice.upper(), "green")
-            else:
-                prompt.append(choice)
-        prompt.append("] ")
-        return prompt
-
-
-class RequiredPrompt(RichInputMixin, Prompt):
-    def process_response(self, value: str):
-        from rich.prompt import InvalidResponse
-
-        ret = super().process_response(value)
-        if not ret:
-            raise InvalidResponse(
-                "[prompt.invalid]Response required. Please try again."
-            )
-        return ret
-
-    def render_default(self, default):
-        from rich.text import Text
-
-        return Text(f"{default!s}", "green")
-
-
-class SkippablePrompt(RequiredPrompt):
-    skip_value: str = "n"
-
-    def process_response(self, value: str):
-        ret = super().process_response(value)
-        return None if ret == self.skip_value else ret
-
-    def make_prompt(self, default):
-        prompt = self.prompt.copy()
-        prompt.end = ""
-
-        prompt.append(" [")
-        if (
-            default is not ...
-            and self.show_default
-            and isinstance(default, (str, self.response_type))
-        ):
-            _default = self.render_default(default)
-            prompt.append(_default)
-            prompt.append(", ")
-
-        prompt.append(f"{self.skip_value} to omit", style="italic")
-        prompt.append("]")
-        prompt.append(self.prompt_suffix)
-        return prompt
-
-
-@post_processing(dict)
-def _prompts(keys: Iterable[str], defaults: Dict[str, OptStr]):
-    for key in keys:
-        if key == "cmd":
-            prompt_cls = RequiredPrompt
-        else:
-            prompt_cls = SkippablePrompt
-        kwargs = {"default": defaults[key]} if key in defaults else {}
-        prompt = PROMPTS[key]
-        value = prompt_cls.ask(  # type: ignore[call-overload]
-            prompt, console=ui.error_console, **kwargs
+    defaults = defaults or {}
+    return {
+        key: Prompt.prompt_(
+            PROMPTS[key],
+            console=ui.error_console,
+            default=defaults.get(key),
+            validator=partial(validator, key) if validator else None,
+            allow_omission=allow_omission,
+            stream=stream,
         )
-        yield key, value
+        for key in keys
+    }
 
 
 @contextmanager
@@ -134,65 +80,80 @@ def _disable_logging(highest_level=logging.CRITICAL):
         logging.disable(previous_level)
 
 
-PIPELINE_FILE_LINK = (
-    "https://dvc.org/doc/user-guide/project-structure/pipelines-files"
-)
+def build_workspace_tree(workspace: Dict[str, str]) -> "Tree":
+    from rich.tree import Tree
+
+    tree = Tree(
+        "DVC assumes the following workspace structure:",
+        highlight=True,
+    )
+    for value in sorted(workspace.values()):
+        tree.add(f"[green]{value}[/green]")
+    return tree
+
+
+PIPELINE_FILE_LINK = "https://s.dvc.org/g/pipeline-files"
 
 
 def init_interactive(
     name: str,
     defaults: Dict[str, str],
     provided: Dict[str, str],
-    show_tree: bool = False,
+    validator: Callable[[str, str], Union[str, Tuple[str, str]]] = None,
     live: bool = False,
+    stream: Optional[TextIO] = None,
 ) -> Dict[str, str]:
     command = provided.pop("cmd", None)
     primary = lremove(provided.keys(), ["code", "data", "models", "params"])
     secondary = lremove(
         provided.keys(), ["live"] if live else ["metrics", "plots"]
     )
+    prompts = primary + secondary
+
+    workspace = {**defaults, **provided}
+    if not live and "live" not in provided:
+        workspace.pop("live", None)
+    for key in ("plots", "metrics"):
+        if live and key not in provided:
+            workspace.pop(key, None)
 
     ret: Dict[str, str] = {}
-    if not (command or primary or secondary):
+    if command:
+        ret["cmd"] = command
+
+    if not prompts and command:
         return ret
 
-    message = ui.rich_text.assemble(
-        "This command will guide you to set up a ",
-        (name, "bright_blue"),
-        " stage in ",
-        ("dvc.yaml", "green"),
-        ".",
+    ui.error_write(
+        f"This command will guide you to set up a [bright_blue]{name}[/]",
+        "stage in [green]dvc.yaml[/].",
+        f"\nSee [repr.url]{PIPELINE_FILE_LINK}[/].\n",
+        styled=True,
     )
-    doc_link = ui.rich_text.assemble(
-        "See ", (PIPELINE_FILE_LINK, "repr.url"), "."
-    )
-    ui.error_write(message, doc_link, "", sep="\n", styled=True)
 
     if not command:
-        ret.update(_prompts(["cmd"], defaults))
-        ui.write()
-    else:
-        ret.update({"cmd": command})
-
-    ui.write("Enter the paths for dependencies and outputs of the command.")
-    workspace = {**defaults, **provided}
-    if show_tree and workspace:
-        from rich.tree import Tree
-
-        tree = Tree(
-            "DVC assumes the following workspace structure:",
-            highlight=True,
+        ret.update(
+            compact(_prompts(["cmd"], allow_omission=False, stream=stream))
         )
-        if not live and "live" not in provided:
-            workspace.pop("live", None)
-        for value in sorted(workspace.values()):
-            tree.add(f"[green]{value}[/green]")
-        ui.error_write(tree, styled=True)
+        if prompts:
+            ui.error_write(styled=True)
 
-    ui.error_write()
-    ret.update(_prompts(primary, defaults))
-    ret.update(_prompts(secondary, defaults))
-    return compact(ret)
+    if not prompts:
+        return ret
+
+    ui.error_write(
+        "Enter the paths for dependencies and outputs of the command.",
+        styled=True,
+    )
+    if workspace:
+        ui.error_write(build_workspace_tree(workspace), styled=True)
+    ui.error_write(styled=True)
+    ret.update(
+        compact(
+            _prompts(prompts, defaults, validator=validator, stream=stream)
+        )
+    )
+    return ret
 
 
 def _check_stage_exists(
@@ -207,41 +168,71 @@ def _check_stage_exists(
         )
 
 
+def loadd_params(path: str) -> Dict[str, List[str]]:
+    from dvc.utils.serialize import LOADERS
+
+    _, ext = os.path.splitext(path)
+    return {path: list(LOADERS[ext](path))}
+
+
+def validate_prompts(key: str, value: str) -> Union[Any, Tuple[Any, str]]:
+    from dvc.ui.prompt import InvalidResponse
+
+    if key == "params":
+        assert isinstance(value, str)
+        msg_format = (
+            "[prompt.invalid]'{0}' {1}. "
+            "Please retry with an existing parameters file."
+        )
+        if not os.path.exists(value):
+            raise InvalidResponse(msg_format.format(value, "does not exist"))
+        if os.path.isdir(value):
+            raise InvalidResponse(msg_format.format(value, "is a directory"))
+    elif key in ("code", "data"):
+        if not os.path.exists(value):
+            return value, (
+                f"[yellow]'{value}' does not exist in the workspace. "
+                '"exp run" may fail.[/]'
+            )
+    return value
+
+
 def init(
     repo: "Repo",
-    name: str = None,
+    name: str = "train",
     type: str = "default",  # pylint: disable=redefined-builtin
     defaults: Dict[str, str] = None,
     overrides: Dict[str, str] = None,
     interactive: bool = False,
     force: bool = False,
-) -> "Stage":
+    stream: Optional[TextIO] = None,
+) -> PipelineStage:
     from dvc.dvcfile import make_dvcfile
 
     dvcfile = make_dvcfile(repo, "dvc.yaml")
-    name = name or type
-
     _check_stage_exists(dvcfile, name, force=force)
 
-    defaults = defaults or {}
-    overrides = overrides or {}
+    defaults = defaults.copy() if defaults else {}
+    overrides = overrides.copy() if overrides else {}
+
+    with_live = type == "dl"
 
-    with_live = type == "live"
     if interactive:
         defaults = init_interactive(
             name,
+            validator=validate_prompts,
             defaults=defaults,
             live=with_live,
             provided=overrides,
-            show_tree=True,
+            stream=stream,
         )
     else:
         if with_live:
-            # suppress `metrics`/`params` if live is selected, unless
+            # suppress `metrics`/`plots` if live is selected, unless
             # it is also provided via overrides/cli.
             # This makes output to be a checkpoint as well.
             defaults.pop("metrics", None)
-            defaults.pop("params", None)
+            defaults.pop("plots", None)
         else:
             defaults.pop("live", None)  # suppress live otherwise
 
@@ -251,11 +242,7 @@ def init(
     params_kv = []
     params = context.get("params")
     if params:
-        from dvc.utils.serialize import LOADERS
-
-        assert isinstance(params, str)
-        _, ext = os.path.splitext(params)
-        params_kv = [{params: list(LOADERS[ext](params))}]
+        params_kv.append(loadd_params(params))
 
     checkpoint_out = bool(context.get("live"))
     models = context.get("models")
@@ -272,22 +259,26 @@ def init(
     )
 
     if interactive:
-        ui.write(Rule(style="green"), styled=True)
+        ui.error_write(Rule(style="green"), styled=True)
         _yaml = dumps_yaml(to_pipeline_file(cast(PipelineStage, stage)))
         syn = Syntax(_yaml, "yaml", theme="ansi_dark")
         ui.error_write(syn, styled=True)
 
-    if not interactive or ConfirmationPrompt.ask(
+    from dvc.ui.prompt import Confirm
+
+    if not interactive or Confirm.ask(
         "Do you want to add the above contents to dvc.yaml?",
         console=ui.error_console,
         default=True,
+        stream=stream,
     ):
-        scm = repo.scm
-        with _disable_logging(), scm.track_file_changes(autostage=True):
+        with _disable_logging(), repo.scm_context(autostage=True, quiet=True):
             stage.dump(update_lock=False)
             stage.ignore_outs()
             if params:
-                scm.track_file(params)
+                repo.scm_context.track_file(params)
     else:
         raise DvcException("Aborting ...")
+
+    assert isinstance(stage, PipelineStage)
     return stage

dvc/repo/experiments/ls.py

@@ -3,7 +3,6 @@ from collections import defaultdict
 
 from dvc.repo import locked
 from dvc.repo.scm_context import scm_context
-from dvc.scm.base import RevError
 
 from .utils import (
     exp_refs,
@@ -18,11 +17,13 @@ logger = logging.getLogger(__name__)
 @locked
 @scm_context
 def ls(repo, *args, rev=None, git_remote=None, all_=False, **kwargs):
-    from dvc.scm.git import Git
+    from scmrepo.git import Git
+
+    from dvc.scm import RevError, resolve_rev
 
     if rev:
         try:
-            rev = repo.scm.resolve_rev(rev)
+            rev = resolve_rev(repo.scm, rev)
         except RevError:
             if not (git_remote and Git.is_sha(rev)):
                 # This could be a remote rev that has not been fetched yet

dvc/repo/experiments/pull.py

@@ -31,9 +31,17 @@ def pull(
 
     refspec = f"{exp_ref}:{exp_ref}"
     logger.debug("git pull experiment '%s' -> '%s'", git_remote, refspec)
-    repo.scm.fetch_refspecs(
-        git_remote, [refspec], force=force, on_diverged=on_diverged
-    )
+
+    from dvc.scm import TqdmGit
+
+    with TqdmGit(desc="Fetching git refs") as pbar:
+        repo.scm.fetch_refspecs(
+            git_remote,
+            [refspec],
+            force=force,
+            on_diverged=on_diverged,
+            progress=pbar.update_git,
+        )
 
     if pull_cache:
         _pull_cache(repo, exp_ref, **kwargs)

dvc/repo/experiments/push.py

@@ -4,7 +4,7 @@ from dvc.exceptions import DvcException, InvalidArgumentError
 from dvc.repo import locked
 from dvc.repo.scm_context import scm_context
 
-from .utils import exp_commits, resolve_exp_ref
+from .utils import exp_commits, push_refspec, resolve_exp_ref
 
 logger = logging.getLogger(__name__)
 
@@ -37,9 +37,19 @@ def push(
 
     refname = str(exp_ref)
     logger.debug("git push experiment '%s' -> '%s'", exp_ref, git_remote)
-    repo.scm.push_refspec(
-        git_remote, refname, refname, force=force, on_diverged=on_diverged
-    )
+
+    from dvc.scm import TqdmGit
+
+    with TqdmGit(desc="Pushing git refs") as pbar:
+        push_refspec(
+            repo.scm,
+            git_remote,
+            refname,
+            refname,
+            force=force,
+            on_diverged=on_diverged,
+            progress=pbar.update_git,
+        )
 
     if push_cache:
         _push_cache(repo, exp_ref, **kwargs)

dvc/repo/experiments/remove.py

@@ -4,9 +4,9 @@ from typing import List, Optional
 from dvc.exceptions import InvalidArgumentError
 from dvc.repo import locked
 from dvc.repo.scm_context import scm_context
-from dvc.scm.base import RevError
+from dvc.scm import RevError
 
-from .utils import exp_refs, remove_exp_refs, resolve_exp_ref
+from .utils import exp_refs, push_refspec, remove_exp_refs, resolve_exp_ref
 
 logger = logging.getLogger(__name__)
 
@@ -52,8 +52,11 @@ def _get_exp_stash_index(repo, ref_or_rev: str) -> Optional[int]:
     for _, ref_info in stash_revs.items():
         if ref_info.name == ref_or_rev:
             return ref_info.index
+
+    from dvc.scm import resolve_rev
+
     try:
-        rev = repo.scm.resolve_rev(ref_or_rev)
+        rev = resolve_rev(repo.scm, ref_or_rev)
         if rev in stash_revs:
             return stash_revs.get(rev).index
     except RevError:
@@ -77,8 +80,17 @@ def _remove_commited_exps(
         if not remote:
             remove_exp_refs(repo.scm, remove_list)
         else:
+            from dvc.scm import TqdmGit
+
             for ref_info in remove_list:
-                repo.scm.push_refspec(remote, None, str(ref_info))
+                with TqdmGit(desc="Pushing git refs") as pbar:
+                    push_refspec(
+                        repo.scm,
+                        remote,
+                        None,
+                        str(ref_info),
+                        progress=pbar.update_git,
+                    )
     return remain_list
 
 

dvc/repo/experiments/show.py

@@ -10,7 +10,6 @@ from dvc.repo.experiments.executor.base import ExecutorInfo
 from dvc.repo.experiments.utils import fix_exp_head
 from dvc.repo.metrics.show import _gather_metrics
 from dvc.repo.params.show import _gather_params
-from dvc.scm.base import SCMError
 from dvc.utils import error_handler, onerror_collect
 
 logger = logging.getLogger(__name__)
@@ -72,7 +71,9 @@ def _collect_experiment_commit(
 def _collect_experiment_branch(
     res, repo, branch, baseline, onerror: Optional[Callable] = None, **kwargs
 ):
-    exp_rev = repo.scm.resolve_rev(branch)
+    from dvc.scm import resolve_rev
+
+    exp_rev = resolve_rev(repo.scm, branch)
     prev = None
     revs = list(repo.scm.branch_revs(exp_rev, baseline))
     for rev in revs:
@@ -121,12 +122,15 @@ def show(
         raise InvalidArgumentError(f"Invalid number of commits '{num}'")
 
     if revs is None:
+        from dvc.scm import RevError, resolve_rev
+
         revs = []
         for n in range(num):
             try:
                 head = fix_exp_head(repo.scm, f"HEAD~{n}")
-                revs.append(repo.scm.resolve_rev(head))
-            except SCMError:
+                assert head
+                revs.append(resolve_rev(repo.scm, head))
+            except RevError:
                 break
 
     revs = OrderedDict(

dvc/repo/experiments/utils.py

@@ -1,7 +1,8 @@
-from typing import Generator, Iterable, Optional, Set
+from typing import Callable, Generator, Iterable, Optional, Set
+
+from scmrepo.git import Git
 
 from dvc.exceptions import InvalidArgumentError
-from dvc.scm.git import Git
 
 from .base import (
     EXEC_BASELINE,
@@ -49,11 +50,47 @@ def exp_refs_by_baseline(
         yield ExpRefInfo.from_ref(ref)
 
 
+def iter_remote_refs(
+    scm: "Git", url: str, base: Optional[str] = None, **kwargs
+):
+    from scmrepo.exceptions import AuthError, InvalidRemote
+
+    from dvc.scm import GitAuthError, InvalidRemoteSCMRepo
+
+    try:
+        yield from scm.iter_remote_refs(url, base=base, **kwargs)
+    except InvalidRemote as exc:
+        raise InvalidRemoteSCMRepo(str(exc))
+    except AuthError as exc:
+        raise GitAuthError(str(exc))
+
+
+def push_refspec(
+    scm: "Git",
+    url: str,
+    src: Optional[str],
+    dest: str,
+    force: bool = False,
+    on_diverged: Optional[Callable[[str, str], bool]] = None,
+    **kwargs,
+):
+    from scmrepo.exceptions import AuthError
+
+    from ...scm import GitAuthError
+
+    try:
+        return scm.push_refspec(
+            url, src, dest, force=force, on_diverged=on_diverged, **kwargs
+        )
+    except AuthError as exc:
+        raise GitAuthError(str(exc))
+
+
 def remote_exp_refs(
     scm: "Git", url: str
 ) -> Generator["ExpRefInfo", None, None]:
     """Iterate over all remote experiment refs."""
-    for ref in scm.iter_remote_refs(url, base=EXPS_NAMESPACE):
+    for ref in iter_remote_refs(scm, url, base=EXPS_NAMESPACE):
         if ref.startswith(EXEC_NAMESPACE) or ref == EXPS_STASH:
             continue
         yield ExpRefInfo.from_ref(ref)
@@ -73,7 +110,7 @@ def remote_exp_refs_by_baseline(
 ) -> Generator["ExpRefInfo", None, None]:
     """Iterate over all remote experiment refs with the specified baseline."""
     ref_info = ExpRefInfo(baseline_sha=rev)
-    for ref in scm.iter_remote_refs(url, base=str(ref_info)):
+    for ref in iter_remote_refs(scm, url, base=str(ref_info)):
         if ref.startswith(EXEC_NAMESPACE) or ref == EXPS_STASH:
             continue
         yield ExpRefInfo.from_ref(ref)

dvc/repo/get.py

@@ -2,7 +2,6 @@ import logging
 import os
 
 from dvc.exceptions import DvcException
-from dvc.path_info import PathInfo
 from dvc.utils import resolve_output
 from dvc.utils.fs import remove
 
@@ -50,8 +49,9 @@ def get(url, path, out=None, rev=None, jobs=None):
         with external_repo(
             url=url, rev=rev, cache_dir=tmp_dir, cache_types=cache_types
         ) as repo:
-            from_info = PathInfo(repo.root_dir) / path
-            to_info = PathInfo(out)
-            repo.repo_fs.download(from_info, to_info, jobs=jobs)
+            from_fs_path = os.path.abspath(os.path.join(repo.root_dir, path))
+            repo.repo_fs.download(
+                from_fs_path, os.path.abspath(out), jobs=jobs
+            )
     finally:
         remove(tmp_dir)

dvc/repo/graph.py

@@ -1,5 +1,6 @@
 from typing import TYPE_CHECKING, Iterator, List, Set
 
+from dvc.fs.local import localfs
 from dvc.utils.fs import path_isin
 
 if TYPE_CHECKING:
@@ -94,7 +95,6 @@ def build_graph(stages, outs_trie=None):
 
     from dvc.exceptions import StagePathAsOutputError
 
-    from ..path_info import PathInfo
     from .trie import build_outs_trie
 
     G = nx.DiGraph()
@@ -103,7 +103,7 @@ def build_graph(stages, outs_trie=None):
     outs_trie = outs_trie or build_outs_trie(stages)
 
     for stage in stages:
-        out = outs_trie.shortest_prefix(PathInfo(stage.path).parts).value
+        out = outs_trie.shortest_prefix(localfs.path.parts(stage.path)).value
         if out:
             raise StagePathAsOutputError(stage, str(out))
 
@@ -111,10 +111,10 @@ def build_graph(stages, outs_trie=None):
     G.add_nodes_from(stages)
     for stage in stages:
         for dep in stage.deps:
-            if dep.path_info is None:
+            if dep.fs_path is None:
                 continue
 
-            dep_key = dep.path_info.parts
+            dep_key = dep.fs.path.parts(dep.fs_path)
             overlapping = [n.value for n in outs_trie.prefixes(dep_key)]
             if outs_trie.has_subtrie(dep_key):
                 overlapping.extend(outs_trie.values(prefix=dep_key))
@@ -136,10 +136,10 @@ def build_outs_graph(graph, outs_trie):
     G.add_nodes_from(outs_trie.values())
     for stage in graph.nodes():
         for dep in stage.deps:
-            if dep.path_info is None:
-                # RepoDependency don't have a path_info
+            if dep.fs_path is None:
+                # RepoDependency don't have a path
                 continue
-            dep_key = dep.path_info.parts
+            dep_key = dep.fs.path.parts(dep.fs_path)
             overlapping = [n.value for n in outs_trie.prefixes(dep_key)]
             if outs_trie.has_subtrie(dep_key):
                 overlapping.extend(outs_trie.values(prefix=dep_key))

dvc/repo/index.py

@@ -20,7 +20,7 @@ if TYPE_CHECKING:
     from pygtrie import Trie
 
     from dvc.dependency import Dependency, ParamsDependency
-    from dvc.fs.base import BaseFileSystem
+    from dvc.fs.base import FileSystem
     from dvc.objects import HashInfo, ObjectDB
     from dvc.output import Output
     from dvc.repo.stage import StageLoad
@@ -35,7 +35,7 @@ class Index:
     def __init__(
         self,
         repo: "Repo",  # pylint: disable=redefined-outer-name
-        fs: "BaseFileSystem" = None,
+        fs: "FileSystem" = None,
         stages: List["Stage"] = None,
     ) -> None:
         """Index is an immutable collection of stages.
@@ -58,7 +58,7 @@ class Index:
         """
 
         self.repo: "Repo" = repo
-        self.fs: "BaseFileSystem" = fs or repo.fs
+        self.fs: "FileSystem" = fs or repo.fs
         self.stage_collector: "StageLoad" = repo.stage
         if stages is not None:
             self.stages: List["Stage"] = stages

dvc/repo/init.py

@@ -5,8 +5,7 @@ from dvc.config import Config
 from dvc.exceptions import InitError, InvalidArgumentError
 from dvc.ignore import init as init_dvcignore
 from dvc.repo import Repo
-from dvc.scm import SCM
-from dvc.scm.base import SCMError
+from dvc.scm import SCM, SCMError
 from dvc.utils import relpath
 from dvc.utils.fs import remove
 
@@ -78,14 +77,18 @@ def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False):
 
     proj.plots.templates.init()
 
-    scm.add(
-        [config.files["repo"], dvcignore, proj.plots.templates.templates_dir]
-    )
+    with proj.scm_context(autostage=True) as context:
+        files = [
+            config.files["repo"],
+            dvcignore,
+            proj.plots.templates.templates_dir,
+        ]
+        ignore_file = context.scm.ignore_file
+        if ignore_file:
+            files.extend([os.path.join(dvc_dir, ignore_file)])
+        proj.scm_context.track_file(files)
 
     logger.info("Initialized DVC repository.\n")
-
-    if scm.ignore_file:
-        scm.add([os.path.join(dvc_dir, scm.ignore_file)])
+    if not no_scm:
         logger.info("You can now commit the changes to git.\n")
-
     return proj

dvc/repo/install.py

@@ -1,8 +1,11 @@
 from typing import TYPE_CHECKING
 
+from dvc.exceptions import DvcException
+
 if TYPE_CHECKING:
+    from scmrepo.git import Git
+
     from dvc.repo import Repo
-    from dvc.scm.git import Git
 
 
 def pre_commit_install(scm: "Git") -> None:
@@ -41,9 +44,18 @@ def pre_commit_install(scm: "Git") -> None:
 
 
 def install_hooks(scm: "Git") -> None:
+    from scmrepo.exceptions import GitHookAlreadyExists
+
+    from dvc.utils import format_link
+
     hooks = ["post-checkout", "pre-commit", "pre-push"]
     for hook in hooks:
-        scm.verify_hook(hook)
+        try:
+            scm.verify_hook(hook)
+        except GitHookAlreadyExists as exc:
+            link = format_link("https://man.dvc.org/install")
+            raise DvcException(f"{exc}. Please refer to {link} for more info.")
+
     for hook in hooks:
         scm.install_hook(hook, f"exec dvc git-hook {hook} $@")
 
@@ -54,7 +66,7 @@ def install(self: "Repo", use_pre_commit_tool: bool = False) -> None:
     If use_pre_commit_tool is set and pre-commit is installed it will be used
     to install the hooks.
     """
-    from dvc.scm.git import Git
+    from scmrepo.git import Git
 
     scm = self.scm
     if not isinstance(scm, Git):

dvc/repo/live.py

@@ -1,27 +1,40 @@
 import logging
 from typing import TYPE_CHECKING, List, Optional
 
+from funcy import once_per_args
+
 from dvc.render.utils import render
 
 logger = logging.getLogger(__name__)
 
 if TYPE_CHECKING:
     from dvc.output import Output
-    from dvc.path_info import PathInfo
     from dvc.repo import Repo
 
 
+@once_per_args
+def webbrowser_open(url: str) -> int:
+    from dvc.ui import ui
+
+    return ui.open_browser(url)
+
+
 def create_summary(out):
+
     assert out.live and out.live["html"]
 
-    metrics, plots = out.repo.live.show(str(out.path_info))
+    metrics, plots = out.repo.live.show(out.fs_path)
 
-    html_path = out.path_info.fspath + "_dvc_plots"
+    html_path = out.fs_path + "_dvc_plots"
 
-    render(out.repo, plots, metrics=metrics, path=html_path, refresh_seconds=5)
+    index_path = render(
+        out.repo, plots, metrics=metrics, path=html_path, refresh_seconds=5
+    )
+
+    webbrowser_open(index_path)
 
 
-def summary_path_info(out: "Output") -> Optional["PathInfo"]:
+def summary_fs_path(out: "Output") -> Optional[str]:
     from dvc.output import Output
 
     assert out.live
@@ -29,7 +42,7 @@ def summary_path_info(out: "Output") -> Optional["PathInfo"]:
     if isinstance(out.live, dict):
         has_summary = out.live.get(Output.PARAM_LIVE_SUMMARY, True)
     if has_summary:
-        return out.path_info.with_suffix(".json")
+        return out.fs.path.with_suffix(out.fs_path, ".json")
     return None
 
 

dvc/repo/ls.py

@@ -1,7 +1,7 @@
+import os
 from itertools import chain
 
 from dvc.exceptions import PathMissingError
-from dvc.path_info import PathInfo
 
 
 def ls(url, path=None, rev=None, recursive=None, dvc_only=False):
@@ -29,11 +29,11 @@ def ls(url, path=None, rev=None, recursive=None, dvc_only=False):
     from . import Repo
 
     with Repo.open(url, rev=rev, subrepos=True, uninitialized=True) as repo:
-        path_info = PathInfo(repo.root_dir)
+        fs_path = repo.root_dir
         if path:
-            path_info /= path
+            fs_path = os.path.abspath(repo.fs.path.join(fs_path, path))
 
-        ret = _ls(repo.repo_fs, path_info, recursive, dvc_only)
+        ret = _ls(repo.repo_fs, fs_path, recursive, dvc_only)
 
         if path and not ret:
             raise PathMissingError(path, repo, dvc_only=dvc_only)
@@ -46,21 +46,21 @@ def ls(url, path=None, rev=None, recursive=None, dvc_only=False):
         return ret_list
 
 
-def _ls(fs, path_info, recursive=None, dvc_only=False):
+def _ls(fs, fs_path, recursive=None, dvc_only=False):
     def onerror(exc):
         raise exc
 
     infos = []
     try:
         for root, dirs, files in fs.walk(
-            path_info.fspath, onerror=onerror, dvcfiles=True
+            fs_path, onerror=onerror, dvcfiles=True
         ):
             entries = chain(files, dirs) if not recursive else files
-            infos.extend(PathInfo(root) / entry for entry in entries)
+            infos.extend(fs.path.join(root, entry) for entry in entries)
             if not recursive:
                 break
     except NotADirectoryError:
-        infos.append(path_info)
+        infos.append(fs_path)
     except FileNotFoundError:
         return {}
 
@@ -69,9 +69,9 @@ def _ls(fs, path_info, recursive=None, dvc_only=False):
         metadata = fs.metadata(info)
         if metadata.output_exists or not dvc_only:
             path = (
-                path_info.name
-                if path_info == info
-                else str(info.relative_to(path_info))
+                fs.path.name(fs_path)
+                if fs_path == info
+                else fs.path.relpath(info, fs_path)
             )
             ret[path] = {
                 "isout": metadata.is_output,

dvc/repo/metrics/show.py

@@ -1,12 +1,13 @@
 import logging
+import os
 from typing import List
 
 from dvc.fs.repo import RepoFileSystem
 from dvc.output import Output
 from dvc.repo import locked
-from dvc.repo.collect import DvcPaths, collect
-from dvc.repo.live import summary_path_info
-from dvc.scm.base import SCMError
+from dvc.repo.collect import StrPaths, collect
+from dvc.repo.live import summary_fs_path
+from dvc.scm import NoSCMError
 from dvc.utils import error_handler, errored_revisions, onerror_collect
 from dvc.utils.serialize import load_yaml
 
@@ -17,27 +18,27 @@ def _is_metric(out: Output) -> bool:
     return bool(out.metric) or bool(out.live)
 
 
-def _to_path_infos(metrics: List[Output]) -> DvcPaths:
+def _to_fs_paths(metrics: List[Output]) -> StrPaths:
     result = []
     for out in metrics:
         if out.metric:
-            result.append(out.path_info)
+            result.append(out.fs_path)
         elif out.live:
-            path_info = summary_path_info(out)
-            if path_info:
-                result.append(path_info)
+            fs_path = summary_fs_path(out)
+            if fs_path:
+                result.append(fs_path)
     return result
 
 
 def _collect_metrics(repo, targets, revision, recursive):
-    metrics, path_infos = collect(
+    metrics, fs_paths = collect(
         repo,
         targets=targets,
         output_filter=_is_metric,
         recursive=recursive,
         rev=revision,
     )
-    return _to_path_infos(metrics) + list(path_infos)
+    return _to_fs_paths(metrics) + list(fs_paths)
 
 
 def _extract_metrics(metrics, path, rev):
@@ -80,7 +81,9 @@ def _read_metrics(repo, metrics, rev, onerror=None):
         if not fs.isfile(metric):
             continue
 
-        res[str(metric)] = _read_metric(metric, fs, rev, onerror=onerror)
+        res[fs.path.relpath(metric, os.getcwd())] = _read_metric(
+            metric, fs, rev, onerror=onerror
+        )
 
     return res
 
@@ -118,9 +121,9 @@ def show(
     # Hide workspace metrics if they are the same as in the active branch
     try:
         active_branch = repo.scm.active_branch()
-    except (TypeError, SCMError):
+    except (TypeError, NoSCMError):
         # TypeError - detached head
-        # SCMError - no repo case
+        # NoSCMError - no repo case
         pass
     else:
         if res.get("workspace") == res.get(active_branch):

dvc/repo/params/show.py

@@ -1,13 +1,13 @@
 import logging
+import os
 from collections import defaultdict
 from copy import copy
 from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Tuple
 
 from dvc.dependency.param import ParamsDependency
-from dvc.path_info import PathInfo
 from dvc.repo import locked
 from dvc.repo.collect import collect
-from dvc.scm.base import SCMError
+from dvc.scm import NoSCMError
 from dvc.stage import PipelineStage
 from dvc.ui import ui
 from dvc.utils import error_handler, errored_revisions, onerror_collect
@@ -16,7 +16,6 @@ from dvc.utils.serialize import LOADERS
 if TYPE_CHECKING:
     from dvc.output import Output
     from dvc.repo import Repo
-    from dvc.types import DvcPath
 
 logger = logging.getLogger(__name__)
 
@@ -27,56 +26,58 @@ def _is_params(dep: "Output"):
 
 def _collect_configs(
     repo: "Repo", rev, targets=None
-) -> Tuple[List["Output"], List["DvcPath"]]:
+) -> Tuple[List["Output"], List[str]]:
 
-    params, path_infos = collect(
+    params, fs_paths = collect(
         repo,
         targets=targets or [],
         deps=True,
         output_filter=_is_params,
         rev=rev,
     )
-    all_path_infos = path_infos + [p.path_info for p in params]
+    all_fs_paths = fs_paths + [p.fs_path for p in params]
     if not targets:
-        default_params = (
-            PathInfo(repo.root_dir) / ParamsDependency.DEFAULT_PARAMS_FILE
+        default_params = os.path.join(
+            repo.root_dir, ParamsDependency.DEFAULT_PARAMS_FILE
         )
-        if default_params not in all_path_infos and repo.fs.exists(
+        if default_params not in all_fs_paths and repo.fs.exists(
             default_params
         ):
-            path_infos.append(default_params)
-    return params, path_infos
+            fs_paths.append(default_params)
+    return params, fs_paths
 
 
 @error_handler
-def _read_path_info(fs, path_info, **kwargs):
-    suffix = path_info.suffix.lower()
+def _read_fs_path(fs, fs_path, **kwargs):
+    suffix = fs.path.suffix(fs_path).lower()
     loader = LOADERS[suffix]
-    return loader(path_info, fs=fs)
+    return loader(fs_path, fs=fs)
 
 
 def _read_params(
     repo,
     params,
-    params_path_infos,
+    params_fs_paths,
     deps=False,
     onerror: Optional[Callable] = None,
 ):
     res: Dict[str, Dict] = defaultdict(dict)
-    path_infos = copy(params_path_infos)
+    fs_paths = copy(params_fs_paths)
 
     if deps:
         for param in params:
             params_dict = error_handler(param.read_params_d)(onerror=onerror)
             if params_dict:
-                res[str(param.path_info)] = params_dict
+                res[
+                    repo.fs.path.relpath(param.fs_path, os.getcwd())
+                ] = params_dict
     else:
-        path_infos += [param.path_info for param in params]
+        fs_paths += [param.fs_path for param in params]
 
-    for path_info in path_infos:
-        from_path = _read_path_info(repo.fs, path_info, onerror=onerror)
+    for fs_path in fs_paths:
+        from_path = _read_fs_path(repo.fs, fs_path, onerror=onerror)
         if from_path:
-            res[str(path_info)] = from_path
+            res[repo.fs.path.relpath(fs_path, os.getcwd())] = from_path
 
     return res
 
@@ -112,9 +113,9 @@ def show(repo, revs=None, targets=None, deps=False, onerror: Callable = None):
     # Hide workspace params if they are the same as in the active branch
     try:
         active_branch = repo.scm.active_branch()
-    except (TypeError, SCMError):
+    except (TypeError, NoSCMError):
         # TypeError - detached head
-        # SCMError - no repo case
+        # NoSCMError - no repo case
         pass
     else:
         if res.get("workspace") == res.get(active_branch):
@@ -131,13 +132,11 @@ def show(repo, revs=None, targets=None, deps=False, onerror: Callable = None):
 
 
 def _gather_params(repo, rev, targets=None, deps=False, onerror=None):
-    param_outs, params_path_infos = _collect_configs(
-        repo, rev, targets=targets
-    )
+    param_outs, params_fs_paths = _collect_configs(repo, rev, targets=targets)
     params = _read_params(
         repo,
         params=param_outs,
-        params_path_infos=params_path_infos,
+        params_fs_paths=params_fs_paths,
         deps=deps,
         onerror=onerror,
     )

dvc/repo/plots/__init__.py

@@ -3,12 +3,20 @@ import io
 import logging
 import os
 from collections import OrderedDict
-from typing import TYPE_CHECKING, Callable, Dict, Generator, List, Optional
+from functools import partial
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Dict,
+    Generator,
+    List,
+    Optional,
+)
 
 from funcy import cached_property, first, project
 
 from dvc.exceptions import DvcException
-from dvc.render.vega import PlotMetricTypeError
 from dvc.utils import (
     error_handler,
     errored_revisions,
@@ -20,11 +28,18 @@ from dvc.utils.serialize import LOADERS
 if TYPE_CHECKING:
     from dvc.output import Output
     from dvc.repo import Repo
-    from dvc.types import DvcPath
 
 logger = logging.getLogger(__name__)
 
 
+class PlotMetricTypeError(DvcException):
+    def __init__(self, file):
+        super().__init__(
+            "'{}' - file type error\n"
+            "Only JSON, YAML, CSV and TSV formats are supported.".format(file)
+        )
+
+
 class NotAPlotError(DvcException):
     def __init__(self, out):
         super().__init__(
@@ -88,17 +103,14 @@ class Plots:
 
         fs = RepoFileSystem(self.repo)
         plots = _collect_plots(self.repo, targets, revision, recursive)
-        res = {}
-        for path_info, rev_props in plots.items():
-
-            if fs.isdir(path_info):
+        res: Dict[str, Any] = {}
+        for fs_path, rev_props in plots.items():
+            if fs.isdir(fs_path):
                 plot_files = []
-                for pi in fs.walk_files(path_info):
+                for pi in fs.find(fs_path):
                     plot_files.append((pi, relpath(pi, self.repo.root_dir)))
             else:
-                plot_files = [
-                    (path_info, relpath(path_info, self.repo.root_dir))
-                ]
+                plot_files = [(fs_path, relpath(fs_path, self.repo.root_dir))]
 
             props = props or {}
 
@@ -106,12 +118,15 @@ class Plots:
                 joined_props = {**rev_props, **props}
                 res[repo_path] = {"props": joined_props}
                 res[repo_path].update(
-                    parse(
-                        fs,
-                        path,
-                        props=joined_props,
-                        onerror=onerror,
-                    )
+                    {
+                        "data_source": partial(
+                            parse,
+                            fs,
+                            path,
+                            props=joined_props,
+                            onerror=onerror,
+                        )
+                    }
                 )
         return res
 
@@ -126,13 +141,20 @@ class Plots:
         if onerror is None:
             onerror = onerror_collect
 
-        data: Dict[str, Dict] = {}
-        for rev_data in self.collect(
+        result: Dict[str, Dict] = {}
+        for data in self.collect(
             targets, revs, recursive, onerror=onerror, props=props
         ):
-            data.update(rev_data)
+            assert len(data) == 1
+            revision_data = first(data.values())
+            if "data" in revision_data:
+                for path_data in revision_data["data"].values():
+                    result_source = path_data.pop("data_source", None)
+                    if result_source:
+                        path_data.update(result_source())
+            result.update(data)
 
-        errored = errored_revisions(data)
+        errored = errored_revisions(result)
         if errored:
             from dvc.ui import ui
 
@@ -141,7 +163,7 @@ class Plots:
                 f"'{', '.join(errored)}'."
             )
 
-        return data
+        return result
 
     def diff(self, *args, **kwargs):
         from .diff import diff
@@ -165,7 +187,7 @@ class Plots:
         props = props or {}
         template = props.get("template")
         if template:
-            self.templates.load(template)
+            self.templates.get_template(template)
 
         (out,) = self.repo.find_outs_by_path(path)
         if not out.plot and unset is not None:
@@ -205,10 +227,10 @@ def _collect_plots(
     targets: List[str] = None,
     rev: str = None,
     recursive: bool = False,
-) -> Dict["DvcPath", Dict]:
+) -> Dict["List[str]", Dict]:
     from dvc.repo.collect import collect
 
-    plots, path_infos = collect(
+    plots, fs_paths = collect(
         repo,
         output_filter=_is_plot,
         targets=targets,
@@ -216,8 +238,8 @@ def _collect_plots(
         recursive=recursive,
     )
 
-    result = {plot.path_info: _plot_props(plot) for plot in plots}
-    result.update({path_info: {} for path_info in path_infos})
+    result = {plot.fs_path: _plot_props(plot) for plot in plots}
+    result.update({fs_path: {} for fs_path in fs_paths})
     return result
 
 

dvc/repo/plots/template.py

@@ -1,27 +1,15 @@
 import json
 import os
-from typing import TYPE_CHECKING, Iterable, Optional
-
-try:
-    import importlib_resources
-except ImportError:
-    import importlib.resources as importlib_resources  # type: ignore[no-redef]
+from typing import Any, Dict, Optional
 
 from funcy import cached_property
 
 from dvc.exceptions import DvcException
 
-if TYPE_CHECKING:
-    from dvc.types import StrPath
-
 
 class TemplateNotFoundError(DvcException):
-    def __init__(self, name):
-        super().__init__(f"Template '{name}' not found.")
-
-
-class BadTemplateError(DvcException):
-    pass
+    def __init__(self, path):
+        super().__init__(f"Template '{path}' not found.")
 
 
 class NoFieldInDataError(DvcException):
@@ -37,147 +25,510 @@ class Template:
     EXTENSION = ".json"
     ANCHOR = "<DVC_METRIC_{}>"
 
-    def __init__(self, content, name):
-        self.content = content
-        self.name = name
+    DEFAULT_CONTENT: Optional[Dict[str, Any]] = None
+    DEFAULT_NAME: Optional[str] = None
 
-    def render(self, data, props=None):
-        props = props or {}
-
-        if self._anchor_str("data") not in self.content:
-            anchor = self.anchor("data")
-            raise BadTemplateError(
-                f"Template '{self.name}' is not using '{anchor}' anchor"
+    def __init__(self, content=None, name=None):
+        if content:
+            self.content = content
+        else:
+            self.content = (
+                json.dumps(
+                    self.DEFAULT_CONTENT,
+                    indent=self.INDENT,
+                    separators=self.SEPARATORS,
+                )
+                + "\n"
             )
 
-        if props.get("x"):
-            Template._check_field_exists(data, props.get("x"))
-        if props.get("y"):
-            Template._check_field_exists(data, props.get("y"))
-
-        content = self._fill_anchor(self.content, "data", data)
-        content = self._fill_metadata(content, props)
-
-        return content
+        self.name = name or self.DEFAULT_NAME
+        assert self.content and self.name
+        self.filename = self.name + self.EXTENSION
 
     @classmethod
     def anchor(cls, name):
         return cls.ANCHOR.format(name.upper())
 
     def has_anchor(self, name):
-        return self._anchor_str(name) in self.content
+        return self.anchor_str(name) in self.content
 
     @classmethod
-    def _fill_anchor(cls, content, name, value):
+    def fill_anchor(cls, content, name, value):
         value_str = json.dumps(
             value, indent=cls.INDENT, separators=cls.SEPARATORS, sort_keys=True
         )
-        return content.replace(cls._anchor_str(name), value_str)
+        return content.replace(cls.anchor_str(name), value_str)
 
     @classmethod
-    def _anchor_str(cls, name):
+    def anchor_str(cls, name):
         return f'"{cls.anchor(name)}"'
 
-    @classmethod
-    def _fill_metadata(cls, content, props):
-        props.setdefault("title", "")
-        props.setdefault("x_label", props.get("x"))
-        props.setdefault("y_label", props.get("y"))
-
-        names = ["title", "x", "y", "x_label", "y_label"]
-        for name in names:
-            value = props.get(name)
-            if value is not None:
-                content = cls._fill_anchor(content, name, value)
-
-        return content
-
     @staticmethod
-    def _check_field_exists(data, field):
+    def check_field_exists(data, field):
         if not any(field in row for row in data):
             raise NoFieldInDataError(field)
 
 
+class SimpleLinearTemplate(Template):
+    DEFAULT_NAME = "simple"
+
+    DEFAULT_CONTENT = {
+        "$schema": "https://vega.github.io/schema/vega-lite/v4.json",
+        "data": {"values": Template.anchor("data")},
+        "title": Template.anchor("title"),
+        "width": 300,
+        "height": 300,
+        "mark": {"type": "line"},
+        "encoding": {
+            "x": {
+                "field": Template.anchor("x"),
+                "type": "quantitative",
+                "title": Template.anchor("x_label"),
+            },
+            "y": {
+                "field": Template.anchor("y"),
+                "type": "quantitative",
+                "title": Template.anchor("y_label"),
+                "scale": {"zero": False},
+            },
+            "color": {"field": "rev", "type": "nominal"},
+        },
+    }
+
+
+class ConfusionTemplate(Template):
+    DEFAULT_NAME = "confusion"
+    DEFAULT_CONTENT = {
+        "$schema": "https://vega.github.io/schema/vega-lite/v4.json",
+        "data": {"values": Template.anchor("data")},
+        "title": Template.anchor("title"),
+        "facet": {"field": "rev", "type": "nominal"},
+        "spec": {
+            "transform": [
+                {
+                    "aggregate": [{"op": "count", "as": "xy_count"}],
+                    "groupby": [Template.anchor("y"), Template.anchor("x")],
+                },
+                {
+                    "impute": "xy_count",
+                    "groupby": ["rev", Template.anchor("y")],
+                    "key": Template.anchor("x"),
+                    "value": 0,
+                },
+                {
+                    "impute": "xy_count",
+                    "groupby": ["rev", Template.anchor("x")],
+                    "key": Template.anchor("y"),
+                    "value": 0,
+                },
+                {
+                    "joinaggregate": [
+                        {"op": "max", "field": "xy_count", "as": "max_count"}
+                    ],
+                    "groupby": [],
+                },
+                {
+                    "calculate": "datum.xy_count / datum.max_count",
+                    "as": "percent_of_max",
+                },
+            ],
+            "encoding": {
+                "x": {
+                    "field": Template.anchor("x"),
+                    "type": "nominal",
+                    "sort": "ascending",
+                    "title": Template.anchor("x_label"),
+                },
+                "y": {
+                    "field": Template.anchor("y"),
+                    "type": "nominal",
+                    "sort": "ascending",
+                    "title": Template.anchor("y_label"),
+                },
+            },
+            "layer": [
+                {
+                    "mark": "rect",
+                    "width": 300,
+                    "height": 300,
+                    "encoding": {
+                        "color": {
+                            "field": "xy_count",
+                            "type": "quantitative",
+                            "title": "",
+                            "scale": {"domainMin": 0, "nice": True},
+                        }
+                    },
+                },
+                {
+                    "mark": "text",
+                    "encoding": {
+                        "text": {"field": "xy_count", "type": "quantitative"},
+                        "color": {
+                            "condition": {
+                                "test": "datum.percent_of_max > 0.5",
+                                "value": "white",
+                            },
+                            "value": "black",
+                        },
+                    },
+                },
+            ],
+        },
+    }
+
+
+class NormalizedConfusionTemplate(Template):
+    DEFAULT_NAME = "confusion_normalized"
+    DEFAULT_CONTENT = {
+        "$schema": "https://vega.github.io/schema/vega-lite/v4.json",
+        "data": {"values": Template.anchor("data")},
+        "title": Template.anchor("title"),
+        "facet": {"field": "rev", "type": "nominal"},
+        "spec": {
+            "transform": [
+                {
+                    "aggregate": [{"op": "count", "as": "xy_count"}],
+                    "groupby": [Template.anchor("y"), Template.anchor("x")],
+                },
+                {
+                    "impute": "xy_count",
+                    "groupby": ["rev", Template.anchor("y")],
+                    "key": Template.anchor("x"),
+                    "value": 0,
+                },
+                {
+                    "impute": "xy_count",
+                    "groupby": ["rev", Template.anchor("x")],
+                    "key": Template.anchor("y"),
+                    "value": 0,
+                },
+                {
+                    "joinaggregate": [
+                        {"op": "sum", "field": "xy_count", "as": "sum_y"}
+                    ],
+                    "groupby": [Template.anchor("y")],
+                },
+                {
+                    "calculate": "datum.xy_count / datum.sum_y",
+                    "as": "percent_of_y",
+                },
+            ],
+            "encoding": {
+                "x": {
+                    "field": Template.anchor("x"),
+                    "type": "nominal",
+                    "sort": "ascending",
+                    "title": Template.anchor("x_label"),
+                },
+                "y": {
+                    "field": Template.anchor("y"),
+                    "type": "nominal",
+                    "sort": "ascending",
+                    "title": Template.anchor("y_label"),
+                },
+            },
+            "layer": [
+                {
+                    "mark": "rect",
+                    "width": 300,
+                    "height": 300,
+                    "encoding": {
+                        "color": {
+                            "field": "percent_of_y",
+                            "type": "quantitative",
+                            "title": "",
+                            "scale": {"domain": [0, 1]},
+                        }
+                    },
+                },
+                {
+                    "mark": "text",
+                    "encoding": {
+                        "text": {
+                            "field": "percent_of_y",
+                            "type": "quantitative",
+                            "format": ".2f",
+                        },
+                        "color": {
+                            "condition": {
+                                "test": "datum.percent_of_y > 0.5",
+                                "value": "white",
+                            },
+                            "value": "black",
+                        },
+                    },
+                },
+            ],
+        },
+    }
+
+
+class ScatterTemplate(Template):
+    DEFAULT_NAME = "scatter"
+
+    DEFAULT_CONTENT = {
+        "$schema": "https://vega.github.io/schema/vega-lite/v4.json",
+        "data": {"values": Template.anchor("data")},
+        "title": Template.anchor("title"),
+        "width": 300,
+        "height": 300,
+        "layer": [
+            {
+                "encoding": {
+                    "x": {
+                        "field": Template.anchor("x"),
+                        "type": "quantitative",
+                        "title": Template.anchor("x_label"),
+                    },
+                    "y": {
+                        "field": Template.anchor("y"),
+                        "type": "quantitative",
+                        "title": Template.anchor("y_label"),
+                        "scale": {"zero": False},
+                    },
+                    "color": {"field": "rev", "type": "nominal"},
+                },
+                "layer": [
+                    {"mark": "point"},
+                    {
+                        "selection": {
+                            "label": {
+                                "type": "single",
+                                "nearest": True,
+                                "on": "mouseover",
+                                "encodings": ["x"],
+                                "empty": "none",
+                                "clear": "mouseout",
+                            }
+                        },
+                        "mark": "point",
+                        "encoding": {
+                            "opacity": {
+                                "condition": {
+                                    "selection": "label",
+                                    "value": 1,
+                                },
+                                "value": 0,
+                            }
+                        },
+                    },
+                ],
+            },
+            {
+                "transform": [{"filter": {"selection": "label"}}],
+                "layer": [
+                    {
+                        "encoding": {
+                            "text": {
+                                "type": "quantitative",
+                                "field": Template.anchor("y"),
+                            },
+                            "x": {
+                                "field": Template.anchor("x"),
+                                "type": "quantitative",
+                            },
+                            "y": {
+                                "field": Template.anchor("y"),
+                                "type": "quantitative",
+                            },
+                        },
+                        "layer": [
+                            {
+                                "mark": {
+                                    "type": "text",
+                                    "align": "left",
+                                    "dx": 5,
+                                    "dy": -5,
+                                },
+                                "encoding": {
+                                    "color": {
+                                        "type": "nominal",
+                                        "field": "rev",
+                                    }
+                                },
+                            }
+                        ],
+                    }
+                ],
+            },
+        ],
+    }
+
+
+class SmoothLinearTemplate(Template):
+    DEFAULT_NAME = "smooth"
+
+    DEFAULT_CONTENT = {
+        "$schema": "https://vega.github.io/schema/vega-lite/v4.json",
+        "data": {"values": Template.anchor("data")},
+        "title": Template.anchor("title"),
+        "mark": {"type": "line"},
+        "encoding": {
+            "x": {
+                "field": Template.anchor("x"),
+                "type": "quantitative",
+                "title": Template.anchor("x_label"),
+            },
+            "y": {
+                "field": Template.anchor("y"),
+                "type": "quantitative",
+                "title": Template.anchor("y_label"),
+                "scale": {"zero": False},
+            },
+            "color": {"field": "rev", "type": "nominal"},
+        },
+        "transform": [
+            {
+                "loess": Template.anchor("y"),
+                "on": Template.anchor("x"),
+                "groupby": ["rev"],
+                "bandwidth": 0.3,
+            }
+        ],
+    }
+
+
+class LinearTemplate(Template):
+    DEFAULT_NAME = "linear"
+
+    DEFAULT_CONTENT = {
+        "$schema": "https://vega.github.io/schema/vega-lite/v4.json",
+        "data": {"values": Template.anchor("data")},
+        "title": Template.anchor("title"),
+        "width": 300,
+        "height": 300,
+        "layer": [
+            {
+                "encoding": {
+                    "x": {
+                        "field": Template.anchor("x"),
+                        "type": "quantitative",
+                        "title": Template.anchor("x_label"),
+                    },
+                    "y": {
+                        "field": Template.anchor("y"),
+                        "type": "quantitative",
+                        "title": Template.anchor("y_label"),
+                        "scale": {"zero": False},
+                    },
+                    "color": {"field": "rev", "type": "nominal"},
+                },
+                "layer": [
+                    {"mark": "line"},
+                    {
+                        "selection": {
+                            "label": {
+                                "type": "single",
+                                "nearest": True,
+                                "on": "mouseover",
+                                "encodings": ["x"],
+                                "empty": "none",
+                                "clear": "mouseout",
+                            }
+                        },
+                        "mark": "point",
+                        "encoding": {
+                            "opacity": {
+                                "condition": {
+                                    "selection": "label",
+                                    "value": 1,
+                                },
+                                "value": 0,
+                            }
+                        },
+                    },
+                ],
+            },
+            {
+                "transform": [{"filter": {"selection": "label"}}],
+                "layer": [
+                    {
+                        "mark": {"type": "rule", "color": "gray"},
+                        "encoding": {
+                            "x": {
+                                "field": Template.anchor("x"),
+                                "type": "quantitative",
+                            }
+                        },
+                    },
+                    {
+                        "encoding": {
+                            "text": {
+                                "type": "quantitative",
+                                "field": Template.anchor("y"),
+                            },
+                            "x": {
+                                "field": Template.anchor("x"),
+                                "type": "quantitative",
+                            },
+                            "y": {
+                                "field": Template.anchor("y"),
+                                "type": "quantitative",
+                            },
+                        },
+                        "layer": [
+                            {
+                                "mark": {
+                                    "type": "text",
+                                    "align": "left",
+                                    "dx": 5,
+                                    "dy": -5,
+                                },
+                                "encoding": {
+                                    "color": {
+                                        "type": "nominal",
+                                        "field": "rev",
+                                    }
+                                },
+                            }
+                        ],
+                    },
+                ],
+            },
+        ],
+    }
+
+
 class PlotTemplates:
     TEMPLATES_DIR = "plots"
-    PKG_TEMPLATES_DIR = "templates"
+    TEMPLATES = [
+        SimpleLinearTemplate,
+        LinearTemplate,
+        ConfusionTemplate,
+        NormalizedConfusionTemplate,
+        ScatterTemplate,
+        SmoothLinearTemplate,
+    ]
 
     @cached_property
     def templates_dir(self):
         return os.path.join(self.dvc_dir, self.TEMPLATES_DIR)
 
-    @staticmethod
-    def _find(templates, template_name):
-        for template in templates:
-            if template.endswith(template_name) or template.endswith(
-                template_name + ".json"
-            ):
-                return template
-        return None
+    def get_template(self, template_name: str):
+        template_path = os.path.abspath(template_name)
+        if os.path.exists(template_path):
+            return os.path.abspath(template_path)
 
-    def _find_in_project(self, name: str) -> Optional["StrPath"]:
-        full_path = os.path.abspath(name)
-        if os.path.exists(full_path):
-            return full_path
+        if self.dvc_dir and os.path.exists(self.dvc_dir):
+            template_path = os.path.join(self.templates_dir, template_name)
+            if os.path.exists(template_path):
+                return template_path
 
-        if os.path.exists(self.templates_dir):
-            templates = [
+            all_templates = [
                 os.path.join(root, file)
                 for root, _, files in os.walk(self.templates_dir)
                 for file in files
             ]
-            found = self._find(templates, name)
-            if found:
-                return os.path.join(self.templates_dir, found)
-        return None
-
-    @staticmethod
-    def _get_templates() -> Iterable[str]:
-        if (
-            importlib_resources.files(__package__)
-            .joinpath(PlotTemplates.PKG_TEMPLATES_DIR)
-            .is_dir()
-        ):
-            entries = (
-                importlib_resources.files(__package__)
-                .joinpath(PlotTemplates.PKG_TEMPLATES_DIR)
-                .iterdir()
-            )
-            return [entry.name for entry in entries]
-        return []
-
-    @staticmethod
-    def _load_from_pkg(name):
-        templates = PlotTemplates._get_templates()
-        found = PlotTemplates._find(templates, name)
-        if found:
-            return (
-                (
-                    importlib_resources.files(__package__)
-                    / PlotTemplates.PKG_TEMPLATES_DIR
-                    / found
-                )
-                .read_bytes()
-                .decode("utf-8")
-            )
-        return None
-
-    def load(self, name: str = None) -> Template:
-
-        if name is not None:
-            template_path = self._find_in_project(name)
-            if template_path:
-                with open(template_path, "r", encoding="utf-8") as fd:
-                    content = fd.read()
-                return Template(content, name)
-        else:
-            name = "linear"
-
-        content = self._load_from_pkg(name)
-        if content:
-            return Template(content, name)
-
-        raise TemplateNotFoundError(name)
+            matches = [
+                template
+                for template in all_templates
+                if os.path.splitext(template)[0] == template_path
+            ]
+            if matches:
+                assert len(matches) == 1
+                return matches[0]
+        raise TemplateNotFoundError(template_name)
 
     def __init__(self, dvc_dir):
         self.dvc_dir = dvc_dir
@@ -186,18 +537,27 @@ class PlotTemplates:
         from dvc.utils.fs import makedirs
 
         makedirs(self.templates_dir, exist_ok=True)
+        for t in self.TEMPLATES:
+            self._dump(t())
 
-        templates = self._get_templates()
-        for template in templates:
-            content = (
-                importlib_resources.files(__package__)
-                .joinpath(PlotTemplates.PKG_TEMPLATES_DIR)
-                .joinpath(template)
-                .read_text()
-            )
-            with open(
-                os.path.join(self.templates_dir, template),
-                "w",
-                encoding="utf-8",
-            ) as fd:
-                fd.write(content)
+    def _dump(self, template):
+        path = os.path.join(self.templates_dir, template.filename)
+        with open(path, "w", encoding="utf-8") as fd:
+            fd.write(template.content)
+
+    def load(self, template_name=None):
+        if not template_name:
+            template_name = "linear"
+
+        try:
+            path = self.get_template(template_name)
+
+            with open(path, "r", encoding="utf-8") as fd:
+                content = fd.read()
+
+            return Template(content, name=template_name)
+        except TemplateNotFoundError:
+            for template in self.TEMPLATES:
+                if template.DEFAULT_NAME == template_name:
+                    return template()
+            raise

dvc/repo/reproduce.py

@@ -8,12 +8,14 @@ from dvc.repo.scm_context import scm_context
 from . import locked
 
 if typing.TYPE_CHECKING:
+    from dvc.stage import Stage
+
     from . import Repo
 
 logger = logging.getLogger(__name__)
 
 
-def _reproduce_stage(stage, **kwargs):
+def _reproduce_stage(stage: "Stage", **kwargs):
     def _run_callback(repro_callback):
         _dump_stage(stage)
         _track_stage(stage)
@@ -56,27 +58,33 @@ def _dump_stage(stage):
     dvcfile.dump(stage, update_pipeline=False)
 
 
-def _track_stage(stage):
-    from dvc.utils import relpath
-
-    stage.repo.scm.track_file(stage.dvcfile.relpath)
+def _get_stage_files(stage: "Stage") -> typing.Iterator[str]:
+    yield stage.dvcfile.relpath
     for dep in stage.deps:
         if (
             not dep.use_scm_ignore
             and dep.is_in_repo
-            and not stage.repo.repo_fs.isdvc(dep.path_info)
+            and not stage.repo.repo_fs.isdvc(dep.fs_path)
         ):
-            stage.repo.scm.track_file(relpath(dep.path_info))
+            yield dep.fs_path
     for out in stage.outs:
         if not out.use_scm_ignore and out.is_in_repo:
-            stage.repo.scm.track_file(relpath(out.path_info))
+            yield out.fs_path
         if out.live:
-            from dvc.repo.live import summary_path_info
+            from dvc.repo.live import summary_fs_path
 
-            summary = summary_path_info(out)
+            summary = summary_fs_path(out)
             if summary:
-                stage.repo.scm.track_file(relpath(summary))
-    stage.repo.scm.track_changed_files()
+                yield summary
+
+
+def _track_stage(stage: "Stage") -> None:
+    from dvc.utils import relpath
+
+    context = stage.repo.scm_context
+    for path in _get_stage_files(stage):
+        context.track_file(relpath(path))
+    return context.track_changed_files()
 
 
 @locked

dvc/repo/scm_context.py

@@ -1,16 +1,154 @@
+import logging
+import shlex
+from contextlib import contextmanager
 from functools import wraps
-from typing import TYPE_CHECKING
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    Iterable,
+    Iterator,
+    List,
+    Set,
+    Union,
+)
+
+from dvc.utils import relpath
+from dvc.utils.collections import ensure_list
 
 if TYPE_CHECKING:
+    from scmrepo.base import Base
+
     from dvc.repo import Repo
 
 
-def scm_context(method):
+logger = logging.getLogger(__name__)
+
+
+class SCMContext:
+    def __init__(self, scm: "Base", config: Dict[str, Any] = None) -> None:
+        from funcy import get_in
+
+        self.scm: "Base" = scm
+        self.autostage: bool = get_in(
+            config or {}, ["core", "autostage"], default=False
+        )
+        self.ignored_paths: List[str] = []
+        self.files_to_track: Set[str] = set()
+        self.quiet: bool = False
+
+    def track_file(
+        self, paths: Union[str, Iterable[str], None] = None
+    ) -> None:
+        """Track file to remind user to track new files or autostage later."""
+        return self.files_to_track.update(ensure_list(paths))
+
+    @staticmethod
+    def _make_git_add_cmd(paths: Union[str, Iterable[str]]) -> str:
+        files = " ".join(map(shlex.quote, ensure_list(paths)))
+        return f"\tgit add {files}".expandtabs(4)
+
+    def add(self, paths: Union[str, Iterable[str]]) -> None:
+        from scmrepo.exceptions import UnsupportedIndexFormat
+
+        try:
+            return self.scm.add(paths)
+        except UnsupportedIndexFormat:
+            link = "https://github.com/iterative/dvc/issues/610"
+            add_cmd = self._make_git_add_cmd([relpath(path) for path in paths])
+            logger.info("")
+            msg = (
+                f"failed to add, add manually using:\n\n{add_cmd}\n"
+                f"\nSee {link} for more details.\n"
+            )
+            logger.warning(msg)
+
+    def track_changed_files(self) -> None:
+        """Stage files that have changed."""
+        if not self.files_to_track:
+            return
+        logger.debug("Staging files: %s", self.files_to_track)
+        return self.add(self.files_to_track)
+
+    def ignore(self, path: str) -> None:
+        from scmrepo.exceptions import FileNotInRepoError
+
+        from dvc.scm import SCMError
+
+        logger.debug("Adding '%s' to gitignore file.", path)
+        try:
+            gitignore_file = self.scm.ignore(path)
+        except FileNotInRepoError as exc:
+            raise SCMError(str(exc))
+
+        if gitignore_file:
+            self.track_file(relpath(gitignore_file))
+            return self.ignored_paths.append(path)
+
+    def ignore_remove(self, path: str) -> None:
+        from scmrepo.exceptions import FileNotInRepoError
+
+        from dvc.scm import SCMError
+
+        logger.debug("Removing '%s' from gitignore file.", path)
+        try:
+            gitignore_file = self.scm.ignore_remove(path)
+        except FileNotInRepoError as exc:
+            raise SCMError(str(exc))
+
+        if gitignore_file:
+            return self.track_file(relpath(gitignore_file))
+
+    @contextmanager
+    def __call__(
+        self, autostage: bool = None, quiet: bool = None
+    ) -> Iterator["SCMContext"]:
+        try:
+            yield self
+        except Exception:
+            for path in self.ignored_paths:
+                self.ignore_remove(path)
+            raise
+        finally:
+            self.ignored_paths = []
+
+        if not self.files_to_track:
+            return
+
+        if autostage is None:
+            autostage = self.autostage
+        if quiet is None:
+            quiet = self.quiet
+
+        from dvc.scm import NoSCM
+
+        if autostage:
+            self.track_changed_files()
+        elif not quiet and not isinstance(self.scm, NoSCM):
+            add_cmd = self._make_git_add_cmd(self.files_to_track)
+            logger.info(
+                f"\nTo track the changes with git, run:\n" f"\n{add_cmd}"
+            )
+            logger.info(
+                "\nTo enable auto staging, run:\n\n"
+                "\tdvc config core.autostage true"
+            )
+
+        self.files_to_track = set()
+
+    def __enter__(self) -> "SCMContext":
+        self._cm = self()  # pylint: disable=attribute-defined-outside-init
+        return self._cm.__enter__()  # pylint: disable=no-member
+
+    def __exit__(self, *exc_args) -> None:
+        assert self._cm
+        self._cm.__exit__(*exc_args)  # pylint: disable=no-member
+
+
+def scm_context(method, autostage: bool = None, quiet: bool = None):
     @wraps(method)
     def run(repo: "Repo", *args, **kw):
-        scm = repo.scm
-
-        with scm.track_file_changes(config=repo.config):
+        with repo.scm_context(autostage=autostage, quiet=quiet):
             return method(repo, *args, **kw)
 
     return run