Merge pull request #2128 from pared/2119

pathinfo: extract relative path method to standalone method
This commit is contained in:
Ruslan Kuprieiev
2019-06-18 14:07:40 +03:00
committed by GitHub
24 changed files with 120 additions and 88 deletions
+3 -3
View File
@@ -1,9 +1,9 @@
from __future__ import unicode_literals
from dvc.utils import relpath
from dvc.utils.compat import str
import argparse
import os
import logging
from dvc.exceptions import DvcException
@@ -21,7 +21,7 @@ class CmdPipelineShow(CmdBase):
stage = Stage.load(self.repo, target)
G = self.repo.graph()[0]
stages = networkx.get_node_attributes(G, "stage")
node = os.path.relpath(stage.path, self.repo.root_dir)
node = relpath(stage.path, self.repo.root_dir)
nodes = networkx.dfs_postorder_nodes(G, node)
if locked:
@@ -41,7 +41,7 @@ class CmdPipelineShow(CmdBase):
from dvc.stage import Stage
stage = Stage.load(self.repo, target)
node = os.path.relpath(stage.path, self.repo.root_dir)
node = relpath(stage.path, self.repo.root_dir)
pipelines = list(
filter(lambda g: node in g.nodes(), self.repo.pipelines())
+2 -3
View File
@@ -1,19 +1,18 @@
from __future__ import unicode_literals
import argparse
import os
import logging
from dvc.repo import Repo
from dvc.command.base import CmdBaseNoRepo, append_doc_link
from dvc.utils import relpath
logger = logging.getLogger(__name__)
class CmdRoot(CmdBaseNoRepo):
def run(self):
logger.info(os.path.relpath(Repo.find_root()))
logger.info(relpath(Repo.find_root()))
return 0
+5 -4
View File
@@ -4,9 +4,10 @@ from __future__ import unicode_literals
from dvc.utils.compat import str, builtin_str
import os
import traceback
from dvc.utils import relpath
class DvcException(Exception):
"""Base class for all dvc exceptions.
@@ -62,7 +63,7 @@ class OutputNotFoundError(DvcException):
def __init__(self, output):
super(OutputNotFoundError, self).__init__(
"unable to find stage file with output '{path}'".format(
path=os.path.relpath(output)
path=relpath(output)
)
)
@@ -198,7 +199,7 @@ class BadMetricError(DvcException):
def __init__(self, path):
super(BadMetricError, self).__init__(
"'{}' does not exist, not a metric or is malformed".format(
os.path.relpath(path)
relpath(path)
)
)
@@ -213,7 +214,7 @@ class NoMetricsError(DvcException):
class StageFileCorruptedError(DvcException):
def __init__(self, path, cause=None):
path = os.path.relpath(path)
path = relpath(path)
super(StageFileCorruptedError, self).__init__(
"unable to read stage file: {} "
"YAML file structure is corrupted".format(path),
+6 -5
View File
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
import os
from dulwich.ignore import match_pattern, read_ignore_patterns
from dvc.utils import relpath
from dvc.utils.compat import cast_bytes
from dvc.utils.fs import get_parent_dirs_up_to
@@ -55,15 +56,15 @@ class DvcIgnoreFromFile(DvcIgnore):
return dirs, files
def get_match(self, abs_path):
rel_path = os.path.relpath(abs_path, self.dirname)
relative_path = relpath(abs_path, self.dirname)
if os.name == "nt":
rel_path = rel_path.replace("\\", "/")
rel_path = cast_bytes(rel_path, "utf-8")
relative_path = relative_path.replace("\\", "/")
relative_path = cast_bytes(relative_path, "utf-8")
for pattern in self.patterns:
if match_pattern(
rel_path, pattern
) and self._no_negate_pattern_matches(rel_path):
relative_path, pattern
) and self._no_negate_pattern_matches(relative_path):
return (abs_path, pattern, self.ignore_file_path)
return None
+5 -8
View File
@@ -6,12 +6,14 @@ import posixpath
from funcy import cached_property
from dvc.utils.compat import str, builtin_str, basestring, is_py2
from dvc.utils.compat import pathlib, fspath_py35, urlparse
from dvc.utils.compat import pathlib, urlparse
# On Python 2.7/Windows sys.getfilesystemencoding() is set to mbcs,
# which is lossy, thus we can't use that,
# see https://github.com/mcmtroffaes/pathlib2/issues/56.
from dvc.utils import relpath
if is_py2:
fs_encoding = "utf-8"
@@ -31,10 +33,7 @@ class PathInfo(pathlib.PurePath):
def __str__(self):
path = self.__fspath__()
# Windows path on different drive than curdir doesn't have relpath
if os.name == "nt" and not os.path.commonprefix([os.getcwd(), path]):
return path
return os.path.relpath(path)
return relpath(path)
def __repr__(self):
return builtin_str("{}: '{}'").format(type(self).__name__, self)
@@ -51,9 +50,7 @@ class PathInfo(pathlib.PurePath):
url = fspath
def relpath(self, other):
return self.__class__(
os.path.relpath(fspath_py35(self), fspath_py35(other))
)
return self.__class__(relpath(self, other))
def isin(self, other):
if isinstance(other, basestring):
+8 -8
View File
@@ -16,7 +16,7 @@ import dvc.prompt as prompt
from dvc.config import Config
from dvc.exceptions import DvcException, ConfirmRemoveError
from dvc.progress import progress, ProgressCallback
from dvc.utils import LARGE_DIR_SIZE, tmp_fname, to_chunks, move
from dvc.utils import LARGE_DIR_SIZE, tmp_fname, to_chunks, move, relpath
from dvc.state import StateBase
from dvc.path_info import PathInfo, URLInfo
@@ -148,7 +148,7 @@ class RemoteBASE(object):
for fname in files:
file_info = self.path_cls(root) / fname
relpath = file_info.relative_to(path_info)
relative_path = file_info.relative_to(path_info)
dir_info.append(
{
# NOTE: this is lossy transformation:
@@ -159,7 +159,7 @@ class RemoteBASE(object):
#
# Yes, this is a BUG, as long as we permit "/" in
# filenames on Windows and "\" on Unix
self.PARAM_RELPATH: relpath.as_posix(),
self.PARAM_RELPATH: relative_path.as_posix(),
self.PARAM_CHECKSUM: self.get_file_checksum(file_info),
}
)
@@ -221,13 +221,13 @@ class RemoteBASE(object):
if not isinstance(d, list):
msg = "dir cache file format error '{}' [skipping the file]"
logger.error(msg.format(os.path.relpath(path)))
logger.error(msg.format(relpath(path)))
return []
for info in d:
# NOTE: here is a BUG, see comment to .as_posix() below
relpath = self.path_cls.from_posix(info[self.PARAM_RELPATH])
info[self.PARAM_RELPATH] = relpath.fspath
relative_path = self.path_cls.from_posix(info[self.PARAM_RELPATH])
info[self.PARAM_RELPATH] = relative_path.fspath
return d
@@ -699,10 +699,10 @@ class RemoteBASE(object):
logger.debug("Linking directory '{}'.".format(path_info))
for entry in dir_info:
relpath = entry[self.PARAM_RELPATH]
relative_path = entry[self.PARAM_RELPATH]
entry_checksum = entry[self.PARAM_CHECKSUM]
entry_cache_info = self.checksum_to_path_info(entry_checksum)
entry_info = path_info / relpath
entry_info = path_info / relative_path
entry_checksum_info = {self.PARAM_CHECKSUM: entry_checksum}
if self.changed(entry_info, entry_checksum_info):
+2 -1
View File
@@ -2,6 +2,7 @@ import os
import logging
import posixpath
from dvc.utils import relpath
from dvc.utils.compat import urlparse
from dvc.config import Config, ConfigError
@@ -85,7 +86,7 @@ class RemoteConfig(object):
"""
if os.path.isabs(path):
return path
return os.path.relpath(path, os.path.dirname(config_file))
return relpath(path, os.path.dirname(config_file))
def add(self, name, url, default=False, force=False, level=None):
from dvc.remote import _get, RemoteLOCAL
+4 -7
View File
@@ -28,6 +28,7 @@ from dvc.utils import (
tmp_fname,
file_md5,
walk_files,
relpath,
)
from dvc.config import Config
from dvc.exceptions import DvcException
@@ -464,11 +465,7 @@ class RemoteLOCAL(RemoteBASE):
# would get only the part of file. So, at first, the file should be
# copied with the temporary name, and then original file should be
# replaced by new.
copyfile(
path,
tmp,
name="Unprotecting '{}'".format(os.path.relpath(path)),
)
copyfile(path, tmp, name="Unprotecting '{}'".format(relpath(path)))
remove(path)
os.rename(tmp, path)
@@ -536,9 +533,9 @@ class RemoteLOCAL(RemoteBASE):
entry_cache_info = self.checksum_to_path_info(
entry[self.PARAM_CHECKSUM]
)
relpath = entry[self.PARAM_RELPATH]
relative_path = entry[self.PARAM_RELPATH]
self.link(
entry_cache_info, unpacked_dir_info / relpath, "hardlink"
entry_cache_info, unpacked_dir_info / relative_path, "hardlink"
)
self.state.save(unpacked_dir_info, checksum)
+4 -3
View File
@@ -12,6 +12,7 @@ from dvc.exceptions import (
from dvc.ignore import DvcIgnoreFileHandler
from dvc.path_info import PathInfo
from dvc.utils.compat import open as _open
from dvc.utils import relpath
logger = logging.getLogger(__name__)
@@ -167,7 +168,7 @@ class Repo(object):
if not with_deps:
return [stage]
node = os.path.relpath(stage.path, self.root_dir)
node = relpath(stage.path, self.root_dir)
G = self._get_pipeline(node)
ret = []
@@ -324,7 +325,7 @@ class Repo(object):
raise StagePathAsOutputError(stage.wdir, stage.relpath)
for stage in stages:
node = os.path.relpath(stage.path, self.root_dir)
node = relpath(stage.path, self.root_dir)
G.add_node(node, stage=stage)
G_active.add_node(node, stage=stage)
@@ -339,7 +340,7 @@ class Repo(object):
continue
dep_stage = out.stage
dep_node = os.path.relpath(dep_stage.path, self.root_dir)
dep_node = relpath(dep_stage.path, self.root_dir)
G.add_node(dep_node, stage=dep_stage)
G.add_edge(node, dep_node)
if not stage.locked:
+2 -2
View File
@@ -8,7 +8,7 @@ from dvc.scm import SCM, NoSCM
from dvc.config import Config
from dvc.exceptions import InitError
from dvc.utils import boxify
from dvc.utils import relpath
logger = logging.getLogger(__name__)
@@ -74,7 +74,7 @@ def init(root_dir=os.curdir, no_scm=False, force=False):
if not force:
raise InitError(
"'{repo}' exists. Use '-f' to force.".format(
repo=os.path.relpath(dvc_dir)
repo=relpath(dvc_dir)
)
)
+3 -3
View File
@@ -5,7 +5,7 @@ import logging
from dvc.exceptions import ReproductionError
from dvc.repo.scm_context import scm_context
from dvc.utils import relpath
logger = logging.getLogger(__name__)
@@ -67,7 +67,7 @@ def reproduce(
elif pipeline or all_pipelines:
if pipeline:
stage = Stage.load(self, target)
node = os.path.relpath(stage.path, self.root_dir)
node = relpath(stage.path, self.root_dir)
pipelines = [self._get_pipeline(node)]
else:
pipelines = self.pipelines()
@@ -115,7 +115,7 @@ def _reproduce(
stage = Stage.load(self, target)
G = self.graph()[1]
stages = nx.get_node_attributes(G, "stage")
node = os.path.relpath(stage.path, self.root_dir)
node = relpath(stage.path, self.root_dir)
if single_item:
ret = _reproduce_stage(
+6 -8
View File
@@ -6,7 +6,7 @@ import os
import logging
from dvc.utils.compat import str, open
from dvc.utils import fix_env
from dvc.utils import fix_env, relpath
from dvc.scm.base import (
Base,
SCMError,
@@ -92,7 +92,7 @@ class Git(Base):
msg.format(self.GITIGNORE, path, ignore_file_dir)
)
entry = os.path.relpath(path, ignore_file_dir).replace(os.sep, "/")
entry = relpath(path, ignore_file_dir).replace(os.sep, "/")
# NOTE: using '/' prefix to make path unambiguous
if len(entry) > 0 and entry[0] != "/":
entry = "/" + entry
@@ -125,14 +125,12 @@ class Git(Base):
if self._ignored(entry, gitignore):
return
msg = "Adding '{}' to '{}'.".format(
os.path.relpath(path), os.path.relpath(gitignore)
)
msg = "Adding '{}' to '{}'.".format(relpath(path), relpath(gitignore))
logger.info(msg)
self._add_entry_to_gitignore(entry, gitignore)
self.track_file(os.path.relpath(gitignore))
self.track_file(relpath(gitignore))
self.ignored_paths.append(path)
@@ -163,7 +161,7 @@ class Git(Base):
with open(gitignore, "w") as fobj:
fobj.writelines(filtered)
self.track_file(os.path.relpath(gitignore))
self.track_file(relpath(gitignore))
def add(self, paths):
# NOTE: GitPython is not currently able to handle index version >= 3.
@@ -202,7 +200,7 @@ class Git(Base):
def is_tracked(self, path):
# it is equivalent to `bool(self.git.git.ls_files(path))` by
# functionality, but ls_files fails on unicode filenames
path = os.path.relpath(path, self.root_dir)
path = relpath(path, self.root_dir)
return path in [i[0] for i in self.git.index.entries]
def is_dirty(self):
+5 -4
View File
@@ -2,6 +2,7 @@ import errno
import os
from dvc.ignore import DvcIgnoreFilter
from dvc.utils import relpath
from dvc.utils.compat import StringIO, BytesIO
from dvc.exceptions import DvcException
@@ -32,14 +33,14 @@ class GitTree(BaseTree):
def open(self, path, binary=False):
relpath = os.path.relpath(path, self.git.working_dir)
relative_path = relpath(path, self.git.working_dir)
obj = self.git_object_by_path(path)
if obj is None:
msg = "No such file in branch '{}'".format(self.rev)
raise IOError(errno.ENOENT, msg, relpath)
raise IOError(errno.ENOENT, msg, relative_path)
if obj.mode == GIT_MODE_DIR:
raise IOError(errno.EISDIR, "Is a directory", relpath)
raise IOError(errno.EISDIR, "Is a directory", relative_path)
# GitPython's obj.data_stream is a fragile thing, it is better to
# read it immediately, also it needs to be to decoded if we follow
@@ -98,7 +99,7 @@ class GitTree(BaseTree):
def git_object_by_path(self, path):
import git
path = os.path.relpath(os.path.realpath(path), self.git.working_dir)
path = relpath(os.path.realpath(path), self.git.working_dir)
assert path.split(os.sep, 1)[0] != ".."
self._try_fetch_from_remote()
+7 -8
View File
@@ -8,6 +8,7 @@ import os
import subprocess
import logging
from dvc.utils import relpath
from dvc.utils.compat import pathlib
from dvc.utils.fs import contains_symlink_up_to
from schema import Schema, SchemaError, Optional, Or, And
@@ -176,7 +177,7 @@ class Stage(object):
@property
def relpath(self):
return os.path.relpath(self.path)
return relpath(self.path)
@property
def is_data_source(self):
@@ -553,7 +554,7 @@ class Stage(object):
raise StageFileBadNameError(
"bad stage filename '{}'. Stage files should be named"
" 'Dvcfile' or have a '.dvc' suffix (e.g. '{}.dvc').".format(
os.path.relpath(fname), os.path.basename(fname)
relpath(fname), os.path.basename(fname)
)
)
@@ -593,7 +594,7 @@ class Stage(object):
# looses keys in deps and outs load
state = copy.deepcopy(d)
Stage.validate(d, fname=os.path.relpath(fname))
Stage.validate(d, fname=relpath(fname))
path = os.path.abspath(fname)
stage = Stage(
@@ -617,7 +618,7 @@ class Stage(object):
return stage
def dumpd(self):
rel_wdir = os.path.relpath(self.wdir, os.path.dirname(self.path))
rel_wdir = relpath(self.wdir, os.path.dirname(self.path))
return {
key: value
for key, value in {
@@ -638,15 +639,13 @@ class Stage(object):
self._check_dvc_filename(fname)
logger.info(
"Saving information to '{file}'.".format(
file=os.path.relpath(fname)
)
"Saving information to '{file}'.".format(file=relpath(fname))
)
d = self.dumpd()
apply_diff(d, self._state)
dump_stage_file(fname, self._state)
self.repo.scm.track_file(os.path.relpath(fname))
self.repo.scm.track_file(relpath(fname))
def _compute_md5(self):
from dvc.output.base import OutputBase
+6 -3
View File
@@ -7,7 +7,7 @@ import sqlite3
import logging
from dvc.config import Config
from dvc.utils import remove, current_timestamp
from dvc.utils import remove, current_timestamp, relpath
from dvc.exceptions import DvcException
from dvc.utils.fs import get_mtime_and_size, get_inode
from dvc.utils.compat import fspath_py35
@@ -416,12 +416,15 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes
mtime, _ = get_mtime_and_size(path)
inode = get_inode(path)
relpath = os.path.relpath(path, self.root_dir)
relative_path = relpath(path, self.root_dir)
cmd = (
"REPLACE INTO {}(path, inode, mtime) "
'VALUES ("{}", {}, "{}")'.format(
self.LINK_STATE_TABLE, relpath, self._to_sqlite(inode), mtime
self.LINK_STATE_TABLE,
relative_path,
self._to_sqlite(inode),
mtime,
)
)
self._execute(cmd)
+23 -4
View File
@@ -2,7 +2,14 @@
from __future__ import unicode_literals
from dvc.utils.compat import str, builtin_str, open, cast_bytes_py2, StringIO
from dvc.utils.compat import (
str,
builtin_str,
open,
cast_bytes_py2,
StringIO,
fspath_py35,
)
from dvc.utils.compat import fspath
import os
@@ -46,8 +53,8 @@ def file_md5(fname):
if size >= LARGE_FILE_SIZE:
bar = True
msg = "Computing md5 for a large file {}. This is only done once."
logger.info(msg.format(os.path.relpath(fname)))
name = os.path.relpath(fname)
logger.info(msg.format(relpath(fname)))
name = relpath(fname)
total = 0
with open(fname, "rb") as fobj:
@@ -170,7 +177,7 @@ def _chmod(func, p, excinfo):
def remove(path):
logger.debug("Removing '{}'".format(os.path.relpath(path)))
logger.debug("Removing '{}'".format(relpath(path)))
try:
if os.path.isdir(path):
@@ -369,3 +376,15 @@ def _visual_center(line, width):
right_padding = spaces - left_padding
return (left_padding * " ") + line + (right_padding * " ")
def relpath(path, start=os.curdir):
path = fspath_py35(path)
start = fspath_py35(os.path.abspath(start))
# Windows path on different drive than curdir doesn't have relpath
if os.name == "nt" and not os.path.commonprefix(
[start, os.path.abspath(path)]
):
return path
return os.path.relpath(path, start)
+4 -2
View File
@@ -83,10 +83,12 @@ class TestDirFixture(object):
)
@staticmethod
def mkdtemp():
def mkdtemp(base_directory=None):
prefix = "dvc-test.{}.".format(os.getpid())
suffix = ".{}".format(uuid.uuid4())
return tempfile.mkdtemp(prefix=prefix, suffix=suffix)
return tempfile.mkdtemp(
prefix=prefix, suffix=suffix, dir=base_directory
)
def setUp(self):
self._pushd(self._root_dir)
+3 -1
View File
@@ -126,7 +126,9 @@ def temporary_windows_drive(repo_dir):
if set_up_result == 0:
raise RuntimeError("Failed to mount windows drive!")
yield new_drive
# NOTE: new_drive has form of `A:` and joining it with some relative
# path might result in non-existing path (A:path\\to)
yield os.path.join(new_drive, os.sep)
tear_down_result = windll.kernel32.DefineDosDeviceW(
DDD_REMOVE_DEFINITION, new_drive, target_path
+2 -2
View File
@@ -16,7 +16,7 @@ from dvc.system import System
from mock import patch
from dvc.main import main
from dvc.utils import file_md5, LARGE_DIR_SIZE
from dvc.utils import file_md5, LARGE_DIR_SIZE, relpath
from dvc.utils.stage import load_stage_file
from dvc.utils.compat import range
from dvc.stage import Stage
@@ -421,7 +421,7 @@ class TestShouldThrowProperExceptionOnCorruptedStageFile(TestDvc):
ret = main(["add", self.FOO])
assert 0 == ret
foo_stage = os.path.relpath(self.FOO + Stage.STAGE_FILE_SUFFIX)
foo_stage = relpath(self.FOO + Stage.STAGE_FILE_SUFFIX)
# corrupt stage file
with open(foo_stage, "a+") as file:
+2 -1
View File
@@ -4,6 +4,7 @@ import configobj
from dvc.cache import Cache
from dvc.main import main
from dvc.utils import relpath
from tests.basic_env import TestDvc, TestDir
@@ -152,7 +153,7 @@ class TestCmdCacheDir(TestDvc):
def test_relative_path(self):
tmpdir = self.mkdtemp()
dname = os.path.relpath(tmpdir)
dname = relpath(tmpdir)
ret = main(["cache", "dir", dname])
self.assertEqual(ret, 0)
+5 -5
View File
@@ -11,7 +11,7 @@ from dvc.main import main
from dvc import progress
from dvc.repo import Repo as DvcRepo
from dvc.system import System
from dvc.utils import walk_files
from dvc.utils import walk_files, relpath
from dvc.utils.stage import load_stage_file, dump_stage_file
from tests.basic_env import TestDvc
from tests.func.test_repro import TestRepro
@@ -566,11 +566,11 @@ class TestCheckoutMovedCacheDirWithSymlinks(TestDvc):
new_data_link = readlink(self.DATA)
self.assertEqual(
os.path.relpath(old_foo_link, old_cache_dir),
os.path.relpath(new_foo_link, new_cache_dir),
relpath(old_foo_link, old_cache_dir),
relpath(new_foo_link, new_cache_dir),
)
self.assertEqual(
os.path.relpath(old_data_link, old_cache_dir),
os.path.relpath(new_data_link, new_cache_dir),
relpath(old_data_link, old_cache_dir),
relpath(new_data_link, new_cache_dir),
)
+2 -1
View File
@@ -10,6 +10,7 @@ from dvc.main import main
from dvc.exceptions import DvcException, BadMetricError, NoMetricsError
from dvc.repo.metrics.show import NO_METRICS_FILE_AT_REFERENCE_WARNING
from dvc.stage import Stage
from dvc.utils import relpath
from tests.basic_env import TestDvc
@@ -342,7 +343,7 @@ class TestMetrics(TestMetricsBase):
self.assertMetricsHaveRelativePaths(metrics)
def assertMetricsHaveRelativePaths(self, metrics):
root_relpath = os.path.relpath(self.dvc.root_dir)
root_relpath = relpath(self.dvc.root_dir)
metric_path = os.path.join(root_relpath, "metric")
metric_json_path = os.path.join(root_relpath, "metric_json")
metric_tsv_path = os.path.join(root_relpath, "metric_tsv")
+2 -2
View File
@@ -18,7 +18,7 @@ import pytest
from dvc.main import main
from dvc.repo import Repo as DvcRepo
from dvc.utils import file_md5
from dvc.utils import file_md5, relpath
from dvc.utils.stage import load_stage_file, dump_stage_file
from dvc.remote.local import RemoteLOCAL
from dvc.stage import Stage, StageFileDoesNotExistError
@@ -142,7 +142,7 @@ class TestReproWorkingDirectoryAsOutput(TestDvc):
os.mkdir(dir2)
nested_dir = os.path.join(dir2, "nested")
out_dir = os.path.relpath(nested_dir, dir1)
out_dir = relpath(nested_dir, dir1)
nested_stage = self.dvc.run(
cwd=dir1, # b
+9
View File
@@ -4,6 +4,7 @@ from unittest import TestCase
import dvc
import pytest
from dvc.system import System
from dvc.utils import relpath
from dvc.utils.compat import str
from dvc.utils.fs import (
get_mtime_and_size,
@@ -135,3 +136,11 @@ def test_get_parent_dirs_up_to(path1, path2, expected_dirs):
result = get_parent_dirs_up_to(path1, path2)
assert set(result) == set(expected_dirs)
@pytest.mark.skipif(os.name != "nt", reason="Windows specific")
def test_relpath_windows_different_drives():
path1 = os.path.join("A:", os.sep, "some", "path")
path2 = os.path.join("B:", os.sep, "other", "path")
assert relpath(path1, path2) == path1