system: change System class to a dvc.fs.system module

This commit is contained in:
Saugat Pachhai (सौगात)
2022-05-12 18:02:42 +03:00
committed by Ruslan Kuprieiev
parent 4d6df989f2
commit b4faa5616f
15 changed files with 150 additions and 156 deletions
+6 -6
View File
@@ -5,7 +5,7 @@ import threading
from fsspec import AbstractFileSystem
from funcy import cached_property, wrap_prop
from dvc.fs.system import System
from dvc.fs import system
from dvc.scheme import Schemes
from dvc.utils import tmp_fname
from dvc.utils.fs import copy_fobj_to_file, copyfile, makedirs, move, remove
@@ -123,15 +123,15 @@ class FsspecLocalFileSystem(AbstractFileSystem):
return open(path, mode=mode, encoding=encoding)
def symlink(self, path1, path2):
return System.symlink(path1, path2)
return system.symlink(path1, path2)
@staticmethod
def is_symlink(path):
return System.is_symlink(path)
return system.is_symlink(path)
@staticmethod
def is_hardlink(path):
return System.is_hardlink(path)
return system.is_hardlink(path)
def hardlink(self, path1, path2):
# If there are a lot of empty files (which happens a lot in datasets),
@@ -153,10 +153,10 @@ class FsspecLocalFileSystem(AbstractFileSystem):
logger.debug("Created empty file: %s -> %s", path1, path2)
return
return System.hardlink(path1, path2)
return system.hardlink(path1, path2)
def reflink(self, path1, path2):
return System.reflink(path1, path2)
return system.reflink(path1, path2)
def created(self, path):
return self.fs.created(path)
+93 -93
View File
@@ -21,112 +21,112 @@ else:
realpath = os.path.realpath
class System:
@staticmethod
def hardlink(source, link_name):
# NOTE: we should really be using `os.link()` here with
# `follow_symlinks=True`, but unfortunately the implementation is
# buggy across platforms, so until it is fixed, we just dereference
# the symlink ourselves here.
#
# See https://bugs.python.org/issue41355 for more info.
st = os.lstat(source)
if stat.S_ISLNK(st.st_mode):
src = realpath(source)
else:
src = source
def hardlink(source, link_name):
# NOTE: we should really be using `os.link()` here with
# `follow_symlinks=True`, but unfortunately the implementation is
# buggy across platforms, so until it is fixed, we just dereference
# the symlink ourselves here.
#
# See https://bugs.python.org/issue41355 for more info.
st = os.lstat(source)
if stat.S_ISLNK(st.st_mode):
src = realpath(source)
else:
src = source
os.link(src, link_name)
os.link(src, link_name)
@staticmethod
def symlink(source, link_name):
os.symlink(source, link_name)
@staticmethod
def _reflink_darwin(src, dst):
import ctypes
def symlink(source, link_name):
os.symlink(source, link_name)
def _cdll(name):
return ctypes.CDLL(name, use_errno=True)
LIBC = "libc.dylib"
LIBC_FALLBACK = "/usr/lib/libSystem.dylib"
try:
clib = _cdll(LIBC)
except OSError as exc:
logger.debug(
"unable to access '{}' (errno '{}'). "
"Falling back to '{}'.".format(LIBC, exc.errno, LIBC_FALLBACK)
)
if exc.errno != errno.ENOENT:
raise
# NOTE: trying to bypass System Integrity Protection (SIP)
clib = _cdll(LIBC_FALLBACK)
def _reflink_darwin(src, dst):
import ctypes
if not hasattr(clib, "clonefile"):
raise OSError(
errno.ENOTSUP,
"'clonefile' not supported by the standard library",
)
def _cdll(name):
return ctypes.CDLL(name, use_errno=True)
clonefile = clib.clonefile
clonefile.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_int]
clonefile.restype = ctypes.c_int
ret = clonefile(
ctypes.c_char_p(os.fsencode(src)),
ctypes.c_char_p(os.fsencode(dst)),
ctypes.c_int(0),
LIBC = "libc.dylib"
LIBC_FALLBACK = "/usr/lib/libSystem.dylib"
try:
clib = _cdll(LIBC)
except OSError as exc:
logger.debug(
"unable to access '{}' (errno '{}'). "
"Falling back to '{}'.".format(LIBC, exc.errno, LIBC_FALLBACK)
)
if ret:
err = ctypes.get_errno()
msg = os.strerror(err)
raise OSError(err, msg)
@staticmethod
def _reflink_linux(src, dst):
import fcntl # pylint: disable=import-error
from contextlib import suppress
FICLONE = 0x40049409
try:
with open(src, "rb") as s, open(dst, "wb+") as d:
fcntl.ioctl(d.fileno(), FICLONE, s.fileno())
except OSError:
with suppress(OSError):
os.unlink(dst)
if exc.errno != errno.ENOENT:
raise
# NOTE: trying to bypass System Integrity Protection (SIP)
clib = _cdll(LIBC_FALLBACK)
@staticmethod
def reflink(source, link_name):
from dvc.utils.fs import umask
if not hasattr(clib, "clonefile"):
raise OSError(
errno.ENOTSUP,
"'clonefile' not supported by the standard library",
)
source, link_name = os.fspath(source), os.fspath(link_name)
clonefile = clib.clonefile
clonefile.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_int]
clonefile.restype = ctypes.c_int
system = platform.system()
if system == "Darwin":
System._reflink_darwin(source, link_name)
elif system == "Linux":
System._reflink_linux(source, link_name)
else:
raise OSError(
errno.ENOTSUP,
f"reflink is not supported on '{system}'",
)
ret = clonefile(
ctypes.c_char_p(os.fsencode(src)),
ctypes.c_char_p(os.fsencode(dst)),
ctypes.c_int(0),
)
if ret:
err = ctypes.get_errno()
msg = os.strerror(err)
raise OSError(err, msg)
# NOTE: reflink has a new inode, but has the same mode as the src,
# so we need to chmod it to look like a normal copy.
os.chmod(link_name, 0o666 & ~umask)
@staticmethod
def inode(path):
return os.lstat(path).st_ino
def _reflink_linux(src, dst):
import fcntl # pylint: disable=import-error
from contextlib import suppress
@staticmethod
def is_symlink(path):
return os.path.islink(path)
FICLONE = 0x40049409
@staticmethod
def is_hardlink(path):
return os.stat(path).st_nlink > 1
try:
with open(src, "rb") as s, open(dst, "wb+") as d:
fcntl.ioctl(d.fileno(), FICLONE, s.fileno())
except OSError:
with suppress(OSError):
os.unlink(dst)
raise
def reflink(source, link_name):
from dvc.utils.fs import umask
source, link_name = os.fspath(source), os.fspath(link_name)
system = platform.system()
if system == "Darwin":
_reflink_darwin(source, link_name)
elif system == "Linux":
_reflink_linux(source, link_name)
else:
raise OSError(
errno.ENOTSUP,
f"reflink is not supported on '{system}'",
)
# NOTE: reflink has a new inode, but has the same mode as the src,
# so we need to chmod it to look like a normal copy.
os.chmod(link_name, 0o666 & ~umask)
def inode(path):
ino = os.lstat(path).st_ino
logger.trace("Path '%s' inode '%d'", path, ino)
return ino
def is_symlink(path):
return os.path.islink(path)
def is_hardlink(path):
return os.stat(path).st_nlink > 1
+2 -1
View File
@@ -5,9 +5,10 @@ import os
from abc import ABC, abstractmethod
from dvc.fs.local import LocalFileSystem
from dvc.fs.system import inode as get_inode
from dvc.utils import relpath
from dvc.utils.decorators import with_diskcache
from dvc.utils.fs import get_inode, get_mtime_and_size, remove
from dvc.utils.fs import get_mtime_and_size, remove
from .hash_info import HashInfo
+2 -2
View File
@@ -370,8 +370,8 @@ def resolve_output(inp, out):
def resolve_paths(repo, out, always_local=False):
from urllib.parse import urlparse
from dvc.fs import system
from dvc.fs.local import localfs
from dvc.fs.system import System
from ..dvcfile import DVC_FILE_SUFFIX
from ..exceptions import DvcException
@@ -390,7 +390,7 @@ def resolve_paths(repo, out, always_local=False):
if scheme or not localfs.path.isin_or_eq(abspath, repo.root_dir):
wdir = os.getcwd()
elif contains_symlink_up_to(dirname, repo.root_dir) or (
os.path.isdir(abspath) and System.is_symlink(abspath)
os.path.isdir(abspath) and system.is_symlink(abspath)
):
msg = (
"Cannot add files inside symlinked directories to DVC. "
+3 -9
View File
@@ -8,7 +8,7 @@ from contextlib import contextmanager, suppress
from typing import TYPE_CHECKING
from dvc.exceptions import DvcException
from dvc.fs.system import System
from dvc.fs import system
from dvc.utils import dict_md5
if TYPE_CHECKING:
@@ -22,12 +22,6 @@ umask = os.umask(0)
os.umask(umask)
def get_inode(path):
inode = System.inode(path)
logger.trace("Path '%s' inode '%d'", path, inode)
return inode
def get_mtime_and_size(path, fs, dvcignore=None):
import nanotime
@@ -78,7 +72,7 @@ def contains_symlink_up_to(path: "StrPath", base_path: "StrPath"):
if path == base_path:
return False
if System.is_symlink(path):
if system.is_symlink(path):
return True
if os.path.dirname(path) == path:
return False
@@ -198,7 +192,7 @@ def copyfile(src, dest, callback=None, no_progress_bar=False, name=None):
callback.set_size(total)
try:
System.reflink(src, dest)
system.reflink(src, dest)
except OSError:
from dvc.fs._callback import FsspecCallback
+11 -11
View File
@@ -21,8 +21,8 @@ from dvc.exceptions import (
OverlappingOutputPathsError,
RecursiveAddingWhileUsingFilename,
)
from dvc.fs import system
from dvc.fs.local import LocalFileSystem
from dvc.fs.system import System
from dvc.objects.hash_info import HashInfo
from dvc.output import (
OutputAlreadyTrackedError,
@@ -478,7 +478,7 @@ class TestShouldPlaceStageInDataDirIfRepositoryBelowSymlink(TestDvc):
return False
with patch.object(
System, "is_symlink", side_effect=is_symlink_true_below_dvc_root
system, "is_symlink", side_effect=is_symlink_true_below_dvc_root
):
ret = main(["add", self.DATA])
@@ -582,7 +582,7 @@ class TestAddUnprotected(TestDvc):
self.assertEqual(ret, 0)
self.assertFalse(os.access(self.FOO, os.W_OK))
self.assertTrue(System.is_hardlink(self.FOO))
self.assertTrue(system.is_hardlink(self.FOO))
ret = main(["unprotect", self.FOO])
self.assertEqual(ret, 0)
@@ -591,7 +591,7 @@ class TestAddUnprotected(TestDvc):
self.assertEqual(ret, 0)
self.assertFalse(os.access(self.FOO, os.W_OK))
self.assertTrue(System.is_hardlink(self.FOO))
self.assertTrue(system.is_hardlink(self.FOO))
@pytest.fixture
@@ -663,7 +663,7 @@ def test_readding_dir_should_not_unprotect_all(tmp_dir, dvc, mocker):
dvc.add("dir")
assert not unprotect_spy.mock.called
assert System.is_symlink(os.path.join("dir", "new_file"))
assert system.is_symlink(os.path.join("dir", "new_file"))
def test_should_not_checkout_when_adding_cached_copy(tmp_dir, dvc, mocker):
@@ -683,10 +683,10 @@ def test_should_not_checkout_when_adding_cached_copy(tmp_dir, dvc, mocker):
@pytest.mark.parametrize(
"link,new_link,link_test_func",
[
("hardlink", "copy", lambda path: not System.is_hardlink(path)),
("symlink", "copy", lambda path: not System.is_symlink(path)),
("copy", "hardlink", System.is_hardlink),
("copy", "symlink", System.is_symlink),
("hardlink", "copy", lambda path: not system.is_hardlink(path)),
("symlink", "copy", lambda path: not system.is_symlink(path)),
("copy", "hardlink", system.is_hardlink),
("copy", "symlink", system.is_symlink),
],
)
def test_should_relink_on_repeated_add(
@@ -789,10 +789,10 @@ def test_add_optimization_for_hardlink_on_empty_files(tmp_dir, dvc, mocker):
for stage in stages[:2]:
# hardlinks are not created for empty files
assert not System.is_hardlink(stage.outs[0].fs_path)
assert not system.is_hardlink(stage.outs[0].fs_path)
for stage in stages[2:]:
assert System.is_hardlink(stage.outs[0].fs_path)
assert system.is_hardlink(stage.outs[0].fs_path)
for stage in stages:
assert os.path.exists(stage.path)
+7 -7
View File
@@ -18,8 +18,8 @@ from dvc.exceptions import (
DvcException,
NoOutputOrStageError,
)
from dvc.fs import system
from dvc.fs.local import LocalFileSystem
from dvc.fs.system import System
from dvc.stage import Stage
from dvc.stage.exceptions import StageFileDoesNotExistError
from dvc.utils import relpath
@@ -147,7 +147,7 @@ class CheckoutBase(TestDvcGit):
]
return [
FileInfo(path=path, inode=System.inode(path)) for path in paths
FileInfo(path=path, inode=system.inode(path)) for path in paths
]
@@ -459,10 +459,10 @@ class TestCheckoutMovedCacheDirWithSymlinks(TestDvc):
ret = main(["add", self.DATA_DIR])
self.assertEqual(ret, 0)
self.assertTrue(System.is_symlink(self.FOO))
self.assertTrue(system.is_symlink(self.FOO))
old_foo_link = os.path.realpath(self.FOO)
self.assertTrue(System.is_symlink(self.DATA))
self.assertTrue(system.is_symlink(self.DATA))
old_data_link = os.path.realpath(self.DATA)
old_cache_dir = self.dvc.odb.local.cache_dir
@@ -475,10 +475,10 @@ class TestCheckoutMovedCacheDirWithSymlinks(TestDvc):
ret = main(["checkout", "-f"])
self.assertEqual(ret, 0)
self.assertTrue(System.is_symlink(self.FOO))
self.assertTrue(system.is_symlink(self.FOO))
new_foo_link = os.path.realpath(self.FOO)
self.assertTrue(System.is_symlink(self.DATA))
self.assertTrue(system.is_symlink(self.DATA))
new_data_link = os.path.realpath(self.DATA)
self.assertEqual(
@@ -506,7 +506,7 @@ def test_checkout_no_checksum(tmp_dir, dvc):
@pytest.mark.parametrize(
"link, link_test_func",
[("hardlink", System.is_hardlink), ("symlink", System.is_symlink)],
[("hardlink", system.is_hardlink), ("symlink", system.is_symlink)],
)
def test_checkout_relink(tmp_dir, dvc, link, link_test_func):
dvc.odb.local.cache_types = [link]
+3 -3
View File
@@ -5,7 +5,7 @@ import pytest
from dvc.cli import main
from dvc.data.db import ODBManager
from dvc.fs.system import System
from dvc.fs import system
from dvc.repo import Repo
from dvc.repo.get import GetDVCFileError
from dvc.utils.fs import makedirs
@@ -86,11 +86,11 @@ def test_cache_type_is_properly_overridden(tmp_dir, erepo_dir):
[erepo_dir.dvc.config.files["repo"]], "set cache type to symlinks"
)
erepo_dir.dvc_gen("file", "contents", "create file")
assert System.is_symlink(erepo_dir / "file")
assert system.is_symlink(erepo_dir / "file")
Repo.get(os.fspath(erepo_dir), "file", "file_imported")
assert not System.is_symlink("file_imported")
assert not system.is_symlink("file_imported")
assert (tmp_dir / "file_imported").read_text() == "contents"
+3 -3
View File
@@ -10,7 +10,7 @@ from dvc.config import NoRemoteError
from dvc.data.db import ODBManager
from dvc.dvcfile import Dvcfile
from dvc.exceptions import DownloadError, PathMissingError
from dvc.fs.system import System
from dvc.fs import system
from dvc.stage.exceptions import StagePathNotFoundError
from dvc.utils.fs import makedirs, remove
from tests.unit.fs.test_dvc import make_subrepo
@@ -227,11 +227,11 @@ def test_cache_type_is_properly_overridden(tmp_dir, scm, dvc, erepo_dir):
"set source repo cache type to symlink",
)
erepo_dir.dvc_gen("foo", "foo content", "create foo")
assert System.is_symlink(erepo_dir / "foo")
assert system.is_symlink(erepo_dir / "foo")
dvc.imp(os.fspath(erepo_dir), "foo", "foo_imported")
assert not System.is_symlink("foo_imported")
assert not system.is_symlink("foo_imported")
assert (tmp_dir / "foo_imported").read_text() == "foo content"
assert scm.is_ignored("foo_imported")
+2 -2
View File
@@ -573,10 +573,10 @@ def test_subrepo(dvc_top_level, erepo):
def test_broken_symlink(tmp_dir, dvc):
from dvc.fs.system import System
from dvc.fs import system
tmp_dir.gen("file", "content")
System.symlink("file", "link")
system.symlink("file", "link")
os.remove("file")
+2 -2
View File
@@ -4,7 +4,7 @@ import pytest
from dvc.cli import main
from dvc.exceptions import DvcException
from dvc.fs.system import System
from dvc.fs import system
from dvc.stage.exceptions import StageFileDoesNotExistError
from dvc.utils.fs import remove
from tests.utils import get_gitignore_content
@@ -45,7 +45,7 @@ def test_remove_broken_symlink(tmp_dir, dvc):
(stage,) = dvc.add("foo")
remove(dvc.odb.local.cache_dir)
assert System.is_symlink("foo")
assert system.is_symlink("foo")
with pytest.raises(DvcException):
dvc.remove(stage.addressing)
+2 -2
View File
@@ -1,6 +1,6 @@
from dvc.data.db import ODBManager
from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
from dvc.fs.system import System
from dvc.fs import system
def test_destroy(tmp_dir, dvc, run_copy):
@@ -50,4 +50,4 @@ def test_destroy(tmp_dir, dvc, run_copy):
"dir/subdir",
"dir/subdir/file",
]:
assert not System.is_symlink(tmp_dir / path)
assert not system.is_symlink(tmp_dir / path)
+2 -2
View File
@@ -14,8 +14,8 @@ from dvc.exceptions import (
ReproductionError,
StagePathAsOutputError,
)
from dvc.fs import system
from dvc.fs.local import LocalFileSystem
from dvc.fs.system import System
from dvc.output import Output
from dvc.stage import Stage
from dvc.stage.exceptions import StageFileDoesNotExistError
@@ -729,7 +729,7 @@ class TestReproChangedDirData(SingleStageRun, TestDvc):
self.assertTrue(stages[0] is not None)
# Check that dvc registers mtime change for the directory.
System.hardlink(self.DATA_SUB, self.DATA_SUB + ".lnk")
system.hardlink(self.DATA_SUB, self.DATA_SUB + ".lnk")
stages = self.dvc.reproduce(target)
self.assertEqual(len(stages), 1)
self.assertTrue(stages[0] is not None)
+5 -5
View File
@@ -19,7 +19,7 @@ from dvc.exceptions import (
OverlappingOutputPathsError,
StagePathAsOutputError,
)
from dvc.fs.system import System
from dvc.fs import system
from dvc.output import Output, OutputIsStageFileError
from dvc.repo import Repo as DvcRepo
from dvc.stage import Stage
@@ -401,7 +401,7 @@ class TestRunUnprotectOutsSymlink(TestDvc):
else:
self.assertFalse(os.access(self.FOO, os.W_OK))
self.assertTrue(System.is_symlink(self.FOO))
self.assertTrue(system.is_symlink(self.FOO))
with open(self.FOO, encoding="utf-8") as fd:
self.assertEqual(fd.read(), "foo")
@@ -428,7 +428,7 @@ class TestRunUnprotectOutsSymlink(TestDvc):
else:
self.assertFalse(os.access(self.FOO, os.W_OK))
self.assertTrue(System.is_symlink(self.FOO))
self.assertTrue(system.is_symlink(self.FOO))
with open(self.FOO, encoding="utf-8") as fd:
self.assertEqual(fd.read(), "foo")
@@ -460,7 +460,7 @@ class TestRunUnprotectOutsHardlink(TestDvc):
)
self.assertEqual(ret, 0)
self.assertFalse(os.access(self.FOO, os.W_OK))
self.assertTrue(System.is_hardlink(self.FOO))
self.assertTrue(system.is_hardlink(self.FOO))
with open(self.FOO, encoding="utf-8") as fd:
self.assertEqual(fd.read(), "foo")
@@ -481,7 +481,7 @@ class TestRunUnprotectOutsHardlink(TestDvc):
)
self.assertEqual(ret, 0)
self.assertFalse(os.access(self.FOO, os.W_OK))
self.assertTrue(System.is_hardlink(self.FOO))
self.assertTrue(system.is_hardlink(self.FOO))
with open(self.FOO, encoding="utf-8") as fd:
self.assertEqual(fd.read(), "foo")
+7 -8
View File
@@ -4,15 +4,14 @@ import os
import pytest
import dvc
from dvc.fs import system
from dvc.fs.local import LocalFileSystem
from dvc.fs.system import System
from dvc.utils import relpath
from dvc.utils.fs import (
BasePathNotInCheckedPathException,
contains_symlink_up_to,
copy_fobj_to_file,
copyfile,
get_inode,
get_mtime_and_size,
makedirs,
move,
@@ -47,20 +46,20 @@ def test_should_raise_exception_on_base_path_not_in_path():
def test_should_return_true_on_symlink_in_path(mocker):
mocker.patch.object(System, "is_symlink", return_value=True)
mocker.patch.object(system, "is_symlink", return_value=True)
base_path = "foo"
path = os.path.join(base_path, "bar")
assert contains_symlink_up_to(path, base_path)
def test_should_return_false_on_path_eq_to_base_path(mocker):
mocker.patch.object(System, "is_symlink", return_value=False)
mocker.patch.object(system, "is_symlink", return_value=False)
path = "path"
assert not contains_symlink_up_to(path, path)
def test_should_return_false_on_no_more_dirs_below_path(mocker):
mocker.patch.object(System, "is_symlink", return_value=False)
mocker.patch.object(system, "is_symlink", return_value=False)
dirname_patch = mocker.patch.object(
os.path, "dirname", side_effect=lambda arg: arg
)
@@ -78,7 +77,7 @@ def test_should_return_false_when_base_path_is_symlink(mocker):
return False
mocker.patch.object(
System,
system,
"is_symlink",
return_value=True,
side_effect=base_path_is_symlink,
@@ -94,7 +93,7 @@ def test_path_object_and_str_are_valid_arg_types():
def test_should_call_recursive_on_no_condition_matched(mocker):
mocker.patch.object(System, "is_symlink", return_value=False)
mocker.patch.object(system, "is_symlink", return_value=False)
contains_symlink_spy = mocker.spy(dvc.utils.fs, "contains_symlink_up_to")
@@ -119,7 +118,7 @@ def test_relpath_windows_different_drives():
def test_get_inode(tmp_dir):
tmp_dir.gen("foo", "foo content")
assert get_inode("foo") == get_inode("foo")
assert system.inode("foo") == system.inode("foo")
def test_path_object_and_str_are_valid_types_get_mtime_and_size(tmp_dir):