mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 07:18:25 +01:00
dvc: remove unused utils and redundant tests
The tests should be in dvc-objects repo npw, and will be migrated soon. The nanotime and diskcache are specified in dvc-objects, so we don't need those here.
This commit is contained in:
Saugat Pachhai (सौगात)
committed by
skshetry
skshetry
parent
0ecd47b333
commit
75fa487406
@@ -1,6 +1,7 @@
|
||||
import os
|
||||
import shutil
|
||||
|
||||
from dvc.utils.fs import move, remove
|
||||
from dvc.utils.fs import remove
|
||||
|
||||
|
||||
class TestRemote:
|
||||
@@ -41,7 +42,7 @@ class TestRemote:
|
||||
# Move cache and check status
|
||||
# See issue https://github.com/iterative/dvc/issues/4383 for details
|
||||
backup_dir = dvc.odb.local.cache_dir + ".backup"
|
||||
move(dvc.odb.local.cache_dir, backup_dir)
|
||||
shutil.move(dvc.odb.local.cache_dir, backup_dir)
|
||||
status = dvc.cloud.status(foo_hashes)
|
||||
_check_status(status, missing={foo_hash})
|
||||
|
||||
@@ -50,7 +51,7 @@ class TestRemote:
|
||||
|
||||
# Restore original cache:
|
||||
remove(dvc.odb.local.cache_dir)
|
||||
move(backup_dir, dvc.odb.local.cache_dir)
|
||||
shutil.move(backup_dir, dvc.odb.local.cache_dir)
|
||||
|
||||
# Push and check status
|
||||
dvc.cloud.push(foo_hashes)
|
||||
|
||||
@@ -7,7 +7,6 @@ import math
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
import colorama
|
||||
@@ -72,16 +71,6 @@ def _to_chunks_by_chunks_number(list_to_split, num_chunks):
|
||||
return _split(list_to_split, chunk_size)
|
||||
|
||||
|
||||
def to_chunks(list_to_split, num_chunks=None, chunk_size=None):
|
||||
if (num_chunks and chunk_size) or (not num_chunks and not chunk_size):
|
||||
raise ValueError(
|
||||
"Only one among `num_chunks` or `chunk_size` must be defined."
|
||||
)
|
||||
if chunk_size:
|
||||
return _split(list_to_split, chunk_size)
|
||||
return _to_chunks_by_chunks_number(list_to_split, num_chunks)
|
||||
|
||||
|
||||
# NOTE: Check if we are in a bundle
|
||||
# https://pythonhosted.org/PyInstaller/runtime-information.html
|
||||
def is_binary():
|
||||
@@ -158,19 +147,6 @@ def fix_env(env=None):
|
||||
return env
|
||||
|
||||
|
||||
def tmp_fname(fname=""):
|
||||
"""Temporary name for a partial download"""
|
||||
from shortuuid import uuid
|
||||
|
||||
return os.fspath(fname) + "." + uuid() + ".tmp"
|
||||
|
||||
|
||||
def current_timestamp():
|
||||
import nanotime
|
||||
|
||||
return int(nanotime.timestamp(time.time()))
|
||||
|
||||
|
||||
def colorize(message, color=None, style=None):
|
||||
"""Returns a message in a specified color."""
|
||||
if not color:
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
# https://github.com/aws/aws-cli/blob/5aa599949f60b6af554fd5714d7161aa272716f7/awscli/customizations/s3/utils.py
|
||||
|
||||
MULTIPLIERS = {
|
||||
"kb": 1024,
|
||||
"mb": 1024**2,
|
||||
"gb": 1024**3,
|
||||
"tb": 1024**4,
|
||||
"kib": 1024,
|
||||
"mib": 1024**2,
|
||||
"gib": 1024**3,
|
||||
"tib": 1024**4,
|
||||
}
|
||||
|
||||
|
||||
def human_readable_to_bytes(value):
|
||||
value = value.lower()
|
||||
suffix = None
|
||||
if value.endswith(tuple(MULTIPLIERS.keys())):
|
||||
size = 2
|
||||
size += value[-2] == "i" # KiB, MiB etc
|
||||
value, suffix = value[:-size], value[-size:]
|
||||
|
||||
multiplier = MULTIPLIERS.get(suffix, 1)
|
||||
return int(value) * multiplier
|
||||
+2
-94
@@ -4,19 +4,15 @@ import os
|
||||
import shutil
|
||||
import stat
|
||||
import sys
|
||||
from contextlib import contextmanager, suppress
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.fs import system
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from dvc.types import StrPath
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
LOCAL_CHUNK_SIZE = 2**20 # 1 MB
|
||||
|
||||
|
||||
class BasePathNotInCheckedPathException(DvcException):
|
||||
def __init__(self, path, base_path):
|
||||
@@ -27,6 +23,8 @@ class BasePathNotInCheckedPathException(DvcException):
|
||||
|
||||
|
||||
def contains_symlink_up_to(path: "StrPath", base_path: "StrPath"):
|
||||
from dvc.fs import system
|
||||
|
||||
base_path = os.path.normcase(os.fspath(base_path))
|
||||
path = os.path.normcase(os.fspath(path))
|
||||
|
||||
@@ -42,27 +40,6 @@ def contains_symlink_up_to(path: "StrPath", base_path: "StrPath"):
|
||||
return contains_symlink_up_to(os.path.dirname(path), base_path)
|
||||
|
||||
|
||||
def move(src, dst):
|
||||
"""Atomically move src to dst and chmod it with mode.
|
||||
|
||||
Moving is performed in two stages to make the whole operation atomic in
|
||||
case src and dst are on different filesystems and actual physical copying
|
||||
of data is happening.
|
||||
"""
|
||||
from shortuuid import uuid
|
||||
|
||||
dst = os.path.abspath(dst)
|
||||
tmp = f"{dst}.{uuid()}"
|
||||
|
||||
if os.path.islink(src):
|
||||
shutil.copy(src, tmp)
|
||||
_unlink(src, _chmod)
|
||||
else:
|
||||
shutil.move(src, tmp)
|
||||
|
||||
shutil.move(tmp, dst)
|
||||
|
||||
|
||||
def _chmod(func, p, excinfo): # pylint: disable=unused-argument
|
||||
perm = os.lstat(p).st_mode
|
||||
perm |= stat.S_IWRITE
|
||||
@@ -141,72 +118,3 @@ def makedirs(path, exist_ok=False, mode=None):
|
||||
os.chmod(path, mode)
|
||||
except OSError:
|
||||
logger.trace("failed to chmod '%o' '%s'", mode, path, exc_info=True)
|
||||
|
||||
|
||||
def copyfile(src, dest, callback=None, no_progress_bar=False, name=None):
|
||||
"""Copy file with progress bar"""
|
||||
name = name if name else os.path.basename(dest)
|
||||
total = os.stat(src).st_size
|
||||
|
||||
if os.path.isdir(dest):
|
||||
dest = os.path.join(dest, os.path.basename(src))
|
||||
|
||||
if callback:
|
||||
callback.set_size(total)
|
||||
|
||||
try:
|
||||
system.reflink(src, dest)
|
||||
except OSError:
|
||||
from dvc.fs.callbacks import Callback
|
||||
|
||||
with open(src, "rb") as fsrc, open(dest, "wb+") as fdest:
|
||||
with Callback.as_tqdm_callback(
|
||||
callback,
|
||||
size=total,
|
||||
bytes=True,
|
||||
disable=no_progress_bar,
|
||||
desc=name,
|
||||
) as cb:
|
||||
wrapped = cb.wrap_attr(fdest, "write")
|
||||
while True:
|
||||
buf = fsrc.read(LOCAL_CHUNK_SIZE)
|
||||
if not buf:
|
||||
break
|
||||
wrapped.write(buf)
|
||||
|
||||
if callback:
|
||||
callback.absolute_update(total)
|
||||
|
||||
|
||||
def copy_fobj_to_file(fsrc, dest):
|
||||
"""Copy contents of open file object to destination path."""
|
||||
with open(dest, "wb+") as fdest:
|
||||
shutil.copyfileobj(fsrc, fdest)
|
||||
|
||||
|
||||
def walk_files(directory):
|
||||
for root, _, files in os.walk(directory):
|
||||
for f in files:
|
||||
yield os.path.join(root, f)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def as_atomic(fs, to_info, create_parents=False):
|
||||
from dvc.utils import tmp_fname
|
||||
|
||||
parent = fs.path.parent(to_info)
|
||||
if create_parents:
|
||||
fs.makedirs(parent, exist_ok=True)
|
||||
|
||||
tmp_info = fs.path.join(parent, tmp_fname())
|
||||
try:
|
||||
yield tmp_info
|
||||
except BaseException:
|
||||
# Handle stuff like KeyboardInterrupt
|
||||
# as well as other errors that might
|
||||
# arise during file transfer.
|
||||
with suppress(FileNotFoundError):
|
||||
fs.remove(tmp_info)
|
||||
raise
|
||||
else:
|
||||
fs.move(tmp_info, to_info)
|
||||
|
||||
@@ -37,7 +37,6 @@ include_package_data = True
|
||||
install_requires =
|
||||
colorama>=0.3.9
|
||||
configobj>=5.0.6
|
||||
nanotime>=0.5.2
|
||||
voluptuous>=0.11.7
|
||||
requests>=2.22.0
|
||||
# See https://github.com/bdcht/grandalf/issues/26
|
||||
@@ -64,13 +63,11 @@ install_requires =
|
||||
dpath>=2.0.2,<3
|
||||
shtab>=1.3.4,<2
|
||||
rich>=10.13.0
|
||||
dictdiffer>=0.8.1
|
||||
python-benedict>=0.24.2
|
||||
pyparsing>=2.4.7
|
||||
typing-extensions>=3.7.4
|
||||
fsspec[http]>=2021.10.1
|
||||
aiohttp-retry>=2.4.5
|
||||
diskcache>=5.2.1
|
||||
scmrepo==0.0.22
|
||||
dvc-render==0.0.5
|
||||
dvclive>=0.7.3
|
||||
|
||||
@@ -22,7 +22,7 @@ from dvc.fs import LocalFileSystem, system
|
||||
from dvc.stage import Stage
|
||||
from dvc.stage.exceptions import StageFileDoesNotExistError
|
||||
from dvc.utils import relpath
|
||||
from dvc.utils.fs import remove, walk_files
|
||||
from dvc.utils.fs import remove
|
||||
from dvc.utils.serialize import dump_yaml, load_yaml
|
||||
from tests.basic_env import TestDvc, TestDvcGit
|
||||
from tests.func.test_repro import TestRepro
|
||||
@@ -30,6 +30,12 @@ from tests.func.test_repro import TestRepro
|
||||
logger = logging.getLogger("dvc")
|
||||
|
||||
|
||||
def walk_files(directory):
|
||||
for root, _, files in os.walk(directory):
|
||||
for f in files:
|
||||
yield os.path.join(root, f)
|
||||
|
||||
|
||||
class TestCheckout(TestRepro):
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
|
||||
+1
-260
@@ -1,166 +1,10 @@
|
||||
import os
|
||||
from io import BytesIO
|
||||
from operator import itemgetter
|
||||
from os.path import join
|
||||
|
||||
import pytest
|
||||
|
||||
from dvc.fs import LocalFileSystem, get_cloud_fs
|
||||
from dvc.fs.callbacks import DEFAULT_CALLBACK, Callback
|
||||
from dvc.fs.callbacks import Callback
|
||||
from dvc.repo import Repo
|
||||
|
||||
|
||||
def test_local_fs_open(tmp_dir):
|
||||
tmp_dir.gen(
|
||||
{
|
||||
"foo": "foo",
|
||||
"bar": "bar",
|
||||
"тест": "проверка",
|
||||
"code.py": "import sys\nimport shutil\n"
|
||||
"shutil.copyfile(sys.argv[1], sys.argv[2])",
|
||||
"data_dir": {
|
||||
"data": "data",
|
||||
"data_sub_dir": {"data_sub": "data_sub"},
|
||||
},
|
||||
}
|
||||
)
|
||||
fs = LocalFileSystem()
|
||||
|
||||
with fs.open("foo", encoding="utf-8") as fobj:
|
||||
assert fobj.read() == "foo"
|
||||
with fs.open("тест", encoding="utf-8") as fobj:
|
||||
assert fobj.read() == "проверка"
|
||||
|
||||
|
||||
def test_local_fs_exists(tmp_dir):
|
||||
tmp_dir.gen(
|
||||
{
|
||||
"foo": "foo",
|
||||
"bar": "bar",
|
||||
"тест": "проверка",
|
||||
"code.py": "import sys\nimport shutil\n"
|
||||
"shutil.copyfile(sys.argv[1], sys.argv[2])",
|
||||
"data_dir": {
|
||||
"data": "data",
|
||||
"data_sub_dir": {"data_sub": "data_sub"},
|
||||
},
|
||||
}
|
||||
)
|
||||
fs = LocalFileSystem()
|
||||
|
||||
assert fs.exists("foo")
|
||||
assert fs.exists("тест")
|
||||
assert not fs.exists("not-existing-file")
|
||||
|
||||
|
||||
def test_local_fs_isdir(tmp_dir):
|
||||
tmp_dir.gen(
|
||||
{
|
||||
"foo": "foo",
|
||||
"bar": "bar",
|
||||
"тест": "проверка",
|
||||
"code.py": "import sys\nimport shutil\n"
|
||||
"shutil.copyfile(sys.argv[1], sys.argv[2])",
|
||||
"data_dir": {
|
||||
"data": "data",
|
||||
"data_sub_dir": {"data_sub": "data_sub"},
|
||||
},
|
||||
}
|
||||
)
|
||||
fs = LocalFileSystem()
|
||||
|
||||
assert fs.isdir("data_dir")
|
||||
assert not fs.isdir("foo")
|
||||
assert not fs.isdir("not-existing-file")
|
||||
|
||||
|
||||
def test_local_fs_isfile(tmp_dir):
|
||||
tmp_dir.gen(
|
||||
{
|
||||
"foo": "foo",
|
||||
"bar": "bar",
|
||||
"тест": "проверка",
|
||||
"code.py": "import sys\nimport shutil\n"
|
||||
"shutil.copyfile(sys.argv[1], sys.argv[2])",
|
||||
"data_dir": {
|
||||
"data": "data",
|
||||
"data_sub_dir": {"data_sub": "data_sub"},
|
||||
},
|
||||
}
|
||||
)
|
||||
fs = LocalFileSystem()
|
||||
|
||||
assert fs.isfile("foo")
|
||||
assert not fs.isfile("data_dir")
|
||||
assert not fs.isfile("not-existing-file")
|
||||
|
||||
|
||||
def convert_to_sets(walk_results):
|
||||
return [
|
||||
(root, set(dirs), set(nondirs)) for root, dirs, nondirs in walk_results
|
||||
]
|
||||
|
||||
|
||||
def test_walk_no_scm(tmp_dir):
|
||||
tmp_dir.gen(
|
||||
{
|
||||
"foo": "foo",
|
||||
"bar": "bar",
|
||||
"тест": "проверка",
|
||||
"code.py": "import sys\nimport shutil\n"
|
||||
"shutil.copyfile(sys.argv[1], sys.argv[2])",
|
||||
"data_dir": {
|
||||
"data": "data",
|
||||
"data_sub_dir": {"data_sub": "data_sub"},
|
||||
},
|
||||
}
|
||||
)
|
||||
fs = LocalFileSystem()
|
||||
walk_results = fs.walk(str(tmp_dir))
|
||||
assert convert_to_sets(walk_results) == [
|
||||
(str(tmp_dir), {"data_dir"}, {"code.py", "bar", "тест", "foo"}),
|
||||
(str(tmp_dir / "data_dir"), {"data_sub_dir"}, {"data"}),
|
||||
(str(tmp_dir / "data_dir" / "data_sub_dir"), set(), {"data_sub"}),
|
||||
]
|
||||
|
||||
walk_results = fs.walk(join("data_dir", "data_sub_dir"))
|
||||
assert convert_to_sets(walk_results) == [
|
||||
(join("data_dir", "data_sub_dir"), set(), {"data_sub"}),
|
||||
]
|
||||
|
||||
|
||||
def test_walk_fs_with_git(tmp_dir, scm):
|
||||
tmp_dir.gen(
|
||||
{
|
||||
"foo": "foo",
|
||||
"bar": "bar",
|
||||
"тест": "проверка",
|
||||
"code.py": "import sys\nimport shutil\n"
|
||||
"shutil.copyfile(sys.argv[1], sys.argv[2])",
|
||||
"data_dir": {
|
||||
"data": "data",
|
||||
"data_sub_dir": {"data_sub": "data_sub"},
|
||||
},
|
||||
}
|
||||
)
|
||||
fs = LocalFileSystem(url=str(tmp_dir))
|
||||
walk_result = []
|
||||
for root, dirs, files in fs.walk("."):
|
||||
dirs[:] = [i for i in dirs if i != ".git"]
|
||||
walk_result.append((root, dirs, files))
|
||||
|
||||
assert convert_to_sets(walk_result) == [
|
||||
(".", {"data_dir"}, {"bar", "тест", "code.py", "foo"}),
|
||||
(join("data_dir"), {"data_sub_dir"}, {"data"}),
|
||||
(join("data_dir", "data_sub_dir"), set(), {"data_sub"}),
|
||||
]
|
||||
|
||||
walk_result = fs.walk(join("data_dir", "data_sub_dir"))
|
||||
assert convert_to_sets(walk_result) == [
|
||||
(join("data_dir", "data_sub_dir"), set(), {"data_sub"})
|
||||
]
|
||||
|
||||
|
||||
def test_cleanfs_subrepo(tmp_dir, dvc, scm, monkeypatch):
|
||||
tmp_dir.gen({"subdir": {}})
|
||||
subrepo_dir = tmp_dir / "subdir"
|
||||
@@ -200,78 +44,6 @@ def test_walk_dont_ignore_subrepos(tmp_dir, scm, dvc):
|
||||
assert set(get_dirs(next(scm_fs.walk("/")))) == {".dvc", "subdir"}
|
||||
|
||||
|
||||
def test_fs_size(dvc, cloud):
|
||||
cloud.gen({"data": {"foo": "foo"}, "baz": "baz baz"})
|
||||
cls, config, path = get_cloud_fs(dvc, **cloud.config)
|
||||
fs = cls(**config)
|
||||
|
||||
assert fs.size(fs.path.join(path, "baz")) == 7
|
||||
assert fs.size(fs.path.join(path, "data", "foo")) == 3
|
||||
|
||||
|
||||
def test_fs_upload_fobj(dvc, tmp_dir, cloud):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
cls, config, path = get_cloud_fs(dvc, **cloud.config)
|
||||
fs = cls(**config)
|
||||
|
||||
from_info = tmp_dir / "foo"
|
||||
to_info = fs.path.join(path, "foo")
|
||||
|
||||
with open(from_info, "rb") as stream:
|
||||
fs.upload_fobj(stream, to_info)
|
||||
|
||||
assert fs.exists(to_info)
|
||||
with fs.open(to_info, "rb") as stream:
|
||||
assert stream.read() == b"foo"
|
||||
|
||||
|
||||
def test_fs_makedirs_on_upload_and_copy(dvc, cloud):
|
||||
cls, config, _ = get_cloud_fs(dvc, **cloud.config)
|
||||
fs = cls(**config)
|
||||
with BytesIO(b"foo") as stream:
|
||||
fs.put_file(stream, (cloud / "dir" / "foo").fs_path)
|
||||
|
||||
assert fs.isdir((cloud / "dir").fs_path)
|
||||
assert fs.exists((cloud / "dir" / "foo").fs_path)
|
||||
|
||||
fs.makedirs((cloud / "dir2").fs_path)
|
||||
fs.copy((cloud / "dir" / "foo").fs_path, (cloud / "dir2" / "foo").fs_path)
|
||||
assert fs.isdir((cloud / "dir2").fs_path)
|
||||
assert fs.exists((cloud / "dir2" / "foo").fs_path)
|
||||
|
||||
|
||||
def test_upload_callback(tmp_dir, dvc, cloud):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
cls, config, _ = get_cloud_fs(dvc, **cloud.config)
|
||||
fs = cls(**config)
|
||||
expected_size = os.path.getsize(tmp_dir / "foo")
|
||||
|
||||
callback = Callback()
|
||||
fs.put_file(
|
||||
(tmp_dir / "foo").fs_path,
|
||||
(cloud / "foo").fs_path,
|
||||
callback=callback,
|
||||
)
|
||||
|
||||
assert callback.size == expected_size
|
||||
assert callback.value == expected_size
|
||||
|
||||
|
||||
def test_get_dir_callback(tmp_dir, dvc, cloud):
|
||||
cls, config, _ = get_cloud_fs(dvc, **cloud.config)
|
||||
fs = cls(**config)
|
||||
cloud.gen({"dir": {"foo": "foo", "bar": "bar"}})
|
||||
|
||||
callback = Callback()
|
||||
fs.get(
|
||||
(cloud / "dir").fs_path, (tmp_dir / "dir").fs_path, callback=callback
|
||||
)
|
||||
|
||||
assert callback.size == 2
|
||||
assert callback.value == 2
|
||||
assert (tmp_dir / "dir").read_text() == {"foo": "foo", "bar": "bar"}
|
||||
|
||||
|
||||
def test_callback_on_dvcfs(tmp_dir, dvc, scm, mocker):
|
||||
tmp_dir.dvc_gen({"dir": {"bar": "bar"}}, commit="dvc")
|
||||
tmp_dir.scm_gen({"dir": {"foo": "foo"}}, commit="git")
|
||||
@@ -324,34 +96,3 @@ def test_callback_on_dvcfs(tmp_dir, dvc, scm, mocker):
|
||||
assert branch.call_count == 1
|
||||
assert branch.spy_return.size == size
|
||||
assert branch.spy_return.value == size
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"api", ["set_size", "relative_update", "absolute_update"]
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"callback_factory, kwargs",
|
||||
[
|
||||
(Callback.as_callback, {}),
|
||||
(Callback.as_tqdm_callback, {"desc": "test"}),
|
||||
],
|
||||
)
|
||||
def test_callback_with_none(request, api, callback_factory, kwargs, mocker):
|
||||
"""
|
||||
Test that callback don't fail if they receive None.
|
||||
|
||||
The callbacks should not receive None, but there may be some
|
||||
filesystems that are not compliant, we may want to maintain
|
||||
maximum compatibility, and not break UI in these edge-cases.
|
||||
See https://github.com/iterative/dvc/issues/7704.
|
||||
"""
|
||||
callback = callback_factory(**kwargs)
|
||||
request.addfinalizer(callback.close)
|
||||
|
||||
call_mock = mocker.spy(callback, "call")
|
||||
method = getattr(callback, api)
|
||||
method(None)
|
||||
call_mock.assert_called_once_with()
|
||||
if callback is not DEFAULT_CALLBACK:
|
||||
assert callback.size is None
|
||||
assert callback.value == 0
|
||||
|
||||
@@ -3,7 +3,7 @@ import stat
|
||||
|
||||
import pytest
|
||||
|
||||
from dvc.utils.fs import copyfile, makedirs
|
||||
from dvc.utils.fs import makedirs
|
||||
|
||||
|
||||
@pytest.mark.skipif(os.name == "nt", reason="Not supported for Windows.")
|
||||
@@ -18,19 +18,3 @@ def test_makedirs_permissions(tmp_dir):
|
||||
|
||||
assert stat.S_IMODE(os.stat(test_dir).st_mode) == dir_mode
|
||||
assert stat.S_IMODE(os.stat(intermediate_dir).st_mode) == dir_mode
|
||||
|
||||
|
||||
def test_copyfile(tmp_dir):
|
||||
src = "file1"
|
||||
dest = "file2"
|
||||
dest_dir = "testdir"
|
||||
|
||||
tmp_dir.gen(src, "file1contents")
|
||||
|
||||
os.mkdir(dest_dir)
|
||||
|
||||
copyfile(src, dest)
|
||||
assert (tmp_dir / dest).read_text() == "file1contents"
|
||||
|
||||
copyfile(src, dest_dir)
|
||||
assert (tmp_dir / dest_dir / src).read_text() == "file1contents"
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
import pytest
|
||||
|
||||
from dvc.utils.conversions import human_readable_to_bytes
|
||||
|
||||
KB = 1024
|
||||
MB = KB**2
|
||||
GB = KB**3
|
||||
TB = KB**4
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"test_input, expected",
|
||||
[
|
||||
("10", 10),
|
||||
("10 ", 10),
|
||||
("1kb", 1 * KB),
|
||||
("2kb", 2 * KB),
|
||||
("1000mib", 1000 * MB),
|
||||
("20gB", 20 * GB),
|
||||
("10Tib", 10 * TB),
|
||||
],
|
||||
)
|
||||
def test_conversions_human_readable_to_bytes(test_input, expected):
|
||||
assert human_readable_to_bytes(test_input) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("invalid_input", ["foo", "10XB", "1000Pb", "fooMiB"])
|
||||
def test_conversions_human_readable_to_bytes_invalid(invalid_input):
|
||||
with pytest.raises(ValueError):
|
||||
human_readable_to_bytes(invalid_input)
|
||||
@@ -1,42 +1,17 @@
|
||||
import filecmp
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
import dvc
|
||||
from dvc.fs import LocalFileSystem, system
|
||||
from dvc.fs import system
|
||||
from dvc.utils import relpath
|
||||
from dvc.utils.fs import (
|
||||
BasePathNotInCheckedPathException,
|
||||
contains_symlink_up_to,
|
||||
copy_fobj_to_file,
|
||||
copyfile,
|
||||
makedirs,
|
||||
move,
|
||||
path_isin,
|
||||
remove,
|
||||
walk_files,
|
||||
)
|
||||
from dvc_objects.utils import get_mtime_and_size
|
||||
|
||||
|
||||
def test_mtime_and_size(tmp_dir):
|
||||
tmp_dir.gen({"dir/data": "data", "dir/subdir/subdata": "subdata"})
|
||||
fs = LocalFileSystem(url=tmp_dir)
|
||||
file_time, file_size = get_mtime_and_size("dir/data", fs)
|
||||
dir_time, dir_size = get_mtime_and_size("dir", fs)
|
||||
|
||||
actual_file_size = os.path.getsize("dir/data")
|
||||
actual_dir_size = os.path.getsize("dir/data") + os.path.getsize(
|
||||
"dir/subdir/subdata"
|
||||
)
|
||||
|
||||
assert isinstance(file_time, str)
|
||||
assert isinstance(file_size, int)
|
||||
assert file_size == actual_file_size
|
||||
assert isinstance(dir_time, str)
|
||||
assert isinstance(dir_size, int)
|
||||
assert dir_size == actual_dir_size
|
||||
|
||||
|
||||
def test_should_raise_exception_on_base_path_not_in_path():
|
||||
@@ -114,41 +89,6 @@ def test_relpath_windows_different_drives():
|
||||
assert rel == path1
|
||||
|
||||
|
||||
def test_get_inode(tmp_dir):
|
||||
tmp_dir.gen("foo", "foo content")
|
||||
|
||||
assert system.inode("foo") == system.inode("foo")
|
||||
|
||||
|
||||
def test_path_object_and_str_are_valid_types_get_mtime_and_size(tmp_dir):
|
||||
tmp_dir.gen(
|
||||
{"dir": {"dir_file": "dir file content"}, "file": "file_content"}
|
||||
)
|
||||
fs = LocalFileSystem(url=os.fspath(tmp_dir))
|
||||
|
||||
time, size = get_mtime_and_size("dir", fs)
|
||||
object_time, object_size = get_mtime_and_size("dir", fs)
|
||||
assert time == object_time
|
||||
assert size == object_size
|
||||
|
||||
time, size = get_mtime_and_size("file", fs)
|
||||
object_time, object_size = get_mtime_and_size("file", fs)
|
||||
assert time == object_time
|
||||
assert size == object_size
|
||||
|
||||
|
||||
def test_move(tmp_dir):
|
||||
tmp_dir.gen({"foo": "foo content"})
|
||||
src = "foo"
|
||||
dest = os.path.join("some", "directory")
|
||||
|
||||
os.makedirs(dest)
|
||||
assert len(os.listdir(dest)) == 0
|
||||
move(src, dest)
|
||||
assert not os.path.isfile(src)
|
||||
assert len(os.listdir(dest)) == 1
|
||||
|
||||
|
||||
def test_remove(tmp_dir):
|
||||
tmp_dir.gen({"foo": "foo content"})
|
||||
path = "foo"
|
||||
@@ -217,38 +157,3 @@ def test_makedirs(tmp_dir):
|
||||
|
||||
makedirs(path)
|
||||
assert os.path.isdir(path)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["file", "dir"])
|
||||
def test_copyfile(path, tmp_dir):
|
||||
tmp_dir.gen(
|
||||
{
|
||||
"foo": "foo content",
|
||||
"file": "file content",
|
||||
"dir": {},
|
||||
}
|
||||
)
|
||||
src = "foo"
|
||||
dest = path
|
||||
|
||||
copyfile(src, dest)
|
||||
if os.path.isdir(dest):
|
||||
assert filecmp.cmp(
|
||||
src, os.path.join(dest, os.path.basename(src)), shallow=False
|
||||
)
|
||||
else:
|
||||
assert filecmp.cmp(src, dest, shallow=False)
|
||||
|
||||
|
||||
def test_copy_fobj_to_file(tmp_dir):
|
||||
tmp_dir.gen({"foo": "foo content"})
|
||||
src = tmp_dir / "foo"
|
||||
dest = "path"
|
||||
|
||||
with open(src, "rb") as fobj:
|
||||
copy_fobj_to_file(fobj, dest)
|
||||
assert filecmp.cmp(src, dest)
|
||||
|
||||
|
||||
def test_walk_files(tmp_dir):
|
||||
assert list(walk_files(".")) == list(walk_files(tmp_dir))
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import os
|
||||
import re
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -9,38 +8,9 @@ from dvc.utils import (
|
||||
parse_target,
|
||||
relpath,
|
||||
resolve_output,
|
||||
tmp_fname,
|
||||
to_chunks,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"chunk_size, expected_chunks",
|
||||
[(1, [[1], [2], [3], [4]]), (2, [[1, 2], [3, 4]]), (3, [[1, 2, 3], [4]])],
|
||||
)
|
||||
def test_to_chunks_chunk_size(chunk_size, expected_chunks):
|
||||
list_to_chunk = [1, 2, 3, 4]
|
||||
result = list(to_chunks(list_to_chunk, chunk_size=chunk_size))
|
||||
assert result == expected_chunks
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_chunks, chunk_size", [(1, 2), (None, None)])
|
||||
def test_to_chunks_should_raise(num_chunks, chunk_size):
|
||||
list_to_chunk = [1, 2, 3]
|
||||
with pytest.raises(ValueError):
|
||||
to_chunks(list_to_chunk, num_chunks, chunk_size)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"num_chunks, expected_chunks",
|
||||
[(4, [[1], [2], [3], [4]]), (3, [[1, 2], [3, 4]]), (2, [[1, 2], [3, 4]])],
|
||||
)
|
||||
def test_to_chunks_num_chunks(num_chunks, expected_chunks):
|
||||
list_to_chunk = [1, 2, 3, 4]
|
||||
result = to_chunks(list_to_chunk, num_chunks=num_chunks)
|
||||
assert result == expected_chunks
|
||||
|
||||
|
||||
@pytest.mark.skipif(os.name == "nt", reason="pyenv-win is not supported")
|
||||
@pytest.mark.parametrize(
|
||||
"path, orig",
|
||||
@@ -81,20 +51,6 @@ def test_fix_env_pyenv(path, orig):
|
||||
assert fix_env(env)["PATH"] == orig
|
||||
|
||||
|
||||
def test_tmp_fname():
|
||||
file_path = os.path.join("path", "to", "file")
|
||||
|
||||
def pattern(path):
|
||||
return r"^" + re.escape(path) + r"\.[a-z0-9]{22}\.tmp$"
|
||||
|
||||
assert re.search(pattern(file_path), tmp_fname(file_path), re.IGNORECASE)
|
||||
assert re.search(
|
||||
pattern(file_path),
|
||||
tmp_fname(file_path),
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(os.name != "nt", reason="Windows specific")
|
||||
def test_relpath_windows(monkeypatch):
|
||||
"""test that relpath correctly generated when run on a
|
||||
|
||||
Reference in New Issue
Block a user