From 75fa487406d6b6db6017629fb4ccac9a4243ed37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Saugat=20Pachhai=20=28=E0=A4=B8=E0=A5=8C=E0=A4=97=E0=A4=BE?= =?UTF-8?q?=E0=A4=A4=29?= Date: Fri, 20 May 2022 17:08:21 +0545 Subject: [PATCH] dvc: remove unused utils and redundant tests The tests should be in dvc-objects repo npw, and will be migrated soon. The nanotime and diskcache are specified in dvc-objects, so we don't need those here. --- dvc/testing/test_remote.py | 7 +- dvc/utils/__init__.py | 24 --- dvc/utils/conversions.py | 24 --- dvc/utils/fs.py | 96 +--------- setup.cfg | 3 - tests/func/test_checkout.py | 8 +- tests/func/test_fs.py | 261 +-------------------------- tests/func/utils/test_fs.py | 18 +- tests/unit/utils/test_conversions.py | 30 --- tests/unit/utils/test_fs.py | 97 +--------- tests/unit/utils/test_utils.py | 44 ----- 11 files changed, 16 insertions(+), 596 deletions(-) delete mode 100644 dvc/utils/conversions.py delete mode 100644 tests/unit/utils/test_conversions.py diff --git a/dvc/testing/test_remote.py b/dvc/testing/test_remote.py index cb311b7e3..ea366a487 100644 --- a/dvc/testing/test_remote.py +++ b/dvc/testing/test_remote.py @@ -1,6 +1,7 @@ import os +import shutil -from dvc.utils.fs import move, remove +from dvc.utils.fs import remove class TestRemote: @@ -41,7 +42,7 @@ class TestRemote: # Move cache and check status # See issue https://github.com/iterative/dvc/issues/4383 for details backup_dir = dvc.odb.local.cache_dir + ".backup" - move(dvc.odb.local.cache_dir, backup_dir) + shutil.move(dvc.odb.local.cache_dir, backup_dir) status = dvc.cloud.status(foo_hashes) _check_status(status, missing={foo_hash}) @@ -50,7 +51,7 @@ class TestRemote: # Restore original cache: remove(dvc.odb.local.cache_dir) - move(backup_dir, dvc.odb.local.cache_dir) + shutil.move(backup_dir, dvc.odb.local.cache_dir) # Push and check status dvc.cloud.push(foo_hashes) diff --git a/dvc/utils/__init__.py b/dvc/utils/__init__.py index a170c5b9e..f633f2dac 100644 --- a/dvc/utils/__init__.py +++ b/dvc/utils/__init__.py @@ -7,7 +7,6 @@ import math import os import re import sys -import time from typing import Dict, List, Optional, Tuple import colorama @@ -72,16 +71,6 @@ def _to_chunks_by_chunks_number(list_to_split, num_chunks): return _split(list_to_split, chunk_size) -def to_chunks(list_to_split, num_chunks=None, chunk_size=None): - if (num_chunks and chunk_size) or (not num_chunks and not chunk_size): - raise ValueError( - "Only one among `num_chunks` or `chunk_size` must be defined." - ) - if chunk_size: - return _split(list_to_split, chunk_size) - return _to_chunks_by_chunks_number(list_to_split, num_chunks) - - # NOTE: Check if we are in a bundle # https://pythonhosted.org/PyInstaller/runtime-information.html def is_binary(): @@ -158,19 +147,6 @@ def fix_env(env=None): return env -def tmp_fname(fname=""): - """Temporary name for a partial download""" - from shortuuid import uuid - - return os.fspath(fname) + "." + uuid() + ".tmp" - - -def current_timestamp(): - import nanotime - - return int(nanotime.timestamp(time.time())) - - def colorize(message, color=None, style=None): """Returns a message in a specified color.""" if not color: diff --git a/dvc/utils/conversions.py b/dvc/utils/conversions.py deleted file mode 100644 index 3fa0097ba..000000000 --- a/dvc/utils/conversions.py +++ /dev/null @@ -1,24 +0,0 @@ -# https://github.com/aws/aws-cli/blob/5aa599949f60b6af554fd5714d7161aa272716f7/awscli/customizations/s3/utils.py - -MULTIPLIERS = { - "kb": 1024, - "mb": 1024**2, - "gb": 1024**3, - "tb": 1024**4, - "kib": 1024, - "mib": 1024**2, - "gib": 1024**3, - "tib": 1024**4, -} - - -def human_readable_to_bytes(value): - value = value.lower() - suffix = None - if value.endswith(tuple(MULTIPLIERS.keys())): - size = 2 - size += value[-2] == "i" # KiB, MiB etc - value, suffix = value[:-size], value[-size:] - - multiplier = MULTIPLIERS.get(suffix, 1) - return int(value) * multiplier diff --git a/dvc/utils/fs.py b/dvc/utils/fs.py index ceeb77d27..0fcb44b39 100644 --- a/dvc/utils/fs.py +++ b/dvc/utils/fs.py @@ -4,19 +4,15 @@ import os import shutil import stat import sys -from contextlib import contextmanager, suppress from typing import TYPE_CHECKING from dvc.exceptions import DvcException -from dvc.fs import system if TYPE_CHECKING: from dvc.types import StrPath logger = logging.getLogger(__name__) -LOCAL_CHUNK_SIZE = 2**20 # 1 MB - class BasePathNotInCheckedPathException(DvcException): def __init__(self, path, base_path): @@ -27,6 +23,8 @@ class BasePathNotInCheckedPathException(DvcException): def contains_symlink_up_to(path: "StrPath", base_path: "StrPath"): + from dvc.fs import system + base_path = os.path.normcase(os.fspath(base_path)) path = os.path.normcase(os.fspath(path)) @@ -42,27 +40,6 @@ def contains_symlink_up_to(path: "StrPath", base_path: "StrPath"): return contains_symlink_up_to(os.path.dirname(path), base_path) -def move(src, dst): - """Atomically move src to dst and chmod it with mode. - - Moving is performed in two stages to make the whole operation atomic in - case src and dst are on different filesystems and actual physical copying - of data is happening. - """ - from shortuuid import uuid - - dst = os.path.abspath(dst) - tmp = f"{dst}.{uuid()}" - - if os.path.islink(src): - shutil.copy(src, tmp) - _unlink(src, _chmod) - else: - shutil.move(src, tmp) - - shutil.move(tmp, dst) - - def _chmod(func, p, excinfo): # pylint: disable=unused-argument perm = os.lstat(p).st_mode perm |= stat.S_IWRITE @@ -141,72 +118,3 @@ def makedirs(path, exist_ok=False, mode=None): os.chmod(path, mode) except OSError: logger.trace("failed to chmod '%o' '%s'", mode, path, exc_info=True) - - -def copyfile(src, dest, callback=None, no_progress_bar=False, name=None): - """Copy file with progress bar""" - name = name if name else os.path.basename(dest) - total = os.stat(src).st_size - - if os.path.isdir(dest): - dest = os.path.join(dest, os.path.basename(src)) - - if callback: - callback.set_size(total) - - try: - system.reflink(src, dest) - except OSError: - from dvc.fs.callbacks import Callback - - with open(src, "rb") as fsrc, open(dest, "wb+") as fdest: - with Callback.as_tqdm_callback( - callback, - size=total, - bytes=True, - disable=no_progress_bar, - desc=name, - ) as cb: - wrapped = cb.wrap_attr(fdest, "write") - while True: - buf = fsrc.read(LOCAL_CHUNK_SIZE) - if not buf: - break - wrapped.write(buf) - - if callback: - callback.absolute_update(total) - - -def copy_fobj_to_file(fsrc, dest): - """Copy contents of open file object to destination path.""" - with open(dest, "wb+") as fdest: - shutil.copyfileobj(fsrc, fdest) - - -def walk_files(directory): - for root, _, files in os.walk(directory): - for f in files: - yield os.path.join(root, f) - - -@contextmanager -def as_atomic(fs, to_info, create_parents=False): - from dvc.utils import tmp_fname - - parent = fs.path.parent(to_info) - if create_parents: - fs.makedirs(parent, exist_ok=True) - - tmp_info = fs.path.join(parent, tmp_fname()) - try: - yield tmp_info - except BaseException: - # Handle stuff like KeyboardInterrupt - # as well as other errors that might - # arise during file transfer. - with suppress(FileNotFoundError): - fs.remove(tmp_info) - raise - else: - fs.move(tmp_info, to_info) diff --git a/setup.cfg b/setup.cfg index 617a43999..343112e56 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,7 +37,6 @@ include_package_data = True install_requires = colorama>=0.3.9 configobj>=5.0.6 - nanotime>=0.5.2 voluptuous>=0.11.7 requests>=2.22.0 # See https://github.com/bdcht/grandalf/issues/26 @@ -64,13 +63,11 @@ install_requires = dpath>=2.0.2,<3 shtab>=1.3.4,<2 rich>=10.13.0 - dictdiffer>=0.8.1 python-benedict>=0.24.2 pyparsing>=2.4.7 typing-extensions>=3.7.4 fsspec[http]>=2021.10.1 aiohttp-retry>=2.4.5 - diskcache>=5.2.1 scmrepo==0.0.22 dvc-render==0.0.5 dvclive>=0.7.3 diff --git a/tests/func/test_checkout.py b/tests/func/test_checkout.py index 42417aca1..8a7401536 100644 --- a/tests/func/test_checkout.py +++ b/tests/func/test_checkout.py @@ -22,7 +22,7 @@ from dvc.fs import LocalFileSystem, system from dvc.stage import Stage from dvc.stage.exceptions import StageFileDoesNotExistError from dvc.utils import relpath -from dvc.utils.fs import remove, walk_files +from dvc.utils.fs import remove from dvc.utils.serialize import dump_yaml, load_yaml from tests.basic_env import TestDvc, TestDvcGit from tests.func.test_repro import TestRepro @@ -30,6 +30,12 @@ from tests.func.test_repro import TestRepro logger = logging.getLogger("dvc") +def walk_files(directory): + for root, _, files in os.walk(directory): + for f in files: + yield os.path.join(root, f) + + class TestCheckout(TestRepro): def setUp(self): super().setUp() diff --git a/tests/func/test_fs.py b/tests/func/test_fs.py index 8dd399b83..13aba98fe 100644 --- a/tests/func/test_fs.py +++ b/tests/func/test_fs.py @@ -1,166 +1,10 @@ import os -from io import BytesIO from operator import itemgetter -from os.path import join -import pytest - -from dvc.fs import LocalFileSystem, get_cloud_fs -from dvc.fs.callbacks import DEFAULT_CALLBACK, Callback +from dvc.fs.callbacks import Callback from dvc.repo import Repo -def test_local_fs_open(tmp_dir): - tmp_dir.gen( - { - "foo": "foo", - "bar": "bar", - "тест": "проверка", - "code.py": "import sys\nimport shutil\n" - "shutil.copyfile(sys.argv[1], sys.argv[2])", - "data_dir": { - "data": "data", - "data_sub_dir": {"data_sub": "data_sub"}, - }, - } - ) - fs = LocalFileSystem() - - with fs.open("foo", encoding="utf-8") as fobj: - assert fobj.read() == "foo" - with fs.open("тест", encoding="utf-8") as fobj: - assert fobj.read() == "проверка" - - -def test_local_fs_exists(tmp_dir): - tmp_dir.gen( - { - "foo": "foo", - "bar": "bar", - "тест": "проверка", - "code.py": "import sys\nimport shutil\n" - "shutil.copyfile(sys.argv[1], sys.argv[2])", - "data_dir": { - "data": "data", - "data_sub_dir": {"data_sub": "data_sub"}, - }, - } - ) - fs = LocalFileSystem() - - assert fs.exists("foo") - assert fs.exists("тест") - assert not fs.exists("not-existing-file") - - -def test_local_fs_isdir(tmp_dir): - tmp_dir.gen( - { - "foo": "foo", - "bar": "bar", - "тест": "проверка", - "code.py": "import sys\nimport shutil\n" - "shutil.copyfile(sys.argv[1], sys.argv[2])", - "data_dir": { - "data": "data", - "data_sub_dir": {"data_sub": "data_sub"}, - }, - } - ) - fs = LocalFileSystem() - - assert fs.isdir("data_dir") - assert not fs.isdir("foo") - assert not fs.isdir("not-existing-file") - - -def test_local_fs_isfile(tmp_dir): - tmp_dir.gen( - { - "foo": "foo", - "bar": "bar", - "тест": "проверка", - "code.py": "import sys\nimport shutil\n" - "shutil.copyfile(sys.argv[1], sys.argv[2])", - "data_dir": { - "data": "data", - "data_sub_dir": {"data_sub": "data_sub"}, - }, - } - ) - fs = LocalFileSystem() - - assert fs.isfile("foo") - assert not fs.isfile("data_dir") - assert not fs.isfile("not-existing-file") - - -def convert_to_sets(walk_results): - return [ - (root, set(dirs), set(nondirs)) for root, dirs, nondirs in walk_results - ] - - -def test_walk_no_scm(tmp_dir): - tmp_dir.gen( - { - "foo": "foo", - "bar": "bar", - "тест": "проверка", - "code.py": "import sys\nimport shutil\n" - "shutil.copyfile(sys.argv[1], sys.argv[2])", - "data_dir": { - "data": "data", - "data_sub_dir": {"data_sub": "data_sub"}, - }, - } - ) - fs = LocalFileSystem() - walk_results = fs.walk(str(tmp_dir)) - assert convert_to_sets(walk_results) == [ - (str(tmp_dir), {"data_dir"}, {"code.py", "bar", "тест", "foo"}), - (str(tmp_dir / "data_dir"), {"data_sub_dir"}, {"data"}), - (str(tmp_dir / "data_dir" / "data_sub_dir"), set(), {"data_sub"}), - ] - - walk_results = fs.walk(join("data_dir", "data_sub_dir")) - assert convert_to_sets(walk_results) == [ - (join("data_dir", "data_sub_dir"), set(), {"data_sub"}), - ] - - -def test_walk_fs_with_git(tmp_dir, scm): - tmp_dir.gen( - { - "foo": "foo", - "bar": "bar", - "тест": "проверка", - "code.py": "import sys\nimport shutil\n" - "shutil.copyfile(sys.argv[1], sys.argv[2])", - "data_dir": { - "data": "data", - "data_sub_dir": {"data_sub": "data_sub"}, - }, - } - ) - fs = LocalFileSystem(url=str(tmp_dir)) - walk_result = [] - for root, dirs, files in fs.walk("."): - dirs[:] = [i for i in dirs if i != ".git"] - walk_result.append((root, dirs, files)) - - assert convert_to_sets(walk_result) == [ - (".", {"data_dir"}, {"bar", "тест", "code.py", "foo"}), - (join("data_dir"), {"data_sub_dir"}, {"data"}), - (join("data_dir", "data_sub_dir"), set(), {"data_sub"}), - ] - - walk_result = fs.walk(join("data_dir", "data_sub_dir")) - assert convert_to_sets(walk_result) == [ - (join("data_dir", "data_sub_dir"), set(), {"data_sub"}) - ] - - def test_cleanfs_subrepo(tmp_dir, dvc, scm, monkeypatch): tmp_dir.gen({"subdir": {}}) subrepo_dir = tmp_dir / "subdir" @@ -200,78 +44,6 @@ def test_walk_dont_ignore_subrepos(tmp_dir, scm, dvc): assert set(get_dirs(next(scm_fs.walk("/")))) == {".dvc", "subdir"} -def test_fs_size(dvc, cloud): - cloud.gen({"data": {"foo": "foo"}, "baz": "baz baz"}) - cls, config, path = get_cloud_fs(dvc, **cloud.config) - fs = cls(**config) - - assert fs.size(fs.path.join(path, "baz")) == 7 - assert fs.size(fs.path.join(path, "data", "foo")) == 3 - - -def test_fs_upload_fobj(dvc, tmp_dir, cloud): - tmp_dir.gen("foo", "foo") - cls, config, path = get_cloud_fs(dvc, **cloud.config) - fs = cls(**config) - - from_info = tmp_dir / "foo" - to_info = fs.path.join(path, "foo") - - with open(from_info, "rb") as stream: - fs.upload_fobj(stream, to_info) - - assert fs.exists(to_info) - with fs.open(to_info, "rb") as stream: - assert stream.read() == b"foo" - - -def test_fs_makedirs_on_upload_and_copy(dvc, cloud): - cls, config, _ = get_cloud_fs(dvc, **cloud.config) - fs = cls(**config) - with BytesIO(b"foo") as stream: - fs.put_file(stream, (cloud / "dir" / "foo").fs_path) - - assert fs.isdir((cloud / "dir").fs_path) - assert fs.exists((cloud / "dir" / "foo").fs_path) - - fs.makedirs((cloud / "dir2").fs_path) - fs.copy((cloud / "dir" / "foo").fs_path, (cloud / "dir2" / "foo").fs_path) - assert fs.isdir((cloud / "dir2").fs_path) - assert fs.exists((cloud / "dir2" / "foo").fs_path) - - -def test_upload_callback(tmp_dir, dvc, cloud): - tmp_dir.gen("foo", "foo") - cls, config, _ = get_cloud_fs(dvc, **cloud.config) - fs = cls(**config) - expected_size = os.path.getsize(tmp_dir / "foo") - - callback = Callback() - fs.put_file( - (tmp_dir / "foo").fs_path, - (cloud / "foo").fs_path, - callback=callback, - ) - - assert callback.size == expected_size - assert callback.value == expected_size - - -def test_get_dir_callback(tmp_dir, dvc, cloud): - cls, config, _ = get_cloud_fs(dvc, **cloud.config) - fs = cls(**config) - cloud.gen({"dir": {"foo": "foo", "bar": "bar"}}) - - callback = Callback() - fs.get( - (cloud / "dir").fs_path, (tmp_dir / "dir").fs_path, callback=callback - ) - - assert callback.size == 2 - assert callback.value == 2 - assert (tmp_dir / "dir").read_text() == {"foo": "foo", "bar": "bar"} - - def test_callback_on_dvcfs(tmp_dir, dvc, scm, mocker): tmp_dir.dvc_gen({"dir": {"bar": "bar"}}, commit="dvc") tmp_dir.scm_gen({"dir": {"foo": "foo"}}, commit="git") @@ -324,34 +96,3 @@ def test_callback_on_dvcfs(tmp_dir, dvc, scm, mocker): assert branch.call_count == 1 assert branch.spy_return.size == size assert branch.spy_return.value == size - - -@pytest.mark.parametrize( - "api", ["set_size", "relative_update", "absolute_update"] -) -@pytest.mark.parametrize( - "callback_factory, kwargs", - [ - (Callback.as_callback, {}), - (Callback.as_tqdm_callback, {"desc": "test"}), - ], -) -def test_callback_with_none(request, api, callback_factory, kwargs, mocker): - """ - Test that callback don't fail if they receive None. - - The callbacks should not receive None, but there may be some - filesystems that are not compliant, we may want to maintain - maximum compatibility, and not break UI in these edge-cases. - See https://github.com/iterative/dvc/issues/7704. - """ - callback = callback_factory(**kwargs) - request.addfinalizer(callback.close) - - call_mock = mocker.spy(callback, "call") - method = getattr(callback, api) - method(None) - call_mock.assert_called_once_with() - if callback is not DEFAULT_CALLBACK: - assert callback.size is None - assert callback.value == 0 diff --git a/tests/func/utils/test_fs.py b/tests/func/utils/test_fs.py index b73b5b681..be5dcb3ac 100644 --- a/tests/func/utils/test_fs.py +++ b/tests/func/utils/test_fs.py @@ -3,7 +3,7 @@ import stat import pytest -from dvc.utils.fs import copyfile, makedirs +from dvc.utils.fs import makedirs @pytest.mark.skipif(os.name == "nt", reason="Not supported for Windows.") @@ -18,19 +18,3 @@ def test_makedirs_permissions(tmp_dir): assert stat.S_IMODE(os.stat(test_dir).st_mode) == dir_mode assert stat.S_IMODE(os.stat(intermediate_dir).st_mode) == dir_mode - - -def test_copyfile(tmp_dir): - src = "file1" - dest = "file2" - dest_dir = "testdir" - - tmp_dir.gen(src, "file1contents") - - os.mkdir(dest_dir) - - copyfile(src, dest) - assert (tmp_dir / dest).read_text() == "file1contents" - - copyfile(src, dest_dir) - assert (tmp_dir / dest_dir / src).read_text() == "file1contents" diff --git a/tests/unit/utils/test_conversions.py b/tests/unit/utils/test_conversions.py deleted file mode 100644 index b6fb596ee..000000000 --- a/tests/unit/utils/test_conversions.py +++ /dev/null @@ -1,30 +0,0 @@ -import pytest - -from dvc.utils.conversions import human_readable_to_bytes - -KB = 1024 -MB = KB**2 -GB = KB**3 -TB = KB**4 - - -@pytest.mark.parametrize( - "test_input, expected", - [ - ("10", 10), - ("10 ", 10), - ("1kb", 1 * KB), - ("2kb", 2 * KB), - ("1000mib", 1000 * MB), - ("20gB", 20 * GB), - ("10Tib", 10 * TB), - ], -) -def test_conversions_human_readable_to_bytes(test_input, expected): - assert human_readable_to_bytes(test_input) == expected - - -@pytest.mark.parametrize("invalid_input", ["foo", "10XB", "1000Pb", "fooMiB"]) -def test_conversions_human_readable_to_bytes_invalid(invalid_input): - with pytest.raises(ValueError): - human_readable_to_bytes(invalid_input) diff --git a/tests/unit/utils/test_fs.py b/tests/unit/utils/test_fs.py index 1a6c4b9f2..3a73375fd 100644 --- a/tests/unit/utils/test_fs.py +++ b/tests/unit/utils/test_fs.py @@ -1,42 +1,17 @@ -import filecmp import os import pytest import dvc -from dvc.fs import LocalFileSystem, system +from dvc.fs import system from dvc.utils import relpath from dvc.utils.fs import ( BasePathNotInCheckedPathException, contains_symlink_up_to, - copy_fobj_to_file, - copyfile, makedirs, - move, path_isin, remove, - walk_files, ) -from dvc_objects.utils import get_mtime_and_size - - -def test_mtime_and_size(tmp_dir): - tmp_dir.gen({"dir/data": "data", "dir/subdir/subdata": "subdata"}) - fs = LocalFileSystem(url=tmp_dir) - file_time, file_size = get_mtime_and_size("dir/data", fs) - dir_time, dir_size = get_mtime_and_size("dir", fs) - - actual_file_size = os.path.getsize("dir/data") - actual_dir_size = os.path.getsize("dir/data") + os.path.getsize( - "dir/subdir/subdata" - ) - - assert isinstance(file_time, str) - assert isinstance(file_size, int) - assert file_size == actual_file_size - assert isinstance(dir_time, str) - assert isinstance(dir_size, int) - assert dir_size == actual_dir_size def test_should_raise_exception_on_base_path_not_in_path(): @@ -114,41 +89,6 @@ def test_relpath_windows_different_drives(): assert rel == path1 -def test_get_inode(tmp_dir): - tmp_dir.gen("foo", "foo content") - - assert system.inode("foo") == system.inode("foo") - - -def test_path_object_and_str_are_valid_types_get_mtime_and_size(tmp_dir): - tmp_dir.gen( - {"dir": {"dir_file": "dir file content"}, "file": "file_content"} - ) - fs = LocalFileSystem(url=os.fspath(tmp_dir)) - - time, size = get_mtime_and_size("dir", fs) - object_time, object_size = get_mtime_and_size("dir", fs) - assert time == object_time - assert size == object_size - - time, size = get_mtime_and_size("file", fs) - object_time, object_size = get_mtime_and_size("file", fs) - assert time == object_time - assert size == object_size - - -def test_move(tmp_dir): - tmp_dir.gen({"foo": "foo content"}) - src = "foo" - dest = os.path.join("some", "directory") - - os.makedirs(dest) - assert len(os.listdir(dest)) == 0 - move(src, dest) - assert not os.path.isfile(src) - assert len(os.listdir(dest)) == 1 - - def test_remove(tmp_dir): tmp_dir.gen({"foo": "foo content"}) path = "foo" @@ -217,38 +157,3 @@ def test_makedirs(tmp_dir): makedirs(path) assert os.path.isdir(path) - - -@pytest.mark.parametrize("path", ["file", "dir"]) -def test_copyfile(path, tmp_dir): - tmp_dir.gen( - { - "foo": "foo content", - "file": "file content", - "dir": {}, - } - ) - src = "foo" - dest = path - - copyfile(src, dest) - if os.path.isdir(dest): - assert filecmp.cmp( - src, os.path.join(dest, os.path.basename(src)), shallow=False - ) - else: - assert filecmp.cmp(src, dest, shallow=False) - - -def test_copy_fobj_to_file(tmp_dir): - tmp_dir.gen({"foo": "foo content"}) - src = tmp_dir / "foo" - dest = "path" - - with open(src, "rb") as fobj: - copy_fobj_to_file(fobj, dest) - assert filecmp.cmp(src, dest) - - -def test_walk_files(tmp_dir): - assert list(walk_files(".")) == list(walk_files(tmp_dir)) diff --git a/tests/unit/utils/test_utils.py b/tests/unit/utils/test_utils.py index 5971b5fff..e0b3ef836 100644 --- a/tests/unit/utils/test_utils.py +++ b/tests/unit/utils/test_utils.py @@ -1,5 +1,4 @@ import os -import re import pytest @@ -9,38 +8,9 @@ from dvc.utils import ( parse_target, relpath, resolve_output, - tmp_fname, - to_chunks, ) -@pytest.mark.parametrize( - "chunk_size, expected_chunks", - [(1, [[1], [2], [3], [4]]), (2, [[1, 2], [3, 4]]), (3, [[1, 2, 3], [4]])], -) -def test_to_chunks_chunk_size(chunk_size, expected_chunks): - list_to_chunk = [1, 2, 3, 4] - result = list(to_chunks(list_to_chunk, chunk_size=chunk_size)) - assert result == expected_chunks - - -@pytest.mark.parametrize("num_chunks, chunk_size", [(1, 2), (None, None)]) -def test_to_chunks_should_raise(num_chunks, chunk_size): - list_to_chunk = [1, 2, 3] - with pytest.raises(ValueError): - to_chunks(list_to_chunk, num_chunks, chunk_size) - - -@pytest.mark.parametrize( - "num_chunks, expected_chunks", - [(4, [[1], [2], [3], [4]]), (3, [[1, 2], [3, 4]]), (2, [[1, 2], [3, 4]])], -) -def test_to_chunks_num_chunks(num_chunks, expected_chunks): - list_to_chunk = [1, 2, 3, 4] - result = to_chunks(list_to_chunk, num_chunks=num_chunks) - assert result == expected_chunks - - @pytest.mark.skipif(os.name == "nt", reason="pyenv-win is not supported") @pytest.mark.parametrize( "path, orig", @@ -81,20 +51,6 @@ def test_fix_env_pyenv(path, orig): assert fix_env(env)["PATH"] == orig -def test_tmp_fname(): - file_path = os.path.join("path", "to", "file") - - def pattern(path): - return r"^" + re.escape(path) + r"\.[a-z0-9]{22}\.tmp$" - - assert re.search(pattern(file_path), tmp_fname(file_path), re.IGNORECASE) - assert re.search( - pattern(file_path), - tmp_fname(file_path), - re.IGNORECASE, - ) - - @pytest.mark.skipif(os.name != "nt", reason="Windows specific") def test_relpath_windows(monkeypatch): """test that relpath correctly generated when run on a