mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 07:18:25 +01:00
Ilya PekelnyandRuslan Kuprieiev
2f620e9226
* Job count expected in a core config Besides defaulted 4 * core_count number of workers sourced in the "tree", the configuration file based value expected and prioritised. * Multiply the configured value by cpu cores count. * Don't multiply by the core count twice. * A couple more usages and a question note. * Don't multiply a number of jobs to a cpu count Number of jobs set in a config file directly used to spawn workers. * Remove the obsolete note. * Disallow 0 jobs set in a config * `jobs` config-based value is optional * Apply suggestions from code review Co-authored-by: Ruslan Kuprieiev <[email protected]> * CPU count based default * Update dvc/tree/base.py * Happy path unit test * Update tests/unit/remote/test_remote.py * Fix the accidentally removed blank line * Update dvc/tree/base.py Co-authored-by: Ruslan Kuprieiev <[email protected]>
55 lines
1.6 KiB
Python
55 lines
1.6 KiB
Python
import pytest
|
|
|
|
from dvc.tree import get_cloud_tree
|
|
from dvc.tree.gs import GSTree
|
|
from dvc.tree.s3 import S3Tree
|
|
|
|
|
|
def test_remote_with_hash_jobs(dvc):
|
|
dvc.config["remote"]["with_hash_jobs"] = {
|
|
"url": "s3://bucket/name",
|
|
"checksum_jobs": 100,
|
|
}
|
|
dvc.config["core"]["checksum_jobs"] = 200
|
|
|
|
tree = get_cloud_tree(dvc, name="with_hash_jobs")
|
|
assert tree.hash_jobs == 100
|
|
|
|
|
|
def test_remote_with_jobs(dvc):
|
|
dvc.config["remote"]["with_jobs"] = {
|
|
"url": "s3://bucket/name",
|
|
"jobs": 100,
|
|
}
|
|
dvc.config["core"]["jobs"] = 200
|
|
|
|
tree = get_cloud_tree(dvc, name="with_jobs")
|
|
assert tree.jobs == 100
|
|
|
|
|
|
def test_remote_without_hash_jobs(dvc):
|
|
dvc.config["remote"]["without_hash_jobs"] = {"url": "s3://bucket/name"}
|
|
dvc.config["core"]["checksum_jobs"] = 200
|
|
|
|
tree = get_cloud_tree(dvc, name="without_hash_jobs")
|
|
assert tree.hash_jobs == 200
|
|
|
|
|
|
def test_remote_without_hash_jobs_default(dvc):
|
|
dvc.config["remote"]["without_hash_jobs"] = {"url": "s3://bucket/name"}
|
|
|
|
tree = get_cloud_tree(dvc, name="without_hash_jobs")
|
|
assert tree.hash_jobs == tree.HASH_JOBS
|
|
|
|
|
|
@pytest.mark.parametrize("tree_cls", [GSTree, S3Tree])
|
|
def test_makedirs_not_create_for_top_level_path(tree_cls, dvc, mocker):
|
|
url = f"{tree_cls.scheme}://bucket/"
|
|
tree = tree_cls(dvc, {"url": url})
|
|
mocked_client = mocker.PropertyMock()
|
|
# we use remote clients with same name as scheme to interact with remote
|
|
mocker.patch.object(tree_cls, tree.scheme, mocked_client)
|
|
|
|
tree.makedirs(tree.path_info)
|
|
assert not mocked_client.called
|