Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-06 07:18:28 +01:00
Code Issues Packages Projects Releases Wiki Activity

data cloud: verify is config option now

Browse Source
This commit is contained in:
pawel
2020-01-22 10:54:07 +01:00
committed by Paweł Redzyński
parent f4e0079ec1
commit a148d16a95
11 changed files with 19 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/command/data_sync.py
-14
View File
@@ -28,7 +28,6 @@ class CmdDataPull(CmdDataBase):
with_deps=self.args.with_deps,
force=self.args.force,
recursive=self.args.recursive,
verify=self.args.verify,
)
except DvcException:
logger.exception("failed to pull data from the cloud")
@@ -67,7 +66,6 @@ class CmdDataFetch(CmdDataBase):
all_tags=self.args.all_tags,
with_deps=self.args.with_deps,
recursive=self.args.recursive,
verify=self.args.verify,
)
except DvcException:
logger.exception("failed to fetch data from the cloud")
@@ -147,12 +145,6 @@ def add_parser(subparsers, _parent_parser):
default=False,
help="Pull cache for subdirectories of the specified directory.",
)
pull_parser.add_argument(
"--verify",
action="store_true",
default=False,
help="Verify downloaded files checksums.",
)
pull_parser.set_defaults(func=CmdDataPull)
# Push
@@ -239,12 +231,6 @@ def add_parser(subparsers, _parent_parser):
default=False,
help="Fetch cache for subdirectories of specified directory.",
)
fetch_parser.add_argument(
"--verify",
action="store_true",
default=False,
help="Verify downloaded files checksums.",
)
fetch_parser.set_defaults(func=CmdDataFetch)
# Status
dvc/config.py
+2
View File
@@ -198,6 +198,7 @@ class Config(object): # pylint: disable=too-many-instance-attributes
SECTION_REMOTE_ASK_PASSWORD = "ask_password"
SECTION_REMOTE_GSS_AUTH = "gss_auth"
SECTION_REMOTE_NO_TRAVERSE = "no_traverse"
SECTION_REMOTE_VERIFY = "verify"
SECTION_REMOTE_SCHEMA = {
Required(SECTION_REMOTE_URL): str,
SECTION_AWS_REGION: str,
@@ -232,6 +233,7 @@ class Config(object): # pylint: disable=too-many-instance-attributes
SECTION_GDRIVE_USER_CREDENTIALS_FILE: str,
PRIVATE_CWD: str,
SECTION_REMOTE_NO_TRAVERSE: Bool,
SECTION_REMOTE_VERIFY: Bool,
}
SECTION_STATE = "state"
dvc/data_cloud.py
+4 -8
View File
@@ -62,9 +62,7 @@ class DataCloud(object):
show_checksums=show_checksums,
)
def pull(
self, cache, jobs=None, remote=None, show_checksums=False, verify=False
):
def pull(self, cache, jobs=None, remote=None, show_checksums=False):
"""Pull data items in a cloud-agnostic way.
Args:
@@ -75,14 +73,12 @@ class DataCloud(object):
show_checksums (bool): show checksums instead of file names in
information messages.
"""
remote = self.get_remote(remote, "pull")
downloaded_items_num = self.repo.cache.local.pull(
cache,
jobs=jobs,
remote=self.get_remote(remote, "pull"),
show_checksums=show_checksums,
cache, jobs=jobs, remote=remote, show_checksums=show_checksums
)
if not verify:
if not remote.verify:
self._save_pulled_checksums(cache)
return downloaded_items_num
dvc/output/base.py
+1 -7
View File
@@ -340,12 +340,7 @@ class OutputBase(object):
self.remote.unprotect(self.path_info)
def _collect_used_dir_cache(
self,
remote=None,
force=False,
jobs=None,
filter_info=None,
verify=False,
self, remote=None, force=False, jobs=None, filter_info=None
):
"""Get a list of `info`s related to the given directory.
@@ -375,7 +370,6 @@ class OutputBase(object):
jobs=jobs,
remote=remote,
show_checksums=False,
verify=verify,
)
except DvcException:
logger.debug("failed to pull cache for '{}'".format(self))
dvc/remote/base.py
+4
View File
@@ -80,6 +80,7 @@ class RemoteBASE(object):
CHECKSUM_JOBS = max(1, min(4, cpu_count() // 2))
DEFAULT_CACHE_TYPES = ["copy"]
DEFAULT_NO_TRAVERSE = True
DEFAULT_VERIFY = False
state = StateNoop()
@@ -93,6 +94,9 @@ class RemoteBASE(object):
self.no_traverse = config.get(
Config.SECTION_REMOTE_NO_TRAVERSE, self.DEFAULT_NO_TRAVERSE
)
self.verify = config.get(
Config.SECTION_REMOTE_VERIFY, self.DEFAULT_VERIFY
)
self._dir_info = {}
types = config.get(Config.SECTION_CACHE_TYPE, None)
dvc/remote/gdrive.py
+1
View File
@@ -63,6 +63,7 @@ class RemoteGDrive(RemoteBASE):
path_cls = CloudURLInfo
REQUIRES = {"pydrive2": "pydrive2"}
DEFAULT_NO_TRAVERSE = False
DEFAULT_VERIFY = True
GDRIVE_USER_CREDENTIALS_DATA = "GDRIVE_USER_CREDENTIALS_DATA"
DEFAULT_USER_CREDENTIALS_FILE = "gdrive-user-credentials.json"
dvc/repo/__init__.py
-2
View File
@@ -223,7 +223,6 @@ class Repo(object):
force=False,
jobs=None,
recursive=False,
verify=False,
):
"""Get the stages related to the given target and collect
the `info` of its outputs.
@@ -263,7 +262,6 @@ class Repo(object):
force=force,
jobs=jobs,
filter_info=filter_info,
verify=verify,
)
cache.update(used_cache, suffix=suffix)
dvc/repo/fetch.py
+1 -7
View File
@@ -20,7 +20,6 @@ def _fetch(
with_deps=False,
all_tags=False,
recursive=False,
verify=False,
):
"""Download data items from a cloud and imported repositories
@@ -43,7 +42,6 @@ def _fetch(
remote=remote,
jobs=jobs,
recursive=recursive,
verify=verify,
)
downloaded = 0
@@ -51,11 +49,7 @@ def _fetch(
try:
downloaded += self.cloud.pull(
used,
jobs,
remote=remote,
show_checksums=show_checksums,
verify=verify,
used, jobs, remote=remote, show_checksums=show_checksums
)
except NoRemoteError:
if not used.external and used["local"]:
dvc/repo/pull.py
-2
View File
@@ -17,7 +17,6 @@ def pull(
all_tags=False,
force=False,
recursive=False,
verify=False,
):
processed_files_count = self._fetch(
targets,
@@ -27,7 +26,6 @@ def pull(
all_tags=all_tags,
with_deps=with_deps,
recursive=recursive,
verify=verify,
)
self._checkout(
targets=targets, with_deps=with_deps, force=force, recursive=recursive
tests/func/test_remote.py
+6 -1
View File
@@ -280,6 +280,11 @@ def test_verify_checksums(tmp_dir, erepo_dir, mocker):
# Removing cache will invalidate existing state entries
shutil.rmtree(dvc.cache.local.cache_dir)
dvc.config.set(
Config.SECTION_REMOTE_FMT.format("upstream"),
Config.SECTION_REMOTE_VERIFY,
"True",
)
dvc.pull(verify=True)
dvc.pull()
assert file_md5_spy.call_count == 3
tests/unit/command/test_data_sync.py
-74
View File
@@ -1,74 +0,0 @@
from dvc.cli import parse_args
from dvc.command.data_sync import CmdDataFetch, CmdDataPull
def test_fetch(mocker):
cli_args = parse_args(
[
"fetch",
"target.dvc",
"-r",
"remote",
"-a",
"-T",
"-d",
"-R",
"--verify",
"-j",
"2",
]
)
assert cli_args.func == CmdDataFetch
cmd = cli_args.func(cli_args)
m = mocker.patch("dvc.repo.Repo.fetch")
assert cmd.run() == 0
m.assert_called_once_with(
targets=["target.dvc"],
remote="remote",
all_branches=True,
all_tags=True,
with_deps=True,
recursive=True,
verify=True,
jobs=2,
)
def test_pull(mocker):
cli_args = parse_args(
[
"pull",
"target.dvc",
"-r",
"remote",
"-a",
"-T",
"-d",
"-R",
"-f",
"--verify",
"-j",
"2",
]
)
assert cli_args.func == CmdDataPull
cmd = cli_args.func(cli_args)
m = mocker.patch("dvc.repo.Repo.pull")
assert cmd.run() == 0
m.assert_called_once_with(
targets=["target.dvc"],
remote="remote",
all_branches=True,
all_tags=True,
with_deps=True,
recursive=True,
force=True,
verify=True,
jobs=2,
)