mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 07:18:27 +01:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7c6351d039 | ||
|
|
613ca0bdb5 | ||
|
|
0fb9870de6 | ||
|
|
73e60a8c13 | ||
|
|
9d1dde4051 | ||
|
|
38dc0f85d4 | ||
|
|
d2dbe9649c | ||
|
|
ece6f2d3ee | ||
|
|
b672622324 | ||
|
|
c9391df313 | ||
|
|
a0c229cf9b | ||
|
|
a59f90fac9 | ||
|
|
bba502314d | ||
|
|
bc4bf66d17 | ||
|
|
f7348af1e0 | ||
|
|
4d140a9944 | ||
|
|
2ba73daedc |
+9
-2
@@ -107,7 +107,6 @@ jobs:
|
||||
- sudo /snap/bin/lxd.migrate -yes
|
||||
- sudo /snap/bin/lxd waitready
|
||||
- sudo /snap/bin/lxd init --auto
|
||||
- git fetch --tags
|
||||
script:
|
||||
- ./scripts/build_snap.sh
|
||||
after_failure:
|
||||
@@ -158,6 +157,14 @@ deploy:
|
||||
script: "(echo $SNAP_TOKEN | snapcraft login --with -) && timeout 600 snapcraft push dvc_*.snap --release $SNAP_CHANNEL || echo timed out"
|
||||
on:
|
||||
all_branches: true
|
||||
condition: "$(./scripts/ci/deploy_condition.sh dvc_*.snap) && ($TRAVIS_BRANCH = master || -n $TRAVIS_TAG)"
|
||||
condition: "$(./scripts/ci/deploy_condition.sh dvc_*.snap) && ($TRAVIS_BRANCH = master || -n $TRAVIS_TAG) && -n $SNAP_CHANNEL"
|
||||
repo: iterative/dvc
|
||||
stage: build
|
||||
- provider: script
|
||||
skip_cleanup: true
|
||||
script: "(echo $SNAP_TOKEN | snapcraft login --with -) && timeout 600 snapcraft push dvc_*.snap --release $SNAP_CHANNEL_MAJOR || echo timed out"
|
||||
on:
|
||||
all_branches: true
|
||||
condition: "$(./scripts/ci/deploy_condition.sh dvc_*.snap) && ($TRAVIS_BRANCH = master || -n $TRAVIS_TAG) && -n $SNAP_CHANNEL_MAJOR"
|
||||
repo: iterative/dvc
|
||||
stage: build
|
||||
|
||||
+1
-1
@@ -30,7 +30,7 @@ def get_url(path, repo=None, rev=None, remote=None):
|
||||
raise UrlNotDvcRepoError(_repo.url)
|
||||
out = _repo.find_out_by_relpath(path)
|
||||
remote_obj = _repo.cloud.get_remote(remote)
|
||||
return str(remote_obj.checksum_to_path_info(out.checksum))
|
||||
return str(remote_obj.hash_to_path_info(out.checksum))
|
||||
|
||||
|
||||
def open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
|
||||
|
||||
+8
-4
@@ -28,13 +28,15 @@ def _make_remote_property(name):
|
||||
"""
|
||||
|
||||
def getter(self):
|
||||
from dvc.remote import Cache as CloudCache
|
||||
from dvc.remote import get_cloud_tree
|
||||
from dvc.remote.base import CloudCache
|
||||
|
||||
remote = self.config.get(name)
|
||||
if not remote:
|
||||
return None
|
||||
|
||||
return CloudCache(self.repo, name=remote)
|
||||
tree = get_cloud_tree(self.repo, name=remote)
|
||||
return CloudCache(tree)
|
||||
|
||||
getter.__name__ = name
|
||||
return cached_property(getter)
|
||||
@@ -50,7 +52,8 @@ class Cache:
|
||||
CACHE_DIR = "cache"
|
||||
|
||||
def __init__(self, repo):
|
||||
from dvc.remote import Cache as CloudCache
|
||||
from dvc.remote import get_cloud_tree
|
||||
from dvc.remote.local import LocalCache
|
||||
|
||||
self.repo = repo
|
||||
self.config = config = repo.config["cache"]
|
||||
@@ -62,7 +65,8 @@ class Cache:
|
||||
else:
|
||||
settings = {**config, "url": config["dir"]}
|
||||
|
||||
self.local = CloudCache(repo, **settings)
|
||||
tree = get_cloud_tree(repo, **settings)
|
||||
self.local = LocalCache(tree)
|
||||
|
||||
s3 = _make_remote_property("s3")
|
||||
gs = _make_remote_property("gs")
|
||||
|
||||
+19
-16
@@ -165,7 +165,10 @@ def add_parser(subparsers, parent_parser):
|
||||
metrics_show_parser.add_argument(
|
||||
"targets",
|
||||
nargs="*",
|
||||
help="Metric files or directories (see -R) to show",
|
||||
help=(
|
||||
"Limit command scope to these metric files. Using -R, "
|
||||
"directories to search metric files in can also be given."
|
||||
),
|
||||
)
|
||||
metrics_show_parser.add_argument(
|
||||
"-a",
|
||||
@@ -187,6 +190,12 @@ def add_parser(subparsers, parent_parser):
|
||||
default=False,
|
||||
help="Show metrics for all commits.",
|
||||
)
|
||||
metrics_show_parser.add_argument(
|
||||
"--show-json",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Show output in JSON format.",
|
||||
)
|
||||
metrics_show_parser.add_argument(
|
||||
"-R",
|
||||
"--recursive",
|
||||
@@ -197,12 +206,6 @@ def add_parser(subparsers, parent_parser):
|
||||
"metric files."
|
||||
),
|
||||
)
|
||||
metrics_show_parser.add_argument(
|
||||
"--show-json",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Show output in JSON format.",
|
||||
)
|
||||
metrics_show_parser.set_defaults(func=CmdMetricsShow)
|
||||
|
||||
METRICS_DIFF_HELP = (
|
||||
@@ -222,14 +225,14 @@ def add_parser(subparsers, parent_parser):
|
||||
metrics_diff_parser.add_argument(
|
||||
"b_rev",
|
||||
nargs="?",
|
||||
help=("New Git commit to compare (defaults to the current workspace)"),
|
||||
help="New Git commit to compare (defaults to the current workspace)",
|
||||
)
|
||||
metrics_diff_parser.add_argument(
|
||||
"--targets",
|
||||
nargs="*",
|
||||
help=(
|
||||
"Metric files or directories (see -R) to show diff for. "
|
||||
"Shows diff for all metric files by default."
|
||||
"Limit command scope to these metric files. Using -R, "
|
||||
"directories to search metric files in can also be given."
|
||||
),
|
||||
metavar="<paths>",
|
||||
)
|
||||
@@ -261,18 +264,18 @@ def add_parser(subparsers, parent_parser):
|
||||
default=False,
|
||||
help="Show tabulated output in the Markdown format (GFM).",
|
||||
)
|
||||
metrics_diff_parser.add_argument(
|
||||
"--no-path",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Don't show metric path.",
|
||||
)
|
||||
metrics_diff_parser.add_argument(
|
||||
"--old",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Show old metric value.",
|
||||
)
|
||||
metrics_diff_parser.add_argument(
|
||||
"--no-path",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Don't show metric path.",
|
||||
)
|
||||
metrics_diff_parser.add_argument(
|
||||
"--precision",
|
||||
type=int,
|
||||
|
||||
+23
-13
@@ -98,8 +98,8 @@ class CmdPlotsModify(CmdPlots):
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
PLOTS_HELP = (
|
||||
"Generating plots for metrics stored in structured files "
|
||||
"(JSON, CSV, TSV)."
|
||||
"Commands to visualize and compare plot metrics in structured files "
|
||||
"(JSON, YAML, CSV, TSV)"
|
||||
)
|
||||
|
||||
plots_parser = subparsers.add_parser(
|
||||
@@ -116,7 +116,7 @@ def add_parser(subparsers, parent_parser):
|
||||
|
||||
fix_subparsers(plots_subparsers)
|
||||
|
||||
SHOW_HELP = "Generate a plots image file from a metrics file."
|
||||
SHOW_HELP = "Generate plots from metric files."
|
||||
plots_show_parser = plots_subparsers.add_parser(
|
||||
"show",
|
||||
parents=[parent_parser],
|
||||
@@ -134,8 +134,8 @@ def add_parser(subparsers, parent_parser):
|
||||
plots_show_parser.set_defaults(func=CmdPlotsShow)
|
||||
|
||||
PLOTS_DIFF_HELP = (
|
||||
"Plot differences in metrics between commits in the DVC "
|
||||
"repository, or between the last commit and the workspace."
|
||||
"Show multiple versions of plot metrics "
|
||||
"by plotting them in a single image."
|
||||
)
|
||||
plots_diff_parser = plots_subparsers.add_parser(
|
||||
"diff",
|
||||
@@ -148,6 +148,7 @@ def add_parser(subparsers, parent_parser):
|
||||
"--targets",
|
||||
nargs="*",
|
||||
help="Plots file to visualize. Shows all plots by default.",
|
||||
metavar="<path>",
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"revisions", nargs="*", default=None, help="Git commits to plot from",
|
||||
@@ -156,7 +157,7 @@ def add_parser(subparsers, parent_parser):
|
||||
_add_output_arguments(plots_diff_parser)
|
||||
plots_diff_parser.set_defaults(func=CmdPlotsDiff)
|
||||
|
||||
PLOTS_MODIFY_HELP = "Modify plot properties associated with a target file."
|
||||
PLOTS_MODIFY_HELP = "Modify display properties of plot metric files."
|
||||
plots_modify_parser = plots_subparsers.add_parser(
|
||||
"modify",
|
||||
parents=[parent_parser],
|
||||
@@ -165,14 +166,14 @@ def add_parser(subparsers, parent_parser):
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
plots_modify_parser.add_argument(
|
||||
"target", help="Plot file to set properties to.",
|
||||
"target", help="Metric file to set properties to",
|
||||
)
|
||||
_add_props_arguments(plots_modify_parser)
|
||||
plots_modify_parser.add_argument(
|
||||
"--unset",
|
||||
nargs="*",
|
||||
metavar="<property>",
|
||||
help="Properties to unset.",
|
||||
help="Unset one or more display properties.",
|
||||
)
|
||||
plots_modify_parser.set_defaults(func=CmdPlotsModify)
|
||||
|
||||
@@ -189,9 +190,14 @@ def _add_props_arguments(parser):
|
||||
format_link("https://man.dvc.org/plots#plot-templates")
|
||||
)
|
||||
),
|
||||
metavar="<path>",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-x", default=None, help="Field name for X axis.", metavar="<field>"
|
||||
)
|
||||
parser.add_argument(
|
||||
"-y", default=None, help="Field name for Y axis.", metavar="<field>"
|
||||
)
|
||||
parser.add_argument("-x", default=None, help="Field name for x axis.")
|
||||
parser.add_argument("-y", default=None, help="Field name for y axis.")
|
||||
parser.add_argument(
|
||||
"--no-header",
|
||||
action="store_false",
|
||||
@@ -203,16 +209,20 @@ def _add_props_arguments(parser):
|
||||
"--title", default=None, metavar="<text>", help="Plot title."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--x-label", default=None, metavar="<text>", help="X axis label."
|
||||
"--x-label", default=None, help="X axis label", metavar="<text>"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--y-label", default=None, metavar="<text>", help="Y axis lebel."
|
||||
"--y-label", default=None, help="Y axis label", metavar="<text>"
|
||||
)
|
||||
|
||||
|
||||
def _add_output_arguments(parser):
|
||||
parser.add_argument(
|
||||
"-o", "--out", default=None, help="Destination path to save plots to.",
|
||||
"-o",
|
||||
"--out",
|
||||
default=None,
|
||||
help="Destination path to save plots to",
|
||||
metavar="<path>",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--show-vega",
|
||||
|
||||
@@ -122,12 +122,12 @@ class CmdVersion(CmdBaseNoRepo):
|
||||
|
||||
@staticmethod
|
||||
def get_supported_remotes():
|
||||
from dvc.remote import REMOTES
|
||||
from dvc.remote import TREES
|
||||
|
||||
supported_remotes = []
|
||||
for remote in REMOTES:
|
||||
if not remote.get_missing_deps():
|
||||
supported_remotes.append(remote.scheme)
|
||||
for tree_cls in TREES:
|
||||
if not tree_cls.get_missing_deps():
|
||||
supported_remotes.append(tree_cls.scheme)
|
||||
|
||||
return ", ".join(supported_remotes)
|
||||
|
||||
|
||||
+4
-4
@@ -3,7 +3,7 @@
|
||||
import logging
|
||||
|
||||
from dvc.config import NoRemoteError
|
||||
from dvc.remote import Remote
|
||||
from dvc.remote import get_remote
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -45,7 +45,7 @@ class DataCloud:
|
||||
raise NoRemoteError(error_msg)
|
||||
|
||||
def _init_remote(self, name):
|
||||
return Remote(self.repo, name=name)
|
||||
return get_remote(self.repo, name=name)
|
||||
|
||||
def push(
|
||||
self, cache, jobs=None, remote=None, show_checksums=False,
|
||||
@@ -85,14 +85,14 @@ class DataCloud:
|
||||
cache, jobs=jobs, remote=remote, show_checksums=show_checksums
|
||||
)
|
||||
|
||||
if not remote.verify:
|
||||
if not remote.tree.verify:
|
||||
self._save_pulled_checksums(cache)
|
||||
|
||||
return downloaded_items_num
|
||||
|
||||
def _save_pulled_checksums(self, cache):
|
||||
for checksum in cache.scheme_keys("local"):
|
||||
cache_file = self.repo.cache.local.checksum_to_path_info(checksum)
|
||||
cache_file = self.repo.cache.local.hash_to_path_info(checksum)
|
||||
if self.repo.cache.local.tree.exists(cache_file):
|
||||
# We can safely save here, as existing corrupted files will
|
||||
# be removed upon status, while files corrupted during
|
||||
|
||||
@@ -12,7 +12,7 @@ from dvc.dependency.param import ParamsDependency
|
||||
from dvc.dependency.s3 import S3Dependency
|
||||
from dvc.dependency.ssh import SSHDependency
|
||||
from dvc.output.base import BaseOutput
|
||||
from dvc.remote import Remote
|
||||
from dvc.remote import get_remote
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
from .repo import RepoDependency
|
||||
@@ -54,7 +54,7 @@ SCHEMA.update(ParamsDependency.PARAM_SCHEMA)
|
||||
def _get(stage, p, info):
|
||||
parsed = urlparse(p) if p else None
|
||||
if parsed and parsed.scheme == "remote":
|
||||
remote = Remote(stage.repo, name=parsed.netloc)
|
||||
remote = get_remote(stage.repo, name=parsed.netloc)
|
||||
return DEP_MAP[remote.scheme](stage, p, info, remote=remote)
|
||||
|
||||
if info and info.get(RepoDependency.PARAM_REPO):
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from dvc.dependency.base import BaseDependency
|
||||
from dvc.output.base import BaseOutput
|
||||
from dvc.remote.azure import AzureRemote
|
||||
from dvc.remote.azure import AzureRemoteTree
|
||||
|
||||
|
||||
class AzureDependency(BaseDependency, BaseOutput):
|
||||
REMOTE = AzureRemote
|
||||
TREE_CLS = AzureRemoteTree
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from dvc.dependency.base import BaseDependency
|
||||
from dvc.output.base import BaseOutput
|
||||
from dvc.remote.http import HTTPRemote
|
||||
from dvc.remote.http import HTTPRemoteTree
|
||||
|
||||
|
||||
class HTTPDependency(BaseDependency, BaseOutput):
|
||||
REMOTE = HTTPRemote
|
||||
TREE_CLS = HTTPRemoteTree
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from dvc.remote.https import HTTPSRemote
|
||||
from dvc.remote.https import HTTPSRemoteTree
|
||||
|
||||
from .http import HTTPDependency
|
||||
|
||||
|
||||
class HTTPSDependency(HTTPDependency):
|
||||
REMOTE = HTTPSRemote
|
||||
TREE_CLS = HTTPSRemoteTree
|
||||
|
||||
@@ -64,8 +64,8 @@ class RepoDependency(LocalDependency):
|
||||
|
||||
# We are polluting our repo cache with some dir listing here
|
||||
if tree.isdir(path):
|
||||
return self.repo.cache.local.get_checksum(path, tree)
|
||||
return tree.get_file_checksum(path)
|
||||
return self.repo.cache.local.get_hash(path, tree)
|
||||
return tree.get_file_hash(path)
|
||||
|
||||
def status(self):
|
||||
current_checksum = self._get_checksum(locked=True)
|
||||
|
||||
+5
-2
@@ -222,9 +222,12 @@ class ETagMismatchError(DvcException):
|
||||
|
||||
|
||||
class FileMissingError(DvcException):
|
||||
def __init__(self, path):
|
||||
def __init__(self, path, hint=None):
|
||||
self.path = path
|
||||
super().__init__(f"Can't find '{path}' neither locally nor on remote")
|
||||
hint = "" if hint is None else f". {hint}"
|
||||
super().__init__(
|
||||
f"Can't find '{path}' neither locally nor on remote{hint}"
|
||||
)
|
||||
|
||||
|
||||
class DvcIgnoreInCollectedDirError(DvcException):
|
||||
|
||||
+24
-3
@@ -1,12 +1,15 @@
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from itertools import groupby
|
||||
|
||||
from funcy import cached_property
|
||||
from pathspec import PathSpec
|
||||
from pathspec.patterns import GitWildMatchPattern
|
||||
from pathspec.util import normalize_file
|
||||
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.scm.tree import BaseTree
|
||||
from dvc.system import System
|
||||
from dvc.utils import relpath
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -27,7 +30,16 @@ class DvcIgnorePatterns(DvcIgnore):
|
||||
self.dirname = os.path.normpath(os.path.dirname(ignore_file_path))
|
||||
|
||||
with tree.open(ignore_file_path, encoding="utf-8") as fobj:
|
||||
self.ignore_spec = PathSpec.from_lines(GitWildMatchPattern, fobj)
|
||||
path_spec_lines = fobj.readlines()
|
||||
regex_pattern_list = map(
|
||||
GitWildMatchPattern.pattern_to_regex, path_spec_lines
|
||||
)
|
||||
self.ignore_spec = [
|
||||
(ignore, re.compile("|".join(item[0] for item in group)))
|
||||
for ignore, group in groupby(
|
||||
regex_pattern_list, lambda x: x[1]
|
||||
)
|
||||
]
|
||||
|
||||
def __call__(self, root, dirs, files):
|
||||
files = [f for f in files if not self.matches(root, f)]
|
||||
@@ -48,7 +60,16 @@ class DvcIgnorePatterns(DvcIgnore):
|
||||
else:
|
||||
return False
|
||||
|
||||
return self.ignore_spec.match_file(path)
|
||||
if not System.is_unix():
|
||||
path = normalize_file(path)
|
||||
return self.ignore(path)
|
||||
|
||||
def ignore(self, path):
|
||||
result = False
|
||||
for ignore, pattern in self.ignore_spec:
|
||||
if pattern.match(path):
|
||||
result = ignore
|
||||
return result
|
||||
|
||||
def __hash__(self):
|
||||
return hash(self.ignore_file_path)
|
||||
|
||||
@@ -10,10 +10,10 @@ from dvc.output.hdfs import HDFSOutput
|
||||
from dvc.output.local import LocalOutput
|
||||
from dvc.output.s3 import S3Output
|
||||
from dvc.output.ssh import SSHOutput
|
||||
from dvc.remote import Remote
|
||||
from dvc.remote.hdfs import HDFSRemote
|
||||
from dvc.remote.local import LocalRemote
|
||||
from dvc.remote.s3 import S3Remote
|
||||
from dvc.remote import get_remote
|
||||
from dvc.remote.hdfs import HDFSRemoteTree
|
||||
from dvc.remote.local import LocalRemoteTree
|
||||
from dvc.remote.s3 import S3RemoteTree
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
OUTS = [
|
||||
@@ -47,9 +47,9 @@ CHECKSUM_SCHEMA = Any(
|
||||
# so when a few types of outputs share the same name, we only need
|
||||
# specify it once.
|
||||
CHECKSUMS_SCHEMA = {
|
||||
LocalRemote.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
|
||||
S3Remote.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
|
||||
HDFSRemote.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
|
||||
LocalRemoteTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
|
||||
S3RemoteTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
|
||||
HDFSRemoteTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
|
||||
}
|
||||
|
||||
SCHEMA = CHECKSUMS_SCHEMA.copy()
|
||||
@@ -66,7 +66,7 @@ def _get(
|
||||
parsed = urlparse(p)
|
||||
|
||||
if parsed.scheme == "remote":
|
||||
remote = Remote(stage.repo, name=parsed.netloc)
|
||||
remote = get_remote(stage.repo, name=parsed.netloc)
|
||||
return OUTS_MAP[remote.scheme](
|
||||
stage,
|
||||
p,
|
||||
|
||||
+20
-15
@@ -12,7 +12,7 @@ from dvc.exceptions import (
|
||||
DvcException,
|
||||
RemoteCacheRequiredError,
|
||||
)
|
||||
from dvc.remote.base import BaseRemote
|
||||
from dvc.remote.base import BaseRemoteTree, Remote
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -47,7 +47,8 @@ class OutputIsStageFileError(DvcException):
|
||||
class BaseOutput:
|
||||
IS_DEPENDENCY = False
|
||||
|
||||
REMOTE = BaseRemote
|
||||
REMOTE_CLS = Remote
|
||||
TREE_CLS = BaseRemoteTree
|
||||
|
||||
PARAM_PATH = "path"
|
||||
PARAM_CACHE = "cache"
|
||||
@@ -105,7 +106,11 @@ class BaseOutput:
|
||||
self.repo = stage.repo if stage else None
|
||||
self.def_path = path
|
||||
self.info = info
|
||||
self.remote = remote or self.REMOTE(self.repo, {})
|
||||
if remote:
|
||||
self.remote = remote
|
||||
else:
|
||||
tree = self.TREE_CLS(self.repo, {})
|
||||
self.remote = self.REMOTE_CLS(tree)
|
||||
self.use_cache = False if self.IS_DEPENDENCY else cache
|
||||
self.metric = False if self.IS_DEPENDENCY else metric
|
||||
self.plot = False if self.IS_DEPENDENCY else plot
|
||||
@@ -119,7 +124,7 @@ class BaseOutput:
|
||||
if remote:
|
||||
parsed = urlparse(path)
|
||||
return remote.path_info / parsed.path.lstrip("/")
|
||||
return self.REMOTE.TREE_CLS.PATH_CLS(path)
|
||||
return self.TREE_CLS.PATH_CLS(path)
|
||||
|
||||
def __repr__(self):
|
||||
return "{class_name}: '{def_path}'".format(
|
||||
@@ -131,7 +136,7 @@ class BaseOutput:
|
||||
|
||||
@property
|
||||
def scheme(self):
|
||||
return self.REMOTE.scheme
|
||||
return self.TREE_CLS.scheme
|
||||
|
||||
@property
|
||||
def is_in_repo(self):
|
||||
@@ -154,30 +159,30 @@ class BaseOutput:
|
||||
|
||||
@classmethod
|
||||
def supported(cls, url):
|
||||
return cls.REMOTE.supported(url)
|
||||
return cls.TREE_CLS.supported(url)
|
||||
|
||||
@property
|
||||
def cache_path(self):
|
||||
return self.cache.checksum_to_path_info(self.checksum).url
|
||||
return self.cache.hash_to_path_info(self.checksum).url
|
||||
|
||||
@property
|
||||
def checksum_type(self):
|
||||
return self.remote.PARAM_CHECKSUM
|
||||
return self.remote.tree.PARAM_CHECKSUM
|
||||
|
||||
@property
|
||||
def checksum(self):
|
||||
return self.info.get(self.remote.PARAM_CHECKSUM)
|
||||
return self.info.get(self.remote.tree.PARAM_CHECKSUM)
|
||||
|
||||
@checksum.setter
|
||||
def checksum(self, checksum):
|
||||
self.info[self.remote.PARAM_CHECKSUM] = checksum
|
||||
self.info[self.remote.tree.PARAM_CHECKSUM] = checksum
|
||||
|
||||
def get_checksum(self):
|
||||
return self.remote.get_checksum(self.path_info)
|
||||
return self.remote.get_hash(self.path_info)
|
||||
|
||||
@property
|
||||
def is_dir_checksum(self):
|
||||
return self.remote.is_dir_checksum(self.checksum)
|
||||
return self.remote.is_dir_hash(self.checksum)
|
||||
|
||||
@property
|
||||
def exists(self):
|
||||
@@ -357,7 +362,7 @@ class BaseOutput:
|
||||
|
||||
def unprotect(self):
|
||||
if self.exists:
|
||||
self.remote.unprotect(self.path_info)
|
||||
self.remote.tree.unprotect(self.path_info)
|
||||
|
||||
def get_dir_cache(self, **kwargs):
|
||||
if not self.is_dir_checksum:
|
||||
@@ -417,8 +422,8 @@ class BaseOutput:
|
||||
filter_path = str(filter_info) if filter_info else None
|
||||
is_win = os.name == "nt"
|
||||
for entry in self.dir_cache:
|
||||
checksum = entry[self.remote.PARAM_CHECKSUM]
|
||||
entry_relpath = entry[self.remote.PARAM_RELPATH]
|
||||
checksum = entry[self.remote.tree.PARAM_CHECKSUM]
|
||||
entry_relpath = entry[self.remote.tree.PARAM_RELPATH]
|
||||
if is_win:
|
||||
entry_relpath = entry_relpath.replace("/", os.sep)
|
||||
entry_path = os.path.join(path, entry_relpath)
|
||||
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
from dvc.output.s3 import S3Output
|
||||
from dvc.remote.gs import GSRemote
|
||||
from dvc.remote.gs import GSRemoteTree
|
||||
|
||||
|
||||
class GSOutput(S3Output):
|
||||
REMOTE = GSRemote
|
||||
TREE_CLS = GSRemoteTree
|
||||
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
from dvc.output.base import BaseOutput
|
||||
from dvc.remote.hdfs import HDFSRemote
|
||||
from dvc.remote.hdfs import HDFSRemoteTree
|
||||
|
||||
|
||||
class HDFSOutput(BaseOutput):
|
||||
REMOTE = HDFSRemote
|
||||
TREE_CLS = HDFSRemoteTree
|
||||
|
||||
+7
-6
@@ -5,7 +5,7 @@ from urllib.parse import urlparse
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.istextfile import istextfile
|
||||
from dvc.output.base import BaseOutput
|
||||
from dvc.remote.local import LocalRemote
|
||||
from dvc.remote.local import LocalRemote, LocalRemoteTree
|
||||
from dvc.utils import relpath
|
||||
from dvc.utils.fs import path_isin
|
||||
|
||||
@@ -13,7 +13,8 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LocalOutput(BaseOutput):
|
||||
REMOTE = LocalRemote
|
||||
REMOTE_CLS = LocalRemote
|
||||
TREE_CLS = LocalRemoteTree
|
||||
sep = os.sep
|
||||
|
||||
def __init__(self, stage, path, *args, **kwargs):
|
||||
@@ -33,15 +34,15 @@ class LocalOutput(BaseOutput):
|
||||
#
|
||||
# FIXME: if we have Windows path containing / or posix one with \
|
||||
# then we have #2059 bug and can't really handle that.
|
||||
p = self.REMOTE.TREE_CLS.PATH_CLS(path)
|
||||
if not p.is_absolute():
|
||||
p = self.TREE_CLS.PATH_CLS(path)
|
||||
if self.stage and not p.is_absolute():
|
||||
p = self.stage.wdir / p
|
||||
|
||||
abs_p = os.path.abspath(os.path.normpath(p))
|
||||
return self.REMOTE.TREE_CLS.PATH_CLS(abs_p)
|
||||
return self.TREE_CLS.PATH_CLS(abs_p)
|
||||
|
||||
def __str__(self):
|
||||
if not self.is_in_repo:
|
||||
if not self.repo or not self.is_in_repo:
|
||||
return str(self.def_path)
|
||||
|
||||
cur_dir = os.getcwd()
|
||||
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
from dvc.output.base import BaseOutput
|
||||
from dvc.remote.s3 import S3Remote
|
||||
from dvc.remote.s3 import S3RemoteTree
|
||||
|
||||
|
||||
class S3Output(BaseOutput):
|
||||
REMOTE = S3Remote
|
||||
TREE_CLS = S3RemoteTree
|
||||
|
||||
+3
-2
@@ -1,6 +1,7 @@
|
||||
from dvc.output.base import BaseOutput
|
||||
from dvc.remote.ssh import SSHRemote
|
||||
from dvc.remote.ssh import SSHRemote, SSHRemoteTree
|
||||
|
||||
|
||||
class SSHOutput(BaseOutput):
|
||||
REMOTE = SSHRemote
|
||||
REMOTE_CLS = SSHRemote
|
||||
TREE_CLS = SSHRemoteTree
|
||||
|
||||
+41
-45
@@ -1,45 +1,37 @@
|
||||
import posixpath
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from dvc.remote.azure import AzureCache, AzureRemote
|
||||
from dvc.remote.gdrive import GDriveRemote
|
||||
from dvc.remote.gs import GSCache, GSRemote
|
||||
from dvc.remote.hdfs import HDFSCache, HDFSRemote
|
||||
from dvc.remote.http import HTTPRemote
|
||||
from dvc.remote.https import HTTPSRemote
|
||||
from dvc.remote.local import LocalCache, LocalRemote
|
||||
from dvc.remote.oss import OSSRemote
|
||||
from dvc.remote.s3 import S3Cache, S3Remote
|
||||
from dvc.remote.ssh import SSHCache, SSHRemote
|
||||
from dvc.remote.azure import AzureRemoteTree
|
||||
from dvc.remote.base import Remote
|
||||
from dvc.remote.gdrive import GDriveRemoteTree
|
||||
from dvc.remote.gs import GSRemoteTree
|
||||
from dvc.remote.hdfs import HDFSRemoteTree
|
||||
from dvc.remote.http import HTTPRemoteTree
|
||||
from dvc.remote.https import HTTPSRemoteTree
|
||||
from dvc.remote.local import LocalRemote, LocalRemoteTree
|
||||
from dvc.remote.oss import OSSRemoteTree
|
||||
from dvc.remote.s3 import S3RemoteTree
|
||||
from dvc.remote.ssh import SSHRemote, SSHRemoteTree
|
||||
|
||||
CACHES = [
|
||||
AzureCache,
|
||||
GSCache,
|
||||
HDFSCache,
|
||||
S3Cache,
|
||||
SSHCache,
|
||||
# LocalCache is the default
|
||||
]
|
||||
|
||||
REMOTES = [
|
||||
AzureRemote,
|
||||
GDriveRemote,
|
||||
GSRemote,
|
||||
HDFSRemote,
|
||||
HTTPRemote,
|
||||
HTTPSRemote,
|
||||
S3Remote,
|
||||
SSHRemote,
|
||||
OSSRemote,
|
||||
# NOTE: LocalRemote is the default
|
||||
TREES = [
|
||||
AzureRemoteTree,
|
||||
GDriveRemoteTree,
|
||||
GSRemoteTree,
|
||||
HDFSRemoteTree,
|
||||
HTTPRemoteTree,
|
||||
HTTPSRemoteTree,
|
||||
S3RemoteTree,
|
||||
SSHRemoteTree,
|
||||
OSSRemoteTree,
|
||||
# NOTE: LocalRemoteTree is the default
|
||||
]
|
||||
|
||||
|
||||
def _get(remote_conf, remotes, default):
|
||||
for remote in remotes:
|
||||
if remote.supported(remote_conf):
|
||||
return remote
|
||||
return default
|
||||
def _get_tree(remote_conf):
|
||||
for tree_cls in TREES:
|
||||
if tree_cls.supported(remote_conf):
|
||||
return tree_cls
|
||||
return LocalRemoteTree
|
||||
|
||||
|
||||
def _get_conf(repo, **kwargs):
|
||||
@@ -51,16 +43,6 @@ def _get_conf(repo, **kwargs):
|
||||
return _resolve_remote_refs(repo.config, remote_conf)
|
||||
|
||||
|
||||
def Remote(repo, **kwargs):
|
||||
remote_conf = _get_conf(repo, **kwargs)
|
||||
return _get(remote_conf, REMOTES, LocalRemote)(repo, remote_conf)
|
||||
|
||||
|
||||
def Cache(repo, **kwargs):
|
||||
remote_conf = _get_conf(repo, **kwargs)
|
||||
return _get(remote_conf, CACHES, LocalCache)(repo, remote_conf)
|
||||
|
||||
|
||||
def _resolve_remote_refs(config, remote_conf):
|
||||
# Support for cross referenced remotes.
|
||||
# This will merge the settings, shadowing base ref with remote_conf.
|
||||
@@ -90,3 +72,17 @@ def _resolve_remote_refs(config, remote_conf):
|
||||
base = config["remote"][parsed.netloc]
|
||||
url = posixpath.join(base["url"], parsed.path.lstrip("/"))
|
||||
return {**base, **remote_conf, "url": url}
|
||||
|
||||
|
||||
def get_cloud_tree(repo, **kwargs):
|
||||
remote_conf = _get_conf(repo, **kwargs)
|
||||
return _get_tree(remote_conf)(repo, remote_conf)
|
||||
|
||||
|
||||
def get_remote(repo, **kwargs):
|
||||
tree = get_cloud_tree(repo, **kwargs)
|
||||
if tree.scheme == "local":
|
||||
return LocalRemote(tree)
|
||||
if tree.scheme == "ssh":
|
||||
return SSHRemote(tree)
|
||||
return Remote(tree)
|
||||
|
||||
+9
-17
@@ -7,17 +7,22 @@ from funcy import cached_property, wrap_prop
|
||||
|
||||
from dvc.path_info import CloudURLInfo
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.remote.base import BaseRemote, BaseRemoteTree, CacheMixin
|
||||
from dvc.remote.base import BaseRemoteTree
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AzureRemoteTree(BaseRemoteTree):
|
||||
scheme = Schemes.AZURE
|
||||
PATH_CLS = CloudURLInfo
|
||||
REQUIRES = {"azure-storage-blob": "azure.storage.blob"}
|
||||
PARAM_CHECKSUM = "etag"
|
||||
COPY_POLL_SECONDS = 5
|
||||
LIST_OBJECT_PAGE_SIZE = 5000
|
||||
|
||||
def __init__(self, remote, config):
|
||||
super().__init__(remote, config)
|
||||
def __init__(self, repo, config):
|
||||
super().__init__(repo, config)
|
||||
|
||||
url = config.get("url", "azure://")
|
||||
self.path_info = self.PATH_CLS(url)
|
||||
@@ -108,7 +113,7 @@ class AzureRemoteTree(BaseRemoteTree):
|
||||
logger.debug(f"Removing {path_info}")
|
||||
self.blob_service.delete_blob(path_info.bucket, path_info.path)
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
def get_file_hash(self, path_info):
|
||||
return self.get_etag(path_info)
|
||||
|
||||
def _upload(
|
||||
@@ -132,16 +137,3 @@ class AzureRemoteTree(BaseRemoteTree):
|
||||
to_file,
|
||||
progress_callback=pbar.update_to,
|
||||
)
|
||||
|
||||
|
||||
class AzureRemote(BaseRemote):
|
||||
scheme = Schemes.AZURE
|
||||
REQUIRES = {"azure-storage-blob": "azure.storage.blob"}
|
||||
TREE_CLS = AzureRemoteTree
|
||||
PARAM_CHECKSUM = "etag"
|
||||
COPY_POLL_SECONDS = 5
|
||||
LIST_OBJECT_PAGE_SIZE = 5000
|
||||
|
||||
|
||||
class AzureCache(AzureRemote, CacheMixin):
|
||||
pass
|
||||
|
||||
+538
-510
File diff suppressed because it is too large
Load Diff
+36
-27
@@ -1,38 +1,35 @@
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import posixpath
|
||||
import re
|
||||
import threading
|
||||
from collections import defaultdict
|
||||
from contextlib import contextmanager
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from funcy import cached_property, retry, wrap_prop, wrap_with
|
||||
from funcy.py3 import cat
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.exceptions import DvcException, FileMissingError
|
||||
from dvc.path_info import CloudURLInfo
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.remote.base import BaseRemote, BaseRemoteTree
|
||||
from dvc.remote.base import BaseRemoteTree
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.utils import format_link, tmp_fname
|
||||
from dvc.utils.stream import IterStream
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"
|
||||
|
||||
|
||||
class GDrivePathNotFound(DvcException):
|
||||
def __init__(self, path_info, hint):
|
||||
hint = "" if hint is None else f" {hint}"
|
||||
super().__init__(f"GDrive path '{path_info}' not found.{hint}")
|
||||
|
||||
|
||||
class GDriveAuthError(DvcException):
|
||||
def __init__(self, cred_location):
|
||||
|
||||
if cred_location:
|
||||
message = (
|
||||
"GDrive remote auth failed with credentials in '{}'.\n"
|
||||
"Backup first, remove of fix them, and run DVC again.\n"
|
||||
"Backup first, remove or fix them, and run DVC again.\n"
|
||||
"It should do auth again and refresh the credentials.\n\n"
|
||||
"Details:".format(cred_location)
|
||||
)
|
||||
@@ -88,7 +85,13 @@ class GDriveURLInfo(CloudURLInfo):
|
||||
|
||||
|
||||
class GDriveRemoteTree(BaseRemoteTree):
|
||||
scheme = Schemes.GDRIVE
|
||||
PATH_CLS = GDriveURLInfo
|
||||
REQUIRES = {"pydrive2": "pydrive2"}
|
||||
DEFAULT_VERIFY = True
|
||||
# Always prefer traverse for GDrive since API usage quotas are a concern.
|
||||
TRAVERSE_WEIGHT_MULTIPLIER = 1
|
||||
TRAVERSE_PREFIX_LEN = 2
|
||||
|
||||
GDRIVE_CREDENTIALS_DATA = "GDRIVE_CREDENTIALS_DATA"
|
||||
DEFAULT_USER_CREDENTIALS_FILE = "gdrive-user-credentials.json"
|
||||
@@ -96,8 +99,8 @@ class GDriveRemoteTree(BaseRemoteTree):
|
||||
DEFAULT_GDRIVE_CLIENT_ID = "710796635688-iivsgbgsb6uv1fap6635dhvuei09o66c.apps.googleusercontent.com" # noqa: E501
|
||||
DEFAULT_GDRIVE_CLIENT_SECRET = "a1Fz59uTpVNeG_VGuSKDLJXv"
|
||||
|
||||
def __init__(self, remote, config):
|
||||
super().__init__(remote, config)
|
||||
def __init__(self, repo, config):
|
||||
super().__init__(repo, config)
|
||||
|
||||
self.path_info = self.PATH_CLS(config["url"])
|
||||
|
||||
@@ -125,13 +128,12 @@ class GDriveRemoteTree(BaseRemoteTree):
|
||||
self._client_secret = config.get("gdrive_client_secret")
|
||||
self._validate_config()
|
||||
self._gdrive_user_credentials_path = (
|
||||
tmp_fname(os.path.join(self.remote.repo.tmp_dir, ""))
|
||||
tmp_fname(os.path.join(self.repo.tmp_dir, ""))
|
||||
if os.getenv(GDriveRemoteTree.GDRIVE_CREDENTIALS_DATA)
|
||||
else config.get(
|
||||
"gdrive_user_credentials_file",
|
||||
os.path.join(
|
||||
self.remote.repo.tmp_dir,
|
||||
self.DEFAULT_USER_CREDENTIALS_FILE,
|
||||
self.repo.tmp_dir, self.DEFAULT_USER_CREDENTIALS_FILE,
|
||||
),
|
||||
)
|
||||
)
|
||||
@@ -389,6 +391,23 @@ class GDriveRemoteTree(BaseRemoteTree):
|
||||
) as pbar:
|
||||
gdrive_file.GetContentFile(to_file, callback=pbar.update_to)
|
||||
|
||||
@contextmanager
|
||||
@_gdrive_retry
|
||||
def open(self, path_info, mode="r", encoding=None):
|
||||
assert mode in {"r", "rt", "rb"}
|
||||
|
||||
item_id = self._get_item_id(path_info)
|
||||
param = {"id": item_id}
|
||||
# it does not create a file on the remote
|
||||
gdrive_file = self._drive.CreateFile(param)
|
||||
fd = gdrive_file.GetContentIOBuffer()
|
||||
stream = IterStream(iter(fd))
|
||||
|
||||
if mode != "rb":
|
||||
stream = io.TextIOWrapper(stream, encoding=encoding)
|
||||
|
||||
yield stream
|
||||
|
||||
@_gdrive_retry
|
||||
def gdrive_delete_file(self, item_id):
|
||||
from pydrive2.files import ApiRequestError
|
||||
@@ -502,12 +521,12 @@ class GDriveRemoteTree(BaseRemoteTree):
|
||||
return min(item_ids)
|
||||
|
||||
assert not create
|
||||
raise GDrivePathNotFound(path_info, hint)
|
||||
raise FileMissingError(path_info, hint)
|
||||
|
||||
def exists(self, path_info):
|
||||
try:
|
||||
self._get_item_id(path_info)
|
||||
except GDrivePathNotFound:
|
||||
except FileMissingError:
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
@@ -545,7 +564,7 @@ class GDriveRemoteTree(BaseRemoteTree):
|
||||
item_id = self._get_item_id(path_info)
|
||||
self.gdrive_delete_file(item_id)
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
def get_file_hash(self, path_info):
|
||||
raise NotImplementedError
|
||||
|
||||
def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
|
||||
@@ -560,13 +579,3 @@ class GDriveRemoteTree(BaseRemoteTree):
|
||||
def _download(self, from_info, to_file, name=None, no_progress_bar=False):
|
||||
item_id = self._get_item_id(from_info)
|
||||
self._gdrive_download_file(item_id, to_file, name, no_progress_bar)
|
||||
|
||||
|
||||
class GDriveRemote(BaseRemote):
|
||||
scheme = Schemes.GDRIVE
|
||||
REQUIRES = {"pydrive2": "pydrive2"}
|
||||
TREE_CLS = GDriveRemoteTree
|
||||
DEFAULT_VERIFY = True
|
||||
# Always prefer traverse for GDrive since API usage quotas are a concern.
|
||||
TRAVERSE_WEIGHT_MULTIPLIER = 1
|
||||
TRAVERSE_PREFIX_LEN = 2
|
||||
|
||||
+7
-15
@@ -9,7 +9,7 @@ from funcy import cached_property, wrap_prop
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.path_info import CloudURLInfo
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.remote.base import BaseRemote, BaseRemoteTree, CacheMixin
|
||||
from dvc.remote.base import BaseRemoteTree
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -65,10 +65,13 @@ def _upload_to_bucket(
|
||||
|
||||
|
||||
class GSRemoteTree(BaseRemoteTree):
|
||||
scheme = Schemes.GS
|
||||
PATH_CLS = CloudURLInfo
|
||||
REQUIRES = {"google-cloud-storage": "google.cloud.storage"}
|
||||
PARAM_CHECKSUM = "md5"
|
||||
|
||||
def __init__(self, remote, config):
|
||||
super().__init__(remote, config)
|
||||
def __init__(self, repo, config):
|
||||
super().__init__(repo, config)
|
||||
|
||||
url = config.get("url", "gs:///")
|
||||
self.path_info = self.PATH_CLS(url)
|
||||
@@ -157,7 +160,7 @@ class GSRemoteTree(BaseRemoteTree):
|
||||
to_bucket = self.gs.bucket(to_info.bucket)
|
||||
from_bucket.copy_blob(blob, to_bucket, new_name=to_info.path)
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
def get_file_hash(self, path_info):
|
||||
import base64
|
||||
import codecs
|
||||
|
||||
@@ -193,14 +196,3 @@ class GSRemoteTree(BaseRemoteTree):
|
||||
disable=no_progress_bar,
|
||||
) as wrapped:
|
||||
blob.download_to_file(wrapped)
|
||||
|
||||
|
||||
class GSRemote(BaseRemote):
|
||||
scheme = Schemes.GS
|
||||
REQUIRES = {"google-cloud-storage": "google.cloud.storage"}
|
||||
TREE_CLS = GSRemoteTree
|
||||
PARAM_CHECKSUM = "md5"
|
||||
|
||||
|
||||
class GSCache(GSRemote, CacheMixin):
|
||||
pass
|
||||
|
||||
+10
-17
@@ -11,15 +11,21 @@ from urllib.parse import urlparse
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.utils import fix_env, tmp_fname
|
||||
|
||||
from .base import BaseRemote, BaseRemoteTree, CacheMixin, RemoteCmdError
|
||||
from .base import BaseRemoteTree, RemoteCmdError
|
||||
from .pool import get_connection
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HDFSRemoteTree(BaseRemoteTree):
|
||||
def __init__(self, remote, config):
|
||||
super().__init__(remote, config)
|
||||
scheme = Schemes.HDFS
|
||||
REQUIRES = {"pyarrow": "pyarrow"}
|
||||
REGEX = r"^hdfs://((?P<user>.*)@)?.*$"
|
||||
PARAM_CHECKSUM = "checksum"
|
||||
TRAVERSE_PREFIX_LEN = 2
|
||||
|
||||
def __init__(self, repo, config):
|
||||
super().__init__(repo, config)
|
||||
|
||||
self.path_info = None
|
||||
url = config.get("url")
|
||||
@@ -153,7 +159,7 @@ class HDFSRemoteTree(BaseRemoteTree):
|
||||
assert match is not None
|
||||
return match.group(gname)
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
def get_file_hash(self, path_info):
|
||||
# NOTE: pyarrow doesn't support checksum, so we need to use hadoop
|
||||
regex = r".*\t.*\t(?P<checksum>.*)"
|
||||
stdout = self.hadoop_fs(
|
||||
@@ -173,16 +179,3 @@ class HDFSRemoteTree(BaseRemoteTree):
|
||||
with self.hdfs(from_info) as hdfs:
|
||||
with open(to_file, "wb+") as fobj:
|
||||
hdfs.download(from_info.path, fobj)
|
||||
|
||||
|
||||
class HDFSRemote(BaseRemote):
|
||||
scheme = Schemes.HDFS
|
||||
REGEX = r"^hdfs://((?P<user>.*)@)?.*$"
|
||||
PARAM_CHECKSUM = "checksum"
|
||||
REQUIRES = {"pyarrow": "pyarrow"}
|
||||
TREE_CLS = HDFSRemoteTree
|
||||
TRAVERSE_PREFIX_LEN = 2
|
||||
|
||||
|
||||
class HDFSCache(HDFSRemote, CacheMixin):
|
||||
pass
|
||||
|
||||
+7
-17
@@ -8,7 +8,7 @@ import dvc.prompt as prompt
|
||||
from dvc.exceptions import DvcException, HTTPError
|
||||
from dvc.path_info import HTTPURLInfo
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.remote.base import BaseRemote, BaseRemoteTree
|
||||
from dvc.remote.base import BaseRemoteTree
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -24,15 +24,18 @@ def ask_password(host, user):
|
||||
|
||||
|
||||
class HTTPRemoteTree(BaseRemoteTree):
|
||||
scheme = Schemes.HTTP
|
||||
PATH_CLS = HTTPURLInfo
|
||||
PARAM_CHECKSUM = "etag"
|
||||
CAN_TRAVERSE = False
|
||||
|
||||
SESSION_RETRIES = 5
|
||||
SESSION_BACKOFF_FACTOR = 0.1
|
||||
REQUEST_TIMEOUT = 10
|
||||
CHUNK_SIZE = 2 ** 16
|
||||
|
||||
def __init__(self, remote, config):
|
||||
super().__init__(remote, config)
|
||||
def __init__(self, repo, config):
|
||||
super().__init__(repo, config)
|
||||
|
||||
url = config.get("url")
|
||||
if url:
|
||||
@@ -121,7 +124,7 @@ class HTTPRemoteTree(BaseRemoteTree):
|
||||
def exists(self, path_info):
|
||||
return bool(self.request("HEAD", path_info.url))
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
def get_file_hash(self, path_info):
|
||||
url = path_info.url
|
||||
headers = self.request("HEAD", url).headers
|
||||
etag = headers.get("ETag") or headers.get("Content-MD5")
|
||||
@@ -185,16 +188,3 @@ class HTTPRemoteTree(BaseRemoteTree):
|
||||
def _content_length(response):
|
||||
res = response.headers.get("Content-Length")
|
||||
return int(res) if res else None
|
||||
|
||||
|
||||
class HTTPRemote(BaseRemote):
|
||||
scheme = Schemes.HTTP
|
||||
PARAM_CHECKSUM = "etag"
|
||||
CAN_TRAVERSE = False
|
||||
TREE_CLS = HTTPRemoteTree
|
||||
|
||||
def list_paths(self, prefix=None, progress_callback=None):
|
||||
raise NotImplementedError
|
||||
|
||||
def gc(self):
|
||||
raise NotImplementedError
|
||||
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
from .http import HTTPRemote
|
||||
from .http import HTTPRemoteTree
|
||||
|
||||
|
||||
class HTTPSRemote(HTTPRemote):
|
||||
class HTTPSRemoteTree(HTTPRemoteTree):
|
||||
scheme = Schemes.HTTPS
|
||||
|
||||
+27
-31
@@ -25,15 +25,15 @@ class RemoteIndexNoop:
|
||||
def __iter__(self):
|
||||
return iter([])
|
||||
|
||||
def __contains__(self, checksum):
|
||||
def __contains__(self, hash_):
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def checksums():
|
||||
def hashes():
|
||||
return []
|
||||
|
||||
@staticmethod
|
||||
def dir_checksums():
|
||||
def dir_hashes():
|
||||
return []
|
||||
|
||||
def load(self):
|
||||
@@ -54,13 +54,13 @@ class RemoteIndexNoop:
|
||||
|
||||
|
||||
class RemoteIndex:
|
||||
"""Class for indexing remote checksums in a sqlite3 database.
|
||||
"""Class for indexing remote hashes in a sqlite3 database.
|
||||
|
||||
Args:
|
||||
repo: repo for this remote index.
|
||||
name: name for this index. Index db will be loaded from and saved to
|
||||
``.dvc/tmp/index/{name}.idx``.
|
||||
dir_suffix: suffix used for naming directory checksums
|
||||
dir_suffix: suffix used for naming directory hashes
|
||||
"""
|
||||
|
||||
INDEX_SUFFIX = ".idx"
|
||||
@@ -79,8 +79,8 @@ class RemoteIndex:
|
||||
|
||||
def __iter__(self):
|
||||
cmd = f"SELECT checksum FROM {self.INDEX_TABLE}"
|
||||
for (checksum,) in self._execute(cmd):
|
||||
yield checksum
|
||||
for (hash_,) in self._execute(cmd):
|
||||
yield hash_
|
||||
|
||||
def __enter__(self):
|
||||
self.lock.acquire()
|
||||
@@ -90,25 +90,25 @@ class RemoteIndex:
|
||||
self.dump()
|
||||
self.lock.release()
|
||||
|
||||
def __contains__(self, checksum):
|
||||
def __contains__(self, hash_):
|
||||
cmd = "SELECT checksum FROM {} WHERE checksum = (?)".format(
|
||||
self.INDEX_TABLE
|
||||
)
|
||||
self._execute(cmd, (checksum,))
|
||||
self._execute(cmd, (hash_,))
|
||||
return self.cursor.fetchone() is not None
|
||||
|
||||
def checksums(self):
|
||||
"""Iterate over checksums stored in the index."""
|
||||
def hashes(self):
|
||||
"""Iterate over hashes stored in the index."""
|
||||
return iter(self)
|
||||
|
||||
def dir_checksums(self):
|
||||
"""Iterate over .dir checksums stored in the index."""
|
||||
def dir_hashes(self):
|
||||
"""Iterate over .dir hashes stored in the index."""
|
||||
cmd = f"SELECT checksum FROM {self.INDEX_TABLE} WHERE dir = 1"
|
||||
for (checksum,) in self._execute(cmd):
|
||||
yield checksum
|
||||
for (hash_,) in self._execute(cmd):
|
||||
yield hash_
|
||||
|
||||
def is_dir_checksum(self, checksum):
|
||||
return checksum.endswith(self.dir_suffix)
|
||||
def is_dir_hash(self, hash_):
|
||||
return hash_.endswith(self.dir_suffix)
|
||||
|
||||
def _execute(self, cmd, parameters=()):
|
||||
return self.cursor.execute(cmd, parameters)
|
||||
@@ -185,28 +185,24 @@ class RemoteIndex:
|
||||
cmd = f"DELETE FROM {self.INDEX_TABLE}"
|
||||
self._execute(cmd)
|
||||
|
||||
def update(self, dir_checksums, file_checksums):
|
||||
"""Update this index, adding the specified checksums.
|
||||
def update(self, dir_hashes, file_hashes):
|
||||
"""Update this index, adding the specified hashes.
|
||||
|
||||
Changes to the index will not committed until dump() is called.
|
||||
"""
|
||||
cmd = "INSERT OR IGNORE INTO {} (checksum, dir) VALUES (?, ?)".format(
|
||||
self.INDEX_TABLE
|
||||
)
|
||||
self._executemany(
|
||||
cmd, ((checksum, True) for checksum in dir_checksums)
|
||||
)
|
||||
self._executemany(
|
||||
cmd, ((checksum, False) for checksum in file_checksums)
|
||||
)
|
||||
self._executemany(cmd, ((hash_, True) for hash_ in dir_hashes))
|
||||
self._executemany(cmd, ((hash_, False) for hash_ in file_hashes))
|
||||
|
||||
def intersection(self, checksums):
|
||||
"""Iterate over values from `checksums` which exist in the index."""
|
||||
def intersection(self, hashes):
|
||||
"""Iterate over values from `hashes` which exist in the index."""
|
||||
# sqlite has a compile time limit of 999, see:
|
||||
# https://www.sqlite.org/c3ref/c_limit_attached.html#sqlitelimitvariablenumber
|
||||
for chunk in lchunks(999, checksums):
|
||||
for chunk in lchunks(999, hashes):
|
||||
cmd = "SELECT checksum FROM {} WHERE checksum IN ({})".format(
|
||||
self.INDEX_TABLE, ",".join("?" for checksum in chunk)
|
||||
self.INDEX_TABLE, ",".join("?" for hash_ in chunk)
|
||||
)
|
||||
for (checksum,) in self._execute(cmd, chunk):
|
||||
yield checksum
|
||||
for (hash_,) in self._execute(cmd, chunk):
|
||||
yield hash_
|
||||
|
||||
+148
-156
@@ -16,9 +16,9 @@ from dvc.remote.base import (
|
||||
STATUS_MAP,
|
||||
STATUS_MISSING,
|
||||
STATUS_NEW,
|
||||
BaseRemote,
|
||||
BaseRemoteTree,
|
||||
CacheMixin,
|
||||
CloudCache,
|
||||
Remote,
|
||||
index_locked,
|
||||
)
|
||||
from dvc.remote.index import RemoteIndexNoop
|
||||
@@ -39,17 +39,26 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LocalRemoteTree(BaseRemoteTree):
|
||||
SHARED_MODE_MAP = {None: (0o644, 0o755), "group": (0o664, 0o775)}
|
||||
scheme = Schemes.LOCAL
|
||||
PATH_CLS = PathInfo
|
||||
PARAM_CHECKSUM = "md5"
|
||||
PARAM_PATH = "path"
|
||||
TRAVERSE_PREFIX_LEN = 2
|
||||
UNPACKED_DIR_SUFFIX = ".unpacked"
|
||||
|
||||
def __init__(self, remote, config):
|
||||
super().__init__(remote, config)
|
||||
CACHE_MODE = 0o444
|
||||
SHARED_MODE_MAP = {None: (0o644, 0o755), "group": (0o664, 0o775)}
|
||||
|
||||
def __init__(self, repo, config):
|
||||
super().__init__(repo, config)
|
||||
url = config.get("url")
|
||||
self.path_info = self.PATH_CLS(url) if url else None
|
||||
|
||||
@property
|
||||
def repo(self):
|
||||
return self.remote.repo
|
||||
def state(self):
|
||||
from dvc.state import StateNoop
|
||||
|
||||
return self.repo.state if self.repo else StateNoop()
|
||||
|
||||
@cached_property
|
||||
def work_tree(self):
|
||||
@@ -162,7 +171,7 @@ class LocalRemoteTree(BaseRemoteTree):
|
||||
# and the cache type is `hardlink`, we might reach link limits and
|
||||
# will get something like: `too many links error`
|
||||
#
|
||||
# This is because all those empty files will have the same checksum
|
||||
# This is because all those empty files will have the same hash
|
||||
# (i.e. 68b329da9893e34099c7d8ad5cb9c940), therefore, they will be
|
||||
# linked to the same file in the cache.
|
||||
#
|
||||
@@ -195,78 +204,6 @@ class LocalRemoteTree(BaseRemoteTree):
|
||||
os.chmod(tmp_info, self.file_mode)
|
||||
os.rename(tmp_info, to_info)
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
return file_md5(path_info)[0]
|
||||
|
||||
@staticmethod
|
||||
def getsize(path_info):
|
||||
return os.path.getsize(path_info)
|
||||
|
||||
def _upload(
|
||||
self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
|
||||
):
|
||||
makedirs(to_info.parent, exist_ok=True)
|
||||
|
||||
tmp_file = tmp_fname(to_info)
|
||||
copyfile(
|
||||
from_file, tmp_file, name=name, no_progress_bar=no_progress_bar
|
||||
)
|
||||
|
||||
self.remote.protect(tmp_file)
|
||||
os.rename(tmp_file, to_info)
|
||||
|
||||
@staticmethod
|
||||
def _download(
|
||||
from_info, to_file, name=None, no_progress_bar=False, **_kwargs
|
||||
):
|
||||
copyfile(
|
||||
from_info, to_file, no_progress_bar=no_progress_bar, name=name
|
||||
)
|
||||
|
||||
|
||||
def _log_exceptions(func, operation):
|
||||
@wraps(func)
|
||||
def wrapper(from_info, to_info, *args, **kwargs):
|
||||
try:
|
||||
func(from_info, to_info, *args, **kwargs)
|
||||
return 0
|
||||
except Exception as exc:
|
||||
# NOTE: this means we ran out of file descriptors and there is no
|
||||
# reason to try to proceed, as we will hit this error anyways.
|
||||
if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
|
||||
raise
|
||||
|
||||
logger.exception(
|
||||
"failed to %s '%s' to '%s'", operation, from_info, to_info
|
||||
)
|
||||
return 1
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
class LocalRemote(BaseRemote):
|
||||
scheme = Schemes.LOCAL
|
||||
INDEX_CLS = RemoteIndexNoop
|
||||
TREE_CLS = LocalRemoteTree
|
||||
|
||||
PARAM_CHECKSUM = "md5"
|
||||
PARAM_PATH = "path"
|
||||
DEFAULT_CACHE_TYPES = ["reflink", "copy"]
|
||||
TRAVERSE_PREFIX_LEN = 2
|
||||
UNPACKED_DIR_SUFFIX = ".unpacked"
|
||||
|
||||
CACHE_MODE = 0o444
|
||||
|
||||
@property
|
||||
def state(self):
|
||||
return self.repo.state
|
||||
|
||||
def get(self, md5):
|
||||
if not md5:
|
||||
return None
|
||||
|
||||
return self.checksum_to_path_info(md5).url
|
||||
|
||||
def _unprotect_file(self, path):
|
||||
if System.is_symlink(path) or System.is_hardlink(path):
|
||||
logger.debug(f"Unprotecting '{path}'")
|
||||
@@ -287,7 +224,7 @@ class LocalRemote(BaseRemote):
|
||||
"a symlink or a hardlink.".format(path)
|
||||
)
|
||||
|
||||
os.chmod(path, self.tree.file_mode)
|
||||
os.chmod(path, self.file_mode)
|
||||
|
||||
def _unprotect_dir(self, path):
|
||||
assert is_working_tree(self.repo.tree)
|
||||
@@ -333,14 +270,44 @@ class LocalRemote(BaseRemote):
|
||||
|
||||
return stat.S_IMODE(mode) == self.CACHE_MODE
|
||||
|
||||
def get_file_hash(self, path_info):
|
||||
return file_md5(path_info)[0]
|
||||
|
||||
@staticmethod
|
||||
def getsize(path_info):
|
||||
return os.path.getsize(path_info)
|
||||
|
||||
def _upload(
|
||||
self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
|
||||
):
|
||||
makedirs(to_info.parent, exist_ok=True)
|
||||
|
||||
tmp_file = tmp_fname(to_info)
|
||||
copyfile(
|
||||
from_file, tmp_file, name=name, no_progress_bar=no_progress_bar
|
||||
)
|
||||
|
||||
self.protect(tmp_file)
|
||||
os.rename(tmp_file, to_info)
|
||||
|
||||
@staticmethod
|
||||
def _download(
|
||||
from_info, to_file, name=None, no_progress_bar=False, **_kwargs
|
||||
):
|
||||
copyfile(
|
||||
from_info, to_file, no_progress_bar=no_progress_bar, name=name
|
||||
)
|
||||
|
||||
def list_paths(self, prefix=None, progress_callback=None):
|
||||
assert self.path_info is not None
|
||||
if prefix:
|
||||
path_info = self.path_info / prefix[:2]
|
||||
if not self.tree.exists(path_info):
|
||||
if not self.exists(path_info):
|
||||
return
|
||||
else:
|
||||
path_info = self.path_info
|
||||
# NOTE: use utils.fs walk_files since tree.walk_files will not follow
|
||||
# symlinks
|
||||
if progress_callback:
|
||||
for path in walk_files(path_info):
|
||||
progress_callback()
|
||||
@@ -348,16 +315,43 @@ class LocalRemote(BaseRemote):
|
||||
else:
|
||||
yield from walk_files(path_info)
|
||||
|
||||
def _remove_unpacked_dir(self, checksum):
|
||||
info = self.checksum_to_path_info(checksum)
|
||||
def _remove_unpacked_dir(self, hash_):
|
||||
info = self.hash_to_path_info(hash_)
|
||||
path_info = info.with_name(info.name + self.UNPACKED_DIR_SUFFIX)
|
||||
self.tree.remove(path_info)
|
||||
self.remove(path_info)
|
||||
|
||||
|
||||
class LocalCache(LocalRemote, CacheMixin):
|
||||
def __init__(self, repo, config):
|
||||
super().__init__(repo, config)
|
||||
self.cache_dir = config.get("url")
|
||||
def _log_exceptions(func, operation):
|
||||
@wraps(func)
|
||||
def wrapper(from_info, to_info, *args, **kwargs):
|
||||
try:
|
||||
func(from_info, to_info, *args, **kwargs)
|
||||
return 0
|
||||
except Exception as exc:
|
||||
# NOTE: this means we ran out of file descriptors and there is no
|
||||
# reason to try to proceed, as we will hit this error anyways.
|
||||
if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
|
||||
raise
|
||||
|
||||
logger.exception(
|
||||
"failed to %s '%s' to '%s'", operation, from_info, to_info
|
||||
)
|
||||
return 1
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
class LocalRemote(Remote):
|
||||
INDEX_CLS = RemoteIndexNoop
|
||||
|
||||
|
||||
class LocalCache(CloudCache):
|
||||
DEFAULT_CACHE_TYPES = ["reflink", "copy"]
|
||||
CACHE_MODE = LocalRemoteTree.CACHE_MODE
|
||||
|
||||
def __init__(self, tree):
|
||||
super().__init__(tree)
|
||||
self.cache_dir = tree.config.get("url")
|
||||
|
||||
@property
|
||||
def cache_dir(self):
|
||||
@@ -375,30 +369,28 @@ class LocalCache(LocalRemote, CacheMixin):
|
||||
def cache_path(self):
|
||||
return os.path.abspath(self.cache_dir)
|
||||
|
||||
def checksum_to_path(self, checksum):
|
||||
def hash_to_path(self, hash_):
|
||||
# NOTE: `self.cache_path` is already normalized so we can simply use
|
||||
# `os.sep` instead of `os.path.join`. This results in this helper
|
||||
# being ~5.5 times faster.
|
||||
return (
|
||||
f"{self.cache_path}{os.sep}{checksum[0:2]}{os.sep}{checksum[2:]}"
|
||||
)
|
||||
return f"{self.cache_path}{os.sep}{hash_[0:2]}{os.sep}{hash_[2:]}"
|
||||
|
||||
def checksums_exist(self, checksums, jobs=None, name=None):
|
||||
def hashes_exist(self, hashes, jobs=None, name=None):
|
||||
return [
|
||||
checksum
|
||||
for checksum in Tqdm(
|
||||
checksums,
|
||||
hash_
|
||||
for hash_ in Tqdm(
|
||||
hashes,
|
||||
unit="file",
|
||||
desc="Querying "
|
||||
+ ("cache in " + name if name else "local cache"),
|
||||
)
|
||||
if not self.changed_cache_file(checksum)
|
||||
if not self.changed_cache_file(hash_)
|
||||
]
|
||||
|
||||
def already_cached(self, path_info):
|
||||
assert path_info.scheme in ["", "local"]
|
||||
|
||||
current_md5 = self.get_checksum(path_info)
|
||||
current_md5 = self.get_hash(path_info)
|
||||
|
||||
if not current_md5:
|
||||
return False
|
||||
@@ -442,7 +434,7 @@ class LocalCache(LocalRemote, CacheMixin):
|
||||
|
||||
dir_status_info contains status for .dir files, file_status_info
|
||||
contains status for all other files, and dir_contents is a dict of
|
||||
{dir_checksum: set(file_checksum, ...)} which can be used to map
|
||||
{dir_hash: set(file_hash, ...)} which can be used to map
|
||||
a .dir file to its file contents.
|
||||
"""
|
||||
logger.debug(f"Preparing to collect status from {remote.path_info}")
|
||||
@@ -450,7 +442,7 @@ class LocalCache(LocalRemote, CacheMixin):
|
||||
|
||||
logger.debug("Collecting information from local cache...")
|
||||
local_exists = frozenset(
|
||||
self.checksums_exist(md5s, jobs=jobs, name=self.cache_dir)
|
||||
self.hashes_exist(md5s, jobs=jobs, name=self.cache_dir)
|
||||
)
|
||||
|
||||
# This is a performance optimization. We can safely assume that,
|
||||
@@ -465,12 +457,12 @@ class LocalCache(LocalRemote, CacheMixin):
|
||||
dir_md5s = set(named_cache.dir_keys(self.scheme))
|
||||
if dir_md5s:
|
||||
remote_exists.update(
|
||||
self._indexed_dir_checksums(named_cache, remote, dir_md5s)
|
||||
self._indexed_dir_hashes(named_cache, remote, dir_md5s)
|
||||
)
|
||||
md5s.difference_update(remote_exists)
|
||||
if md5s:
|
||||
remote_exists.update(
|
||||
remote.checksums_exist(
|
||||
remote.hashes_exist(
|
||||
md5s, jobs=jobs, name=str(remote.path_info)
|
||||
)
|
||||
)
|
||||
@@ -481,23 +473,23 @@ class LocalCache(LocalRemote, CacheMixin):
|
||||
def _make_status(
|
||||
self, named_cache, show_checksums, local_exists, remote_exists
|
||||
):
|
||||
def make_names(checksum, names):
|
||||
return {"name": checksum if show_checksums else " ".join(names)}
|
||||
def make_names(hash_, names):
|
||||
return {"name": hash_ if show_checksums else " ".join(names)}
|
||||
|
||||
dir_status = {}
|
||||
file_status = {}
|
||||
dir_contents = {}
|
||||
for checksum, item in named_cache[self.scheme].items():
|
||||
for hash_, item in named_cache[self.scheme].items():
|
||||
if item.children:
|
||||
dir_status[checksum] = make_names(checksum, item.names)
|
||||
dir_contents[checksum] = set()
|
||||
for child_checksum, child in item.children.items():
|
||||
file_status[child_checksum] = make_names(
|
||||
child_checksum, child.names
|
||||
dir_status[hash_] = make_names(hash_, item.names)
|
||||
dir_contents[hash_] = set()
|
||||
for child_hash, child in item.children.items():
|
||||
file_status[child_hash] = make_names(
|
||||
child_hash, child.names
|
||||
)
|
||||
dir_contents[checksum].add(child_checksum)
|
||||
dir_contents[hash_].add(child_hash)
|
||||
else:
|
||||
file_status[checksum] = make_names(checksum, item.names)
|
||||
file_status[hash_] = make_names(hash_, item.names)
|
||||
|
||||
self._fill_statuses(dir_status, local_exists, remote_exists)
|
||||
self._fill_statuses(file_status, local_exists, remote_exists)
|
||||
@@ -506,50 +498,50 @@ class LocalCache(LocalRemote, CacheMixin):
|
||||
|
||||
return dir_status, file_status, dir_contents
|
||||
|
||||
def _indexed_dir_checksums(self, named_cache, remote, dir_md5s):
|
||||
# Validate our index by verifying all indexed .dir checksums
|
||||
def _indexed_dir_hashes(self, named_cache, remote, dir_md5s):
|
||||
# Validate our index by verifying all indexed .dir hashes
|
||||
# still exist on the remote
|
||||
indexed_dirs = set(remote.index.dir_checksums())
|
||||
indexed_dirs = set(remote.index.dir_hashes())
|
||||
indexed_dir_exists = set()
|
||||
if indexed_dirs:
|
||||
indexed_dir_exists.update(
|
||||
remote._list_checksums_exists(indexed_dirs)
|
||||
remote.tree.list_hashes_exists(indexed_dirs)
|
||||
)
|
||||
missing_dirs = indexed_dirs.difference(indexed_dir_exists)
|
||||
if missing_dirs:
|
||||
logger.debug(
|
||||
"Remote cache missing indexed .dir checksums '{}', "
|
||||
"Remote cache missing indexed .dir hashes '{}', "
|
||||
"clearing remote index".format(", ".join(missing_dirs))
|
||||
)
|
||||
remote.index.clear()
|
||||
|
||||
# Check if non-indexed (new) dir checksums exist on remote
|
||||
# Check if non-indexed (new) dir hashes exist on remote
|
||||
dir_exists = dir_md5s.intersection(indexed_dir_exists)
|
||||
dir_exists.update(remote._list_checksums_exists(dir_md5s - dir_exists))
|
||||
dir_exists.update(
|
||||
remote.tree.list_hashes_exists(dir_md5s - dir_exists)
|
||||
)
|
||||
|
||||
# If .dir checksum exists on the remote, assume directory contents
|
||||
# If .dir hash exists on the remote, assume directory contents
|
||||
# still exists on the remote
|
||||
for dir_checksum in dir_exists:
|
||||
file_checksums = list(
|
||||
named_cache.child_keys(self.scheme, dir_checksum)
|
||||
)
|
||||
if dir_checksum not in remote.index:
|
||||
for dir_hash in dir_exists:
|
||||
file_hashes = list(named_cache.child_keys(self.scheme, dir_hash))
|
||||
if dir_hash not in remote.index:
|
||||
logger.debug(
|
||||
"Indexing new .dir '{}' with '{}' nested files".format(
|
||||
dir_checksum, len(file_checksums)
|
||||
dir_hash, len(file_hashes)
|
||||
)
|
||||
)
|
||||
remote.index.update([dir_checksum], file_checksums)
|
||||
yield dir_checksum
|
||||
yield from file_checksums
|
||||
remote.index.update([dir_hash], file_hashes)
|
||||
yield dir_hash
|
||||
yield from file_hashes
|
||||
|
||||
@staticmethod
|
||||
def _fill_statuses(checksum_info_dir, local_exists, remote_exists):
|
||||
def _fill_statuses(hash_info_dir, local_exists, remote_exists):
|
||||
# Using sets because they are way faster for lookups
|
||||
local = set(local_exists)
|
||||
remote = set(remote_exists)
|
||||
|
||||
for md5, info in checksum_info_dir.items():
|
||||
for md5, info in hash_info_dir.items():
|
||||
status = STATUS_MAP[(md5 in local, md5 in remote)]
|
||||
info["status"] = status
|
||||
|
||||
@@ -557,15 +549,15 @@ class LocalCache(LocalRemote, CacheMixin):
|
||||
cache = []
|
||||
path_infos = []
|
||||
names = []
|
||||
checksums = []
|
||||
hashes = []
|
||||
for md5, info in Tqdm(
|
||||
status_info.items(), desc="Analysing status", unit="file"
|
||||
):
|
||||
if info["status"] == status:
|
||||
cache.append(self.checksum_to_path_info(md5))
|
||||
path_infos.append(remote.checksum_to_path_info(md5))
|
||||
cache.append(self.hash_to_path_info(md5))
|
||||
path_infos.append(remote.hash_to_path_info(md5))
|
||||
names.append(info["name"])
|
||||
checksums.append(md5)
|
||||
hashes.append(md5)
|
||||
|
||||
if download:
|
||||
to_infos = cache
|
||||
@@ -574,7 +566,7 @@ class LocalCache(LocalRemote, CacheMixin):
|
||||
to_infos = path_infos
|
||||
from_infos = cache
|
||||
|
||||
return from_infos, to_infos, names, checksums
|
||||
return from_infos, to_infos, names, hashes
|
||||
|
||||
def _process(
|
||||
self,
|
||||
@@ -605,7 +597,7 @@ class LocalCache(LocalRemote, CacheMixin):
|
||||
desc = "Uploading"
|
||||
|
||||
if jobs is None:
|
||||
jobs = remote.JOBS
|
||||
jobs = remote.tree.JOBS
|
||||
|
||||
dir_status, file_status, dir_contents = self._status(
|
||||
named_cache,
|
||||
@@ -626,26 +618,28 @@ class LocalCache(LocalRemote, CacheMixin):
|
||||
func = pbar.wrap_fn(func)
|
||||
with ThreadPoolExecutor(max_workers=jobs) as executor:
|
||||
if download:
|
||||
fails = sum(executor.map(func, *dir_plans))
|
||||
fails += sum(executor.map(func, *file_plans))
|
||||
from_infos, to_infos, names, _ = (
|
||||
d + f for d, f in zip(dir_plans, file_plans)
|
||||
)
|
||||
fails = sum(
|
||||
executor.map(func, from_infos, to_infos, names)
|
||||
)
|
||||
else:
|
||||
# for uploads, push files first, and any .dir files last
|
||||
|
||||
file_futures = {}
|
||||
for from_info, to_info, name, checksum in zip(*file_plans):
|
||||
file_futures[checksum] = executor.submit(
|
||||
for from_info, to_info, name, hash_ in zip(*file_plans):
|
||||
file_futures[hash_] = executor.submit(
|
||||
func, from_info, to_info, name
|
||||
)
|
||||
dir_futures = {}
|
||||
for from_info, to_info, name, dir_checksum in zip(
|
||||
*dir_plans
|
||||
):
|
||||
for from_info, to_info, name, dir_hash in zip(*dir_plans):
|
||||
wait_futures = {
|
||||
future
|
||||
for file_checksum, future in file_futures.items()
|
||||
if file_checksum in dir_contents[dir_checksum]
|
||||
for file_hash, future in file_futures.items()
|
||||
if file_hash in dir_contents[dir_hash]
|
||||
}
|
||||
dir_futures[dir_checksum] = executor.submit(
|
||||
dir_futures[dir_hash] = executor.submit(
|
||||
self._dir_upload,
|
||||
func,
|
||||
wait_futures,
|
||||
@@ -668,16 +662,14 @@ class LocalCache(LocalRemote, CacheMixin):
|
||||
|
||||
if not download:
|
||||
# index successfully pushed dirs
|
||||
for dir_checksum, future in dir_futures.items():
|
||||
for dir_hash, future in dir_futures.items():
|
||||
if future.result() == 0:
|
||||
file_checksums = dir_contents[dir_checksum]
|
||||
file_hashes = dir_contents[dir_hash]
|
||||
logger.debug(
|
||||
"Indexing pushed dir '{}' with "
|
||||
"'{}' nested files".format(
|
||||
dir_checksum, len(file_checksums)
|
||||
)
|
||||
"'{}' nested files".format(dir_hash, len(file_hashes))
|
||||
)
|
||||
remote.index.update([dir_checksum], file_checksums)
|
||||
remote.index.update([dir_hash], file_hashes)
|
||||
|
||||
return len(dir_plans[0]) + len(file_plans[0])
|
||||
|
||||
@@ -716,10 +708,10 @@ class LocalCache(LocalRemote, CacheMixin):
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _log_missing_caches(checksum_info_dict):
|
||||
def _log_missing_caches(hash_info_dict):
|
||||
missing_caches = [
|
||||
(md5, info)
|
||||
for md5, info in checksum_info_dict.items()
|
||||
for md5, info in hash_info_dict.items()
|
||||
if info["status"] == STATUS_MISSING
|
||||
]
|
||||
if missing_caches:
|
||||
|
||||
+24
-28
@@ -6,14 +6,37 @@ from funcy import cached_property, wrap_prop
|
||||
|
||||
from dvc.path_info import CloudURLInfo
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.remote.base import BaseRemote, BaseRemoteTree
|
||||
from dvc.remote.base import BaseRemoteTree
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class OSSRemoteTree(BaseRemoteTree):
|
||||
"""
|
||||
oss2 document:
|
||||
https://www.alibabacloud.com/help/doc-detail/32026.htm
|
||||
|
||||
|
||||
Examples
|
||||
----------
|
||||
$ dvc remote add myremote oss://my-bucket/path
|
||||
Set key id, key secret and endpoint using modify command
|
||||
$ dvc remote modify myremote oss_key_id my-key-id
|
||||
$ dvc remote modify myremote oss_key_secret my-key-secret
|
||||
$ dvc remote modify myremote oss_endpoint endpoint
|
||||
or environment variables
|
||||
$ export OSS_ACCESS_KEY_ID="my-key-id"
|
||||
$ export OSS_ACCESS_KEY_SECRET="my-key-secret"
|
||||
$ export OSS_ENDPOINT="endpoint"
|
||||
"""
|
||||
|
||||
scheme = Schemes.OSS
|
||||
PATH_CLS = CloudURLInfo
|
||||
REQUIRES = {"oss2": "oss2"}
|
||||
PARAM_CHECKSUM = "etag"
|
||||
COPY_POLL_SECONDS = 5
|
||||
LIST_OBJECT_PAGE_SIZE = 100
|
||||
|
||||
def __init__(self, repo, config):
|
||||
super().__init__(repo, config)
|
||||
@@ -105,30 +128,3 @@ class OSSRemoteTree(BaseRemoteTree):
|
||||
self.oss_service.get_object_to_file(
|
||||
from_info.path, to_file, progress_callback=pbar.update_to
|
||||
)
|
||||
|
||||
|
||||
class OSSRemote(BaseRemote):
|
||||
"""
|
||||
oss2 document:
|
||||
https://www.alibabacloud.com/help/doc-detail/32026.htm
|
||||
|
||||
|
||||
Examples
|
||||
----------
|
||||
$ dvc remote add myremote oss://my-bucket/path
|
||||
Set key id, key secret and endpoint using modify command
|
||||
$ dvc remote modify myremote oss_key_id my-key-id
|
||||
$ dvc remote modify myremote oss_key_secret my-key-secret
|
||||
$ dvc remote modify myremote oss_endpoint endpoint
|
||||
or environment variables
|
||||
$ export OSS_ACCESS_KEY_ID="my-key-id"
|
||||
$ export OSS_ACCESS_KEY_SECRET="my-key-secret"
|
||||
$ export OSS_ENDPOINT="endpoint"
|
||||
"""
|
||||
|
||||
scheme = Schemes.OSS
|
||||
REQUIRES = {"oss2": "oss2"}
|
||||
PARAM_CHECKSUM = "etag"
|
||||
COPY_POLL_SECONDS = 5
|
||||
LIST_OBJECT_PAGE_SIZE = 100
|
||||
TREE_CLS = OSSRemoteTree
|
||||
|
||||
+5
-13
@@ -8,14 +8,17 @@ from dvc.config import ConfigError
|
||||
from dvc.exceptions import DvcException, ETagMismatchError
|
||||
from dvc.path_info import CloudURLInfo
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.remote.base import BaseRemote, BaseRemoteTree, CacheMixin
|
||||
from dvc.remote.base import BaseRemoteTree
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class S3RemoteTree(BaseRemoteTree):
|
||||
scheme = Schemes.S3
|
||||
PATH_CLS = CloudURLInfo
|
||||
REQUIRES = {"boto3": "boto3"}
|
||||
PARAM_CHECKSUM = "etag"
|
||||
|
||||
def __init__(self, repo, config):
|
||||
super().__init__(repo, config)
|
||||
@@ -305,7 +308,7 @@ class S3RemoteTree(BaseRemoteTree):
|
||||
if etag != cached_etag:
|
||||
raise ETagMismatchError(etag, cached_etag)
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
def get_file_hash(self, path_info):
|
||||
return self.get_etag(self.s3, path_info.bucket, path_info.path)
|
||||
|
||||
def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
|
||||
@@ -334,14 +337,3 @@ class S3RemoteTree(BaseRemoteTree):
|
||||
self.s3.download_file(
|
||||
from_info.bucket, from_info.path, to_file, Callback=pbar.update
|
||||
)
|
||||
|
||||
|
||||
class S3Remote(BaseRemote):
|
||||
scheme = Schemes.S3
|
||||
REQUIRES = {"boto3": "boto3"}
|
||||
PARAM_CHECKSUM = "etag"
|
||||
TREE_CLS = S3RemoteTree
|
||||
|
||||
|
||||
class S3Cache(S3Remote, CacheMixin):
|
||||
pass
|
||||
|
||||
+27
-30
@@ -14,7 +14,7 @@ from funcy import first, memoize, silent, wrap_with
|
||||
|
||||
import dvc.prompt as prompt
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.remote.base import BaseRemote, BaseRemoteTree, CacheMixin
|
||||
from dvc.remote.base import BaseRemoteTree, Remote
|
||||
from dvc.remote.pool import get_connection
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.utils import to_chunks
|
||||
@@ -34,6 +34,18 @@ def ask_password(host, user, port):
|
||||
|
||||
|
||||
class SSHRemoteTree(BaseRemoteTree):
|
||||
scheme = Schemes.SSH
|
||||
REQUIRES = {"paramiko": "paramiko"}
|
||||
JOBS = 4
|
||||
|
||||
PARAM_CHECKSUM = "md5"
|
||||
# At any given time some of the connections will go over network and
|
||||
# paramiko stuff, so we would ideally have it double of server processors.
|
||||
# We use conservative setting of 4 instead to not exhaust max sessions.
|
||||
CHECKSUM_JOBS = 4
|
||||
DEFAULT_CACHE_TYPES = ["copy"]
|
||||
TRAVERSE_PREFIX_LEN = 2
|
||||
|
||||
DEFAULT_PORT = 22
|
||||
TIMEOUT = 1800
|
||||
|
||||
@@ -225,7 +237,7 @@ class SSHRemoteTree(BaseRemoteTree):
|
||||
with self.ssh(from_info) as ssh:
|
||||
ssh.reflink(from_info.path, to_info.path)
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
def get_file_hash(self, path_info):
|
||||
if path_info.scheme != self.scheme:
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -256,27 +268,12 @@ class SSHRemoteTree(BaseRemoteTree):
|
||||
no_progress_bar=no_progress_bar,
|
||||
)
|
||||
|
||||
|
||||
class SSHRemote(BaseRemote):
|
||||
scheme = Schemes.SSH
|
||||
REQUIRES = {"paramiko": "paramiko"}
|
||||
JOBS = 4
|
||||
TREE_CLS = SSHRemoteTree
|
||||
|
||||
PARAM_CHECKSUM = "md5"
|
||||
# At any given time some of the connections will go over network and
|
||||
# paramiko stuff, so we would ideally have it double of server processors.
|
||||
# We use conservative setting of 4 instead to not exhaust max sessions.
|
||||
CHECKSUM_JOBS = 4
|
||||
DEFAULT_CACHE_TYPES = ["copy"]
|
||||
TRAVERSE_PREFIX_LEN = 2
|
||||
|
||||
def list_paths(self, prefix=None, progress_callback=None):
|
||||
if prefix:
|
||||
root = posixpath.join(self.path_info.path, prefix[:2])
|
||||
else:
|
||||
root = self.path_info.path
|
||||
with self.tree.ssh(self.path_info) as ssh:
|
||||
with self.ssh(self.path_info) as ssh:
|
||||
if prefix and not ssh.exists(root):
|
||||
return
|
||||
# If we simply return an iterator then with above closes instantly
|
||||
@@ -287,6 +284,8 @@ class SSHRemote(BaseRemote):
|
||||
else:
|
||||
yield from ssh.walk_files(root)
|
||||
|
||||
|
||||
class SSHRemote(Remote):
|
||||
def batch_exists(self, path_infos, callback):
|
||||
def _exists(chunk_and_channel):
|
||||
chunk, channel = chunk_and_channel
|
||||
@@ -315,14 +314,14 @@ class SSHRemote(BaseRemote):
|
||||
|
||||
return results
|
||||
|
||||
def checksums_exist(self, checksums, jobs=None, name=None):
|
||||
def hashes_exist(self, hashes, jobs=None, name=None):
|
||||
"""This is older implementation used in remote/base.py
|
||||
We are reusing it in RemoteSSH, because SSH's batch_exists proved to be
|
||||
faster than current approach (relying on exists(path_info)) applied in
|
||||
remote/base.
|
||||
"""
|
||||
if not self.CAN_TRAVERSE:
|
||||
return list(set(checksums) & set(self.all()))
|
||||
if not self.tree.CAN_TRAVERSE:
|
||||
return list(set(hashes) & set(self.tree.all()))
|
||||
|
||||
# possibly prompt for credentials before "Querying" progress output
|
||||
self.tree.ensure_credentials()
|
||||
@@ -330,21 +329,19 @@ class SSHRemote(BaseRemote):
|
||||
with Tqdm(
|
||||
desc="Querying "
|
||||
+ ("cache in " + name if name else "remote cache"),
|
||||
total=len(checksums),
|
||||
total=len(hashes),
|
||||
unit="file",
|
||||
) as pbar:
|
||||
|
||||
def exists_with_progress(chunks):
|
||||
return self.batch_exists(chunks, callback=pbar.update_msg)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=jobs or self.JOBS) as executor:
|
||||
path_infos = [self.checksum_to_path_info(x) for x in checksums]
|
||||
chunks = to_chunks(path_infos, num_chunks=self.JOBS)
|
||||
with ThreadPoolExecutor(
|
||||
max_workers=jobs or self.tree.JOBS
|
||||
) as executor:
|
||||
path_infos = [self.hash_to_path_info(x) for x in hashes]
|
||||
chunks = to_chunks(path_infos, num_chunks=self.tree.JOBS)
|
||||
results = executor.map(exists_with_progress, chunks)
|
||||
in_remote = itertools.chain.from_iterable(results)
|
||||
ret = list(itertools.compress(checksums, in_remote))
|
||||
ret = list(itertools.compress(hashes, in_remote))
|
||||
return ret
|
||||
|
||||
|
||||
class SSHCache(SSHRemote, CacheMixin):
|
||||
pass
|
||||
|
||||
@@ -187,7 +187,7 @@ class Repo:
|
||||
return Repo(root_dir)
|
||||
|
||||
def unprotect(self, target):
|
||||
return self.cache.local.unprotect(PathInfo(target))
|
||||
return self.cache.local.tree.unprotect(PathInfo(target))
|
||||
|
||||
def _ignore(self):
|
||||
flist = [self.config.files["local"], self.tmp_dir]
|
||||
|
||||
+1
-1
@@ -37,7 +37,7 @@ def diff(self, a_rev="HEAD", b_rev=None):
|
||||
|
||||
def _to_checksum(output):
|
||||
if on_working_tree:
|
||||
return self.cache.local.get_checksum(output.path_info)
|
||||
return self.cache.local.get_hash(output.path_info)
|
||||
return output.checksum
|
||||
|
||||
def _exists(output):
|
||||
|
||||
+12
-9
@@ -8,8 +8,10 @@ from . import locked
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _do_gc(typ, func, clist, jobs=None):
|
||||
removed = func(clist, jobs=jobs)
|
||||
def _do_gc(typ, remote, clist, jobs=None):
|
||||
from dvc.remote.base import Remote
|
||||
|
||||
removed = Remote.gc(clist, remote, jobs=jobs)
|
||||
if not removed:
|
||||
logger.info(f"No unused '{typ}' cache to remove.")
|
||||
|
||||
@@ -74,22 +76,23 @@ def gc(
|
||||
)
|
||||
)
|
||||
|
||||
_do_gc("local", self.cache.local.gc, used, jobs)
|
||||
# treat caches as remotes for garbage collection
|
||||
_do_gc("local", self.cache.local, used, jobs)
|
||||
|
||||
if self.cache.s3:
|
||||
_do_gc("s3", self.cache.s3.gc, used, jobs)
|
||||
_do_gc("s3", self.cache.s3, used, jobs)
|
||||
|
||||
if self.cache.gs:
|
||||
_do_gc("gs", self.cache.gs.gc, used, jobs)
|
||||
_do_gc("gs", self.cache.gs, used, jobs)
|
||||
|
||||
if self.cache.ssh:
|
||||
_do_gc("ssh", self.cache.ssh.gc, used, jobs)
|
||||
_do_gc("ssh", self.cache.ssh, used, jobs)
|
||||
|
||||
if self.cache.hdfs:
|
||||
_do_gc("hdfs", self.cache.hdfs.gc, used, jobs)
|
||||
_do_gc("hdfs", self.cache.hdfs, used, jobs)
|
||||
|
||||
if self.cache.azure:
|
||||
_do_gc("azure", self.cache.azure.gc, used, jobs)
|
||||
_do_gc("azure", self.cache.azure, used, jobs)
|
||||
|
||||
if cloud:
|
||||
_do_gc("remote", self.cloud.get_remote(remote, "gc -c").gc, used, jobs)
|
||||
_do_gc("remote", self.cloud.get_remote(remote, "gc -c"), used, jobs)
|
||||
|
||||
@@ -16,4 +16,6 @@ def get_url(url, out=None):
|
||||
|
||||
(dep,) = dependency.loads_from(None, [url])
|
||||
(out,) = output.loads_from(None, [out], use_cache=False)
|
||||
dep.save()
|
||||
dep.download(out)
|
||||
out.save()
|
||||
|
||||
@@ -17,6 +17,10 @@ class NotAPlotError(DvcException):
|
||||
)
|
||||
|
||||
|
||||
class PropsNotFoundError(DvcException):
|
||||
pass
|
||||
|
||||
|
||||
class Plots:
|
||||
def __init__(self, repo):
|
||||
self.repo = repo
|
||||
@@ -93,6 +97,17 @@ class Plots:
|
||||
|
||||
return diff(self.repo, *args, **kwargs)
|
||||
|
||||
@staticmethod
|
||||
def _unset(out, props):
|
||||
missing = list(set(props) - set(out.plot.keys()))
|
||||
if missing:
|
||||
raise PropsNotFoundError(
|
||||
f"display properties {missing} not found in plot '{out}'"
|
||||
)
|
||||
|
||||
for prop in props:
|
||||
out.plot.pop(prop)
|
||||
|
||||
def modify(self, path, props=None, unset=None):
|
||||
from dvc.dvcfile import Dvcfile
|
||||
|
||||
@@ -109,8 +124,9 @@ class Plots:
|
||||
if not isinstance(out.plot, dict):
|
||||
out.plot = {}
|
||||
|
||||
for field in unset or ():
|
||||
out.plot.pop(field, None)
|
||||
if unset:
|
||||
self._unset(out, unset)
|
||||
|
||||
out.plot.update(props)
|
||||
|
||||
# Empty dict will move it to non-plots
|
||||
|
||||
@@ -12,6 +12,10 @@ class TemplateNotFoundError(DvcException):
|
||||
super().__init__(f"Template '{path}' not found.")
|
||||
|
||||
|
||||
class BadTemplateError(DvcException):
|
||||
pass
|
||||
|
||||
|
||||
class NoFieldInDataError(DvcException):
|
||||
def __init__(self, field_name):
|
||||
super().__init__(
|
||||
@@ -39,6 +43,12 @@ class Template:
|
||||
def render(self, data, props=None):
|
||||
props = props or {}
|
||||
|
||||
if self.METRIC_DATA_ANCHOR not in self.content:
|
||||
raise BadTemplateError(
|
||||
f"Template '{self.filename}' is not using "
|
||||
f"'{self.METRIC_DATA_ANCHOR}' anchor"
|
||||
)
|
||||
|
||||
if props.get("x"):
|
||||
Template._check_field_exists(data, props.get("x"))
|
||||
if props.get("y"):
|
||||
@@ -158,12 +168,46 @@ class DefaultScatterTemplate(Template):
|
||||
}
|
||||
|
||||
|
||||
class SmoothLinearTemplate(Template):
|
||||
DEFAULT_NAME = "smooth"
|
||||
|
||||
DEFAULT_CONTENT = {
|
||||
"$schema": "https://vega.github.io/schema/vega-lite/v4.json",
|
||||
"data": {"values": Template.METRIC_DATA_ANCHOR},
|
||||
"title": Template.TITLE_ANCHOR,
|
||||
"mark": {"type": "line"},
|
||||
"encoding": {
|
||||
"x": {
|
||||
"field": Template.X_ANCHOR,
|
||||
"type": "quantitative",
|
||||
"title": Template.X_LABEL_ANCHOR,
|
||||
},
|
||||
"y": {
|
||||
"field": Template.Y_ANCHOR,
|
||||
"type": "quantitative",
|
||||
"title": Template.Y_LABEL_ANCHOR,
|
||||
"scale": {"zero": False},
|
||||
},
|
||||
"color": {"field": "rev", "type": "nominal"},
|
||||
},
|
||||
"transform": [
|
||||
{
|
||||
"loess": Template.Y_ANCHOR,
|
||||
"on": Template.X_ANCHOR,
|
||||
"groupby": ["rev"],
|
||||
"bandwidth": 0.3,
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
class PlotTemplates:
|
||||
TEMPLATES_DIR = "plots"
|
||||
TEMPLATES = [
|
||||
DefaultLinearTemplate,
|
||||
DefaultConfusionTemplate,
|
||||
DefaultScatterTemplate,
|
||||
SmoothLinearTemplate,
|
||||
]
|
||||
|
||||
@cached_property
|
||||
|
||||
+8
-8
@@ -48,11 +48,11 @@ class DvcTree(BaseTree):
|
||||
raise FileNotFoundError
|
||||
dir_cache = out.get_dir_cache(remote=remote)
|
||||
for entry in dir_cache:
|
||||
entry_relpath = entry[out.remote.PARAM_RELPATH]
|
||||
entry_relpath = entry[out.remote.tree.PARAM_RELPATH]
|
||||
if os.name == "nt":
|
||||
entry_relpath = entry_relpath.replace("/", os.sep)
|
||||
if path == out.path_info / entry_relpath:
|
||||
return entry[out.remote.PARAM_CHECKSUM]
|
||||
return entry[out.remote.tree.PARAM_CHECKSUM]
|
||||
raise FileNotFoundError
|
||||
|
||||
def open(self, path, mode="r", encoding="utf-8", remote=None):
|
||||
@@ -82,7 +82,7 @@ class DvcTree(BaseTree):
|
||||
else:
|
||||
checksum = out.checksum
|
||||
try:
|
||||
remote_info = remote_obj.checksum_to_path_info(checksum)
|
||||
remote_info = remote_obj.hash_to_path_info(checksum)
|
||||
return remote_obj.open(
|
||||
remote_info, mode=mode, encoding=encoding
|
||||
)
|
||||
@@ -93,7 +93,7 @@ class DvcTree(BaseTree):
|
||||
|
||||
if out.is_dir_checksum:
|
||||
checksum = self._get_granular_checksum(path, out)
|
||||
cache_path = out.cache.checksum_to_path_info(checksum).url
|
||||
cache_path = out.cache.hash_to_path_info(checksum).url
|
||||
else:
|
||||
cache_path = out.cache_path
|
||||
return open(cache_path, mode=mode, encoding=encoding)
|
||||
@@ -194,7 +194,7 @@ class DvcTree(BaseTree):
|
||||
download_callback(downloaded)
|
||||
|
||||
for entry in dir_cache:
|
||||
entry_relpath = entry[out.remote.PARAM_RELPATH]
|
||||
entry_relpath = entry[out.remote.tree.PARAM_RELPATH]
|
||||
if os.name == "nt":
|
||||
entry_relpath = entry_relpath.replace("/", os.sep)
|
||||
path_info = out.path_info / entry_relpath
|
||||
@@ -212,7 +212,7 @@ class DvcTree(BaseTree):
|
||||
def isexec(self, path):
|
||||
return False
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
def get_file_hash(self, path_info):
|
||||
outs = self._find_outs(path_info, strict=False)
|
||||
if len(outs) != 1:
|
||||
raise OutputNotFoundError
|
||||
@@ -373,7 +373,7 @@ class RepoTree(BaseTree):
|
||||
for fname in files:
|
||||
yield PathInfo(root) / fname
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
def get_file_hash(self, path_info):
|
||||
"""Return file checksum for specified path.
|
||||
|
||||
If path_info is a DVC out, the pre-computed checksum for the file
|
||||
@@ -384,7 +384,7 @@ class RepoTree(BaseTree):
|
||||
raise FileNotFoundError
|
||||
if self.dvctree and self.dvctree.exists(path_info):
|
||||
try:
|
||||
return self.dvctree.get_file_checksum(path_info)
|
||||
return self.dvctree.get_file_hash(path_info)
|
||||
except OutputNotFoundError:
|
||||
pass
|
||||
return file_md5(path_info, self)[0]
|
||||
|
||||
@@ -26,4 +26,5 @@ def sync_import(stage, dry=False, force=False):
|
||||
if not force and stage.already_cached():
|
||||
stage.outs[0].checkout()
|
||||
else:
|
||||
stage.save_deps()
|
||||
stage.deps[0].download(stage.outs[0])
|
||||
|
||||
+4
-3
@@ -8,7 +8,8 @@ from dvc import dependency, output
|
||||
from dvc.utils.fs import path_isin
|
||||
|
||||
from ..dependency import ParamsDependency
|
||||
from ..remote import LocalRemote, S3Remote
|
||||
from ..remote.local import LocalRemoteTree
|
||||
from ..remote.s3 import S3RemoteTree
|
||||
from ..utils import dict_md5, format_link, relpath
|
||||
from .exceptions import (
|
||||
MissingDataSource,
|
||||
@@ -132,8 +133,8 @@ def stage_dump_eq(stage_cls, old_d, new_d):
|
||||
new_d.pop(stage_cls.PARAM_MD5, None)
|
||||
outs = old_d.get(stage_cls.PARAM_OUTS, [])
|
||||
for out in outs:
|
||||
out.pop(LocalRemote.PARAM_CHECKSUM, None)
|
||||
out.pop(S3Remote.PARAM_CHECKSUM, None)
|
||||
out.pop(LocalRemoteTree.PARAM_CHECKSUM, None)
|
||||
out.pop(S3RemoteTree.PARAM_CHECKSUM, None)
|
||||
|
||||
# outs and deps are lists of dicts. To check equality, we need to make
|
||||
# them independent of the order, so, we convert them to dicts.
|
||||
|
||||
+2
-44
@@ -1,6 +1,8 @@
|
||||
import io
|
||||
from contextlib import contextmanager
|
||||
|
||||
from dvc.utils.stream import IterStream
|
||||
|
||||
|
||||
@contextmanager
|
||||
def open_url(url, mode="r", encoding=None):
|
||||
@@ -61,47 +63,3 @@ def iter_url(url, chunk_size=io.DEFAULT_BUFFER_SIZE):
|
||||
finally:
|
||||
# Ensure connection is closed
|
||||
it.close()
|
||||
|
||||
|
||||
class IterStream(io.RawIOBase):
|
||||
"""Wraps an iterator yielding bytes as a file object"""
|
||||
|
||||
def __init__(self, iterator):
|
||||
self.iterator = iterator
|
||||
self.leftover = None
|
||||
|
||||
def readable(self):
|
||||
return True
|
||||
|
||||
# Python 3 requires only .readinto() method, it still uses other ones
|
||||
# under some circumstances and falls back if those are absent. Since
|
||||
# iterator already constructs byte strings for us, .readinto() is not the
|
||||
# most optimal, so we provide .read1() too.
|
||||
|
||||
def readinto(self, b):
|
||||
try:
|
||||
n = len(b) # We're supposed to return at most this much
|
||||
chunk = self.leftover or next(self.iterator)
|
||||
output, self.leftover = chunk[:n], chunk[n:]
|
||||
|
||||
n_out = len(output)
|
||||
b[:n_out] = output
|
||||
return n_out
|
||||
except StopIteration:
|
||||
return 0 # indicate EOF
|
||||
|
||||
readinto1 = readinto
|
||||
|
||||
def read1(self, n=-1):
|
||||
try:
|
||||
chunk = self.leftover or next(self.iterator)
|
||||
except StopIteration:
|
||||
return b""
|
||||
|
||||
# Return an arbitrary number or bytes
|
||||
if n <= 0:
|
||||
self.leftover = None
|
||||
return chunk
|
||||
|
||||
output, self.leftover = chunk[:n], chunk[n:]
|
||||
return output
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
import io
|
||||
|
||||
|
||||
class IterStream(io.RawIOBase):
|
||||
"""Wraps an iterator yielding bytes as a file object"""
|
||||
|
||||
def __init__(self, iterator):
|
||||
self.iterator = iterator
|
||||
self.leftover = None
|
||||
|
||||
def readable(self):
|
||||
return True
|
||||
|
||||
# Python 3 requires only .readinto() method, it still uses other ones
|
||||
# under some circumstances and falls back if those are absent. Since
|
||||
# iterator already constructs byte strings for us, .readinto() is not the
|
||||
# most optimal, so we provide .read1() too.
|
||||
|
||||
def readinto(self, b):
|
||||
try:
|
||||
n = len(b) # We're supposed to return at most this much
|
||||
chunk = self.leftover or next(self.iterator)
|
||||
output, self.leftover = chunk[:n], chunk[n:]
|
||||
|
||||
n_out = len(output)
|
||||
b[:n_out] = output
|
||||
return n_out
|
||||
except StopIteration:
|
||||
return 0 # indicate EOF
|
||||
|
||||
readinto1 = readinto
|
||||
|
||||
def read1(self, n=-1):
|
||||
try:
|
||||
chunk = self.leftover or next(self.iterator)
|
||||
except StopIteration:
|
||||
return b""
|
||||
|
||||
# Return an arbitrary number or bytes
|
||||
if n <= 0:
|
||||
self.leftover = None
|
||||
return chunk
|
||||
|
||||
output, self.leftover = chunk[:n], chunk[n:]
|
||||
return output
|
||||
+1
-1
@@ -6,7 +6,7 @@
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
_BASE_VERSION = "1.0.0b0"
|
||||
_BASE_VERSION = "1.0.0b4"
|
||||
|
||||
|
||||
def _generate_version(base_version):
|
||||
|
||||
@@ -51,12 +51,27 @@ elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
|
||||
ln -s -f /usr/local/bin/pip3 /usr/local/bin/pip
|
||||
fi
|
||||
|
||||
if [[ -n "$TRAVIS_TAG" ]]; then
|
||||
if [[ $(echo "$TRAVIS_TAG" | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$') ]]; then
|
||||
echo "export SNAP_CHANNEL=stable,v1/stable" >>env.sh
|
||||
if [ "$TRAVIS_OS_NAME" == "linux" ]; then
|
||||
# fetch tags for `git-describe`, since
|
||||
# - can't rely on $TRAVIS_TAG for snapcraft `edge` (master) releases, and
|
||||
# - `snapcraft` also uses `git-describe` for version detection
|
||||
git fetch --tags
|
||||
TAG_MAJOR="$(git describe --tags | sed -r 's/^v?([0-9]+)\.[0-9]+\.[0-9]+.*/\1/')"
|
||||
[[ -n "$TAG_MAJOR" ]] || exit 1 # failed to detect major version
|
||||
|
||||
if [[ -n "$TRAVIS_TAG" ]]; then
|
||||
if [[ $(echo "$TRAVIS_TAG" | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$') ]]; then
|
||||
echo "export SNAP_CHANNEL=stable" >>env.sh
|
||||
echo "export SNAP_CHANNEL_MAJOR=v$TAG_MAJOR/stable" >>env.sh
|
||||
else
|
||||
echo "export SNAP_CHANNEL=beta" >>env.sh
|
||||
echo "export SNAP_CHANNEL_MAJOR=v$TAG_MAJOR/beta" >>env.sh
|
||||
fi
|
||||
else
|
||||
echo "export SNAP_CHANNEL=beta,v1/beta" >>env.sh
|
||||
echo "export SNAP_CHANNEL=edge" >>env.sh
|
||||
echo "export SNAP_CHANNEL_MAJOR=v$TAG_MAJOR/edge" >>env.sh
|
||||
fi
|
||||
else
|
||||
echo "export SNAP_CHANNEL=edge,v1/edge" >>env.sh
|
||||
|
||||
# NOTE: after deprecating this major version branch, uncomment this line
|
||||
# echo "unset SNAP_CHANNEL" >>env.sh
|
||||
fi
|
||||
|
||||
@@ -84,7 +84,7 @@ install_requires = [
|
||||
# Extra dependencies for remote integrations
|
||||
|
||||
gs = ["google-cloud-storage==1.19.0"]
|
||||
gdrive = ["pydrive2>=1.4.13"]
|
||||
gdrive = ["pydrive2>=1.4.14"]
|
||||
s3 = ["boto3>=1.9.201"]
|
||||
azure = ["azure-storage-blob==2.1.0"]
|
||||
oss = ["oss2==2.6.1"]
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def run_copy_metrics(tmp_dir, run_copy):
|
||||
def run(file1, file2, commit=None, tag=None, single_stage=True, **kwargs):
|
||||
stage = tmp_dir.dvc.run(
|
||||
cmd=f"python copy.py {file1} {file2}",
|
||||
deps=[file1],
|
||||
single_stage=single_stage,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
if hasattr(tmp_dir.dvc, "scm"):
|
||||
files = [stage.path]
|
||||
files += [
|
||||
os.fspath(out.path_info)
|
||||
for out in stage.outs
|
||||
if not out.use_cache
|
||||
]
|
||||
tmp_dir.dvc.scm.add(files)
|
||||
if commit:
|
||||
tmp_dir.dvc.scm.commit(commit)
|
||||
if tag:
|
||||
tmp_dir.dvc.scm.tag(tag)
|
||||
return stage
|
||||
|
||||
return run
|
||||
@@ -1 +0,0 @@
|
||||
from tests.func.plots import run_copy_metrics # noqa: F401
|
||||
@@ -1,30 +0,0 @@
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def run_copy_metrics(tmp_dir, run_copy):
|
||||
def run(file1, file2, commit=None, tag=None, single_stage=True, **kwargs):
|
||||
stage = tmp_dir.dvc.run(
|
||||
cmd=f"python copy.py {file1} {file2}",
|
||||
deps=[file1],
|
||||
single_stage=single_stage,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
if hasattr(tmp_dir.dvc, "scm"):
|
||||
files = [stage.path]
|
||||
files += [
|
||||
os.fspath(out.path_info)
|
||||
for out in stage.outs
|
||||
if not out.use_cache
|
||||
]
|
||||
tmp_dir.dvc.scm.add(files)
|
||||
if commit:
|
||||
tmp_dir.dvc.scm.commit(commit)
|
||||
if tag:
|
||||
tmp_dir.dvc.scm.tag(tag)
|
||||
return stage
|
||||
|
||||
return run
|
||||
|
||||
@@ -1 +1,12 @@
|
||||
from . import run_copy_metrics # noqa: F401
|
||||
import shutil
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def custom_template(tmp_dir, dvc):
|
||||
custom_template = tmp_dir / "custom_template.json"
|
||||
shutil.copy(
|
||||
tmp_dir / ".dvc" / "plots" / "default.json", custom_template,
|
||||
)
|
||||
return custom_template
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
import pytest
|
||||
|
||||
from dvc.dvcfile import PIPELINE_LOCK
|
||||
from dvc.repo.plots import PropsNotFoundError
|
||||
from dvc.repo.plots.template import TemplateNotFoundError
|
||||
from dvc.utils import relpath
|
||||
|
||||
from .test_plots import _write_json
|
||||
|
||||
|
||||
def test_plots_modify_existing_template(
|
||||
tmp_dir, dvc, run_copy_metrics, custom_template
|
||||
):
|
||||
metric = [{"a": 1, "b": 2}, {"a": 2, "b": 3}]
|
||||
_write_json(tmp_dir, metric, "metric_t.json")
|
||||
stage = run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
plots_no_cache=["metric.json"],
|
||||
name="copy-metrics",
|
||||
single_stage=False,
|
||||
)
|
||||
dvc.plots.modify(
|
||||
"metric.json", props={"template": relpath(custom_template)}
|
||||
)
|
||||
stage = stage.reload()
|
||||
assert stage.outs[0].plot == {"template": relpath(custom_template)}
|
||||
|
||||
|
||||
def test_plots_modify_should_not_change_lockfile(
|
||||
tmp_dir, dvc, run_copy_metrics, custom_template
|
||||
):
|
||||
_write_json(tmp_dir, [{"a": 1, "b": 2}], "metric_t.json")
|
||||
run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
plots_no_cache=["metric.json"],
|
||||
name="copy-metrics",
|
||||
single_stage=False,
|
||||
)
|
||||
|
||||
(tmp_dir / PIPELINE_LOCK).unlink()
|
||||
dvc.plots.modify(
|
||||
"metric.json", props={"template": relpath(custom_template)}
|
||||
)
|
||||
assert not (tmp_dir / PIPELINE_LOCK).exists()
|
||||
|
||||
|
||||
def test_plots_modify_not_existing_template(dvc):
|
||||
with pytest.raises(TemplateNotFoundError):
|
||||
dvc.plots.modify(
|
||||
"metric.json", props={"template": "not-existing-template.json"}
|
||||
)
|
||||
|
||||
|
||||
def test_unset_nonexistent(tmp_dir, dvc, run_copy_metrics, custom_template):
|
||||
metric = [{"a": 1, "b": 2}, {"a": 2, "b": 3}]
|
||||
_write_json(tmp_dir, metric, "metric_t.json")
|
||||
run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
plots_no_cache=["metric.json"],
|
||||
name="copy-metrics",
|
||||
single_stage=False,
|
||||
)
|
||||
|
||||
with pytest.raises(PropsNotFoundError):
|
||||
dvc.plots.modify(
|
||||
"metric.json", unset=["nonexistent"],
|
||||
)
|
||||
@@ -9,7 +9,6 @@ import pytest
|
||||
import yaml
|
||||
from funcy import first
|
||||
|
||||
from dvc.dvcfile import PIPELINE_LOCK
|
||||
from dvc.repo.plots.data import (
|
||||
JSONPlotData,
|
||||
NoMetricInHistoryError,
|
||||
@@ -17,8 +16,11 @@ from dvc.repo.plots.data import (
|
||||
PlotMetricTypeError,
|
||||
YAMLPlotData,
|
||||
)
|
||||
from dvc.repo.plots.template import NoFieldInDataError, TemplateNotFoundError
|
||||
from dvc.utils import relpath
|
||||
from dvc.repo.plots.template import (
|
||||
BadTemplateError,
|
||||
NoFieldInDataError,
|
||||
TemplateNotFoundError,
|
||||
)
|
||||
|
||||
|
||||
def _write_csv(metric, filename, header=True):
|
||||
@@ -358,15 +360,6 @@ def test_throw_on_no_metric_at_all(tmp_dir, scm, dvc, caplog):
|
||||
assert str(error.value) == "Could not find 'metric.json'."
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def custom_template(tmp_dir, dvc):
|
||||
custom_template = tmp_dir / "custom_template.json"
|
||||
shutil.copy(
|
||||
tmp_dir / ".dvc" / "plots" / "default.json", custom_template,
|
||||
)
|
||||
return custom_template
|
||||
|
||||
|
||||
def test_custom_template(tmp_dir, scm, dvc, custom_template, run_copy_metrics):
|
||||
metric = [{"a": 1, "b": 2}, {"a": 2, "b": 3}]
|
||||
_write_json(tmp_dir, metric, "metric_t.json")
|
||||
@@ -416,6 +409,23 @@ def test_should_raise_on_no_template(tmp_dir, dvc, run_copy_metrics):
|
||||
dvc.plots.show("metric.json", props=props)
|
||||
|
||||
|
||||
def test_bad_template(tmp_dir, dvc, run_copy_metrics):
|
||||
metric = [{"val": 2}, {"val": 3}]
|
||||
_write_json(tmp_dir, metric, "metric_t.json")
|
||||
run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
plots_no_cache=["metric.json"],
|
||||
commit="first run",
|
||||
)
|
||||
|
||||
tmp_dir.gen("template.json", json.dumps({"a": "b", "c": "d"}))
|
||||
|
||||
with pytest.raises(BadTemplateError):
|
||||
props = {"template": "template.json"}
|
||||
dvc.plots.show("metric.json", props=props)
|
||||
|
||||
|
||||
def test_plot_wrong_metric_type(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
tmp_dir.gen("metric_t.txt", "some text")
|
||||
run_copy_metrics(
|
||||
@@ -542,51 +552,6 @@ def test_load_metric_from_dict_yaml(tmp_dir):
|
||||
assert list(map(dict, plot_data.to_datapoints())) == expected
|
||||
|
||||
|
||||
def test_plots_modify_existing_template(
|
||||
tmp_dir, dvc, run_copy_metrics, custom_template
|
||||
):
|
||||
metric = [{"a": 1, "b": 2}, {"a": 2, "b": 3}]
|
||||
_write_json(tmp_dir, metric, "metric_t.json")
|
||||
stage = run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
plots_no_cache=["metric.json"],
|
||||
name="copy-metrics",
|
||||
single_stage=False,
|
||||
)
|
||||
dvc.plots.modify(
|
||||
"metric.json", props={"template": relpath(custom_template)}
|
||||
)
|
||||
stage = stage.reload()
|
||||
assert stage.outs[0].plot == {"template": relpath(custom_template)}
|
||||
|
||||
|
||||
def test_plots_modify_should_not_change_lockfile(
|
||||
tmp_dir, dvc, run_copy_metrics, custom_template
|
||||
):
|
||||
_write_json(tmp_dir, [{"a": 1, "b": 2}], "metric_t.json")
|
||||
run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
plots_no_cache=["metric.json"],
|
||||
name="copy-metrics",
|
||||
single_stage=False,
|
||||
)
|
||||
|
||||
(tmp_dir / PIPELINE_LOCK).unlink()
|
||||
dvc.plots.modify(
|
||||
"metric.json", props={"template": relpath(custom_template)}
|
||||
)
|
||||
assert not (tmp_dir / PIPELINE_LOCK).exists()
|
||||
|
||||
|
||||
def test_plots_modify_not_existing_template(dvc):
|
||||
with pytest.raises(TemplateNotFoundError):
|
||||
dvc.plots.modify(
|
||||
"metric.json", props={"template": "not-existing-template.json"}
|
||||
)
|
||||
|
||||
|
||||
def test_multiple_plots(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
metric1 = [
|
||||
OrderedDict([("first_val", 100), ("second_val", 100), ("val", 2)]),
|
||||
|
||||
@@ -3,7 +3,7 @@ import os
|
||||
import pytest
|
||||
|
||||
from dvc.exceptions import DownloadError, UploadError
|
||||
from dvc.remote.base import BaseRemote
|
||||
from dvc.remote.base import Remote
|
||||
from dvc.remote.index import RemoteIndex
|
||||
from dvc.remote.local import LocalRemote, LocalRemoteTree
|
||||
from dvc.utils.fs import remove
|
||||
@@ -15,12 +15,12 @@ def remote(tmp_dir, dvc, tmp_path_factory, mocker):
|
||||
dvc.config["remote"]["upstream"] = {"url": url}
|
||||
dvc.config["core"]["remote"] = "upstream"
|
||||
|
||||
# patch checksums_exist since the LocalRemote normally overrides
|
||||
# BaseRemote.checksums_exist.
|
||||
def checksums_exist(self, *args, **kwargs):
|
||||
return BaseRemote.checksums_exist(self, *args, **kwargs)
|
||||
# patch hashes_exist since the LocalRemote normally overrides
|
||||
# BaseRemoteTree.hashes_exist.
|
||||
def hashes_exist(self, *args, **kwargs):
|
||||
return Remote.hashes_exist(self, *args, **kwargs)
|
||||
|
||||
mocker.patch.object(LocalRemote, "checksums_exist", checksums_exist)
|
||||
mocker.patch.object(LocalRemote, "hashes_exist", hashes_exist)
|
||||
|
||||
# patch index class since LocalRemote normally overrides index class
|
||||
mocker.patch.object(LocalRemote, "INDEX_CLS", RemoteIndex)
|
||||
@@ -38,9 +38,9 @@ def test_indexed_on_status(tmp_dir, dvc, tmp_path_factory, remote):
|
||||
|
||||
dvc.status(cloud=True)
|
||||
with remote.index:
|
||||
assert {bar.checksum, baz["md5"]} == set(remote.index.checksums())
|
||||
assert [bar.checksum] == list(remote.index.dir_checksums())
|
||||
assert foo.checksum not in remote.index.checksums()
|
||||
assert {bar.checksum, baz["md5"]} == set(remote.index.hashes())
|
||||
assert [bar.checksum] == list(remote.index.dir_hashes())
|
||||
assert foo.checksum not in remote.index.hashes()
|
||||
|
||||
|
||||
def test_indexed_on_push(tmp_dir, dvc, tmp_path_factory, remote):
|
||||
@@ -50,9 +50,9 @@ def test_indexed_on_push(tmp_dir, dvc, tmp_path_factory, remote):
|
||||
|
||||
dvc.push()
|
||||
with remote.index:
|
||||
assert {bar.checksum, baz["md5"]} == set(remote.index.checksums())
|
||||
assert [bar.checksum] == list(remote.index.dir_checksums())
|
||||
assert foo.checksum not in remote.index.checksums()
|
||||
assert {bar.checksum, baz["md5"]} == set(remote.index.hashes())
|
||||
assert [bar.checksum] == list(remote.index.dir_hashes())
|
||||
assert foo.checksum not in remote.index.hashes()
|
||||
|
||||
|
||||
def test_indexed_dir_missing(tmp_dir, dvc, tmp_path_factory, remote):
|
||||
@@ -61,7 +61,7 @@ def test_indexed_dir_missing(tmp_dir, dvc, tmp_path_factory, remote):
|
||||
remote.index.update([bar.checksum], [])
|
||||
dvc.status(cloud=True)
|
||||
with remote.index:
|
||||
assert not list(remote.index.checksums())
|
||||
assert not list(remote.index.hashes())
|
||||
|
||||
|
||||
def test_clear_on_gc(tmp_dir, dvc, tmp_path_factory, remote, mocker):
|
||||
@@ -101,4 +101,4 @@ def test_partial_upload(tmp_dir, dvc, tmp_path_factory, remote, mocker):
|
||||
with pytest.raises(UploadError):
|
||||
dvc.push()
|
||||
with remote.index:
|
||||
assert not list(remote.index.checksums())
|
||||
assert not list(remote.index.hashes())
|
||||
|
||||
@@ -21,7 +21,7 @@ from dvc.exceptions import (
|
||||
)
|
||||
from dvc.main import main
|
||||
from dvc.output.base import OutputAlreadyTrackedError, OutputIsStageFileError
|
||||
from dvc.remote.local import LocalRemote, LocalRemoteTree
|
||||
from dvc.remote.local import LocalRemoteTree
|
||||
from dvc.repo import Repo as DvcRepo
|
||||
from dvc.stage import Stage
|
||||
from dvc.system import System
|
||||
@@ -340,7 +340,7 @@ class TestAddCommit(TestDvc):
|
||||
|
||||
def test_should_collect_dir_cache_only_once(mocker, tmp_dir, dvc):
|
||||
tmp_dir.gen({"data/data": "foo"})
|
||||
get_dir_checksum_counter = mocker.spy(LocalRemoteTree, "get_dir_checksum")
|
||||
get_dir_hash_counter = mocker.spy(LocalRemoteTree, "get_dir_hash")
|
||||
ret = main(["add", "data"])
|
||||
assert ret == 0
|
||||
|
||||
@@ -349,7 +349,7 @@ def test_should_collect_dir_cache_only_once(mocker, tmp_dir, dvc):
|
||||
|
||||
ret = main(["status"])
|
||||
assert ret == 0
|
||||
assert get_dir_checksum_counter.mock.call_count == 1
|
||||
assert get_dir_hash_counter.mock.call_count == 1
|
||||
|
||||
|
||||
class SymlinkAddTestBase(TestDvc):
|
||||
@@ -583,7 +583,7 @@ def test_readding_dir_should_not_unprotect_all(tmp_dir, dvc, mocker):
|
||||
dvc.add("dir")
|
||||
tmp_dir.gen("dir/new_file", "new_file_content")
|
||||
|
||||
unprotect_spy = mocker.spy(LocalRemote, "unprotect")
|
||||
unprotect_spy = mocker.spy(LocalRemoteTree, "unprotect")
|
||||
dvc.add("dir")
|
||||
|
||||
assert not unprotect_spy.mock.called
|
||||
|
||||
+59
-6
@@ -8,9 +8,19 @@ from dvc.exceptions import FileMissingError
|
||||
from dvc.main import main
|
||||
from dvc.path_info import URLInfo
|
||||
from dvc.utils.fs import remove
|
||||
from tests.remotes import GCP, HDFS, OSS, S3, SSH, Azure, Local
|
||||
from tests.remotes import (
|
||||
GCP,
|
||||
HDFS,
|
||||
OSS,
|
||||
S3,
|
||||
SSH,
|
||||
TEST_REMOTE,
|
||||
Azure,
|
||||
GDrive,
|
||||
Local,
|
||||
)
|
||||
|
||||
remote_params = [S3, GCP, Azure, OSS, SSH, HDFS]
|
||||
remote_params = [S3, GCP, Azure, GDrive, OSS, SSH, HDFS]
|
||||
all_remote_params = [Local] + remote_params
|
||||
|
||||
|
||||
@@ -25,9 +35,48 @@ def run_dvc(*argv):
|
||||
assert main(argv) == 0
|
||||
|
||||
|
||||
def ensure_dir(dvc, url):
|
||||
if url.startswith("gdrive://"):
|
||||
GDrive.create_dir(dvc, url)
|
||||
run_dvc(
|
||||
"remote",
|
||||
"modify",
|
||||
TEST_REMOTE,
|
||||
"gdrive_service_account_email",
|
||||
"test",
|
||||
)
|
||||
run_dvc(
|
||||
"remote",
|
||||
"modify",
|
||||
TEST_REMOTE,
|
||||
"gdrive_service_account_p12_file_path",
|
||||
"test.p12",
|
||||
)
|
||||
run_dvc(
|
||||
"remote",
|
||||
"modify",
|
||||
TEST_REMOTE,
|
||||
"gdrive_use_service_account",
|
||||
"True",
|
||||
)
|
||||
|
||||
|
||||
def ensure_dir_scm(dvc, url):
|
||||
if url.startswith("gdrive://"):
|
||||
GDrive.create_dir(dvc, url)
|
||||
with dvc.config.edit() as conf:
|
||||
conf["remote"][TEST_REMOTE].update(
|
||||
gdrive_service_account_email="test",
|
||||
gdrive_service_account_p12_file_path="test.p12",
|
||||
gdrive_use_service_account=True,
|
||||
)
|
||||
dvc.scm.add(dvc.config.files["repo"])
|
||||
dvc.scm.commit(f"modify '{TEST_REMOTE}' remote")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("remote_url", remote_params, indirect=True)
|
||||
def test_get_url(tmp_dir, dvc, remote_url):
|
||||
run_dvc("remote", "add", "-d", "upstream", remote_url)
|
||||
run_dvc("remote", "add", "-d", TEST_REMOTE, remote_url)
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
|
||||
expected_url = URLInfo(remote_url) / "ac/bd18db4cc2f85cedef654fccc4a4d8"
|
||||
@@ -58,7 +107,8 @@ def test_get_url_requires_dvc(tmp_dir, scm):
|
||||
|
||||
@pytest.mark.parametrize("remote_url", all_remote_params, indirect=True)
|
||||
def test_open(remote_url, tmp_dir, dvc):
|
||||
run_dvc("remote", "add", "-d", "upstream", remote_url)
|
||||
run_dvc("remote", "add", "-d", TEST_REMOTE, remote_url)
|
||||
ensure_dir(dvc, remote_url)
|
||||
tmp_dir.dvc_gen("foo", "foo-text")
|
||||
run_dvc("push")
|
||||
|
||||
@@ -72,6 +122,7 @@ def test_open(remote_url, tmp_dir, dvc):
|
||||
@pytest.mark.parametrize("remote_url", all_remote_params, indirect=True)
|
||||
def test_open_external(remote_url, erepo_dir, setup_remote):
|
||||
setup_remote(erepo_dir.dvc, url=remote_url)
|
||||
ensure_dir_scm(erepo_dir.dvc, remote_url)
|
||||
|
||||
with erepo_dir.chdir():
|
||||
erepo_dir.dvc_gen("version", "master", commit="add version")
|
||||
@@ -95,7 +146,8 @@ def test_open_external(remote_url, erepo_dir, setup_remote):
|
||||
|
||||
@pytest.mark.parametrize("remote_url", all_remote_params, indirect=True)
|
||||
def test_open_granular(remote_url, tmp_dir, dvc):
|
||||
run_dvc("remote", "add", "-d", "upstream", remote_url)
|
||||
run_dvc("remote", "add", "-d", TEST_REMOTE, remote_url)
|
||||
ensure_dir(dvc, remote_url)
|
||||
tmp_dir.dvc_gen({"dir": {"foo": "foo-text"}})
|
||||
run_dvc("push")
|
||||
|
||||
@@ -109,7 +161,8 @@ def test_open_granular(remote_url, tmp_dir, dvc):
|
||||
@pytest.mark.parametrize("remote_url", all_remote_params, indirect=True)
|
||||
def test_missing(remote_url, tmp_dir, dvc):
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
run_dvc("remote", "add", "-d", "upstream", remote_url)
|
||||
run_dvc("remote", "add", "-d", TEST_REMOTE, remote_url)
|
||||
ensure_dir(dvc, remote_url)
|
||||
|
||||
# Remove cache to make foo missing
|
||||
remove(dvc.cache.local.cache_dir)
|
||||
|
||||
@@ -30,14 +30,14 @@ class TestCache(TestDvc):
|
||||
self.create(self.cache2, "2")
|
||||
|
||||
def test_all(self):
|
||||
md5_list = list(Cache(self.dvc).local.all())
|
||||
md5_list = list(Cache(self.dvc).local.tree.all())
|
||||
self.assertEqual(len(md5_list), 2)
|
||||
self.assertIn(self.cache1_md5, md5_list)
|
||||
self.assertIn(self.cache2_md5, md5_list)
|
||||
|
||||
def test_get(self):
|
||||
cache = Cache(self.dvc).local.get(self.cache1_md5)
|
||||
self.assertEqual(cache, self.cache1)
|
||||
cache = Cache(self.dvc).local.hash_to_path_info(self.cache1_md5)
|
||||
self.assertEqual(os.fspath(cache), self.cache1)
|
||||
|
||||
|
||||
class TestCacheLoadBadDirCache(TestDvc):
|
||||
@@ -46,16 +46,16 @@ class TestCacheLoadBadDirCache(TestDvc):
|
||||
self.assertEqual(len(ret), 0)
|
||||
|
||||
def test(self):
|
||||
checksum = "123.dir"
|
||||
fname = self.dvc.cache.local.get(checksum)
|
||||
dir_hash = "123.dir"
|
||||
fname = os.fspath(self.dvc.cache.local.hash_to_path_info(dir_hash))
|
||||
self.create(fname, "<clearly>not,json")
|
||||
with pytest.raises(DirCacheError):
|
||||
self.dvc.cache.local.load_dir_cache(checksum)
|
||||
self.dvc.cache.local.load_dir_cache(dir_hash)
|
||||
|
||||
checksum = "234.dir"
|
||||
fname = self.dvc.cache.local.get(checksum)
|
||||
dir_hash = "234.dir"
|
||||
fname = os.fspath(self.dvc.cache.local.hash_to_path_info(dir_hash))
|
||||
self.create(fname, '{"a": "b"}')
|
||||
self._do_test(self.dvc.cache.local.load_dir_cache(checksum))
|
||||
self._do_test(self.dvc.cache.local.load_dir_cache(dir_hash))
|
||||
|
||||
|
||||
class TestExternalCacheDir(TestDvc):
|
||||
|
||||
@@ -16,8 +16,9 @@ from dvc.exceptions import (
|
||||
NoOutputOrStageError,
|
||||
)
|
||||
from dvc.main import main
|
||||
from dvc.remote import S3Cache, S3Remote
|
||||
from dvc.remote.local import LocalRemote
|
||||
from dvc.remote.base import CloudCache, Remote
|
||||
from dvc.remote.local import LocalRemoteTree
|
||||
from dvc.remote.s3 import S3RemoteTree
|
||||
from dvc.repo import Repo as DvcRepo
|
||||
from dvc.stage import Stage
|
||||
from dvc.stage.exceptions import StageFileDoesNotExistError
|
||||
@@ -99,8 +100,8 @@ class TestCheckoutCorruptedCacheDir(TestDvc):
|
||||
# NOTE: modifying cache file for one of the files inside the directory
|
||||
# to check if dvc will detect that the cache is corrupted.
|
||||
entry = self.dvc.cache.local.load_dir_cache(out.checksum)[0]
|
||||
checksum = entry[self.dvc.cache.local.PARAM_CHECKSUM]
|
||||
cache = self.dvc.cache.local.get(checksum)
|
||||
entry_hash = entry[self.dvc.cache.local.tree.PARAM_CHECKSUM]
|
||||
cache = os.fspath(self.dvc.cache.local.hash_to_path_info(entry_hash))
|
||||
|
||||
os.chmod(cache, 0o644)
|
||||
with open(cache, "w+") as fobj:
|
||||
@@ -305,8 +306,8 @@ class TestGitIgnoreWhenCheckout(CheckoutBase):
|
||||
class TestCheckoutMissingMd5InStageFile(TestRepro):
|
||||
def test(self):
|
||||
d = load_yaml(self.file1_stage)
|
||||
del d[Stage.PARAM_OUTS][0][LocalRemote.PARAM_CHECKSUM]
|
||||
del d[Stage.PARAM_DEPS][0][LocalRemote.PARAM_CHECKSUM]
|
||||
del d[Stage.PARAM_OUTS][0][LocalRemoteTree.PARAM_CHECKSUM]
|
||||
del d[Stage.PARAM_DEPS][0][LocalRemoteTree.PARAM_CHECKSUM]
|
||||
dump_yaml(self.file1_stage, d)
|
||||
|
||||
with pytest.raises(CheckoutError):
|
||||
@@ -755,9 +756,9 @@ def test_checkout_recursive(tmp_dir, dvc):
|
||||
not S3.should_test(), reason="Only run with S3 credentials"
|
||||
)
|
||||
def test_checkout_for_external_outputs(tmp_dir, dvc):
|
||||
dvc.cache.s3 = S3Cache(dvc, {"url": S3.get_url()})
|
||||
dvc.cache.s3 = CloudCache(S3RemoteTree(dvc, {"url": S3.get_url()}))
|
||||
|
||||
remote = S3Remote(dvc, {"url": S3.get_url()})
|
||||
remote = Remote(S3RemoteTree(dvc, {"url": S3.get_url()}))
|
||||
file_path = remote.path_info / "foo"
|
||||
remote.tree.s3.put_object(
|
||||
Bucket=remote.path_info.bucket, Key=file_path.path, Body="foo"
|
||||
|
||||
@@ -12,19 +12,16 @@ from dvc.cache import NamedCache
|
||||
from dvc.data_cloud import DataCloud
|
||||
from dvc.external_repo import clean_repos
|
||||
from dvc.main import main
|
||||
from dvc.remote import (
|
||||
AzureRemote,
|
||||
GDriveRemote,
|
||||
GSRemote,
|
||||
HDFSRemote,
|
||||
HTTPRemote,
|
||||
LocalRemote,
|
||||
OSSRemote,
|
||||
S3Remote,
|
||||
SSHRemote,
|
||||
)
|
||||
from dvc.remote.azure import AzureRemoteTree
|
||||
from dvc.remote.base import STATUS_DELETED, STATUS_NEW, STATUS_OK
|
||||
from dvc.remote.gdrive import GDriveRemoteTree
|
||||
from dvc.remote.gs import GSRemoteTree
|
||||
from dvc.remote.hdfs import HDFSRemoteTree
|
||||
from dvc.remote.http import HTTPRemoteTree
|
||||
from dvc.remote.local import LocalRemoteTree
|
||||
from dvc.remote.oss import OSSRemoteTree
|
||||
from dvc.remote.s3 import S3RemoteTree
|
||||
from dvc.remote.ssh import SSHRemoteTree
|
||||
from dvc.stage.exceptions import StageNotFound
|
||||
from dvc.utils import file_md5
|
||||
from dvc.utils.fs import remove
|
||||
@@ -50,19 +47,19 @@ class TestDataCloud(TestDvc):
|
||||
def _test_cloud(self, config, cl):
|
||||
self.dvc.config = config
|
||||
cloud = DataCloud(self.dvc)
|
||||
self.assertIsInstance(cloud.get_remote(), cl)
|
||||
self.assertIsInstance(cloud.get_remote().tree, cl)
|
||||
|
||||
def test(self):
|
||||
config = copy.deepcopy(TEST_CONFIG)
|
||||
|
||||
clist = [
|
||||
("s3://mybucket/", S3Remote),
|
||||
("gs://mybucket/", GSRemote),
|
||||
("ssh://user@localhost:/", SSHRemote),
|
||||
("http://localhost:8000/", HTTPRemote),
|
||||
("azure://ContainerName=mybucket;conn_string;", AzureRemote),
|
||||
("oss://mybucket/", OSSRemote),
|
||||
(TestDvc.mkdtemp(), LocalRemote),
|
||||
("s3://mybucket/", S3RemoteTree),
|
||||
("gs://mybucket/", GSRemoteTree),
|
||||
("ssh://user@localhost:/", SSHRemoteTree),
|
||||
("http://localhost:8000/", HTTPRemoteTree),
|
||||
("azure://ContainerName=mybucket;conn_string;", AzureRemoteTree),
|
||||
("oss://mybucket/", OSSRemoteTree),
|
||||
(TestDvc.mkdtemp(), LocalRemoteTree),
|
||||
]
|
||||
|
||||
for scheme, cl in clist:
|
||||
@@ -101,7 +98,9 @@ class TestDataCloudBase(TestDvc):
|
||||
self.dvc.config = config
|
||||
self.cloud = DataCloud(self.dvc)
|
||||
|
||||
self.assertIsInstance(self.cloud.get_remote(), self._get_cloud_class())
|
||||
self.assertIsInstance(
|
||||
self.cloud.get_remote().tree, self._get_cloud_class()
|
||||
)
|
||||
|
||||
def _test_cloud(self):
|
||||
self._setup_cloud()
|
||||
@@ -187,7 +186,7 @@ class TestDataCloudBase(TestDvc):
|
||||
|
||||
class TestS3Remote(S3, TestDataCloudBase):
|
||||
def _get_cloud_class(self):
|
||||
return S3Remote
|
||||
return S3RemoteTree
|
||||
|
||||
|
||||
class TestGDriveRemote(GDrive, TestDataCloudBase):
|
||||
@@ -208,10 +207,10 @@ class TestGDriveRemote(GDrive, TestDataCloudBase):
|
||||
self.dvc.config = config
|
||||
self.cloud = DataCloud(self.dvc)
|
||||
remote = self.cloud.get_remote()
|
||||
self.assertIsInstance(remote, self._get_cloud_class())
|
||||
self.assertIsInstance(remote.tree, self._get_cloud_class())
|
||||
|
||||
def _get_cloud_class(self):
|
||||
return GDriveRemote
|
||||
return GDriveRemoteTree
|
||||
|
||||
|
||||
class TestGSRemote(GCP, TestDataCloudBase):
|
||||
@@ -228,25 +227,27 @@ class TestGSRemote(GCP, TestDataCloudBase):
|
||||
self.dvc.config = config
|
||||
self.cloud = DataCloud(self.dvc)
|
||||
|
||||
self.assertIsInstance(self.cloud.get_remote(), self._get_cloud_class())
|
||||
self.assertIsInstance(
|
||||
self.cloud.get_remote().tree, self._get_cloud_class()
|
||||
)
|
||||
|
||||
def _get_cloud_class(self):
|
||||
return GSRemote
|
||||
return GSRemoteTree
|
||||
|
||||
|
||||
class TestAzureRemote(Azure, TestDataCloudBase):
|
||||
def _get_cloud_class(self):
|
||||
return AzureRemote
|
||||
return AzureRemoteTree
|
||||
|
||||
|
||||
class TestOSSRemote(OSS, TestDataCloudBase):
|
||||
def _get_cloud_class(self):
|
||||
return OSSRemote
|
||||
return OSSRemoteTree
|
||||
|
||||
|
||||
class TestLocalRemote(Local, TestDataCloudBase):
|
||||
def _get_cloud_class(self):
|
||||
return LocalRemote
|
||||
return LocalRemoteTree
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("ssh_server")
|
||||
@@ -271,7 +272,9 @@ class TestSSHRemoteMocked(SSHMocked, TestDataCloudBase):
|
||||
self.dvc.config = config
|
||||
self.cloud = DataCloud(self.dvc)
|
||||
|
||||
self.assertIsInstance(self.cloud.get_remote(), self._get_cloud_class())
|
||||
self.assertIsInstance(
|
||||
self.cloud.get_remote().tree, self._get_cloud_class()
|
||||
)
|
||||
|
||||
def get_url(self):
|
||||
user = self.ssh_server.test_creds["username"]
|
||||
@@ -281,12 +284,12 @@ class TestSSHRemoteMocked(SSHMocked, TestDataCloudBase):
|
||||
return self.ssh_server.test_creds["key_filename"]
|
||||
|
||||
def _get_cloud_class(self):
|
||||
return SSHRemote
|
||||
return SSHRemoteTree
|
||||
|
||||
|
||||
class TestHDFSRemote(HDFS, TestDataCloudBase):
|
||||
def _get_cloud_class(self):
|
||||
return HDFSRemote
|
||||
return HDFSRemoteTree
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("http_server")
|
||||
@@ -300,7 +303,7 @@ class TestHTTPRemote(HTTP, TestDataCloudBase):
|
||||
return super().get_url(self.http_server.server_port)
|
||||
|
||||
def _get_cloud_class(self):
|
||||
return HTTPRemote
|
||||
return HTTPRemoteTree
|
||||
|
||||
|
||||
class TestDataCloudCLIBase(TestDvc):
|
||||
@@ -544,7 +547,7 @@ class TestRecursiveSyncOperations(Local, TestDataCloudBase):
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
def _get_cloud_class(self):
|
||||
return LocalRemote
|
||||
return LocalRemoteTree
|
||||
|
||||
def _prepare_repo(self):
|
||||
remote = self.cloud.get_remote()
|
||||
@@ -577,8 +580,9 @@ class TestRecursiveSyncOperations(Local, TestDataCloudBase):
|
||||
def _test_recursive_fetch(self, data_md5, data_sub_md5):
|
||||
self._clear_local_cache()
|
||||
|
||||
local_cache_data_path = self.dvc.cache.local.get(data_md5)
|
||||
local_cache_data_sub_path = self.dvc.cache.local.get(data_sub_md5)
|
||||
local_cache = self.dvc.cache.local
|
||||
local_cache_data_path = local_cache.hash_to_path_info(data_md5)
|
||||
local_cache_data_sub_path = local_cache.hash_to_path_info(data_sub_md5)
|
||||
|
||||
self.assertFalse(os.path.exists(local_cache_data_path))
|
||||
self.assertFalse(os.path.exists(local_cache_data_sub_path))
|
||||
@@ -590,8 +594,8 @@ class TestRecursiveSyncOperations(Local, TestDataCloudBase):
|
||||
|
||||
def _test_recursive_push(self, data_md5, data_sub_md5):
|
||||
remote = self.cloud.get_remote()
|
||||
cloud_data_path = remote.get(data_md5)
|
||||
cloud_data_sub_path = remote.get(data_sub_md5)
|
||||
cloud_data_path = remote.hash_to_path_info(data_md5)
|
||||
cloud_data_sub_path = remote.hash_to_path_info(data_sub_md5)
|
||||
|
||||
self.assertFalse(os.path.exists(cloud_data_path))
|
||||
self.assertFalse(os.path.exists(cloud_data_sub_path))
|
||||
@@ -615,9 +619,9 @@ class TestRecursiveSyncOperations(Local, TestDataCloudBase):
|
||||
self._test_recursive_pull()
|
||||
|
||||
|
||||
def test_checksum_recalculation(mocker, dvc, tmp_dir):
|
||||
def test_hash_recalculation(mocker, dvc, tmp_dir):
|
||||
tmp_dir.gen({"foo": "foo"})
|
||||
test_get_file_checksum = mocker.spy(LocalRemoteTree, "get_file_checksum")
|
||||
test_get_file_hash = mocker.spy(LocalRemoteTree, "get_file_hash")
|
||||
url = Local.get_url()
|
||||
ret = main(["remote", "add", "-d", TEST_REMOTE, url])
|
||||
assert ret == 0
|
||||
@@ -629,7 +633,7 @@ def test_checksum_recalculation(mocker, dvc, tmp_dir):
|
||||
assert ret == 0
|
||||
ret = main(["run", "--single-stage", "-d", "foo", "echo foo"])
|
||||
assert ret == 0
|
||||
assert test_get_file_checksum.mock.call_count == 1
|
||||
assert test_get_file_hash.mock.call_count == 1
|
||||
|
||||
|
||||
class TestShouldWarnOnNoChecksumInLocalAndRemoteCache(TestDvc):
|
||||
@@ -682,7 +686,7 @@ class TestShouldWarnOnNoChecksumInLocalAndRemoteCache(TestDvc):
|
||||
assert self.message_bar_part in self._caplog.text
|
||||
|
||||
|
||||
def test_verify_checksums(
|
||||
def test_verify_hashes(
|
||||
tmp_dir, scm, dvc, mocker, tmp_path_factory, setup_remote
|
||||
):
|
||||
|
||||
@@ -696,10 +700,10 @@ def test_verify_checksums(
|
||||
remove("dir")
|
||||
remove(dvc.cache.local.cache_dir)
|
||||
|
||||
checksum_spy = mocker.spy(dvc.cache.local.tree, "get_file_checksum")
|
||||
hash_spy = mocker.spy(dvc.cache.local.tree, "get_file_hash")
|
||||
|
||||
dvc.pull()
|
||||
assert checksum_spy.call_count == 0
|
||||
assert hash_spy.call_count == 0
|
||||
|
||||
# Removing cache will invalidate existing state entries
|
||||
remove(dvc.cache.local.cache_dir)
|
||||
@@ -707,7 +711,7 @@ def test_verify_checksums(
|
||||
dvc.config["remote"]["upstream"]["verify"] = True
|
||||
|
||||
dvc.pull()
|
||||
assert checksum_spy.call_count == 3
|
||||
assert hash_spy.call_count == 3
|
||||
|
||||
|
||||
@flaky(max_runs=3, min_passes=1)
|
||||
|
||||
@@ -8,7 +8,7 @@ from git import Repo
|
||||
|
||||
from dvc.exceptions import CollectCacheError
|
||||
from dvc.main import main
|
||||
from dvc.remote.local import LocalRemote, LocalRemoteTree
|
||||
from dvc.remote.local import LocalRemoteTree
|
||||
from dvc.repo import Repo as DvcRepo
|
||||
from dvc.utils.fs import remove
|
||||
from tests.basic_env import TestDir, TestDvcGit
|
||||
@@ -21,7 +21,8 @@ class TestGC(TestDvcGit):
|
||||
self.dvc.add(self.FOO)
|
||||
self.dvc.add(self.DATA_DIR)
|
||||
self.good_cache = [
|
||||
self.dvc.cache.local.get(md5) for md5 in self.dvc.cache.local.all()
|
||||
self.dvc.cache.local.hash_to_path_info(md5)
|
||||
for md5 in self.dvc.cache.local.tree.all()
|
||||
]
|
||||
|
||||
self.bad_cache = []
|
||||
@@ -216,7 +217,7 @@ def test_gc_no_unpacked_dir(tmp_dir, dvc):
|
||||
|
||||
os.remove("dir.dvc")
|
||||
unpackeddir = (
|
||||
dir_stages[0].outs[0].cache_path + LocalRemote.UNPACKED_DIR_SUFFIX
|
||||
dir_stages[0].outs[0].cache_path + LocalRemoteTree.UNPACKED_DIR_SUFFIX
|
||||
)
|
||||
|
||||
# older (pre 1.0) versions of dvc used to generate this dir
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import pytest
|
||||
|
||||
from dvc.dependency.base import DependencyDoesNotExistError
|
||||
from dvc.repo import Repo
|
||||
|
||||
|
||||
@@ -20,3 +21,8 @@ def test_get_url_to_dir(tmp_dir, dname):
|
||||
|
||||
assert (tmp_dir / dname).is_dir()
|
||||
assert (tmp_dir / dname / "foo").read_text() == "foo contents"
|
||||
|
||||
|
||||
def test_get_url_nonexistent(tmp_dir):
|
||||
with pytest.raises(DependencyDoesNotExistError):
|
||||
Repo.get_url("nonexistent")
|
||||
|
||||
@@ -3,6 +3,7 @@ from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from dvc.dependency.base import DependencyDoesNotExistError
|
||||
from dvc.main import main
|
||||
from dvc.stage import Stage
|
||||
from dvc.utils.fs import makedirs
|
||||
@@ -97,3 +98,8 @@ def test_import_stage_accompanies_target(tmp_dir, dvc, erepo_dir):
|
||||
|
||||
assert (tmp_dir / "dir" / "imported_file").exists()
|
||||
assert (tmp_dir / "dir" / "imported_file.dvc").exists()
|
||||
|
||||
|
||||
def test_import_url_nonexistent(dvc, erepo_dir):
|
||||
with pytest.raises(DependencyDoesNotExistError):
|
||||
dvc.imp_url(os.fspath(erepo_dir / "non-existent"))
|
||||
|
||||
@@ -145,7 +145,7 @@ class TestRemoteShouldHandleUppercaseRemoteName(TestDvc):
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
|
||||
def test_dir_checksum_should_be_key_order_agnostic(tmp_dir, dvc):
|
||||
def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc):
|
||||
tmp_dir.gen({"data": {"1": "1 content", "2": "2 content"}})
|
||||
|
||||
path_info = PathInfo("data")
|
||||
@@ -158,7 +158,7 @@ def test_dir_checksum_should_be_key_order_agnostic(tmp_dir, dvc):
|
||||
{"relpath": "2", "md5": "2"},
|
||||
],
|
||||
):
|
||||
checksum1 = dvc.cache.local.get_checksum(path_info)
|
||||
hash1 = dvc.cache.local.get_hash(path_info)
|
||||
|
||||
with patch.object(
|
||||
BaseRemoteTree,
|
||||
@@ -168,9 +168,9 @@ def test_dir_checksum_should_be_key_order_agnostic(tmp_dir, dvc):
|
||||
{"md5": "2", "relpath": "2"},
|
||||
],
|
||||
):
|
||||
checksum2 = dvc.cache.local.get_checksum(path_info)
|
||||
hash2 = dvc.cache.local.get_hash(path_info)
|
||||
|
||||
assert checksum1 == checksum2
|
||||
assert hash1 == hash2
|
||||
|
||||
|
||||
def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory, setup_remote):
|
||||
@@ -194,13 +194,9 @@ def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory, setup_remote):
|
||||
assert upload_error_info.value.amount == 3
|
||||
|
||||
remote = dvc.cloud.get_remote("upstream")
|
||||
assert not remote.tree.exists(
|
||||
remote.checksum_to_path_info(foo.checksum)
|
||||
)
|
||||
assert remote.tree.exists(remote.checksum_to_path_info(bar.checksum))
|
||||
assert not remote.tree.exists(
|
||||
remote.checksum_to_path_info(baz.checksum)
|
||||
)
|
||||
assert not remote.tree.exists(remote.hash_to_path_info(foo.checksum))
|
||||
assert remote.tree.exists(remote.hash_to_path_info(bar.checksum))
|
||||
assert not remote.tree.exists(remote.hash_to_path_info(baz.checksum))
|
||||
|
||||
# Push everything and delete local cache
|
||||
dvc.push()
|
||||
@@ -397,7 +393,7 @@ def test_protect_local_remote(tmp_dir, dvc, setup_remote):
|
||||
|
||||
dvc.push()
|
||||
remote = dvc.cloud.get_remote("upstream")
|
||||
remote_cache_file = remote.checksum_to_path_info(stage.outs[0].checksum)
|
||||
remote_cache_file = remote.hash_to_path_info(stage.outs[0].checksum)
|
||||
|
||||
assert os.path.exists(remote_cache_file)
|
||||
assert stat.S_IMODE(os.stat(remote_cache_file).st_mode) == 0o444
|
||||
|
||||
@@ -26,7 +26,7 @@ from dvc.exceptions import (
|
||||
from dvc.main import main
|
||||
from dvc.output.base import BaseOutput
|
||||
from dvc.path_info import URLInfo
|
||||
from dvc.remote.local import LocalRemote, LocalRemoteTree
|
||||
from dvc.remote.local import LocalRemoteTree
|
||||
from dvc.repo import Repo as DvcRepo
|
||||
from dvc.stage import Stage
|
||||
from dvc.stage.exceptions import StageFileDoesNotExistError
|
||||
@@ -782,8 +782,8 @@ class TestReproChangedDirData(SingleStageRun, TestDvc):
|
||||
class TestReproMissingMd5InStageFile(TestRepro):
|
||||
def test(self):
|
||||
d = load_yaml(self.file1_stage)
|
||||
del d[Stage.PARAM_OUTS][0][LocalRemote.PARAM_CHECKSUM]
|
||||
del d[Stage.PARAM_DEPS][0][LocalRemote.PARAM_CHECKSUM]
|
||||
del d[Stage.PARAM_OUTS][0][LocalRemoteTree.PARAM_CHECKSUM]
|
||||
del d[Stage.PARAM_DEPS][0][LocalRemoteTree.PARAM_CHECKSUM]
|
||||
dump_yaml(self.file1_stage, d)
|
||||
|
||||
stages = self.dvc.reproduce(self.file1_stage)
|
||||
|
||||
@@ -5,7 +5,8 @@ import moto.s3.models as s3model
|
||||
import pytest
|
||||
from moto import mock_s3
|
||||
|
||||
from dvc.remote.s3 import S3Cache, S3Remote, S3RemoteTree
|
||||
from dvc.remote.base import CloudCache
|
||||
from dvc.remote.s3 import S3RemoteTree
|
||||
from tests.remotes import S3
|
||||
|
||||
# from https://github.com/spulec/moto/blob/v1.3.5/tests/test_s3/test_s3.py#L40
|
||||
@@ -54,7 +55,8 @@ def test_copy_singlepart_preserve_etag():
|
||||
],
|
||||
)
|
||||
def test_link_created_on_non_nested_path(base_info, tmp_dir, dvc, scm):
|
||||
cache = S3Cache(dvc, {"url": str(base_info.parent)})
|
||||
tree = S3RemoteTree(dvc, {"url": str(base_info.parent)})
|
||||
cache = CloudCache(tree)
|
||||
s3 = cache.tree.s3
|
||||
s3.create_bucket(Bucket=base_info.bucket)
|
||||
s3.put_object(
|
||||
@@ -69,8 +71,8 @@ def test_link_created_on_non_nested_path(base_info, tmp_dir, dvc, scm):
|
||||
@mock_s3
|
||||
def test_makedirs_doesnot_try_on_top_level_paths(tmp_dir, dvc, scm):
|
||||
base_info = S3RemoteTree.PATH_CLS("s3://bucket/")
|
||||
remote = S3Remote(dvc, {"url": str(base_info)})
|
||||
remote.tree.makedirs(base_info)
|
||||
tree = S3RemoteTree(dvc, {"url": str(base_info)})
|
||||
tree.makedirs(base_info)
|
||||
|
||||
|
||||
def _upload_multipart(s3, Bucket, Key):
|
||||
|
||||
@@ -6,7 +6,7 @@ import pytest
|
||||
from dvc.dvcfile import SingleStageFile
|
||||
from dvc.main import main
|
||||
from dvc.output.local import LocalOutput
|
||||
from dvc.remote.local import LocalRemote
|
||||
from dvc.remote.local import LocalRemoteTree
|
||||
from dvc.repo import Repo
|
||||
from dvc.stage import PipelineStage, Stage
|
||||
from dvc.stage.exceptions import StageFileFormatError
|
||||
@@ -54,8 +54,8 @@ def test_empty_list():
|
||||
|
||||
def test_list():
|
||||
lst = [
|
||||
{LocalOutput.PARAM_PATH: "foo", LocalRemote.PARAM_CHECKSUM: "123"},
|
||||
{LocalOutput.PARAM_PATH: "bar", LocalRemote.PARAM_CHECKSUM: None},
|
||||
{LocalOutput.PARAM_PATH: "foo", LocalRemoteTree.PARAM_CHECKSUM: "123"},
|
||||
{LocalOutput.PARAM_PATH: "bar", LocalRemoteTree.PARAM_CHECKSUM: None},
|
||||
{LocalOutput.PARAM_PATH: "baz"},
|
||||
]
|
||||
d = {Stage.PARAM_DEPS: lst}
|
||||
|
||||
@@ -192,8 +192,8 @@ def test_repotree_walk_fetch(tmp_dir, dvc, scm, setup_remote):
|
||||
|
||||
assert os.path.exists(out.cache_path)
|
||||
for entry in out.dir_cache:
|
||||
checksum = entry[out.remote.PARAM_CHECKSUM]
|
||||
assert os.path.exists(dvc.cache.local.checksum_to_path_info(checksum))
|
||||
hash_ = entry[out.remote.tree.PARAM_CHECKSUM]
|
||||
assert os.path.exists(dvc.cache.local.hash_to_path_info(hash_))
|
||||
|
||||
|
||||
def test_repotree_cache_save(tmp_dir, dvc, scm, erepo_dir, setup_remote):
|
||||
@@ -211,7 +211,7 @@ def test_repotree_cache_save(tmp_dir, dvc, scm, erepo_dir, setup_remote):
|
||||
# into dvc.cache, not fetched or streamed from a remote
|
||||
tree = RepoTree(erepo_dir.dvc, stream=True)
|
||||
expected = [
|
||||
tree.get_file_checksum(erepo_dir / path)
|
||||
tree.get_file_hash(erepo_dir / path)
|
||||
for path in ("dir/bar", "dir/subdir/foo")
|
||||
]
|
||||
|
||||
@@ -219,8 +219,8 @@ def test_repotree_cache_save(tmp_dir, dvc, scm, erepo_dir, setup_remote):
|
||||
cache = dvc.cache.local
|
||||
with cache.state:
|
||||
cache.save(PathInfo(erepo_dir / "dir"), tree, None)
|
||||
for checksum in expected:
|
||||
assert os.path.exists(cache.checksum_to_path_info(checksum))
|
||||
for hash_ in expected:
|
||||
assert os.path.exists(cache.hash_to_path_info(hash_))
|
||||
|
||||
|
||||
def test_cleantree_subrepo(tmp_dir, dvc, scm, monkeypatch):
|
||||
|
||||
+14
-8
@@ -7,9 +7,10 @@ from subprocess import CalledProcessError, Popen, check_output
|
||||
|
||||
from moto.s3 import mock_s3
|
||||
|
||||
from dvc.remote.gdrive import GDriveRemote, GDriveRemoteTree
|
||||
from dvc.remote.gs import GSRemote
|
||||
from dvc.remote.s3 import S3Remote
|
||||
from dvc.remote.base import Remote
|
||||
from dvc.remote.gdrive import GDriveRemoteTree
|
||||
from dvc.remote.gs import GSRemoteTree
|
||||
from dvc.remote.s3 import S3RemoteTree
|
||||
from dvc.utils import env2bool
|
||||
from tests.basic_env import TestDvc
|
||||
|
||||
@@ -22,6 +23,7 @@ TEST_CONFIG = {
|
||||
|
||||
TEST_AWS_REPO_BUCKET = os.environ.get("DVC_TEST_AWS_REPO_BUCKET", "dvc-temp")
|
||||
TEST_GCP_REPO_BUCKET = os.environ.get("DVC_TEST_GCP_REPO_BUCKET", "dvc-test")
|
||||
TEST_GDRIVE_REPO_BUCKET = "root"
|
||||
TEST_OSS_REPO_BUCKET = "dvc-test"
|
||||
|
||||
TEST_GCP_CREDS_FILE = os.path.abspath(
|
||||
@@ -78,7 +80,7 @@ class S3Mocked(S3):
|
||||
@contextmanager
|
||||
def remote(cls, repo):
|
||||
with mock_s3():
|
||||
yield S3Remote(repo, {"url": cls.get_url()})
|
||||
yield Remote(S3RemoteTree(repo, {"url": cls.get_url()}))
|
||||
|
||||
@staticmethod
|
||||
def put_objects(remote, objects):
|
||||
@@ -126,7 +128,7 @@ class GCP:
|
||||
@classmethod
|
||||
@contextmanager
|
||||
def remote(cls, repo):
|
||||
yield GSRemote(repo, {"url": cls.get_url()})
|
||||
yield Remote(GSRemoteTree(repo, {"url": cls.get_url()}))
|
||||
|
||||
@staticmethod
|
||||
def put_objects(remote, objects):
|
||||
@@ -149,13 +151,17 @@ class GDrive:
|
||||
"gdrive_service_account_p12_file_path": "test.p12",
|
||||
"gdrive_use_service_account": True,
|
||||
}
|
||||
remote = GDriveRemote(dvc, config)
|
||||
remote.tree._gdrive_create_dir("root", remote.path_info.path)
|
||||
tree = GDriveRemoteTree(dvc, config)
|
||||
tree._gdrive_create_dir("root", tree.path_info.path)
|
||||
|
||||
@staticmethod
|
||||
def get_storagepath():
|
||||
return TEST_GDRIVE_REPO_BUCKET + "/" + str(uuid.uuid4())
|
||||
|
||||
@staticmethod
|
||||
def get_url():
|
||||
# NOTE: `get_url` should always return new random url
|
||||
return "gdrive://root/" + str(uuid.uuid4())
|
||||
return "gdrive://" + GDrive.get_storagepath()
|
||||
|
||||
|
||||
class Azure:
|
||||
|
||||
@@ -5,7 +5,7 @@ import sys
|
||||
import pytest
|
||||
from mock import mock_open, patch
|
||||
|
||||
from dvc.remote.ssh import SSHRemote, SSHRemoteTree
|
||||
from dvc.remote.ssh import SSHRemoteTree
|
||||
from dvc.system import System
|
||||
from tests.remotes import SSHMocked
|
||||
|
||||
@@ -20,21 +20,21 @@ def test_url(dvc):
|
||||
url = f"ssh://{user}@{host}:{port}{path}"
|
||||
config = {"url": url}
|
||||
|
||||
remote = SSHRemote(dvc, config)
|
||||
assert remote.path_info == url
|
||||
tree = SSHRemoteTree(dvc, config)
|
||||
assert tree.path_info == url
|
||||
|
||||
# SCP-like URL ssh://[user@]host.xz:/absolute/path
|
||||
url = f"ssh://{user}@{host}:{path}"
|
||||
config = {"url": url}
|
||||
|
||||
remote = SSHRemote(dvc, config)
|
||||
assert remote.tree.path_info == url
|
||||
tree = SSHRemoteTree(dvc, config)
|
||||
assert tree.path_info == url
|
||||
|
||||
|
||||
def test_no_path(dvc):
|
||||
config = {"url": "ssh://127.0.0.1"}
|
||||
remote = SSHRemote(dvc, config)
|
||||
assert remote.tree.path_info.path == ""
|
||||
tree = SSHRemoteTree(dvc, config)
|
||||
assert tree.path_info.path == ""
|
||||
|
||||
|
||||
mock_ssh_config = """
|
||||
@@ -67,11 +67,11 @@ else:
|
||||
def test_ssh_host_override_from_config(
|
||||
mock_file, mock_exists, dvc, config, expected_host
|
||||
):
|
||||
remote = SSHRemote(dvc, config)
|
||||
tree = SSHRemoteTree(dvc, config)
|
||||
|
||||
mock_exists.assert_called_with(SSHRemoteTree.ssh_config_filename())
|
||||
mock_file.assert_called_with(SSHRemoteTree.ssh_config_filename())
|
||||
assert remote.tree.path_info.host == expected_host
|
||||
assert tree.path_info.host == expected_host
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -95,11 +95,11 @@ def test_ssh_host_override_from_config(
|
||||
read_data=mock_ssh_config,
|
||||
)
|
||||
def test_ssh_user(mock_file, mock_exists, dvc, config, expected_user):
|
||||
remote = SSHRemote(dvc, config)
|
||||
tree = SSHRemoteTree(dvc, config)
|
||||
|
||||
mock_exists.assert_called_with(SSHRemoteTree.ssh_config_filename())
|
||||
mock_file.assert_called_with(SSHRemoteTree.ssh_config_filename())
|
||||
assert remote.tree.path_info.user == expected_user
|
||||
assert tree.path_info.user == expected_user
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -120,11 +120,11 @@ def test_ssh_user(mock_file, mock_exists, dvc, config, expected_user):
|
||||
read_data=mock_ssh_config,
|
||||
)
|
||||
def test_ssh_port(mock_file, mock_exists, dvc, config, expected_port):
|
||||
remote = SSHRemote(dvc, config)
|
||||
tree = SSHRemoteTree(dvc, config)
|
||||
|
||||
mock_exists.assert_called_with(SSHRemoteTree.ssh_config_filename())
|
||||
mock_file.assert_called_with(SSHRemoteTree.ssh_config_filename())
|
||||
assert remote.path_info.port == expected_port
|
||||
assert tree.path_info.port == expected_port
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -155,11 +155,11 @@ def test_ssh_port(mock_file, mock_exists, dvc, config, expected_port):
|
||||
read_data=mock_ssh_config,
|
||||
)
|
||||
def test_ssh_keyfile(mock_file, mock_exists, dvc, config, expected_keyfile):
|
||||
remote = SSHRemote(dvc, config)
|
||||
tree = SSHRemoteTree(dvc, config)
|
||||
|
||||
mock_exists.assert_called_with(SSHRemoteTree.ssh_config_filename())
|
||||
mock_file.assert_called_with(SSHRemoteTree.ssh_config_filename())
|
||||
assert remote.tree.keyfile == expected_keyfile
|
||||
assert tree.keyfile == expected_keyfile
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -177,11 +177,11 @@ def test_ssh_keyfile(mock_file, mock_exists, dvc, config, expected_keyfile):
|
||||
read_data=mock_ssh_config,
|
||||
)
|
||||
def test_ssh_gss_auth(mock_file, mock_exists, dvc, config, expected_gss_auth):
|
||||
remote = SSHRemote(dvc, config)
|
||||
tree = SSHRemoteTree(dvc, config)
|
||||
|
||||
mock_exists.assert_called_with(SSHRemoteTree.ssh_config_filename())
|
||||
mock_file.assert_called_with(SSHRemoteTree.ssh_config_filename())
|
||||
assert remote.tree.gss_auth == expected_gss_auth
|
||||
assert tree.gss_auth == expected_gss_auth
|
||||
|
||||
|
||||
def test_hardlink_optimization(dvc, tmp_dir, ssh_server):
|
||||
@@ -194,12 +194,12 @@ def test_hardlink_optimization(dvc, tmp_dir, ssh_server):
|
||||
"user": user,
|
||||
"keyfile": ssh_server.test_creds["key_filename"],
|
||||
}
|
||||
remote = SSHRemote(dvc, config)
|
||||
tree = SSHRemoteTree(dvc, config)
|
||||
|
||||
from_info = remote.path_info / "empty"
|
||||
to_info = remote.path_info / "link"
|
||||
from_info = tree.path_info / "empty"
|
||||
to_info = tree.path_info / "link"
|
||||
|
||||
with remote.open(from_info, "wb"):
|
||||
with tree.open(from_info, "wb"):
|
||||
pass
|
||||
|
||||
if os.name == "nt":
|
||||
@@ -207,5 +207,5 @@ def test_hardlink_optimization(dvc, tmp_dir, ssh_server):
|
||||
else:
|
||||
link_path = to_info.path
|
||||
|
||||
remote.tree.hardlink(from_info, to_info)
|
||||
tree.hardlink(from_info, to_info)
|
||||
assert not System.is_hardlink(link_path)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import pytest
|
||||
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.remote.azure import AzureRemote
|
||||
from dvc.remote.azure import AzureRemoteTree
|
||||
from tests.remotes import Azure
|
||||
|
||||
container_name = "container-name"
|
||||
@@ -18,31 +18,31 @@ def test_init_env_var(monkeypatch, dvc):
|
||||
monkeypatch.setenv("AZURE_STORAGE_CONNECTION_STRING", connection_string)
|
||||
|
||||
config = {"url": "azure://"}
|
||||
remote = AzureRemote(dvc, config)
|
||||
assert remote.tree.path_info == "azure://" + container_name
|
||||
assert remote.tree.connection_string == connection_string
|
||||
tree = AzureRemoteTree(dvc, config)
|
||||
assert tree.path_info == "azure://" + container_name
|
||||
assert tree.connection_string == connection_string
|
||||
|
||||
|
||||
def test_init(dvc):
|
||||
prefix = "some/prefix"
|
||||
url = f"azure://{container_name}/{prefix}"
|
||||
config = {"url": url, "connection_string": connection_string}
|
||||
remote = AzureRemote(dvc, config)
|
||||
assert remote.tree.path_info == url
|
||||
assert remote.tree.connection_string == connection_string
|
||||
tree = AzureRemoteTree(dvc, config)
|
||||
assert tree.path_info == url
|
||||
assert tree.connection_string == connection_string
|
||||
|
||||
|
||||
def test_get_file_checksum(tmp_dir):
|
||||
def test_get_file_hash(tmp_dir):
|
||||
if not Azure.should_test():
|
||||
pytest.skip("no azurite running")
|
||||
|
||||
tmp_dir.gen("foo", "foo")
|
||||
|
||||
remote = AzureRemote(None, {})
|
||||
to_info = remote.tree.PATH_CLS(Azure.get_url())
|
||||
remote.tree.upload(PathInfo("foo"), to_info)
|
||||
assert remote.tree.exists(to_info)
|
||||
checksum = remote.tree.get_file_checksum(to_info)
|
||||
assert checksum
|
||||
assert isinstance(checksum, str)
|
||||
assert checksum.strip("'").strip('"') == checksum
|
||||
tree = AzureRemoteTree(None, {})
|
||||
to_info = tree.PATH_CLS(Azure.get_url())
|
||||
tree.upload(PathInfo("foo"), to_info)
|
||||
assert tree.exists(to_info)
|
||||
hash_ = tree.get_file_hash(to_info)
|
||||
assert hash_
|
||||
assert isinstance(hash_, str)
|
||||
assert hash_.strip("'").strip('"') == hash_
|
||||
|
||||
@@ -4,7 +4,12 @@ import mock
|
||||
import pytest
|
||||
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.remote.base import BaseRemote, RemoteCmdError, RemoteMissingDepsError
|
||||
from dvc.remote.base import (
|
||||
BaseRemoteTree,
|
||||
Remote,
|
||||
RemoteCmdError,
|
||||
RemoteMissingDepsError,
|
||||
)
|
||||
|
||||
|
||||
class _CallableOrNone:
|
||||
@@ -15,14 +20,13 @@ class _CallableOrNone:
|
||||
|
||||
|
||||
CallableOrNone = _CallableOrNone()
|
||||
REMOTE_CLS = BaseRemote
|
||||
|
||||
|
||||
def test_missing_deps(dvc):
|
||||
requires = {"missing": "missing"}
|
||||
with mock.patch.object(REMOTE_CLS, "REQUIRES", requires):
|
||||
with mock.patch.object(BaseRemoteTree, "REQUIRES", requires):
|
||||
with pytest.raises(RemoteMissingDepsError):
|
||||
REMOTE_CLS(dvc, {})
|
||||
BaseRemoteTree(dvc, {})
|
||||
|
||||
|
||||
def test_cmd_error(dvc):
|
||||
@@ -33,63 +37,63 @@ def test_cmd_error(dvc):
|
||||
err = "sed: expression #1, char 2: extra characters after command"
|
||||
|
||||
with mock.patch.object(
|
||||
REMOTE_CLS.TREE_CLS,
|
||||
BaseRemoteTree,
|
||||
"remove",
|
||||
side_effect=RemoteCmdError("base", cmd, ret, err),
|
||||
):
|
||||
with pytest.raises(RemoteCmdError):
|
||||
REMOTE_CLS(dvc, config).tree.remove("file")
|
||||
BaseRemoteTree(dvc, config).remove("file")
|
||||
|
||||
|
||||
@mock.patch.object(BaseRemote, "_list_checksums_traverse")
|
||||
@mock.patch.object(BaseRemote, "_list_checksums_exists")
|
||||
def test_checksums_exist(object_exists, traverse, dvc):
|
||||
remote = BaseRemote(dvc, {})
|
||||
@mock.patch.object(BaseRemoteTree, "list_hashes_traverse")
|
||||
@mock.patch.object(BaseRemoteTree, "list_hashes_exists")
|
||||
def test_hashes_exist(object_exists, traverse, dvc):
|
||||
remote = Remote(BaseRemoteTree(dvc, {}))
|
||||
|
||||
# remote does not support traverse
|
||||
remote.CAN_TRAVERSE = False
|
||||
remote.tree.CAN_TRAVERSE = False
|
||||
with mock.patch.object(
|
||||
remote, "list_checksums", return_value=list(range(256))
|
||||
remote.tree, "list_hashes", return_value=list(range(256))
|
||||
):
|
||||
checksums = set(range(1000))
|
||||
remote.checksums_exist(checksums)
|
||||
object_exists.assert_called_with(checksums, None, None)
|
||||
hashes = set(range(1000))
|
||||
remote.hashes_exist(hashes)
|
||||
object_exists.assert_called_with(hashes, None, None)
|
||||
traverse.assert_not_called()
|
||||
|
||||
remote.CAN_TRAVERSE = True
|
||||
remote.tree.CAN_TRAVERSE = True
|
||||
|
||||
# large remote, small local
|
||||
object_exists.reset_mock()
|
||||
traverse.reset_mock()
|
||||
with mock.patch.object(
|
||||
remote, "list_checksums", return_value=list(range(256))
|
||||
remote.tree, "list_hashes", return_value=list(range(256))
|
||||
):
|
||||
checksums = list(range(1000))
|
||||
remote.checksums_exist(checksums)
|
||||
hashes = list(range(1000))
|
||||
remote.hashes_exist(hashes)
|
||||
# verify that _cache_paths_with_max() short circuits
|
||||
# before returning all 256 remote checksums
|
||||
max_checksums = math.ceil(
|
||||
remote._max_estimation_size(checksums)
|
||||
/ pow(16, remote.TRAVERSE_PREFIX_LEN)
|
||||
# before returning all 256 remote hashes
|
||||
max_hashes = math.ceil(
|
||||
remote.tree._max_estimation_size(hashes)
|
||||
/ pow(16, remote.tree.TRAVERSE_PREFIX_LEN)
|
||||
)
|
||||
assert max_checksums < 256
|
||||
assert max_hashes < 256
|
||||
object_exists.assert_called_with(
|
||||
frozenset(range(max_checksums, 1000)), None, None
|
||||
frozenset(range(max_hashes, 1000)), None, None
|
||||
)
|
||||
traverse.assert_not_called()
|
||||
|
||||
# large remote, large local
|
||||
object_exists.reset_mock()
|
||||
traverse.reset_mock()
|
||||
remote.JOBS = 16
|
||||
remote.tree.JOBS = 16
|
||||
with mock.patch.object(
|
||||
remote, "list_checksums", return_value=list(range(256))
|
||||
remote.tree, "list_hashes", return_value=list(range(256))
|
||||
):
|
||||
checksums = list(range(1000000))
|
||||
remote.checksums_exist(checksums)
|
||||
hashes = list(range(1000000))
|
||||
remote.hashes_exist(hashes)
|
||||
object_exists.assert_not_called()
|
||||
traverse.assert_called_with(
|
||||
256 * pow(16, remote.TRAVERSE_PREFIX_LEN),
|
||||
256 * pow(16, remote.tree.TRAVERSE_PREFIX_LEN),
|
||||
set(range(256)),
|
||||
None,
|
||||
None,
|
||||
@@ -97,50 +101,50 @@ def test_checksums_exist(object_exists, traverse, dvc):
|
||||
|
||||
|
||||
@mock.patch.object(
|
||||
BaseRemote, "list_checksums", return_value=[],
|
||||
BaseRemoteTree, "list_hashes", return_value=[],
|
||||
)
|
||||
@mock.patch.object(
|
||||
BaseRemote, "path_to_checksum", side_effect=lambda x: x,
|
||||
BaseRemoteTree, "path_to_hash", side_effect=lambda x: x,
|
||||
)
|
||||
def test_list_checksums_traverse(path_to_checksum, list_checksums, dvc):
|
||||
remote = BaseRemote(dvc, {})
|
||||
remote.tree.path_info = PathInfo("foo")
|
||||
def test_list_hashes_traverse(path_to_hash, list_hashes, dvc):
|
||||
tree = BaseRemoteTree(dvc, {})
|
||||
tree.path_info = PathInfo("foo")
|
||||
|
||||
# parallel traverse
|
||||
size = 256 / remote.JOBS * remote.LIST_OBJECT_PAGE_SIZE
|
||||
list(remote._list_checksums_traverse(size, {0}))
|
||||
size = 256 / tree.JOBS * tree.LIST_OBJECT_PAGE_SIZE
|
||||
list(tree.list_hashes_traverse(size, {0}))
|
||||
for i in range(1, 16):
|
||||
list_checksums.assert_any_call(
|
||||
list_hashes.assert_any_call(
|
||||
prefix=f"{i:03x}", progress_callback=CallableOrNone
|
||||
)
|
||||
for i in range(1, 256):
|
||||
list_checksums.assert_any_call(
|
||||
list_hashes.assert_any_call(
|
||||
prefix=f"{i:02x}", progress_callback=CallableOrNone
|
||||
)
|
||||
|
||||
# default traverse (small remote)
|
||||
size -= 1
|
||||
list_checksums.reset_mock()
|
||||
list(remote._list_checksums_traverse(size - 1, {0}))
|
||||
list_checksums.assert_called_with(
|
||||
list_hashes.reset_mock()
|
||||
list(tree.list_hashes_traverse(size - 1, {0}))
|
||||
list_hashes.assert_called_with(
|
||||
prefix=None, progress_callback=CallableOrNone
|
||||
)
|
||||
|
||||
|
||||
def test_list_checksums(dvc):
|
||||
remote = BaseRemote(dvc, {})
|
||||
remote.tree.path_info = PathInfo("foo")
|
||||
def test_list_hashes(dvc):
|
||||
tree = BaseRemoteTree(dvc, {})
|
||||
tree.path_info = PathInfo("foo")
|
||||
|
||||
with mock.patch.object(
|
||||
remote, "list_paths", return_value=["12/3456", "bar"]
|
||||
tree, "list_paths", return_value=["12/3456", "bar"]
|
||||
):
|
||||
checksums = list(remote.list_checksums())
|
||||
assert checksums == ["123456"]
|
||||
hashes = list(tree.list_hashes())
|
||||
assert hashes == ["123456"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"checksum, result",
|
||||
"hash_, result",
|
||||
[(None, False), ("", False), ("3456.dir", True), ("3456", False)],
|
||||
)
|
||||
def test_is_dir_checksum(checksum, result):
|
||||
assert BaseRemote.is_dir_checksum(checksum) == result
|
||||
def test_is_dir_hash(hash_, result):
|
||||
assert BaseRemoteTree.is_dir_hash(hash_) == result
|
||||
|
||||
@@ -2,7 +2,7 @@ import os
|
||||
|
||||
import pytest
|
||||
|
||||
from dvc.remote.gdrive import GDriveAuthError, GDriveRemote, GDriveRemoteTree
|
||||
from dvc.remote.gdrive import GDriveAuthError, GDriveRemoteTree
|
||||
|
||||
USER_CREDS_TOKEN_REFRESH_ERROR = '{"access_token": "", "client_id": "", "client_secret": "", "refresh_token": "", "token_expiry": "", "token_uri": "https://oauth2.googleapis.com/token", "user_agent": null, "revoke_uri": "https://oauth2.googleapis.com/revoke", "id_token": null, "id_token_jwt": null, "token_response": {"access_token": "", "expires_in": 3600, "scope": "https://www.googleapis.com/auth/drive.appdata https://www.googleapis.com/auth/drive", "token_type": "Bearer"}, "scopes": ["https://www.googleapis.com/auth/drive", "https://www.googleapis.com/auth/drive.appdata"], "token_info_uri": "https://oauth2.googleapis.com/tokeninfo", "invalid": true, "_class": "OAuth2Credentials", "_module": "oauth2client.client"}' # noqa: E501
|
||||
|
||||
@@ -17,21 +17,21 @@ class TestRemoteGDrive:
|
||||
}
|
||||
|
||||
def test_init(self, dvc):
|
||||
remote = GDriveRemote(dvc, self.CONFIG)
|
||||
assert str(remote.tree.path_info) == self.CONFIG["url"]
|
||||
tree = GDriveRemoteTree(dvc, self.CONFIG)
|
||||
assert str(tree.path_info) == self.CONFIG["url"]
|
||||
|
||||
def test_drive(self, dvc):
|
||||
remote = GDriveRemote(dvc, self.CONFIG)
|
||||
tree = GDriveRemoteTree(dvc, self.CONFIG)
|
||||
os.environ[
|
||||
GDriveRemoteTree.GDRIVE_CREDENTIALS_DATA
|
||||
] = USER_CREDS_TOKEN_REFRESH_ERROR
|
||||
with pytest.raises(GDriveAuthError):
|
||||
remote.tree._drive
|
||||
tree._drive
|
||||
|
||||
os.environ[GDriveRemoteTree.GDRIVE_CREDENTIALS_DATA] = ""
|
||||
remote = GDriveRemote(dvc, self.CONFIG)
|
||||
tree = GDriveRemoteTree(dvc, self.CONFIG)
|
||||
os.environ[
|
||||
GDriveRemoteTree.GDRIVE_CREDENTIALS_DATA
|
||||
] = USER_CREDS_MISSED_KEY_ERROR
|
||||
with pytest.raises(GDriveAuthError):
|
||||
remote.tree._drive
|
||||
tree._drive
|
||||
|
||||
@@ -2,7 +2,7 @@ import mock
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from dvc.remote.gs import GSRemote, dynamic_chunk_size
|
||||
from dvc.remote.gs import GSRemoteTree, dynamic_chunk_size
|
||||
|
||||
BUCKET = "bucket"
|
||||
PREFIX = "prefix"
|
||||
@@ -17,17 +17,17 @@ CONFIG = {
|
||||
|
||||
|
||||
def test_init(dvc):
|
||||
remote = GSRemote(dvc, CONFIG)
|
||||
assert remote.tree.path_info == URL
|
||||
assert remote.tree.projectname == PROJECT
|
||||
assert remote.tree.credentialpath == CREDENTIALPATH
|
||||
tree = GSRemoteTree(dvc, CONFIG)
|
||||
assert tree.path_info == URL
|
||||
assert tree.projectname == PROJECT
|
||||
assert tree.credentialpath == CREDENTIALPATH
|
||||
|
||||
|
||||
@mock.patch("google.cloud.storage.Client.from_service_account_json")
|
||||
def test_gs(mock_client, dvc):
|
||||
remote = GSRemote(dvc, CONFIG)
|
||||
assert remote.tree.credentialpath
|
||||
remote.tree.gs()
|
||||
tree = GSRemoteTree(dvc, CONFIG)
|
||||
assert tree.credentialpath
|
||||
tree.gs()
|
||||
mock_client.assert_called_once_with(CREDENTIALPATH)
|
||||
|
||||
|
||||
@@ -35,8 +35,8 @@ def test_gs(mock_client, dvc):
|
||||
def test_gs_no_credspath(mock_client, dvc):
|
||||
config = CONFIG.copy()
|
||||
del config["credentialpath"]
|
||||
remote = GSRemote(dvc, config)
|
||||
remote.tree.gs()
|
||||
tree = GSRemoteTree(dvc, config)
|
||||
tree.gs()
|
||||
mock_client.assert_called_with(PROJECT)
|
||||
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@ import pytest
|
||||
|
||||
from dvc.exceptions import HTTPError
|
||||
from dvc.path_info import URLInfo
|
||||
from dvc.remote.http import HTTPRemote
|
||||
from dvc.remote.http import HTTPRemoteTree
|
||||
from tests.utils.httpd import StaticFileServer
|
||||
|
||||
|
||||
@@ -11,10 +11,10 @@ def test_download_fails_on_error_code(dvc):
|
||||
url = f"http://localhost:{httpd.server_port}/"
|
||||
config = {"url": url}
|
||||
|
||||
remote = HTTPRemote(dvc, config)
|
||||
tree = HTTPRemoteTree(dvc, config)
|
||||
|
||||
with pytest.raises(HTTPError):
|
||||
remote.tree._download(URLInfo(url) / "missing.txt", "missing.txt")
|
||||
tree._download(URLInfo(url) / "missing.txt", "missing.txt")
|
||||
|
||||
|
||||
def test_public_auth_method(dvc):
|
||||
@@ -25,9 +25,9 @@ def test_public_auth_method(dvc):
|
||||
"password": "",
|
||||
}
|
||||
|
||||
remote = HTTPRemote(dvc, config)
|
||||
tree = HTTPRemoteTree(dvc, config)
|
||||
|
||||
assert remote.tree._auth_method() is None
|
||||
assert tree._auth_method() is None
|
||||
|
||||
|
||||
def test_basic_auth_method(dvc):
|
||||
@@ -44,10 +44,10 @@ def test_basic_auth_method(dvc):
|
||||
"password": password,
|
||||
}
|
||||
|
||||
remote = HTTPRemote(dvc, config)
|
||||
tree = HTTPRemoteTree(dvc, config)
|
||||
|
||||
assert remote.tree._auth_method() == auth
|
||||
assert isinstance(remote.tree._auth_method(), HTTPBasicAuth)
|
||||
assert tree._auth_method() == auth
|
||||
assert isinstance(tree._auth_method(), HTTPBasicAuth)
|
||||
|
||||
|
||||
def test_digest_auth_method(dvc):
|
||||
@@ -64,10 +64,10 @@ def test_digest_auth_method(dvc):
|
||||
"password": password,
|
||||
}
|
||||
|
||||
remote = HTTPRemote(dvc, config)
|
||||
tree = HTTPRemoteTree(dvc, config)
|
||||
|
||||
assert remote.tree._auth_method() == auth
|
||||
assert isinstance(remote.tree._auth_method(), HTTPDigestAuth)
|
||||
assert tree._auth_method() == auth
|
||||
assert isinstance(tree._auth_method(), HTTPDigestAuth)
|
||||
|
||||
|
||||
def test_custom_auth_method(dvc):
|
||||
@@ -81,8 +81,8 @@ def test_custom_auth_method(dvc):
|
||||
"password": password,
|
||||
}
|
||||
|
||||
remote = HTTPRemote(dvc, config)
|
||||
tree = HTTPRemoteTree(dvc, config)
|
||||
|
||||
assert remote.tree._auth_method() is None
|
||||
assert header in remote.tree.headers
|
||||
assert remote.tree.headers[header] == password
|
||||
assert tree._auth_method() is None
|
||||
assert header in tree.headers
|
||||
assert tree.headers[header] == password
|
||||
|
||||
@@ -19,9 +19,9 @@ def test_init(dvc, index):
|
||||
assert str(index.path) == os.path.join(dvc.index_dir, "foo.idx")
|
||||
|
||||
|
||||
def test_is_dir_checksum(dvc, index):
|
||||
assert index.is_dir_checksum("foo.dir")
|
||||
assert not index.is_dir_checksum("foo")
|
||||
def test_is_dir_hash(dvc, index):
|
||||
assert index.is_dir_hash("foo.dir")
|
||||
assert not index.is_dir_hash("foo")
|
||||
|
||||
|
||||
def test_roundtrip(dvc, index):
|
||||
@@ -30,8 +30,8 @@ def test_roundtrip(dvc, index):
|
||||
index.update(expected_dir, expected_file)
|
||||
index.dump()
|
||||
index.load()
|
||||
assert set(index.dir_checksums()) == expected_dir
|
||||
assert set(index.checksums()) == expected_dir | expected_file
|
||||
assert set(index.dir_hashes()) == expected_dir
|
||||
assert set(index.hashes()) == expected_dir | expected_file
|
||||
|
||||
|
||||
def test_clear(dvc, index):
|
||||
@@ -39,19 +39,19 @@ def test_clear(dvc, index):
|
||||
["1234.dir"], ["5678"],
|
||||
)
|
||||
index.clear()
|
||||
assert first(index.checksums()) is None
|
||||
assert first(index.hashes()) is None
|
||||
|
||||
|
||||
def test_update(dvc, index):
|
||||
expected_dir = {"1234.dir"}
|
||||
expected_file = {"5678"}
|
||||
index.update(expected_dir, expected_file)
|
||||
assert set(index.dir_checksums()) == expected_dir
|
||||
assert set(index.checksums()) == expected_dir | expected_file
|
||||
assert set(index.dir_hashes()) == expected_dir
|
||||
assert set(index.hashes()) == expected_dir | expected_file
|
||||
|
||||
|
||||
def test_intersection(dvc, index):
|
||||
checksums = (str(i) for i in range(2000))
|
||||
hashes = (str(i) for i in range(2000))
|
||||
expected = {str(i) for i in range(1000)}
|
||||
index.update([], checksums)
|
||||
index.update([], hashes)
|
||||
assert set(index.intersection(expected)) == expected
|
||||
|
||||
@@ -5,7 +5,8 @@ import pytest
|
||||
|
||||
from dvc.cache import NamedCache
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.remote.local import LocalCache
|
||||
from dvc.remote.index import RemoteIndexNoop
|
||||
from dvc.remote.local import LocalCache, LocalRemoteTree
|
||||
|
||||
|
||||
def test_status_download_optimization(mocker, dvc):
|
||||
@@ -13,28 +14,29 @@ def test_status_download_optimization(mocker, dvc):
|
||||
And the desired files to fetch are already on the local cache,
|
||||
Don't check the existence of the desired files on the remote cache
|
||||
"""
|
||||
cache = LocalCache(dvc, {})
|
||||
cache = LocalCache(LocalRemoteTree(dvc, {}))
|
||||
|
||||
infos = NamedCache()
|
||||
infos.add("local", "acbd18db4cc2f85cedef654fccc4a4d8", "foo")
|
||||
infos.add("local", "37b51d194a7513e45b56f6524f2d51f2", "bar")
|
||||
|
||||
local_exists = list(infos["local"])
|
||||
mocker.patch.object(cache, "checksums_exist", return_value=local_exists)
|
||||
mocker.patch.object(cache, "hashes_exist", return_value=local_exists)
|
||||
|
||||
other_remote = mocker.Mock()
|
||||
other_remote.url = "other_remote"
|
||||
other_remote.checksums_exist.return_value = []
|
||||
other_remote.hashes_exist.return_value = []
|
||||
other_remote.index = RemoteIndexNoop()
|
||||
|
||||
cache.status(infos, other_remote, download=True)
|
||||
|
||||
assert other_remote.checksums_exist.call_count == 0
|
||||
assert other_remote.hashes_exist.call_count == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("link_name", ["hardlink", "symlink"])
|
||||
def test_is_protected(tmp_dir, dvc, link_name):
|
||||
cache = LocalCache(dvc, {})
|
||||
link_method = getattr(cache.tree, link_name)
|
||||
tree = LocalRemoteTree(dvc, {})
|
||||
link_method = getattr(tree, link_name)
|
||||
|
||||
(tmp_dir / "foo").write_text("foo")
|
||||
|
||||
@@ -43,47 +45,47 @@ def test_is_protected(tmp_dir, dvc, link_name):
|
||||
|
||||
link_method(foo, link)
|
||||
|
||||
assert not cache.is_protected(foo)
|
||||
assert not cache.is_protected(link)
|
||||
assert not tree.is_protected(foo)
|
||||
assert not tree.is_protected(link)
|
||||
|
||||
cache.protect(foo)
|
||||
tree.protect(foo)
|
||||
|
||||
assert cache.is_protected(foo)
|
||||
assert cache.is_protected(link)
|
||||
assert tree.is_protected(foo)
|
||||
assert tree.is_protected(link)
|
||||
|
||||
cache.unprotect(link)
|
||||
tree.unprotect(link)
|
||||
|
||||
assert not cache.is_protected(link)
|
||||
assert not tree.is_protected(link)
|
||||
if os.name == "nt" and link_name == "hardlink":
|
||||
# NOTE: NTFS doesn't allow deleting read-only files, which forces us to
|
||||
# set write perms on the link, which propagates to the source.
|
||||
assert not cache.is_protected(foo)
|
||||
assert not tree.is_protected(foo)
|
||||
else:
|
||||
assert cache.is_protected(foo)
|
||||
assert tree.is_protected(foo)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("err", [errno.EPERM, errno.EACCES])
|
||||
def test_protect_ignore_errors(tmp_dir, mocker, err):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
foo = PathInfo("foo")
|
||||
cache = LocalCache(None, {})
|
||||
tree = LocalRemoteTree(None, {})
|
||||
|
||||
cache.protect(foo)
|
||||
tree.protect(foo)
|
||||
|
||||
mock_chmod = mocker.patch(
|
||||
"os.chmod", side_effect=OSError(err, "something")
|
||||
)
|
||||
cache.protect(foo)
|
||||
tree.protect(foo)
|
||||
assert mock_chmod.called
|
||||
|
||||
|
||||
def test_protect_ignore_erofs(tmp_dir, mocker):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
foo = PathInfo("foo")
|
||||
cache = LocalCache(None, {})
|
||||
tree = LocalRemoteTree(None, {})
|
||||
|
||||
mock_chmod = mocker.patch(
|
||||
"os.chmod", side_effect=OSError(errno.EROFS, "read-only fs")
|
||||
)
|
||||
cache.protect(foo)
|
||||
tree.protect(foo)
|
||||
assert mock_chmod.called
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from dvc.remote.oss import OSSRemote
|
||||
from dvc.remote.oss import OSSRemoteTree
|
||||
|
||||
bucket_name = "bucket-name"
|
||||
endpoint = "endpoint"
|
||||
@@ -15,8 +15,8 @@ def test_init(dvc):
|
||||
"oss_key_secret": key_secret,
|
||||
"oss_endpoint": endpoint,
|
||||
}
|
||||
remote = OSSRemote(dvc, config)
|
||||
assert remote.tree.path_info == url
|
||||
assert remote.tree.endpoint == endpoint
|
||||
assert remote.tree.key_id == key_id
|
||||
assert remote.tree.key_secret == key_secret
|
||||
tree = OSSRemoteTree(dvc, config)
|
||||
assert tree.path_info == url
|
||||
assert tree.endpoint == endpoint
|
||||
assert tree.key_id == key_id
|
||||
assert tree.key_secret == key_secret
|
||||
|
||||
@@ -1,41 +1,43 @@
|
||||
import pytest
|
||||
|
||||
from dvc.remote import GSRemote, Remote, S3Remote
|
||||
from dvc.remote import get_cloud_tree
|
||||
from dvc.remote.gs import GSRemoteTree
|
||||
from dvc.remote.s3 import S3RemoteTree
|
||||
|
||||
|
||||
def test_remote_with_checksum_jobs(dvc):
|
||||
dvc.config["remote"]["with_checksum_jobs"] = {
|
||||
def test_remote_with_hash_jobs(dvc):
|
||||
dvc.config["remote"]["with_hash_jobs"] = {
|
||||
"url": "s3://bucket/name",
|
||||
"checksum_jobs": 100,
|
||||
"hash_jobs": 100,
|
||||
}
|
||||
dvc.config["core"]["checksum_jobs"] = 200
|
||||
dvc.config["core"]["hash_jobs"] = 200
|
||||
|
||||
remote = Remote(dvc, name="with_checksum_jobs")
|
||||
assert remote.checksum_jobs == 100
|
||||
tree = get_cloud_tree(dvc, name="with_hash_jobs")
|
||||
assert tree.hash_jobs == 100
|
||||
|
||||
|
||||
def test_remote_without_checksum_jobs(dvc):
|
||||
dvc.config["remote"]["without_checksum_jobs"] = {"url": "s3://bucket/name"}
|
||||
dvc.config["core"]["checksum_jobs"] = 200
|
||||
def test_remote_without_hash_jobs(dvc):
|
||||
dvc.config["remote"]["without_hash_jobs"] = {"url": "s3://bucket/name"}
|
||||
dvc.config["core"]["hash_jobs"] = 200
|
||||
|
||||
remote = Remote(dvc, name="without_checksum_jobs")
|
||||
assert remote.checksum_jobs == 200
|
||||
tree = get_cloud_tree(dvc, name="without_hash_jobs")
|
||||
assert tree.hash_jobs == 200
|
||||
|
||||
|
||||
def test_remote_without_checksum_jobs_default(dvc):
|
||||
dvc.config["remote"]["without_checksum_jobs"] = {"url": "s3://bucket/name"}
|
||||
def test_remote_without_hash_jobs_default(dvc):
|
||||
dvc.config["remote"]["without_hash_jobs"] = {"url": "s3://bucket/name"}
|
||||
|
||||
remote = Remote(dvc, name="without_checksum_jobs")
|
||||
assert remote.checksum_jobs == remote.CHECKSUM_JOBS
|
||||
tree = get_cloud_tree(dvc, name="without_hash_jobs")
|
||||
assert tree.hash_jobs == tree.HASH_JOBS
|
||||
|
||||
|
||||
@pytest.mark.parametrize("remote_cls", [GSRemote, S3Remote])
|
||||
def test_makedirs_not_create_for_top_level_path(remote_cls, dvc, mocker):
|
||||
url = f"{remote_cls.scheme}://bucket/"
|
||||
remote = remote_cls(dvc, {"url": url})
|
||||
@pytest.mark.parametrize("tree_cls", [GSRemoteTree, S3RemoteTree])
|
||||
def test_makedirs_not_create_for_top_level_path(tree_cls, dvc, mocker):
|
||||
url = f"{tree_cls.scheme}://bucket/"
|
||||
tree = tree_cls(dvc, {"url": url})
|
||||
mocked_client = mocker.PropertyMock()
|
||||
# we use remote clients with same name as scheme to interact with remote
|
||||
mocker.patch.object(remote_cls.TREE_CLS, remote.scheme, mocked_client)
|
||||
mocker.patch.object(tree_cls, tree.scheme, mocked_client)
|
||||
|
||||
remote.tree.makedirs(remote.path_info)
|
||||
tree.makedirs(tree.path_info)
|
||||
assert not mocked_client.called
|
||||
|
||||
@@ -3,7 +3,7 @@ import os
|
||||
import pytest
|
||||
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.remote.s3 import S3Remote, S3RemoteTree
|
||||
from dvc.remote.s3 import S3RemoteTree
|
||||
from dvc.utils.fs import walk_files
|
||||
from tests.remotes import GCP, S3Mocked
|
||||
|
||||
@@ -91,7 +91,7 @@ def test_copy_preserve_etag_across_buckets(remote, dvc):
|
||||
s3 = remote.tree.s3
|
||||
s3.create_bucket(Bucket="another")
|
||||
|
||||
another = S3Remote(dvc, {"url": "s3://another", "region": "us-east-1"})
|
||||
another = S3RemoteTree(dvc, {"url": "s3://another", "region": "us-east-1"})
|
||||
|
||||
from_info = remote.path_info / "foo"
|
||||
to_info = another.path_info / "foo"
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import pytest
|
||||
|
||||
from dvc.config import ConfigError
|
||||
from dvc.remote.s3 import S3Remote
|
||||
from dvc.remote.s3 import S3RemoteTree
|
||||
|
||||
bucket_name = "bucket-name"
|
||||
prefix = "some/prefix"
|
||||
@@ -20,9 +20,9 @@ def grants():
|
||||
|
||||
def test_init(dvc):
|
||||
config = {"url": url}
|
||||
remote = S3Remote(dvc, config)
|
||||
tree = S3RemoteTree(dvc, config)
|
||||
|
||||
assert remote.tree.path_info == url
|
||||
assert tree.path_info == url
|
||||
|
||||
|
||||
def test_grants(dvc):
|
||||
@@ -33,8 +33,7 @@ def test_grants(dvc):
|
||||
"grant_write_acp": "id=write-acp-permission-id",
|
||||
"grant_full_control": "id=full-control-permission-id",
|
||||
}
|
||||
remote = S3Remote(dvc, config)
|
||||
tree = remote.tree
|
||||
tree = S3RemoteTree(dvc, config)
|
||||
|
||||
assert (
|
||||
tree.extra_args["GrantRead"]
|
||||
@@ -52,9 +51,9 @@ def test_grants_mutually_exclusive_acl_error(dvc, grants):
|
||||
config = {"url": url, "acl": "public-read", grant_option: grant_value}
|
||||
|
||||
with pytest.raises(ConfigError):
|
||||
S3Remote(dvc, config)
|
||||
S3RemoteTree(dvc, config)
|
||||
|
||||
|
||||
def test_sse_kms_key_id(dvc):
|
||||
remote = S3Remote(dvc, {"url": url, "sse_kms_key_id": "key"})
|
||||
assert remote.tree.extra_args["SSEKMSKeyId"] == "key"
|
||||
tree = S3RemoteTree(dvc, {"url": url, "sse_kms_key_id": "key"})
|
||||
assert tree.extra_args["SSEKMSKeyId"] == "key"
|
||||
|
||||
@@ -22,8 +22,9 @@ def test_open(tmp_dir, dvc):
|
||||
(tmp_dir / "foo").unlink()
|
||||
|
||||
tree = RepoTree(dvc)
|
||||
with tree.open("foo", "r") as fobj:
|
||||
assert fobj.read() == "foo"
|
||||
with dvc.state:
|
||||
with tree.open("foo", "r") as fobj:
|
||||
assert fobj.read() == "foo"
|
||||
|
||||
|
||||
def test_open_dirty_hash(tmp_dir, dvc):
|
||||
|
||||
@@ -22,8 +22,9 @@ def test_open(tmp_dir, dvc):
|
||||
(tmp_dir / "foo").unlink()
|
||||
|
||||
tree = DvcTree(dvc)
|
||||
with tree.open("foo", "r") as fobj:
|
||||
assert fobj.read() == "foo"
|
||||
with dvc.state:
|
||||
with tree.open("foo", "r") as fobj:
|
||||
assert fobj.read() == "foo"
|
||||
|
||||
|
||||
def test_open_dirty_hash(tmp_dir, dvc):
|
||||
|
||||
@@ -99,3 +99,18 @@ def test_should_ignore_dir(omit_dir):
|
||||
new_dirs, _ = ignore(root, dirs, files)
|
||||
|
||||
assert set(new_dirs) == {"dir1", "dir2"}
|
||||
|
||||
|
||||
def test_ignore_order():
|
||||
dvcignore_path = os.path.join(os.path.sep, "ignore_order", ".dvcignore")
|
||||
|
||||
patterns = ["!ac*", "a*", "!ab*"]
|
||||
|
||||
root = os.path.dirname(dvcignore_path)
|
||||
dirs = ["ignore_order"]
|
||||
files = ["ac", "ab", "aa"]
|
||||
|
||||
ignore = mock_dvcignore(dvcignore_path, patterns)
|
||||
_, new_files = ignore(root, dirs, files)
|
||||
|
||||
assert {"ab"} == set(new_files)
|
||||
|
||||
Reference in New Issue
Block a user