mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-18 23:10:14 +01:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a2f1367a9a | ||
|
|
5d97f92bdd | ||
|
|
02b147b6f4 | ||
|
|
fa0f182028 | ||
|
|
15c787f2ac | ||
|
|
4af3919062 | ||
|
|
3a469f2e93 | ||
|
|
51a8c782ae | ||
|
|
b579d06c8e | ||
|
|
039cd49259 | ||
|
|
f516dd252a | ||
|
|
706dd05e8d | ||
|
|
5d957b1830 | ||
|
|
387867fbbe | ||
|
|
7c6351d039 | ||
|
|
613ca0bdb5 | ||
|
|
0fb9870de6 | ||
|
|
73e60a8c13 | ||
|
|
9d1dde4051 | ||
|
|
38dc0f85d4 | ||
|
|
d2dbe9649c | ||
|
|
ece6f2d3ee | ||
|
|
b672622324 | ||
|
|
c9391df313 | ||
|
|
a0c229cf9b | ||
|
|
a59f90fac9 |
+19
-5
@@ -1,5 +1,19 @@
|
||||
languages:
|
||||
Python: true
|
||||
|
||||
exclude_paths:
|
||||
- tests/
|
||||
version: "2"
|
||||
checks:
|
||||
argument-count:
|
||||
config:
|
||||
threshold: 8
|
||||
complex-logic:
|
||||
config:
|
||||
threshold: 6
|
||||
file-lines:
|
||||
config:
|
||||
threshold: 350
|
||||
method-complexity:
|
||||
config:
|
||||
threshold: 15
|
||||
method-count:
|
||||
config:
|
||||
threshold: 30
|
||||
exclude_patterns:
|
||||
- tests
|
||||
|
||||
@@ -4,10 +4,6 @@ test_patterns = [
|
||||
'tests/**'
|
||||
]
|
||||
|
||||
exclude_patterns = [
|
||||
'dvc/utils/compat.py'
|
||||
]
|
||||
|
||||
[[analyzers]]
|
||||
name = 'python'
|
||||
enabled = true
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
* [ ] ❗ I have followed the [Contributing to DVC](https://dvc.org/doc/user-guide/contributing/core) checklist.
|
||||
|
||||
* [ ] 📖 If this PR requires [documentation](https://dvc.org/doc) updates, I have created a separate PR (or issue, at least) in [dvc.org](https://github.com/iterative/dvc.org) and linked it here. If the CLI API is changed, I have updated [tab completion scripts](https://github.com/iterative/dvc/tree/master/scripts/completion).
|
||||
* [ ] 📖 If this PR requires [documentation](https://dvc.org/doc) updates, I have created a separate PR (or issue, at least) in [dvc.org](https://github.com/iterative/dvc.org) and linked it here.
|
||||
|
||||
* [ ] ❌ I will check DeepSource, CodeClimate, and other sanity checks below. (We consider them recommendatory and don't expect everything to be addressed. Please fix things that actually improve code or fix bugs.)
|
||||
|
||||
|
||||
+65
-60
@@ -1,63 +1,68 @@
|
||||
repos:
|
||||
- hooks:
|
||||
- id: black
|
||||
language_version: python3
|
||||
repo: https://github.com/ambv/black
|
||||
rev: 19.10b0
|
||||
- repo: https://github.com/asottile/seed-isort-config
|
||||
- hooks:
|
||||
- id: black
|
||||
language_version: python3
|
||||
repo: https://github.com/ambv/black
|
||||
rev: 19.10b0
|
||||
- repo: https://github.com/asottile/seed-isort-config
|
||||
rev: v2.1.1
|
||||
hooks:
|
||||
- id: seed-isort-config
|
||||
- hooks:
|
||||
- id: isort
|
||||
language_version: python3
|
||||
repo: https://github.com/timothycrosley/isort
|
||||
rev: 4.3.21
|
||||
- hooks:
|
||||
- id: flake8
|
||||
language_version: python3
|
||||
repo: https://gitlab.com/pycqa/flake8
|
||||
rev: 3.7.9
|
||||
- hooks:
|
||||
- args:
|
||||
- -i
|
||||
- '2'
|
||||
id: beautysh
|
||||
language_version: python3
|
||||
repo: https://github.com/lovesegfault/beautysh
|
||||
rev: master
|
||||
- hooks:
|
||||
- args:
|
||||
- git-hook
|
||||
- pre-commit
|
||||
entry: dvc
|
||||
id: dvc-pre-commit
|
||||
language: system
|
||||
name: DVC pre-commit
|
||||
stages:
|
||||
- commit
|
||||
verbose: true
|
||||
require_serial: true
|
||||
- args:
|
||||
- git-hook
|
||||
- pre-push
|
||||
entry: dvc
|
||||
id: dvc-pre-push
|
||||
language: system
|
||||
name: DVC pre-push
|
||||
stages:
|
||||
- push
|
||||
require_serial: true
|
||||
- always_run: true
|
||||
args:
|
||||
- git-hook
|
||||
- post-checkout
|
||||
entry: dvc
|
||||
id: dvc-post-checkout
|
||||
language: system
|
||||
minimum_pre_commit_version: 2.2.0
|
||||
name: DVC post-checkout
|
||||
stages:
|
||||
- post-checkout
|
||||
require_serial: true
|
||||
repo: local
|
||||
- id: seed-isort-config
|
||||
- hooks:
|
||||
- id: isort
|
||||
language_version: python3
|
||||
repo: https://github.com/timothycrosley/isort
|
||||
rev: 4.3.21
|
||||
- hooks:
|
||||
- id: flake8
|
||||
language_version: python3
|
||||
additional_dependencies:
|
||||
- flake8-bugbear
|
||||
- flake8-comprehensions
|
||||
- flake8-debugger
|
||||
- flake8-string-format
|
||||
repo: https://gitlab.com/pycqa/flake8
|
||||
rev: 3.7.9
|
||||
- hooks:
|
||||
- args:
|
||||
- -i
|
||||
- "2"
|
||||
id: beautysh
|
||||
language_version: python3
|
||||
repo: https://github.com/lovesegfault/beautysh
|
||||
rev: master
|
||||
- hooks:
|
||||
- args:
|
||||
- git-hook
|
||||
- pre-commit
|
||||
entry: dvc
|
||||
id: dvc-pre-commit
|
||||
language: system
|
||||
name: DVC pre-commit
|
||||
stages:
|
||||
- commit
|
||||
verbose: true
|
||||
require_serial: true
|
||||
- args:
|
||||
- git-hook
|
||||
- pre-push
|
||||
entry: dvc
|
||||
id: dvc-pre-push
|
||||
language: system
|
||||
name: DVC pre-push
|
||||
stages:
|
||||
- push
|
||||
require_serial: true
|
||||
- always_run: true
|
||||
args:
|
||||
- git-hook
|
||||
- post-checkout
|
||||
entry: dvc
|
||||
id: dvc-post-checkout
|
||||
language: system
|
||||
minimum_pre_commit_version: 2.2.0
|
||||
name: DVC post-checkout
|
||||
stages:
|
||||
- post-checkout
|
||||
require_serial: true
|
||||
repo: local
|
||||
|
||||
+1
-1
@@ -30,7 +30,7 @@ def get_url(path, repo=None, rev=None, remote=None):
|
||||
raise UrlNotDvcRepoError(_repo.url)
|
||||
out = _repo.find_out_by_relpath(path)
|
||||
remote_obj = _repo.cloud.get_remote(remote)
|
||||
return str(remote_obj.checksum_to_path_info(out.checksum))
|
||||
return str(remote_obj.hash_to_path_info(out.checksum))
|
||||
|
||||
|
||||
def open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
|
||||
|
||||
+8
-4
@@ -28,13 +28,15 @@ def _make_remote_property(name):
|
||||
"""
|
||||
|
||||
def getter(self):
|
||||
from dvc.remote import Cache as CloudCache
|
||||
from dvc.remote import get_cloud_tree
|
||||
from dvc.remote.base import CloudCache
|
||||
|
||||
remote = self.config.get(name)
|
||||
if not remote:
|
||||
return None
|
||||
|
||||
return CloudCache(self.repo, name=remote)
|
||||
tree = get_cloud_tree(self.repo, name=remote)
|
||||
return CloudCache(tree)
|
||||
|
||||
getter.__name__ = name
|
||||
return cached_property(getter)
|
||||
@@ -50,7 +52,8 @@ class Cache:
|
||||
CACHE_DIR = "cache"
|
||||
|
||||
def __init__(self, repo):
|
||||
from dvc.remote import Cache as CloudCache
|
||||
from dvc.remote import get_cloud_tree
|
||||
from dvc.remote.local import LocalCache
|
||||
|
||||
self.repo = repo
|
||||
self.config = config = repo.config["cache"]
|
||||
@@ -62,7 +65,8 @@ class Cache:
|
||||
else:
|
||||
settings = {**config, "url": config["dir"]}
|
||||
|
||||
self.local = CloudCache(repo, **settings)
|
||||
tree = get_cloud_tree(repo, **settings)
|
||||
self.local = LocalCache(tree)
|
||||
|
||||
s3 = _make_remote_property("s3")
|
||||
gs = _make_remote_property("gs")
|
||||
|
||||
+16
-10
@@ -8,6 +8,7 @@ from .command import (
|
||||
cache,
|
||||
checkout,
|
||||
commit,
|
||||
completion,
|
||||
config,
|
||||
daemon,
|
||||
dag,
|
||||
@@ -70,6 +71,7 @@ COMMANDS = [
|
||||
dag,
|
||||
daemon,
|
||||
commit,
|
||||
completion,
|
||||
diff,
|
||||
version,
|
||||
update,
|
||||
@@ -140,15 +142,7 @@ def get_parent_parser():
|
||||
return parent_parser
|
||||
|
||||
|
||||
def parse_args(argv=None):
|
||||
"""Parses CLI arguments.
|
||||
|
||||
Args:
|
||||
argv: optional list of arguments to parse. sys.argv is used by default.
|
||||
|
||||
Raises:
|
||||
dvc.exceptions.DvcParserError: raised for argument parsing errors.
|
||||
"""
|
||||
def get_main_parser():
|
||||
parent_parser = get_parent_parser()
|
||||
|
||||
# Main parser
|
||||
@@ -196,6 +190,18 @@ def parse_args(argv=None):
|
||||
for cmd in COMMANDS:
|
||||
cmd.add_parser(subparsers, parent_parser)
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
return parser
|
||||
|
||||
|
||||
def parse_args(argv=None):
|
||||
"""Parses CLI arguments.
|
||||
|
||||
Args:
|
||||
argv: optional list of arguments to parse. sys.argv is used by default.
|
||||
|
||||
Raises:
|
||||
dvc.exceptions.DvcParserError: raised for argument parsing errors.
|
||||
"""
|
||||
parser = get_main_parser()
|
||||
args = parser.parse_args(argv)
|
||||
return args
|
||||
|
||||
+5
-1
@@ -1,6 +1,7 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException, RecursiveAddingWhileUsingFilename
|
||||
|
||||
@@ -62,6 +63,9 @@ def add_parser(subparsers, parent_parser):
|
||||
metavar="<filename>",
|
||||
)
|
||||
parser.add_argument(
|
||||
"targets", nargs="+", help="Input files/directories to add."
|
||||
"targets",
|
||||
nargs="+",
|
||||
help="Input files/directories to add.",
|
||||
choices=completion.Required.FILE,
|
||||
)
|
||||
parser.set_defaults(func=CmdAdd)
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import argparse
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import append_doc_link, fix_subparsers
|
||||
from dvc.command.config import CmdConfig
|
||||
|
||||
@@ -55,5 +56,6 @@ def add_parser(subparsers, parent_parser):
|
||||
help="Path to cache directory. Relative paths are resolved relative "
|
||||
"to the current directory and saved to config relative to the "
|
||||
"config file location.",
|
||||
choices=completion.Required.DIR,
|
||||
)
|
||||
cache_dir_parser.set_defaults(func=CmdCacheDir)
|
||||
|
||||
@@ -4,6 +4,7 @@ import operator
|
||||
|
||||
import colorama
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import CheckoutError
|
||||
from dvc.utils.humanize import get_summary
|
||||
@@ -113,5 +114,6 @@ def add_parser(subparsers, parent_parser):
|
||||
nargs="*",
|
||||
help="DVC-files to checkout. Optional. "
|
||||
"(Finds all DVC-files in the workspace by default.)",
|
||||
choices=completion.Optional.DVC_FILE,
|
||||
)
|
||||
checkout_parser.set_defaults(func=CmdCheckout)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
@@ -66,5 +67,6 @@ def add_parser(subparsers, parent_parser):
|
||||
nargs="*",
|
||||
help="DVC-files to commit. Optional. "
|
||||
"(Finds all DVC-files in the workspace by default.)",
|
||||
choices=completion.Optional.DVC_FILE,
|
||||
)
|
||||
commit_parser.set_defaults(func=CmdCommit)
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
import shtab
|
||||
|
||||
from dvc.command.base import CmdBaseNoRepo, append_doc_link
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
CHOICE_FUNCTIONS = {
|
||||
"bash": {"DVCFile": "_dvc_compgen_DVCFiles"},
|
||||
"zsh": {"DVCFile": "_files -g '(*?.dvc|Dvcfile|dvc.yaml)'"},
|
||||
}
|
||||
PREAMBLE = {
|
||||
"bash": """
|
||||
# $1=COMP_WORDS[1]
|
||||
_dvc_compgen_DVCFiles() {
|
||||
compgen -d -S '/' -- $1 # recurse into subdirs
|
||||
compgen -f -X '!*?.dvc' -- $1
|
||||
compgen -f -X '!*Dvcfile' -- $1
|
||||
compgen -f -X '!*dvc.yaml' -- $1
|
||||
}
|
||||
""",
|
||||
"zsh": "",
|
||||
}
|
||||
|
||||
|
||||
class Optional(shtab.Optional):
|
||||
DVC_FILE = [shtab.Choice("DVCFile", required=False)]
|
||||
|
||||
|
||||
class Required(shtab.Required):
|
||||
DVC_FILE = [shtab.Choice("DVCFile", required=True)]
|
||||
|
||||
|
||||
class CmdCompletion(CmdBaseNoRepo):
|
||||
def run(self):
|
||||
from dvc.cli import get_main_parser
|
||||
|
||||
parser = get_main_parser()
|
||||
shell = self.args.shell
|
||||
script = shtab.complete(
|
||||
parser,
|
||||
shell=shell,
|
||||
preamble=PREAMBLE[shell],
|
||||
choice_functions=CHOICE_FUNCTIONS[shell],
|
||||
)
|
||||
print(script)
|
||||
return 0
|
||||
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
COMPLETION_HELP = "Generate shell tab completion."
|
||||
COMPLETION_DESCRIPTION = "Prints out shell tab completion scripts."
|
||||
completion_parser = subparsers.add_parser(
|
||||
"completion",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(COMPLETION_DESCRIPTION, "completion"),
|
||||
help=COMPLETION_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
completion_parser.add_argument(
|
||||
"-s",
|
||||
"--shell",
|
||||
help="Shell syntax for completions.",
|
||||
default="bash",
|
||||
choices=["bash", "zsh"],
|
||||
)
|
||||
completion_parser.set_defaults(func=CmdCompletion)
|
||||
@@ -1,3 +1,4 @@
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBaseNoRepo, fix_subparsers
|
||||
|
||||
|
||||
@@ -64,5 +65,7 @@ def add_parser(subparsers, parent_parser):
|
||||
description=DAEMON_ANALYTICS_HELP,
|
||||
help=DAEMON_ANALYTICS_HELP,
|
||||
)
|
||||
daemon_analytics_parser.add_argument("target", help="Analytics file.")
|
||||
daemon_analytics_parser.add_argument(
|
||||
"target", help="Analytics file.", choices=completion.Required.FILE
|
||||
)
|
||||
daemon_analytics_parser.set_defaults(func=CmdDaemonAnalytics)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.command.checkout import log_changes
|
||||
from dvc.exceptions import CheckoutError, DvcException
|
||||
@@ -104,6 +105,7 @@ def shared_parent_parser():
|
||||
nargs="*",
|
||||
help="Limit command scope to these DVC-files. "
|
||||
"Using -R, directories to search DVC-files in can also be given.",
|
||||
choices=completion.Optional.DVC_FILE,
|
||||
)
|
||||
|
||||
return parent_parser
|
||||
|
||||
+1
-1
@@ -19,7 +19,7 @@ def _show_md(diff):
|
||||
entries = diff.get(status, [])
|
||||
if not entries:
|
||||
continue
|
||||
paths = sorted([entry["path"] for entry in entries])
|
||||
paths = sorted(entry["path"] for entry in entries)
|
||||
for path in paths:
|
||||
rows.append([status, path])
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
@@ -39,7 +40,10 @@ def add_parser(subparsers, parent_parser):
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
freeze_parser.add_argument(
|
||||
"targets", nargs="+", help="DVC-files to freeze."
|
||||
"targets",
|
||||
nargs="+",
|
||||
help="DVC-files to freeze.",
|
||||
choices=completion.Required.DVC_FILE,
|
||||
)
|
||||
freeze_parser.set_defaults(func=CmdFreeze)
|
||||
|
||||
@@ -52,6 +56,9 @@ def add_parser(subparsers, parent_parser):
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
unfreeze_parser.add_argument(
|
||||
"targets", nargs="+", help="DVC-files to unfreeze."
|
||||
"targets",
|
||||
nargs="+",
|
||||
help="DVC-files to unfreeze.",
|
||||
choices=completion.Required.DVC_FILE,
|
||||
)
|
||||
unfreeze_parser.set_defaults(func=CmdUnfreeze)
|
||||
|
||||
+5
-1
@@ -3,6 +3,7 @@ import logging
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
from . import completion
|
||||
from .base import CmdBaseNoRepo, append_doc_link
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -62,7 +63,9 @@ def add_parser(subparsers, parent_parser):
|
||||
"url", help="Location of DVC or Git repository to download from"
|
||||
)
|
||||
get_parser.add_argument(
|
||||
"path", help="Path to a file or directory within the repository"
|
||||
"path",
|
||||
help="Path to a file or directory within the repository",
|
||||
choices=completion.Required.FILE,
|
||||
)
|
||||
get_parser.add_argument(
|
||||
"-o",
|
||||
@@ -70,6 +73,7 @@ def add_parser(subparsers, parent_parser):
|
||||
nargs="?",
|
||||
help="Destination path to download files to",
|
||||
metavar="<path>",
|
||||
choices=completion.Optional.DIR,
|
||||
)
|
||||
get_parser.add_argument(
|
||||
"--rev",
|
||||
|
||||
@@ -3,6 +3,7 @@ import logging
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
from . import completion
|
||||
from .base import CmdBaseNoRepo, append_doc_link
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -33,6 +34,9 @@ def add_parser(subparsers, parent_parser):
|
||||
"url", help="See `dvc import-url -h` for full list of supported URLs."
|
||||
)
|
||||
get_parser.add_argument(
|
||||
"out", nargs="?", help="Destination path to put data to."
|
||||
"out",
|
||||
nargs="?",
|
||||
help="Destination path to put data to.",
|
||||
choices=completion.Optional.DIR,
|
||||
)
|
||||
get_parser.set_defaults(func=CmdGetUrl)
|
||||
|
||||
+5
-1
@@ -1,6 +1,7 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
@@ -43,7 +44,9 @@ def add_parser(subparsers, parent_parser):
|
||||
"url", help="Location of DVC or Git repository to download from"
|
||||
)
|
||||
import_parser.add_argument(
|
||||
"path", help="Path to a file or directory within the repository"
|
||||
"path",
|
||||
help="Path to a file or directory within the repository",
|
||||
choices=completion.Required.FILE,
|
||||
)
|
||||
import_parser.add_argument(
|
||||
"-o",
|
||||
@@ -51,6 +54,7 @@ def add_parser(subparsers, parent_parser):
|
||||
nargs="?",
|
||||
help="Destination path to download files to",
|
||||
metavar="<path>",
|
||||
choices=completion.Optional.DIR,
|
||||
)
|
||||
import_parser.add_argument(
|
||||
"--rev",
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
@@ -54,11 +55,15 @@ def add_parser(subparsers, parent_parser):
|
||||
"remote://myremote/path/to/file (see `dvc remote`)",
|
||||
)
|
||||
import_parser.add_argument(
|
||||
"out", nargs="?", help="Destination path to put files to."
|
||||
"out",
|
||||
nargs="?",
|
||||
help="Destination path to put files to.",
|
||||
choices=completion.Optional.DIR,
|
||||
)
|
||||
import_parser.add_argument(
|
||||
"--file",
|
||||
help="Specify name of the DVC-file this command will generate.",
|
||||
metavar="<filename>",
|
||||
choices=completion.Optional.DIR,
|
||||
)
|
||||
import_parser.set_defaults(func=CmdImportUrl)
|
||||
|
||||
@@ -2,6 +2,7 @@ import argparse
|
||||
import logging
|
||||
import sys
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBaseNoRepo, append_doc_link
|
||||
from dvc.command.ls.ls_colors import LsColors
|
||||
from dvc.exceptions import DvcException
|
||||
@@ -74,5 +75,6 @@ def add_parser(subparsers, parent_parser):
|
||||
"path",
|
||||
nargs="?",
|
||||
help="Path to directory within the repository to list outputs for",
|
||||
choices=completion.Optional.DIR,
|
||||
)
|
||||
list_parser.set_defaults(func=CmdList)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
|
||||
from dvc.exceptions import BadMetricError, DvcException
|
||||
|
||||
@@ -169,6 +170,7 @@ def add_parser(subparsers, parent_parser):
|
||||
"Limit command scope to these metric files. Using -R, "
|
||||
"directories to search metric files in can also be given."
|
||||
),
|
||||
choices=completion.Optional.FILE,
|
||||
)
|
||||
metrics_show_parser.add_argument(
|
||||
"-a",
|
||||
@@ -235,6 +237,7 @@ def add_parser(subparsers, parent_parser):
|
||||
"directories to search metric files in can also be given."
|
||||
),
|
||||
metavar="<paths>",
|
||||
choices=completion.Optional.FILE,
|
||||
)
|
||||
metrics_diff_parser.add_argument(
|
||||
"-R",
|
||||
|
||||
+7
-2
@@ -1,6 +1,7 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
@@ -36,7 +37,11 @@ def add_parser(subparsers, parent_parser):
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
move_parser.add_argument(
|
||||
"src", help="Source path to a data file or directory."
|
||||
"src",
|
||||
help="Source path to a data file or directory.",
|
||||
choices=completion.Required.FILE,
|
||||
)
|
||||
move_parser.add_argument(
|
||||
"dst", help="Destination path.", choices=completion.Required.FILE,
|
||||
)
|
||||
move_parser.add_argument("dst", help="Destination path.")
|
||||
move_parser.set_defaults(func=CmdMove)
|
||||
|
||||
@@ -2,6 +2,7 @@ import argparse
|
||||
import logging
|
||||
import os
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.schema import PLOT_PROPS
|
||||
@@ -128,6 +129,7 @@ def add_parser(subparsers, parent_parser):
|
||||
"targets",
|
||||
nargs="*",
|
||||
help="Plots files to visualize. Shows all plots by default.",
|
||||
choices=completion.Optional.FILE,
|
||||
)
|
||||
_add_props_arguments(plots_show_parser)
|
||||
_add_output_arguments(plots_show_parser)
|
||||
@@ -149,6 +151,7 @@ def add_parser(subparsers, parent_parser):
|
||||
nargs="*",
|
||||
help="Plots file to visualize. Shows all plots by default.",
|
||||
metavar="<path>",
|
||||
choices=completion.Optional.FILE,
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"revisions", nargs="*", default=None, help="Git commits to plot from",
|
||||
@@ -166,7 +169,9 @@ def add_parser(subparsers, parent_parser):
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
plots_modify_parser.add_argument(
|
||||
"target", help="Metric file to set properties to",
|
||||
"target",
|
||||
help="Metric file to set properties to",
|
||||
choices=completion.Required.FILE,
|
||||
)
|
||||
_add_props_arguments(plots_modify_parser)
|
||||
plots_modify_parser.add_argument(
|
||||
@@ -191,6 +196,7 @@ def _add_props_arguments(parser):
|
||||
)
|
||||
),
|
||||
metavar="<path>",
|
||||
choices=completion.Optional.FILE,
|
||||
)
|
||||
parser.add_argument(
|
||||
"-x", default=None, help="Field name for X axis.", metavar="<field>"
|
||||
@@ -223,6 +229,7 @@ def _add_output_arguments(parser):
|
||||
default=None,
|
||||
help="Destination path to save plots to",
|
||||
metavar="<path>",
|
||||
choices=completion.Optional.DIR,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--show-vega",
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
@@ -34,6 +35,9 @@ def add_parser(subparsers, parent_parser):
|
||||
help="Remove outputs as well.",
|
||||
)
|
||||
remove_parser.add_argument(
|
||||
"targets", nargs="+", help="DVC-files to remove."
|
||||
"targets",
|
||||
nargs="+",
|
||||
help="DVC-files to remove.",
|
||||
choices=completion.Required.DVC_FILE,
|
||||
)
|
||||
remove_parser.set_defaults(func=CmdRemove)
|
||||
|
||||
@@ -2,6 +2,7 @@ import argparse
|
||||
import logging
|
||||
import os
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.command.metrics import show_metrics
|
||||
from dvc.command.status import CmdDataStatus
|
||||
@@ -71,6 +72,7 @@ def add_parser(subparsers, parent_parser):
|
||||
"targets",
|
||||
nargs="*",
|
||||
help=f"Stages to reproduce. '{PIPELINE_FILE}' by default.",
|
||||
choices=completion.Optional.DVC_FILE,
|
||||
)
|
||||
repro_parser.add_argument(
|
||||
"-f",
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
@@ -32,6 +33,9 @@ def add_parser(subparsers, parent_parser):
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
unprotect_parser.add_argument(
|
||||
"targets", nargs="+", help="Data files/directories to unprotect."
|
||||
"targets",
|
||||
nargs="+",
|
||||
help="Data files/directories to unprotect.",
|
||||
choices=completion.Required.FILE,
|
||||
)
|
||||
unprotect_parser.set_defaults(func=CmdUnprotect)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
@@ -32,7 +33,10 @@ def add_parser(subparsers, parent_parser):
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
update_parser.add_argument(
|
||||
"targets", nargs="+", help="DVC-files to update."
|
||||
"targets",
|
||||
nargs="+",
|
||||
help="DVC-files to update.",
|
||||
choices=completion.Required.DVC_FILE,
|
||||
)
|
||||
update_parser.add_argument(
|
||||
"--rev",
|
||||
|
||||
@@ -122,12 +122,12 @@ class CmdVersion(CmdBaseNoRepo):
|
||||
|
||||
@staticmethod
|
||||
def get_supported_remotes():
|
||||
from dvc.remote import REMOTES
|
||||
from dvc.remote import TREES
|
||||
|
||||
supported_remotes = []
|
||||
for remote in REMOTES:
|
||||
if not remote.get_missing_deps():
|
||||
supported_remotes.append(remote.scheme)
|
||||
for tree_cls in TREES:
|
||||
if not tree_cls.get_missing_deps():
|
||||
supported_remotes.append(tree_cls.scheme)
|
||||
|
||||
return ", ".join(supported_remotes)
|
||||
|
||||
|
||||
+1
-1
@@ -222,7 +222,7 @@ def _build_sugiyama_layout(vertexes, edges):
|
||||
vertex.view = VertexViewer(vertex.data)
|
||||
|
||||
# NOTE: determine min box length to create the best layout
|
||||
minw = min([v.view.w for v in vertexes])
|
||||
minw = min(v.view.w for v in vertexes)
|
||||
|
||||
for edge in edges:
|
||||
edge.view = EdgeViewer()
|
||||
|
||||
+4
-4
@@ -3,7 +3,7 @@
|
||||
import logging
|
||||
|
||||
from dvc.config import NoRemoteError
|
||||
from dvc.remote import Remote
|
||||
from dvc.remote import get_remote
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -45,7 +45,7 @@ class DataCloud:
|
||||
raise NoRemoteError(error_msg)
|
||||
|
||||
def _init_remote(self, name):
|
||||
return Remote(self.repo, name=name)
|
||||
return get_remote(self.repo, name=name)
|
||||
|
||||
def push(
|
||||
self, cache, jobs=None, remote=None, show_checksums=False,
|
||||
@@ -85,14 +85,14 @@ class DataCloud:
|
||||
cache, jobs=jobs, remote=remote, show_checksums=show_checksums
|
||||
)
|
||||
|
||||
if not remote.verify:
|
||||
if not remote.tree.verify:
|
||||
self._save_pulled_checksums(cache)
|
||||
|
||||
return downloaded_items_num
|
||||
|
||||
def _save_pulled_checksums(self, cache):
|
||||
for checksum in cache.scheme_keys("local"):
|
||||
cache_file = self.repo.cache.local.checksum_to_path_info(checksum)
|
||||
cache_file = self.repo.cache.local.hash_to_path_info(checksum)
|
||||
if self.repo.cache.local.tree.exists(cache_file):
|
||||
# We can safely save here, as existing corrupted files will
|
||||
# be removed upon status, while files corrupted during
|
||||
|
||||
@@ -12,7 +12,7 @@ from dvc.dependency.param import ParamsDependency
|
||||
from dvc.dependency.s3 import S3Dependency
|
||||
from dvc.dependency.ssh import SSHDependency
|
||||
from dvc.output.base import BaseOutput
|
||||
from dvc.remote import Remote
|
||||
from dvc.remote import get_remote
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
from .repo import RepoDependency
|
||||
@@ -54,7 +54,7 @@ SCHEMA.update(ParamsDependency.PARAM_SCHEMA)
|
||||
def _get(stage, p, info):
|
||||
parsed = urlparse(p) if p else None
|
||||
if parsed and parsed.scheme == "remote":
|
||||
remote = Remote(stage.repo, name=parsed.netloc)
|
||||
remote = get_remote(stage.repo, name=parsed.netloc)
|
||||
return DEP_MAP[remote.scheme](stage, p, info, remote=remote)
|
||||
|
||||
if info and info.get(RepoDependency.PARAM_REPO):
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from dvc.dependency.base import BaseDependency
|
||||
from dvc.output.base import BaseOutput
|
||||
from dvc.remote.azure import AzureRemote
|
||||
from dvc.remote.azure import AzureRemoteTree
|
||||
|
||||
|
||||
class AzureDependency(BaseDependency, BaseOutput):
|
||||
REMOTE = AzureRemote
|
||||
TREE_CLS = AzureRemoteTree
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from dvc.dependency.base import BaseDependency
|
||||
from dvc.output.base import BaseOutput
|
||||
from dvc.remote.http import HTTPRemote
|
||||
from dvc.remote.http import HTTPRemoteTree
|
||||
|
||||
|
||||
class HTTPDependency(BaseDependency, BaseOutput):
|
||||
REMOTE = HTTPRemote
|
||||
TREE_CLS = HTTPRemoteTree
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from dvc.remote.https import HTTPSRemote
|
||||
from dvc.remote.https import HTTPSRemoteTree
|
||||
|
||||
from .http import HTTPDependency
|
||||
|
||||
|
||||
class HTTPSDependency(HTTPDependency):
|
||||
REMOTE = HTTPSRemote
|
||||
TREE_CLS = HTTPSRemoteTree
|
||||
|
||||
@@ -64,8 +64,8 @@ class RepoDependency(LocalDependency):
|
||||
|
||||
# We are polluting our repo cache with some dir listing here
|
||||
if tree.isdir(path):
|
||||
return self.repo.cache.local.get_checksum(path, tree)
|
||||
return tree.get_file_checksum(path)
|
||||
return self.repo.cache.local.get_hash(path, tree)
|
||||
return tree.get_file_hash(path)
|
||||
|
||||
def status(self):
|
||||
current_checksum = self._get_checksum(locked=True)
|
||||
|
||||
+29
-5
@@ -1,12 +1,15 @@
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from itertools import groupby
|
||||
|
||||
from funcy import cached_property
|
||||
from pathspec import PathSpec
|
||||
from pathspec.patterns import GitWildMatchPattern
|
||||
from pathspec.util import normalize_file
|
||||
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.scm.tree import BaseTree
|
||||
from dvc.system import System
|
||||
from dvc.utils import relpath
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -27,7 +30,17 @@ class DvcIgnorePatterns(DvcIgnore):
|
||||
self.dirname = os.path.normpath(os.path.dirname(ignore_file_path))
|
||||
|
||||
with tree.open(ignore_file_path, encoding="utf-8") as fobj:
|
||||
self.ignore_spec = PathSpec.from_lines(GitWildMatchPattern, fobj)
|
||||
path_spec_lines = fobj.readlines()
|
||||
regex_pattern_list = map(
|
||||
GitWildMatchPattern.pattern_to_regex, path_spec_lines
|
||||
)
|
||||
self.ignore_spec = [
|
||||
(ignore, re.compile("|".join(item[0] for item in group)))
|
||||
for ignore, group in groupby(
|
||||
regex_pattern_list, lambda x: x[1]
|
||||
)
|
||||
if ignore is not None
|
||||
]
|
||||
|
||||
def __call__(self, root, dirs, files):
|
||||
files = [f for f in files if not self.matches(root, f)]
|
||||
@@ -48,7 +61,16 @@ class DvcIgnorePatterns(DvcIgnore):
|
||||
else:
|
||||
return False
|
||||
|
||||
return self.ignore_spec.match_file(path)
|
||||
if not System.is_unix():
|
||||
path = normalize_file(path)
|
||||
return self.ignore(path)
|
||||
|
||||
def ignore(self, path):
|
||||
result = False
|
||||
for ignore, pattern in self.ignore_spec:
|
||||
if pattern.match(path):
|
||||
result = ignore
|
||||
return result
|
||||
|
||||
def __hash__(self):
|
||||
return hash(self.ignore_file_path)
|
||||
@@ -208,8 +230,10 @@ class CleanTree(BaseTree):
|
||||
return False
|
||||
return True
|
||||
|
||||
def walk(self, top, topdown=True):
|
||||
for root, dirs, files in self.tree.walk(top, topdown):
|
||||
def walk(self, top, topdown=True, onerror=None):
|
||||
for root, dirs, files in self.tree.walk(
|
||||
top, topdown=topdown, onerror=onerror
|
||||
):
|
||||
dirs[:], files[:] = self.dvcignore(
|
||||
os.path.abspath(root), dirs, files
|
||||
)
|
||||
|
||||
@@ -10,10 +10,10 @@ from dvc.output.hdfs import HDFSOutput
|
||||
from dvc.output.local import LocalOutput
|
||||
from dvc.output.s3 import S3Output
|
||||
from dvc.output.ssh import SSHOutput
|
||||
from dvc.remote import Remote
|
||||
from dvc.remote.hdfs import HDFSRemote
|
||||
from dvc.remote.local import LocalRemote
|
||||
from dvc.remote.s3 import S3Remote
|
||||
from dvc.remote import get_remote
|
||||
from dvc.remote.hdfs import HDFSRemoteTree
|
||||
from dvc.remote.local import LocalRemoteTree
|
||||
from dvc.remote.s3 import S3RemoteTree
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
OUTS = [
|
||||
@@ -47,9 +47,9 @@ CHECKSUM_SCHEMA = Any(
|
||||
# so when a few types of outputs share the same name, we only need
|
||||
# specify it once.
|
||||
CHECKSUMS_SCHEMA = {
|
||||
LocalRemote.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
|
||||
S3Remote.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
|
||||
HDFSRemote.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
|
||||
LocalRemoteTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
|
||||
S3RemoteTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
|
||||
HDFSRemoteTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
|
||||
}
|
||||
|
||||
SCHEMA = CHECKSUMS_SCHEMA.copy()
|
||||
@@ -66,7 +66,7 @@ def _get(
|
||||
parsed = urlparse(p)
|
||||
|
||||
if parsed.scheme == "remote":
|
||||
remote = Remote(stage.repo, name=parsed.netloc)
|
||||
remote = get_remote(stage.repo, name=parsed.netloc)
|
||||
return OUTS_MAP[remote.scheme](
|
||||
stage,
|
||||
p,
|
||||
|
||||
+28
-17
@@ -12,7 +12,7 @@ from dvc.exceptions import (
|
||||
DvcException,
|
||||
RemoteCacheRequiredError,
|
||||
)
|
||||
from dvc.remote.base import BaseRemote
|
||||
from dvc.remote.base import BaseRemoteTree, Remote
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -47,7 +47,8 @@ class OutputIsStageFileError(DvcException):
|
||||
class BaseOutput:
|
||||
IS_DEPENDENCY = False
|
||||
|
||||
REMOTE = BaseRemote
|
||||
REMOTE_CLS = Remote
|
||||
TREE_CLS = BaseRemoteTree
|
||||
|
||||
PARAM_PATH = "path"
|
||||
PARAM_CACHE = "cache"
|
||||
@@ -105,7 +106,11 @@ class BaseOutput:
|
||||
self.repo = stage.repo if stage else None
|
||||
self.def_path = path
|
||||
self.info = info
|
||||
self.remote = remote or self.REMOTE(self.repo, {})
|
||||
if remote:
|
||||
self.remote = remote
|
||||
else:
|
||||
tree = self.TREE_CLS(self.repo, {})
|
||||
self.remote = self.REMOTE_CLS(tree)
|
||||
self.use_cache = False if self.IS_DEPENDENCY else cache
|
||||
self.metric = False if self.IS_DEPENDENCY else metric
|
||||
self.plot = False if self.IS_DEPENDENCY else plot
|
||||
@@ -119,7 +124,7 @@ class BaseOutput:
|
||||
if remote:
|
||||
parsed = urlparse(path)
|
||||
return remote.path_info / parsed.path.lstrip("/")
|
||||
return self.REMOTE.TREE_CLS.PATH_CLS(path)
|
||||
return self.TREE_CLS.PATH_CLS(path)
|
||||
|
||||
def __repr__(self):
|
||||
return "{class_name}: '{def_path}'".format(
|
||||
@@ -131,7 +136,7 @@ class BaseOutput:
|
||||
|
||||
@property
|
||||
def scheme(self):
|
||||
return self.REMOTE.scheme
|
||||
return self.TREE_CLS.scheme
|
||||
|
||||
@property
|
||||
def is_in_repo(self):
|
||||
@@ -154,30 +159,30 @@ class BaseOutput:
|
||||
|
||||
@classmethod
|
||||
def supported(cls, url):
|
||||
return cls.REMOTE.supported(url)
|
||||
return cls.TREE_CLS.supported(url)
|
||||
|
||||
@property
|
||||
def cache_path(self):
|
||||
return self.cache.checksum_to_path_info(self.checksum).url
|
||||
return self.cache.hash_to_path_info(self.checksum).url
|
||||
|
||||
@property
|
||||
def checksum_type(self):
|
||||
return self.remote.PARAM_CHECKSUM
|
||||
return self.remote.tree.PARAM_CHECKSUM
|
||||
|
||||
@property
|
||||
def checksum(self):
|
||||
return self.info.get(self.remote.PARAM_CHECKSUM)
|
||||
return self.info.get(self.remote.tree.PARAM_CHECKSUM)
|
||||
|
||||
@checksum.setter
|
||||
def checksum(self, checksum):
|
||||
self.info[self.remote.PARAM_CHECKSUM] = checksum
|
||||
self.info[self.remote.tree.PARAM_CHECKSUM] = checksum
|
||||
|
||||
def get_checksum(self):
|
||||
return self.remote.get_checksum(self.path_info)
|
||||
return self.remote.get_hash(self.path_info)
|
||||
|
||||
@property
|
||||
def is_dir_checksum(self):
|
||||
return self.remote.is_dir_checksum(self.checksum)
|
||||
return self.remote.is_dir_hash(self.checksum)
|
||||
|
||||
@property
|
||||
def exists(self):
|
||||
@@ -234,6 +239,12 @@ class BaseOutput:
|
||||
|
||||
self.repo.scm.ignore(self.fspath)
|
||||
|
||||
def ignore_remove(self):
|
||||
if not self.use_scm_ignore:
|
||||
return
|
||||
|
||||
self.repo.scm.ignore_remove(self.fspath)
|
||||
|
||||
def save(self):
|
||||
if not self.exists:
|
||||
raise self.DoesNotExistError(self)
|
||||
@@ -331,8 +342,8 @@ class BaseOutput:
|
||||
if self.scheme != "local":
|
||||
return
|
||||
|
||||
if ignore_remove and self.use_scm_ignore:
|
||||
self.repo.scm.ignore_remove(self.fspath)
|
||||
if ignore_remove:
|
||||
self.ignore_remove()
|
||||
|
||||
def move(self, out):
|
||||
if self.scheme == "local" and self.use_scm_ignore:
|
||||
@@ -357,7 +368,7 @@ class BaseOutput:
|
||||
|
||||
def unprotect(self):
|
||||
if self.exists:
|
||||
self.remote.unprotect(self.path_info)
|
||||
self.remote.tree.unprotect(self.path_info)
|
||||
|
||||
def get_dir_cache(self, **kwargs):
|
||||
if not self.is_dir_checksum:
|
||||
@@ -417,8 +428,8 @@ class BaseOutput:
|
||||
filter_path = str(filter_info) if filter_info else None
|
||||
is_win = os.name == "nt"
|
||||
for entry in self.dir_cache:
|
||||
checksum = entry[self.remote.PARAM_CHECKSUM]
|
||||
entry_relpath = entry[self.remote.PARAM_RELPATH]
|
||||
checksum = entry[self.remote.tree.PARAM_CHECKSUM]
|
||||
entry_relpath = entry[self.remote.tree.PARAM_RELPATH]
|
||||
if is_win:
|
||||
entry_relpath = entry_relpath.replace("/", os.sep)
|
||||
entry_path = os.path.join(path, entry_relpath)
|
||||
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
from dvc.output.s3 import S3Output
|
||||
from dvc.remote.gs import GSRemote
|
||||
from dvc.remote.gs import GSRemoteTree
|
||||
|
||||
|
||||
class GSOutput(S3Output):
|
||||
REMOTE = GSRemote
|
||||
TREE_CLS = GSRemoteTree
|
||||
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
from dvc.output.base import BaseOutput
|
||||
from dvc.remote.hdfs import HDFSRemote
|
||||
from dvc.remote.hdfs import HDFSRemoteTree
|
||||
|
||||
|
||||
class HDFSOutput(BaseOutput):
|
||||
REMOTE = HDFSRemote
|
||||
TREE_CLS = HDFSRemoteTree
|
||||
|
||||
+7
-6
@@ -5,7 +5,7 @@ from urllib.parse import urlparse
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.istextfile import istextfile
|
||||
from dvc.output.base import BaseOutput
|
||||
from dvc.remote.local import LocalRemote
|
||||
from dvc.remote.local import LocalRemote, LocalRemoteTree
|
||||
from dvc.utils import relpath
|
||||
from dvc.utils.fs import path_isin
|
||||
|
||||
@@ -13,7 +13,8 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LocalOutput(BaseOutput):
|
||||
REMOTE = LocalRemote
|
||||
REMOTE_CLS = LocalRemote
|
||||
TREE_CLS = LocalRemoteTree
|
||||
sep = os.sep
|
||||
|
||||
def __init__(self, stage, path, *args, **kwargs):
|
||||
@@ -33,15 +34,15 @@ class LocalOutput(BaseOutput):
|
||||
#
|
||||
# FIXME: if we have Windows path containing / or posix one with \
|
||||
# then we have #2059 bug and can't really handle that.
|
||||
p = self.REMOTE.TREE_CLS.PATH_CLS(path)
|
||||
if not p.is_absolute():
|
||||
p = self.TREE_CLS.PATH_CLS(path)
|
||||
if self.stage and not p.is_absolute():
|
||||
p = self.stage.wdir / p
|
||||
|
||||
abs_p = os.path.abspath(os.path.normpath(p))
|
||||
return self.REMOTE.TREE_CLS.PATH_CLS(abs_p)
|
||||
return self.TREE_CLS.PATH_CLS(abs_p)
|
||||
|
||||
def __str__(self):
|
||||
if not self.is_in_repo:
|
||||
if not self.repo or not self.is_in_repo:
|
||||
return str(self.def_path)
|
||||
|
||||
cur_dir = os.getcwd()
|
||||
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
from dvc.output.base import BaseOutput
|
||||
from dvc.remote.s3 import S3Remote
|
||||
from dvc.remote.s3 import S3RemoteTree
|
||||
|
||||
|
||||
class S3Output(BaseOutput):
|
||||
REMOTE = S3Remote
|
||||
TREE_CLS = S3RemoteTree
|
||||
|
||||
+3
-2
@@ -1,6 +1,7 @@
|
||||
from dvc.output.base import BaseOutput
|
||||
from dvc.remote.ssh import SSHRemote
|
||||
from dvc.remote.ssh import SSHRemote, SSHRemoteTree
|
||||
|
||||
|
||||
class SSHOutput(BaseOutput):
|
||||
REMOTE = SSHRemote
|
||||
REMOTE_CLS = SSHRemote
|
||||
TREE_CLS = SSHRemoteTree
|
||||
|
||||
+41
-45
@@ -1,45 +1,37 @@
|
||||
import posixpath
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from dvc.remote.azure import AzureCache, AzureRemote
|
||||
from dvc.remote.gdrive import GDriveRemote
|
||||
from dvc.remote.gs import GSCache, GSRemote
|
||||
from dvc.remote.hdfs import HDFSCache, HDFSRemote
|
||||
from dvc.remote.http import HTTPRemote
|
||||
from dvc.remote.https import HTTPSRemote
|
||||
from dvc.remote.local import LocalCache, LocalRemote
|
||||
from dvc.remote.oss import OSSRemote
|
||||
from dvc.remote.s3 import S3Cache, S3Remote
|
||||
from dvc.remote.ssh import SSHCache, SSHRemote
|
||||
from dvc.remote.azure import AzureRemoteTree
|
||||
from dvc.remote.base import Remote
|
||||
from dvc.remote.gdrive import GDriveRemoteTree
|
||||
from dvc.remote.gs import GSRemoteTree
|
||||
from dvc.remote.hdfs import HDFSRemoteTree
|
||||
from dvc.remote.http import HTTPRemoteTree
|
||||
from dvc.remote.https import HTTPSRemoteTree
|
||||
from dvc.remote.local import LocalRemote, LocalRemoteTree
|
||||
from dvc.remote.oss import OSSRemoteTree
|
||||
from dvc.remote.s3 import S3RemoteTree
|
||||
from dvc.remote.ssh import SSHRemote, SSHRemoteTree
|
||||
|
||||
CACHES = [
|
||||
AzureCache,
|
||||
GSCache,
|
||||
HDFSCache,
|
||||
S3Cache,
|
||||
SSHCache,
|
||||
# LocalCache is the default
|
||||
]
|
||||
|
||||
REMOTES = [
|
||||
AzureRemote,
|
||||
GDriveRemote,
|
||||
GSRemote,
|
||||
HDFSRemote,
|
||||
HTTPRemote,
|
||||
HTTPSRemote,
|
||||
S3Remote,
|
||||
SSHRemote,
|
||||
OSSRemote,
|
||||
# NOTE: LocalRemote is the default
|
||||
TREES = [
|
||||
AzureRemoteTree,
|
||||
GDriveRemoteTree,
|
||||
GSRemoteTree,
|
||||
HDFSRemoteTree,
|
||||
HTTPRemoteTree,
|
||||
HTTPSRemoteTree,
|
||||
S3RemoteTree,
|
||||
SSHRemoteTree,
|
||||
OSSRemoteTree,
|
||||
# NOTE: LocalRemoteTree is the default
|
||||
]
|
||||
|
||||
|
||||
def _get(remote_conf, remotes, default):
|
||||
for remote in remotes:
|
||||
if remote.supported(remote_conf):
|
||||
return remote
|
||||
return default
|
||||
def _get_tree(remote_conf):
|
||||
for tree_cls in TREES:
|
||||
if tree_cls.supported(remote_conf):
|
||||
return tree_cls
|
||||
return LocalRemoteTree
|
||||
|
||||
|
||||
def _get_conf(repo, **kwargs):
|
||||
@@ -51,16 +43,6 @@ def _get_conf(repo, **kwargs):
|
||||
return _resolve_remote_refs(repo.config, remote_conf)
|
||||
|
||||
|
||||
def Remote(repo, **kwargs):
|
||||
remote_conf = _get_conf(repo, **kwargs)
|
||||
return _get(remote_conf, REMOTES, LocalRemote)(repo, remote_conf)
|
||||
|
||||
|
||||
def Cache(repo, **kwargs):
|
||||
remote_conf = _get_conf(repo, **kwargs)
|
||||
return _get(remote_conf, CACHES, LocalCache)(repo, remote_conf)
|
||||
|
||||
|
||||
def _resolve_remote_refs(config, remote_conf):
|
||||
# Support for cross referenced remotes.
|
||||
# This will merge the settings, shadowing base ref with remote_conf.
|
||||
@@ -90,3 +72,17 @@ def _resolve_remote_refs(config, remote_conf):
|
||||
base = config["remote"][parsed.netloc]
|
||||
url = posixpath.join(base["url"], parsed.path.lstrip("/"))
|
||||
return {**base, **remote_conf, "url": url}
|
||||
|
||||
|
||||
def get_cloud_tree(repo, **kwargs):
|
||||
remote_conf = _get_conf(repo, **kwargs)
|
||||
return _get_tree(remote_conf)(repo, remote_conf)
|
||||
|
||||
|
||||
def get_remote(repo, **kwargs):
|
||||
tree = get_cloud_tree(repo, **kwargs)
|
||||
if tree.scheme == "local":
|
||||
return LocalRemote(tree)
|
||||
if tree.scheme == "ssh":
|
||||
return SSHRemote(tree)
|
||||
return Remote(tree)
|
||||
|
||||
+9
-17
@@ -7,17 +7,22 @@ from funcy import cached_property, wrap_prop
|
||||
|
||||
from dvc.path_info import CloudURLInfo
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.remote.base import BaseRemote, BaseRemoteTree, CacheMixin
|
||||
from dvc.remote.base import BaseRemoteTree
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AzureRemoteTree(BaseRemoteTree):
|
||||
scheme = Schemes.AZURE
|
||||
PATH_CLS = CloudURLInfo
|
||||
REQUIRES = {"azure-storage-blob": "azure.storage.blob"}
|
||||
PARAM_CHECKSUM = "etag"
|
||||
COPY_POLL_SECONDS = 5
|
||||
LIST_OBJECT_PAGE_SIZE = 5000
|
||||
|
||||
def __init__(self, remote, config):
|
||||
super().__init__(remote, config)
|
||||
def __init__(self, repo, config):
|
||||
super().__init__(repo, config)
|
||||
|
||||
url = config.get("url", "azure://")
|
||||
self.path_info = self.PATH_CLS(url)
|
||||
@@ -108,7 +113,7 @@ class AzureRemoteTree(BaseRemoteTree):
|
||||
logger.debug(f"Removing {path_info}")
|
||||
self.blob_service.delete_blob(path_info.bucket, path_info.path)
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
def get_file_hash(self, path_info):
|
||||
return self.get_etag(path_info)
|
||||
|
||||
def _upload(
|
||||
@@ -132,16 +137,3 @@ class AzureRemoteTree(BaseRemoteTree):
|
||||
to_file,
|
||||
progress_callback=pbar.update_to,
|
||||
)
|
||||
|
||||
|
||||
class AzureRemote(BaseRemote):
|
||||
scheme = Schemes.AZURE
|
||||
REQUIRES = {"azure-storage-blob": "azure.storage.blob"}
|
||||
TREE_CLS = AzureRemoteTree
|
||||
PARAM_CHECKSUM = "etag"
|
||||
COPY_POLL_SECONDS = 5
|
||||
LIST_OBJECT_PAGE_SIZE = 5000
|
||||
|
||||
|
||||
class AzureCache(AzureRemote, CacheMixin):
|
||||
pass
|
||||
|
||||
+538
-510
File diff suppressed because it is too large
Load Diff
+12
-17
@@ -14,7 +14,7 @@ from funcy.py3 import cat
|
||||
from dvc.exceptions import DvcException, FileMissingError
|
||||
from dvc.path_info import CloudURLInfo
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.remote.base import BaseRemote, BaseRemoteTree
|
||||
from dvc.remote.base import BaseRemoteTree
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.utils import format_link, tmp_fname
|
||||
from dvc.utils.stream import IterStream
|
||||
@@ -85,7 +85,13 @@ class GDriveURLInfo(CloudURLInfo):
|
||||
|
||||
|
||||
class GDriveRemoteTree(BaseRemoteTree):
|
||||
scheme = Schemes.GDRIVE
|
||||
PATH_CLS = GDriveURLInfo
|
||||
REQUIRES = {"pydrive2": "pydrive2"}
|
||||
DEFAULT_VERIFY = True
|
||||
# Always prefer traverse for GDrive since API usage quotas are a concern.
|
||||
TRAVERSE_WEIGHT_MULTIPLIER = 1
|
||||
TRAVERSE_PREFIX_LEN = 2
|
||||
|
||||
GDRIVE_CREDENTIALS_DATA = "GDRIVE_CREDENTIALS_DATA"
|
||||
DEFAULT_USER_CREDENTIALS_FILE = "gdrive-user-credentials.json"
|
||||
@@ -93,8 +99,8 @@ class GDriveRemoteTree(BaseRemoteTree):
|
||||
DEFAULT_GDRIVE_CLIENT_ID = "710796635688-iivsgbgsb6uv1fap6635dhvuei09o66c.apps.googleusercontent.com" # noqa: E501
|
||||
DEFAULT_GDRIVE_CLIENT_SECRET = "a1Fz59uTpVNeG_VGuSKDLJXv"
|
||||
|
||||
def __init__(self, remote, config):
|
||||
super().__init__(remote, config)
|
||||
def __init__(self, repo, config):
|
||||
super().__init__(repo, config)
|
||||
|
||||
self.path_info = self.PATH_CLS(config["url"])
|
||||
|
||||
@@ -122,13 +128,12 @@ class GDriveRemoteTree(BaseRemoteTree):
|
||||
self._client_secret = config.get("gdrive_client_secret")
|
||||
self._validate_config()
|
||||
self._gdrive_user_credentials_path = (
|
||||
tmp_fname(os.path.join(self.remote.repo.tmp_dir, ""))
|
||||
tmp_fname(os.path.join(self.repo.tmp_dir, ""))
|
||||
if os.getenv(GDriveRemoteTree.GDRIVE_CREDENTIALS_DATA)
|
||||
else config.get(
|
||||
"gdrive_user_credentials_file",
|
||||
os.path.join(
|
||||
self.remote.repo.tmp_dir,
|
||||
self.DEFAULT_USER_CREDENTIALS_FILE,
|
||||
self.repo.tmp_dir, self.DEFAULT_USER_CREDENTIALS_FILE,
|
||||
),
|
||||
)
|
||||
)
|
||||
@@ -559,7 +564,7 @@ class GDriveRemoteTree(BaseRemoteTree):
|
||||
item_id = self._get_item_id(path_info)
|
||||
self.gdrive_delete_file(item_id)
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
def get_file_hash(self, path_info):
|
||||
raise NotImplementedError
|
||||
|
||||
def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
|
||||
@@ -574,13 +579,3 @@ class GDriveRemoteTree(BaseRemoteTree):
|
||||
def _download(self, from_info, to_file, name=None, no_progress_bar=False):
|
||||
item_id = self._get_item_id(from_info)
|
||||
self._gdrive_download_file(item_id, to_file, name, no_progress_bar)
|
||||
|
||||
|
||||
class GDriveRemote(BaseRemote):
|
||||
scheme = Schemes.GDRIVE
|
||||
REQUIRES = {"pydrive2": "pydrive2"}
|
||||
TREE_CLS = GDriveRemoteTree
|
||||
DEFAULT_VERIFY = True
|
||||
# Always prefer traverse for GDrive since API usage quotas are a concern.
|
||||
TRAVERSE_WEIGHT_MULTIPLIER = 1
|
||||
TRAVERSE_PREFIX_LEN = 2
|
||||
|
||||
+25
-16
@@ -9,7 +9,7 @@ from funcy import cached_property, wrap_prop
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.path_info import CloudURLInfo
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.remote.base import BaseRemote, BaseRemoteTree, CacheMixin
|
||||
from dvc.remote.base import BaseRemoteTree
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -65,10 +65,13 @@ def _upload_to_bucket(
|
||||
|
||||
|
||||
class GSRemoteTree(BaseRemoteTree):
|
||||
scheme = Schemes.GS
|
||||
PATH_CLS = CloudURLInfo
|
||||
REQUIRES = {"google-cloud-storage": "google.cloud.storage"}
|
||||
PARAM_CHECKSUM = "md5"
|
||||
|
||||
def __init__(self, remote, config):
|
||||
super().__init__(remote, config)
|
||||
def __init__(self, repo, config):
|
||||
super().__init__(repo, config)
|
||||
|
||||
url = config.get("url", "gs:///")
|
||||
self.path_info = self.PATH_CLS(url)
|
||||
@@ -88,13 +91,30 @@ class GSRemoteTree(BaseRemoteTree):
|
||||
)
|
||||
|
||||
def _generate_download_url(self, path_info, expires=3600):
|
||||
import google.auth
|
||||
from google.auth import compute_engine
|
||||
|
||||
expiration = timedelta(seconds=int(expires))
|
||||
|
||||
bucket = self.gs.bucket(path_info.bucket)
|
||||
blob = bucket.get_blob(path_info.path)
|
||||
if blob is None:
|
||||
raise FileNotFoundError
|
||||
return blob.generate_signed_url(expiration=expiration)
|
||||
|
||||
if isinstance(
|
||||
blob.client._credentials, google.auth.credentials.Signing
|
||||
):
|
||||
# sign if we're able to sign with credentials.
|
||||
return blob.generate_signed_url(expiration=expiration)
|
||||
|
||||
auth_request = google.auth.transport.requests.Request()
|
||||
# create signing credentials with the default credentials
|
||||
# for use with Compute Engine and other environments where
|
||||
# Client credentials cannot sign.
|
||||
signing_credentials = compute_engine.IDTokenCredentials(
|
||||
auth_request, ""
|
||||
)
|
||||
return signing_credentials.signer.sign(blob)
|
||||
|
||||
def exists(self, path_info):
|
||||
"""Check if the blob exists. If it does not exist,
|
||||
@@ -157,7 +177,7 @@ class GSRemoteTree(BaseRemoteTree):
|
||||
to_bucket = self.gs.bucket(to_info.bucket)
|
||||
from_bucket.copy_blob(blob, to_bucket, new_name=to_info.path)
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
def get_file_hash(self, path_info):
|
||||
import base64
|
||||
import codecs
|
||||
|
||||
@@ -193,14 +213,3 @@ class GSRemoteTree(BaseRemoteTree):
|
||||
disable=no_progress_bar,
|
||||
) as wrapped:
|
||||
blob.download_to_file(wrapped)
|
||||
|
||||
|
||||
class GSRemote(BaseRemote):
|
||||
scheme = Schemes.GS
|
||||
REQUIRES = {"google-cloud-storage": "google.cloud.storage"}
|
||||
TREE_CLS = GSRemoteTree
|
||||
PARAM_CHECKSUM = "md5"
|
||||
|
||||
|
||||
class GSCache(GSRemote, CacheMixin):
|
||||
pass
|
||||
|
||||
+10
-17
@@ -11,15 +11,21 @@ from urllib.parse import urlparse
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.utils import fix_env, tmp_fname
|
||||
|
||||
from .base import BaseRemote, BaseRemoteTree, CacheMixin, RemoteCmdError
|
||||
from .base import BaseRemoteTree, RemoteCmdError
|
||||
from .pool import get_connection
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HDFSRemoteTree(BaseRemoteTree):
|
||||
def __init__(self, remote, config):
|
||||
super().__init__(remote, config)
|
||||
scheme = Schemes.HDFS
|
||||
REQUIRES = {"pyarrow": "pyarrow"}
|
||||
REGEX = r"^hdfs://((?P<user>.*)@)?.*$"
|
||||
PARAM_CHECKSUM = "checksum"
|
||||
TRAVERSE_PREFIX_LEN = 2
|
||||
|
||||
def __init__(self, repo, config):
|
||||
super().__init__(repo, config)
|
||||
|
||||
self.path_info = None
|
||||
url = config.get("url")
|
||||
@@ -153,7 +159,7 @@ class HDFSRemoteTree(BaseRemoteTree):
|
||||
assert match is not None
|
||||
return match.group(gname)
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
def get_file_hash(self, path_info):
|
||||
# NOTE: pyarrow doesn't support checksum, so we need to use hadoop
|
||||
regex = r".*\t.*\t(?P<checksum>.*)"
|
||||
stdout = self.hadoop_fs(
|
||||
@@ -173,16 +179,3 @@ class HDFSRemoteTree(BaseRemoteTree):
|
||||
with self.hdfs(from_info) as hdfs:
|
||||
with open(to_file, "wb+") as fobj:
|
||||
hdfs.download(from_info.path, fobj)
|
||||
|
||||
|
||||
class HDFSRemote(BaseRemote):
|
||||
scheme = Schemes.HDFS
|
||||
REGEX = r"^hdfs://((?P<user>.*)@)?.*$"
|
||||
PARAM_CHECKSUM = "checksum"
|
||||
REQUIRES = {"pyarrow": "pyarrow"}
|
||||
TREE_CLS = HDFSRemoteTree
|
||||
TRAVERSE_PREFIX_LEN = 2
|
||||
|
||||
|
||||
class HDFSCache(HDFSRemote, CacheMixin):
|
||||
pass
|
||||
|
||||
+7
-17
@@ -8,7 +8,7 @@ import dvc.prompt as prompt
|
||||
from dvc.exceptions import DvcException, HTTPError
|
||||
from dvc.path_info import HTTPURLInfo
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.remote.base import BaseRemote, BaseRemoteTree
|
||||
from dvc.remote.base import BaseRemoteTree
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -24,15 +24,18 @@ def ask_password(host, user):
|
||||
|
||||
|
||||
class HTTPRemoteTree(BaseRemoteTree):
|
||||
scheme = Schemes.HTTP
|
||||
PATH_CLS = HTTPURLInfo
|
||||
PARAM_CHECKSUM = "etag"
|
||||
CAN_TRAVERSE = False
|
||||
|
||||
SESSION_RETRIES = 5
|
||||
SESSION_BACKOFF_FACTOR = 0.1
|
||||
REQUEST_TIMEOUT = 10
|
||||
CHUNK_SIZE = 2 ** 16
|
||||
|
||||
def __init__(self, remote, config):
|
||||
super().__init__(remote, config)
|
||||
def __init__(self, repo, config):
|
||||
super().__init__(repo, config)
|
||||
|
||||
url = config.get("url")
|
||||
if url:
|
||||
@@ -121,7 +124,7 @@ class HTTPRemoteTree(BaseRemoteTree):
|
||||
def exists(self, path_info):
|
||||
return bool(self.request("HEAD", path_info.url))
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
def get_file_hash(self, path_info):
|
||||
url = path_info.url
|
||||
headers = self.request("HEAD", url).headers
|
||||
etag = headers.get("ETag") or headers.get("Content-MD5")
|
||||
@@ -185,16 +188,3 @@ class HTTPRemoteTree(BaseRemoteTree):
|
||||
def _content_length(response):
|
||||
res = response.headers.get("Content-Length")
|
||||
return int(res) if res else None
|
||||
|
||||
|
||||
class HTTPRemote(BaseRemote):
|
||||
scheme = Schemes.HTTP
|
||||
PARAM_CHECKSUM = "etag"
|
||||
CAN_TRAVERSE = False
|
||||
TREE_CLS = HTTPRemoteTree
|
||||
|
||||
def list_paths(self, prefix=None, progress_callback=None):
|
||||
raise NotImplementedError
|
||||
|
||||
def gc(self):
|
||||
raise NotImplementedError
|
||||
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
from .http import HTTPRemote
|
||||
from .http import HTTPRemoteTree
|
||||
|
||||
|
||||
class HTTPSRemote(HTTPRemote):
|
||||
class HTTPSRemoteTree(HTTPRemoteTree):
|
||||
scheme = Schemes.HTTPS
|
||||
|
||||
+27
-31
@@ -25,15 +25,15 @@ class RemoteIndexNoop:
|
||||
def __iter__(self):
|
||||
return iter([])
|
||||
|
||||
def __contains__(self, checksum):
|
||||
def __contains__(self, hash_):
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def checksums():
|
||||
def hashes():
|
||||
return []
|
||||
|
||||
@staticmethod
|
||||
def dir_checksums():
|
||||
def dir_hashes():
|
||||
return []
|
||||
|
||||
def load(self):
|
||||
@@ -54,13 +54,13 @@ class RemoteIndexNoop:
|
||||
|
||||
|
||||
class RemoteIndex:
|
||||
"""Class for indexing remote checksums in a sqlite3 database.
|
||||
"""Class for indexing remote hashes in a sqlite3 database.
|
||||
|
||||
Args:
|
||||
repo: repo for this remote index.
|
||||
name: name for this index. Index db will be loaded from and saved to
|
||||
``.dvc/tmp/index/{name}.idx``.
|
||||
dir_suffix: suffix used for naming directory checksums
|
||||
dir_suffix: suffix used for naming directory hashes
|
||||
"""
|
||||
|
||||
INDEX_SUFFIX = ".idx"
|
||||
@@ -79,8 +79,8 @@ class RemoteIndex:
|
||||
|
||||
def __iter__(self):
|
||||
cmd = f"SELECT checksum FROM {self.INDEX_TABLE}"
|
||||
for (checksum,) in self._execute(cmd):
|
||||
yield checksum
|
||||
for (hash_,) in self._execute(cmd):
|
||||
yield hash_
|
||||
|
||||
def __enter__(self):
|
||||
self.lock.acquire()
|
||||
@@ -90,25 +90,25 @@ class RemoteIndex:
|
||||
self.dump()
|
||||
self.lock.release()
|
||||
|
||||
def __contains__(self, checksum):
|
||||
def __contains__(self, hash_):
|
||||
cmd = "SELECT checksum FROM {} WHERE checksum = (?)".format(
|
||||
self.INDEX_TABLE
|
||||
)
|
||||
self._execute(cmd, (checksum,))
|
||||
self._execute(cmd, (hash_,))
|
||||
return self.cursor.fetchone() is not None
|
||||
|
||||
def checksums(self):
|
||||
"""Iterate over checksums stored in the index."""
|
||||
def hashes(self):
|
||||
"""Iterate over hashes stored in the index."""
|
||||
return iter(self)
|
||||
|
||||
def dir_checksums(self):
|
||||
"""Iterate over .dir checksums stored in the index."""
|
||||
def dir_hashes(self):
|
||||
"""Iterate over .dir hashes stored in the index."""
|
||||
cmd = f"SELECT checksum FROM {self.INDEX_TABLE} WHERE dir = 1"
|
||||
for (checksum,) in self._execute(cmd):
|
||||
yield checksum
|
||||
for (hash_,) in self._execute(cmd):
|
||||
yield hash_
|
||||
|
||||
def is_dir_checksum(self, checksum):
|
||||
return checksum.endswith(self.dir_suffix)
|
||||
def is_dir_hash(self, hash_):
|
||||
return hash_.endswith(self.dir_suffix)
|
||||
|
||||
def _execute(self, cmd, parameters=()):
|
||||
return self.cursor.execute(cmd, parameters)
|
||||
@@ -185,28 +185,24 @@ class RemoteIndex:
|
||||
cmd = f"DELETE FROM {self.INDEX_TABLE}"
|
||||
self._execute(cmd)
|
||||
|
||||
def update(self, dir_checksums, file_checksums):
|
||||
"""Update this index, adding the specified checksums.
|
||||
def update(self, dir_hashes, file_hashes):
|
||||
"""Update this index, adding the specified hashes.
|
||||
|
||||
Changes to the index will not committed until dump() is called.
|
||||
"""
|
||||
cmd = "INSERT OR IGNORE INTO {} (checksum, dir) VALUES (?, ?)".format(
|
||||
self.INDEX_TABLE
|
||||
)
|
||||
self._executemany(
|
||||
cmd, ((checksum, True) for checksum in dir_checksums)
|
||||
)
|
||||
self._executemany(
|
||||
cmd, ((checksum, False) for checksum in file_checksums)
|
||||
)
|
||||
self._executemany(cmd, ((hash_, True) for hash_ in dir_hashes))
|
||||
self._executemany(cmd, ((hash_, False) for hash_ in file_hashes))
|
||||
|
||||
def intersection(self, checksums):
|
||||
"""Iterate over values from `checksums` which exist in the index."""
|
||||
def intersection(self, hashes):
|
||||
"""Iterate over values from `hashes` which exist in the index."""
|
||||
# sqlite has a compile time limit of 999, see:
|
||||
# https://www.sqlite.org/c3ref/c_limit_attached.html#sqlitelimitvariablenumber
|
||||
for chunk in lchunks(999, checksums):
|
||||
for chunk in lchunks(999, hashes):
|
||||
cmd = "SELECT checksum FROM {} WHERE checksum IN ({})".format(
|
||||
self.INDEX_TABLE, ",".join("?" for checksum in chunk)
|
||||
self.INDEX_TABLE, ",".join("?" for hash_ in chunk)
|
||||
)
|
||||
for (checksum,) in self._execute(cmd, chunk):
|
||||
yield checksum
|
||||
for (hash_,) in self._execute(cmd, chunk):
|
||||
yield hash_
|
||||
|
||||
+142
-154
@@ -16,9 +16,9 @@ from dvc.remote.base import (
|
||||
STATUS_MAP,
|
||||
STATUS_MISSING,
|
||||
STATUS_NEW,
|
||||
BaseRemote,
|
||||
BaseRemoteTree,
|
||||
CacheMixin,
|
||||
CloudCache,
|
||||
Remote,
|
||||
index_locked,
|
||||
)
|
||||
from dvc.remote.index import RemoteIndexNoop
|
||||
@@ -39,17 +39,26 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LocalRemoteTree(BaseRemoteTree):
|
||||
SHARED_MODE_MAP = {None: (0o644, 0o755), "group": (0o664, 0o775)}
|
||||
scheme = Schemes.LOCAL
|
||||
PATH_CLS = PathInfo
|
||||
PARAM_CHECKSUM = "md5"
|
||||
PARAM_PATH = "path"
|
||||
TRAVERSE_PREFIX_LEN = 2
|
||||
UNPACKED_DIR_SUFFIX = ".unpacked"
|
||||
|
||||
def __init__(self, remote, config):
|
||||
super().__init__(remote, config)
|
||||
CACHE_MODE = 0o444
|
||||
SHARED_MODE_MAP = {None: (0o644, 0o755), "group": (0o664, 0o775)}
|
||||
|
||||
def __init__(self, repo, config):
|
||||
super().__init__(repo, config)
|
||||
url = config.get("url")
|
||||
self.path_info = self.PATH_CLS(url) if url else None
|
||||
|
||||
@property
|
||||
def repo(self):
|
||||
return self.remote.repo
|
||||
def state(self):
|
||||
from dvc.state import StateNoop
|
||||
|
||||
return self.repo.state if self.repo else StateNoop()
|
||||
|
||||
@cached_property
|
||||
def work_tree(self):
|
||||
@@ -162,7 +171,7 @@ class LocalRemoteTree(BaseRemoteTree):
|
||||
# and the cache type is `hardlink`, we might reach link limits and
|
||||
# will get something like: `too many links error`
|
||||
#
|
||||
# This is because all those empty files will have the same checksum
|
||||
# This is because all those empty files will have the same hash
|
||||
# (i.e. 68b329da9893e34099c7d8ad5cb9c940), therefore, they will be
|
||||
# linked to the same file in the cache.
|
||||
#
|
||||
@@ -195,78 +204,6 @@ class LocalRemoteTree(BaseRemoteTree):
|
||||
os.chmod(tmp_info, self.file_mode)
|
||||
os.rename(tmp_info, to_info)
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
return file_md5(path_info)[0]
|
||||
|
||||
@staticmethod
|
||||
def getsize(path_info):
|
||||
return os.path.getsize(path_info)
|
||||
|
||||
def _upload(
|
||||
self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
|
||||
):
|
||||
makedirs(to_info.parent, exist_ok=True)
|
||||
|
||||
tmp_file = tmp_fname(to_info)
|
||||
copyfile(
|
||||
from_file, tmp_file, name=name, no_progress_bar=no_progress_bar
|
||||
)
|
||||
|
||||
self.remote.protect(tmp_file)
|
||||
os.rename(tmp_file, to_info)
|
||||
|
||||
@staticmethod
|
||||
def _download(
|
||||
from_info, to_file, name=None, no_progress_bar=False, **_kwargs
|
||||
):
|
||||
copyfile(
|
||||
from_info, to_file, no_progress_bar=no_progress_bar, name=name
|
||||
)
|
||||
|
||||
|
||||
def _log_exceptions(func, operation):
|
||||
@wraps(func)
|
||||
def wrapper(from_info, to_info, *args, **kwargs):
|
||||
try:
|
||||
func(from_info, to_info, *args, **kwargs)
|
||||
return 0
|
||||
except Exception as exc:
|
||||
# NOTE: this means we ran out of file descriptors and there is no
|
||||
# reason to try to proceed, as we will hit this error anyways.
|
||||
if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
|
||||
raise
|
||||
|
||||
logger.exception(
|
||||
"failed to %s '%s' to '%s'", operation, from_info, to_info
|
||||
)
|
||||
return 1
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
class LocalRemote(BaseRemote):
|
||||
scheme = Schemes.LOCAL
|
||||
INDEX_CLS = RemoteIndexNoop
|
||||
TREE_CLS = LocalRemoteTree
|
||||
|
||||
PARAM_CHECKSUM = "md5"
|
||||
PARAM_PATH = "path"
|
||||
DEFAULT_CACHE_TYPES = ["reflink", "copy"]
|
||||
TRAVERSE_PREFIX_LEN = 2
|
||||
UNPACKED_DIR_SUFFIX = ".unpacked"
|
||||
|
||||
CACHE_MODE = 0o444
|
||||
|
||||
@property
|
||||
def state(self):
|
||||
return self.repo.state
|
||||
|
||||
def get(self, md5):
|
||||
if not md5:
|
||||
return None
|
||||
|
||||
return self.checksum_to_path_info(md5).url
|
||||
|
||||
def _unprotect_file(self, path):
|
||||
if System.is_symlink(path) or System.is_hardlink(path):
|
||||
logger.debug(f"Unprotecting '{path}'")
|
||||
@@ -287,7 +224,7 @@ class LocalRemote(BaseRemote):
|
||||
"a symlink or a hardlink.".format(path)
|
||||
)
|
||||
|
||||
os.chmod(path, self.tree.file_mode)
|
||||
os.chmod(path, self.file_mode)
|
||||
|
||||
def _unprotect_dir(self, path):
|
||||
assert is_working_tree(self.repo.tree)
|
||||
@@ -333,14 +270,44 @@ class LocalRemote(BaseRemote):
|
||||
|
||||
return stat.S_IMODE(mode) == self.CACHE_MODE
|
||||
|
||||
def get_file_hash(self, path_info):
|
||||
return file_md5(path_info)[0]
|
||||
|
||||
@staticmethod
|
||||
def getsize(path_info):
|
||||
return os.path.getsize(path_info)
|
||||
|
||||
def _upload(
|
||||
self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
|
||||
):
|
||||
makedirs(to_info.parent, exist_ok=True)
|
||||
|
||||
tmp_file = tmp_fname(to_info)
|
||||
copyfile(
|
||||
from_file, tmp_file, name=name, no_progress_bar=no_progress_bar
|
||||
)
|
||||
|
||||
self.protect(tmp_file)
|
||||
os.rename(tmp_file, to_info)
|
||||
|
||||
@staticmethod
|
||||
def _download(
|
||||
from_info, to_file, name=None, no_progress_bar=False, **_kwargs
|
||||
):
|
||||
copyfile(
|
||||
from_info, to_file, no_progress_bar=no_progress_bar, name=name
|
||||
)
|
||||
|
||||
def list_paths(self, prefix=None, progress_callback=None):
|
||||
assert self.path_info is not None
|
||||
if prefix:
|
||||
path_info = self.path_info / prefix[:2]
|
||||
if not self.tree.exists(path_info):
|
||||
if not self.exists(path_info):
|
||||
return
|
||||
else:
|
||||
path_info = self.path_info
|
||||
# NOTE: use utils.fs walk_files since tree.walk_files will not follow
|
||||
# symlinks
|
||||
if progress_callback:
|
||||
for path in walk_files(path_info):
|
||||
progress_callback()
|
||||
@@ -348,16 +315,43 @@ class LocalRemote(BaseRemote):
|
||||
else:
|
||||
yield from walk_files(path_info)
|
||||
|
||||
def _remove_unpacked_dir(self, checksum):
|
||||
info = self.checksum_to_path_info(checksum)
|
||||
def _remove_unpacked_dir(self, hash_):
|
||||
info = self.hash_to_path_info(hash_)
|
||||
path_info = info.with_name(info.name + self.UNPACKED_DIR_SUFFIX)
|
||||
self.tree.remove(path_info)
|
||||
self.remove(path_info)
|
||||
|
||||
|
||||
class LocalCache(LocalRemote, CacheMixin):
|
||||
def __init__(self, repo, config):
|
||||
super().__init__(repo, config)
|
||||
self.cache_dir = config.get("url")
|
||||
def _log_exceptions(func, operation):
|
||||
@wraps(func)
|
||||
def wrapper(from_info, to_info, *args, **kwargs):
|
||||
try:
|
||||
func(from_info, to_info, *args, **kwargs)
|
||||
return 0
|
||||
except Exception as exc:
|
||||
# NOTE: this means we ran out of file descriptors and there is no
|
||||
# reason to try to proceed, as we will hit this error anyways.
|
||||
if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
|
||||
raise
|
||||
|
||||
logger.exception(
|
||||
"failed to %s '%s' to '%s'", operation, from_info, to_info
|
||||
)
|
||||
return 1
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
class LocalRemote(Remote):
|
||||
INDEX_CLS = RemoteIndexNoop
|
||||
|
||||
|
||||
class LocalCache(CloudCache):
|
||||
DEFAULT_CACHE_TYPES = ["reflink", "copy"]
|
||||
CACHE_MODE = LocalRemoteTree.CACHE_MODE
|
||||
|
||||
def __init__(self, tree):
|
||||
super().__init__(tree)
|
||||
self.cache_dir = tree.config.get("url")
|
||||
|
||||
@property
|
||||
def cache_dir(self):
|
||||
@@ -375,30 +369,28 @@ class LocalCache(LocalRemote, CacheMixin):
|
||||
def cache_path(self):
|
||||
return os.path.abspath(self.cache_dir)
|
||||
|
||||
def checksum_to_path(self, checksum):
|
||||
def hash_to_path(self, hash_):
|
||||
# NOTE: `self.cache_path` is already normalized so we can simply use
|
||||
# `os.sep` instead of `os.path.join`. This results in this helper
|
||||
# being ~5.5 times faster.
|
||||
return (
|
||||
f"{self.cache_path}{os.sep}{checksum[0:2]}{os.sep}{checksum[2:]}"
|
||||
)
|
||||
return f"{self.cache_path}{os.sep}{hash_[0:2]}{os.sep}{hash_[2:]}"
|
||||
|
||||
def checksums_exist(self, checksums, jobs=None, name=None):
|
||||
def hashes_exist(self, hashes, jobs=None, name=None):
|
||||
return [
|
||||
checksum
|
||||
for checksum in Tqdm(
|
||||
checksums,
|
||||
hash_
|
||||
for hash_ in Tqdm(
|
||||
hashes,
|
||||
unit="file",
|
||||
desc="Querying "
|
||||
+ ("cache in " + name if name else "local cache"),
|
||||
)
|
||||
if not self.changed_cache_file(checksum)
|
||||
if not self.changed_cache_file(hash_)
|
||||
]
|
||||
|
||||
def already_cached(self, path_info):
|
||||
assert path_info.scheme in ["", "local"]
|
||||
|
||||
current_md5 = self.get_checksum(path_info)
|
||||
current_md5 = self.get_hash(path_info)
|
||||
|
||||
if not current_md5:
|
||||
return False
|
||||
@@ -442,7 +434,7 @@ class LocalCache(LocalRemote, CacheMixin):
|
||||
|
||||
dir_status_info contains status for .dir files, file_status_info
|
||||
contains status for all other files, and dir_contents is a dict of
|
||||
{dir_checksum: set(file_checksum, ...)} which can be used to map
|
||||
{dir_hash: set(file_hash, ...)} which can be used to map
|
||||
a .dir file to its file contents.
|
||||
"""
|
||||
logger.debug(f"Preparing to collect status from {remote.path_info}")
|
||||
@@ -450,7 +442,7 @@ class LocalCache(LocalRemote, CacheMixin):
|
||||
|
||||
logger.debug("Collecting information from local cache...")
|
||||
local_exists = frozenset(
|
||||
self.checksums_exist(md5s, jobs=jobs, name=self.cache_dir)
|
||||
self.hashes_exist(md5s, jobs=jobs, name=self.cache_dir)
|
||||
)
|
||||
|
||||
# This is a performance optimization. We can safely assume that,
|
||||
@@ -465,12 +457,12 @@ class LocalCache(LocalRemote, CacheMixin):
|
||||
dir_md5s = set(named_cache.dir_keys(self.scheme))
|
||||
if dir_md5s:
|
||||
remote_exists.update(
|
||||
self._indexed_dir_checksums(named_cache, remote, dir_md5s)
|
||||
self._indexed_dir_hashes(named_cache, remote, dir_md5s)
|
||||
)
|
||||
md5s.difference_update(remote_exists)
|
||||
if md5s:
|
||||
remote_exists.update(
|
||||
remote.checksums_exist(
|
||||
remote.hashes_exist(
|
||||
md5s, jobs=jobs, name=str(remote.path_info)
|
||||
)
|
||||
)
|
||||
@@ -481,23 +473,23 @@ class LocalCache(LocalRemote, CacheMixin):
|
||||
def _make_status(
|
||||
self, named_cache, show_checksums, local_exists, remote_exists
|
||||
):
|
||||
def make_names(checksum, names):
|
||||
return {"name": checksum if show_checksums else " ".join(names)}
|
||||
def make_names(hash_, names):
|
||||
return {"name": hash_ if show_checksums else " ".join(names)}
|
||||
|
||||
dir_status = {}
|
||||
file_status = {}
|
||||
dir_contents = {}
|
||||
for checksum, item in named_cache[self.scheme].items():
|
||||
for hash_, item in named_cache[self.scheme].items():
|
||||
if item.children:
|
||||
dir_status[checksum] = make_names(checksum, item.names)
|
||||
dir_contents[checksum] = set()
|
||||
for child_checksum, child in item.children.items():
|
||||
file_status[child_checksum] = make_names(
|
||||
child_checksum, child.names
|
||||
dir_status[hash_] = make_names(hash_, item.names)
|
||||
dir_contents[hash_] = set()
|
||||
for child_hash, child in item.children.items():
|
||||
file_status[child_hash] = make_names(
|
||||
child_hash, child.names
|
||||
)
|
||||
dir_contents[checksum].add(child_checksum)
|
||||
dir_contents[hash_].add(child_hash)
|
||||
else:
|
||||
file_status[checksum] = make_names(checksum, item.names)
|
||||
file_status[hash_] = make_names(hash_, item.names)
|
||||
|
||||
self._fill_statuses(dir_status, local_exists, remote_exists)
|
||||
self._fill_statuses(file_status, local_exists, remote_exists)
|
||||
@@ -506,50 +498,50 @@ class LocalCache(LocalRemote, CacheMixin):
|
||||
|
||||
return dir_status, file_status, dir_contents
|
||||
|
||||
def _indexed_dir_checksums(self, named_cache, remote, dir_md5s):
|
||||
# Validate our index by verifying all indexed .dir checksums
|
||||
def _indexed_dir_hashes(self, named_cache, remote, dir_md5s):
|
||||
# Validate our index by verifying all indexed .dir hashes
|
||||
# still exist on the remote
|
||||
indexed_dirs = set(remote.index.dir_checksums())
|
||||
indexed_dirs = set(remote.index.dir_hashes())
|
||||
indexed_dir_exists = set()
|
||||
if indexed_dirs:
|
||||
indexed_dir_exists.update(
|
||||
remote._list_checksums_exists(indexed_dirs)
|
||||
remote.tree.list_hashes_exists(indexed_dirs)
|
||||
)
|
||||
missing_dirs = indexed_dirs.difference(indexed_dir_exists)
|
||||
if missing_dirs:
|
||||
logger.debug(
|
||||
"Remote cache missing indexed .dir checksums '{}', "
|
||||
"Remote cache missing indexed .dir hashes '{}', "
|
||||
"clearing remote index".format(", ".join(missing_dirs))
|
||||
)
|
||||
remote.index.clear()
|
||||
|
||||
# Check if non-indexed (new) dir checksums exist on remote
|
||||
# Check if non-indexed (new) dir hashes exist on remote
|
||||
dir_exists = dir_md5s.intersection(indexed_dir_exists)
|
||||
dir_exists.update(remote._list_checksums_exists(dir_md5s - dir_exists))
|
||||
dir_exists.update(
|
||||
remote.tree.list_hashes_exists(dir_md5s - dir_exists)
|
||||
)
|
||||
|
||||
# If .dir checksum exists on the remote, assume directory contents
|
||||
# If .dir hash exists on the remote, assume directory contents
|
||||
# still exists on the remote
|
||||
for dir_checksum in dir_exists:
|
||||
file_checksums = list(
|
||||
named_cache.child_keys(self.scheme, dir_checksum)
|
||||
)
|
||||
if dir_checksum not in remote.index:
|
||||
for dir_hash in dir_exists:
|
||||
file_hashes = list(named_cache.child_keys(self.scheme, dir_hash))
|
||||
if dir_hash not in remote.index:
|
||||
logger.debug(
|
||||
"Indexing new .dir '{}' with '{}' nested files".format(
|
||||
dir_checksum, len(file_checksums)
|
||||
dir_hash, len(file_hashes)
|
||||
)
|
||||
)
|
||||
remote.index.update([dir_checksum], file_checksums)
|
||||
yield dir_checksum
|
||||
yield from file_checksums
|
||||
remote.index.update([dir_hash], file_hashes)
|
||||
yield dir_hash
|
||||
yield from file_hashes
|
||||
|
||||
@staticmethod
|
||||
def _fill_statuses(checksum_info_dir, local_exists, remote_exists):
|
||||
def _fill_statuses(hash_info_dir, local_exists, remote_exists):
|
||||
# Using sets because they are way faster for lookups
|
||||
local = set(local_exists)
|
||||
remote = set(remote_exists)
|
||||
|
||||
for md5, info in checksum_info_dir.items():
|
||||
for md5, info in hash_info_dir.items():
|
||||
status = STATUS_MAP[(md5 in local, md5 in remote)]
|
||||
info["status"] = status
|
||||
|
||||
@@ -557,15 +549,15 @@ class LocalCache(LocalRemote, CacheMixin):
|
||||
cache = []
|
||||
path_infos = []
|
||||
names = []
|
||||
checksums = []
|
||||
hashes = []
|
||||
for md5, info in Tqdm(
|
||||
status_info.items(), desc="Analysing status", unit="file"
|
||||
):
|
||||
if info["status"] == status:
|
||||
cache.append(self.checksum_to_path_info(md5))
|
||||
path_infos.append(remote.checksum_to_path_info(md5))
|
||||
cache.append(self.hash_to_path_info(md5))
|
||||
path_infos.append(remote.hash_to_path_info(md5))
|
||||
names.append(info["name"])
|
||||
checksums.append(md5)
|
||||
hashes.append(md5)
|
||||
|
||||
if download:
|
||||
to_infos = cache
|
||||
@@ -574,7 +566,7 @@ class LocalCache(LocalRemote, CacheMixin):
|
||||
to_infos = path_infos
|
||||
from_infos = cache
|
||||
|
||||
return from_infos, to_infos, names, checksums
|
||||
return from_infos, to_infos, names, hashes
|
||||
|
||||
def _process(
|
||||
self,
|
||||
@@ -605,7 +597,7 @@ class LocalCache(LocalRemote, CacheMixin):
|
||||
desc = "Uploading"
|
||||
|
||||
if jobs is None:
|
||||
jobs = remote.JOBS
|
||||
jobs = remote.tree.JOBS
|
||||
|
||||
dir_status, file_status, dir_contents = self._status(
|
||||
named_cache,
|
||||
@@ -636,20 +628,18 @@ class LocalCache(LocalRemote, CacheMixin):
|
||||
# for uploads, push files first, and any .dir files last
|
||||
|
||||
file_futures = {}
|
||||
for from_info, to_info, name, checksum in zip(*file_plans):
|
||||
file_futures[checksum] = executor.submit(
|
||||
for from_info, to_info, name, hash_ in zip(*file_plans):
|
||||
file_futures[hash_] = executor.submit(
|
||||
func, from_info, to_info, name
|
||||
)
|
||||
dir_futures = {}
|
||||
for from_info, to_info, name, dir_checksum in zip(
|
||||
*dir_plans
|
||||
):
|
||||
for from_info, to_info, name, dir_hash in zip(*dir_plans):
|
||||
wait_futures = {
|
||||
future
|
||||
for file_checksum, future in file_futures.items()
|
||||
if file_checksum in dir_contents[dir_checksum]
|
||||
for file_hash, future in file_futures.items()
|
||||
if file_hash in dir_contents[dir_hash]
|
||||
}
|
||||
dir_futures[dir_checksum] = executor.submit(
|
||||
dir_futures[dir_hash] = executor.submit(
|
||||
self._dir_upload,
|
||||
func,
|
||||
wait_futures,
|
||||
@@ -672,16 +662,14 @@ class LocalCache(LocalRemote, CacheMixin):
|
||||
|
||||
if not download:
|
||||
# index successfully pushed dirs
|
||||
for dir_checksum, future in dir_futures.items():
|
||||
for dir_hash, future in dir_futures.items():
|
||||
if future.result() == 0:
|
||||
file_checksums = dir_contents[dir_checksum]
|
||||
file_hashes = dir_contents[dir_hash]
|
||||
logger.debug(
|
||||
"Indexing pushed dir '{}' with "
|
||||
"'{}' nested files".format(
|
||||
dir_checksum, len(file_checksums)
|
||||
)
|
||||
"'{}' nested files".format(dir_hash, len(file_hashes))
|
||||
)
|
||||
remote.index.update([dir_checksum], file_checksums)
|
||||
remote.index.update([dir_hash], file_hashes)
|
||||
|
||||
return len(dir_plans[0]) + len(file_plans[0])
|
||||
|
||||
@@ -720,10 +708,10 @@ class LocalCache(LocalRemote, CacheMixin):
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _log_missing_caches(checksum_info_dict):
|
||||
def _log_missing_caches(hash_info_dict):
|
||||
missing_caches = [
|
||||
(md5, info)
|
||||
for md5, info in checksum_info_dict.items()
|
||||
for md5, info in hash_info_dict.items()
|
||||
if info["status"] == STATUS_MISSING
|
||||
]
|
||||
if missing_caches:
|
||||
|
||||
+24
-28
@@ -6,14 +6,37 @@ from funcy import cached_property, wrap_prop
|
||||
|
||||
from dvc.path_info import CloudURLInfo
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.remote.base import BaseRemote, BaseRemoteTree
|
||||
from dvc.remote.base import BaseRemoteTree
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class OSSRemoteTree(BaseRemoteTree):
|
||||
"""
|
||||
oss2 document:
|
||||
https://www.alibabacloud.com/help/doc-detail/32026.htm
|
||||
|
||||
|
||||
Examples
|
||||
----------
|
||||
$ dvc remote add myremote oss://my-bucket/path
|
||||
Set key id, key secret and endpoint using modify command
|
||||
$ dvc remote modify myremote oss_key_id my-key-id
|
||||
$ dvc remote modify myremote oss_key_secret my-key-secret
|
||||
$ dvc remote modify myremote oss_endpoint endpoint
|
||||
or environment variables
|
||||
$ export OSS_ACCESS_KEY_ID="my-key-id"
|
||||
$ export OSS_ACCESS_KEY_SECRET="my-key-secret"
|
||||
$ export OSS_ENDPOINT="endpoint"
|
||||
"""
|
||||
|
||||
scheme = Schemes.OSS
|
||||
PATH_CLS = CloudURLInfo
|
||||
REQUIRES = {"oss2": "oss2"}
|
||||
PARAM_CHECKSUM = "etag"
|
||||
COPY_POLL_SECONDS = 5
|
||||
LIST_OBJECT_PAGE_SIZE = 100
|
||||
|
||||
def __init__(self, repo, config):
|
||||
super().__init__(repo, config)
|
||||
@@ -105,30 +128,3 @@ class OSSRemoteTree(BaseRemoteTree):
|
||||
self.oss_service.get_object_to_file(
|
||||
from_info.path, to_file, progress_callback=pbar.update_to
|
||||
)
|
||||
|
||||
|
||||
class OSSRemote(BaseRemote):
|
||||
"""
|
||||
oss2 document:
|
||||
https://www.alibabacloud.com/help/doc-detail/32026.htm
|
||||
|
||||
|
||||
Examples
|
||||
----------
|
||||
$ dvc remote add myremote oss://my-bucket/path
|
||||
Set key id, key secret and endpoint using modify command
|
||||
$ dvc remote modify myremote oss_key_id my-key-id
|
||||
$ dvc remote modify myremote oss_key_secret my-key-secret
|
||||
$ dvc remote modify myremote oss_endpoint endpoint
|
||||
or environment variables
|
||||
$ export OSS_ACCESS_KEY_ID="my-key-id"
|
||||
$ export OSS_ACCESS_KEY_SECRET="my-key-secret"
|
||||
$ export OSS_ENDPOINT="endpoint"
|
||||
"""
|
||||
|
||||
scheme = Schemes.OSS
|
||||
REQUIRES = {"oss2": "oss2"}
|
||||
PARAM_CHECKSUM = "etag"
|
||||
COPY_POLL_SECONDS = 5
|
||||
LIST_OBJECT_PAGE_SIZE = 100
|
||||
TREE_CLS = OSSRemoteTree
|
||||
|
||||
+5
-13
@@ -8,14 +8,17 @@ from dvc.config import ConfigError
|
||||
from dvc.exceptions import DvcException, ETagMismatchError
|
||||
from dvc.path_info import CloudURLInfo
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.remote.base import BaseRemote, BaseRemoteTree, CacheMixin
|
||||
from dvc.remote.base import BaseRemoteTree
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class S3RemoteTree(BaseRemoteTree):
|
||||
scheme = Schemes.S3
|
||||
PATH_CLS = CloudURLInfo
|
||||
REQUIRES = {"boto3": "boto3"}
|
||||
PARAM_CHECKSUM = "etag"
|
||||
|
||||
def __init__(self, repo, config):
|
||||
super().__init__(repo, config)
|
||||
@@ -305,7 +308,7 @@ class S3RemoteTree(BaseRemoteTree):
|
||||
if etag != cached_etag:
|
||||
raise ETagMismatchError(etag, cached_etag)
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
def get_file_hash(self, path_info):
|
||||
return self.get_etag(self.s3, path_info.bucket, path_info.path)
|
||||
|
||||
def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
|
||||
@@ -334,14 +337,3 @@ class S3RemoteTree(BaseRemoteTree):
|
||||
self.s3.download_file(
|
||||
from_info.bucket, from_info.path, to_file, Callback=pbar.update
|
||||
)
|
||||
|
||||
|
||||
class S3Remote(BaseRemote):
|
||||
scheme = Schemes.S3
|
||||
REQUIRES = {"boto3": "boto3"}
|
||||
PARAM_CHECKSUM = "etag"
|
||||
TREE_CLS = S3RemoteTree
|
||||
|
||||
|
||||
class S3Cache(S3Remote, CacheMixin):
|
||||
pass
|
||||
|
||||
+27
-30
@@ -14,7 +14,7 @@ from funcy import first, memoize, silent, wrap_with
|
||||
|
||||
import dvc.prompt as prompt
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.remote.base import BaseRemote, BaseRemoteTree, CacheMixin
|
||||
from dvc.remote.base import BaseRemoteTree, Remote
|
||||
from dvc.remote.pool import get_connection
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.utils import to_chunks
|
||||
@@ -34,6 +34,18 @@ def ask_password(host, user, port):
|
||||
|
||||
|
||||
class SSHRemoteTree(BaseRemoteTree):
|
||||
scheme = Schemes.SSH
|
||||
REQUIRES = {"paramiko": "paramiko"}
|
||||
JOBS = 4
|
||||
|
||||
PARAM_CHECKSUM = "md5"
|
||||
# At any given time some of the connections will go over network and
|
||||
# paramiko stuff, so we would ideally have it double of server processors.
|
||||
# We use conservative setting of 4 instead to not exhaust max sessions.
|
||||
CHECKSUM_JOBS = 4
|
||||
DEFAULT_CACHE_TYPES = ["copy"]
|
||||
TRAVERSE_PREFIX_LEN = 2
|
||||
|
||||
DEFAULT_PORT = 22
|
||||
TIMEOUT = 1800
|
||||
|
||||
@@ -225,7 +237,7 @@ class SSHRemoteTree(BaseRemoteTree):
|
||||
with self.ssh(from_info) as ssh:
|
||||
ssh.reflink(from_info.path, to_info.path)
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
def get_file_hash(self, path_info):
|
||||
if path_info.scheme != self.scheme:
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -256,27 +268,12 @@ class SSHRemoteTree(BaseRemoteTree):
|
||||
no_progress_bar=no_progress_bar,
|
||||
)
|
||||
|
||||
|
||||
class SSHRemote(BaseRemote):
|
||||
scheme = Schemes.SSH
|
||||
REQUIRES = {"paramiko": "paramiko"}
|
||||
JOBS = 4
|
||||
TREE_CLS = SSHRemoteTree
|
||||
|
||||
PARAM_CHECKSUM = "md5"
|
||||
# At any given time some of the connections will go over network and
|
||||
# paramiko stuff, so we would ideally have it double of server processors.
|
||||
# We use conservative setting of 4 instead to not exhaust max sessions.
|
||||
CHECKSUM_JOBS = 4
|
||||
DEFAULT_CACHE_TYPES = ["copy"]
|
||||
TRAVERSE_PREFIX_LEN = 2
|
||||
|
||||
def list_paths(self, prefix=None, progress_callback=None):
|
||||
if prefix:
|
||||
root = posixpath.join(self.path_info.path, prefix[:2])
|
||||
else:
|
||||
root = self.path_info.path
|
||||
with self.tree.ssh(self.path_info) as ssh:
|
||||
with self.ssh(self.path_info) as ssh:
|
||||
if prefix and not ssh.exists(root):
|
||||
return
|
||||
# If we simply return an iterator then with above closes instantly
|
||||
@@ -287,6 +284,8 @@ class SSHRemote(BaseRemote):
|
||||
else:
|
||||
yield from ssh.walk_files(root)
|
||||
|
||||
|
||||
class SSHRemote(Remote):
|
||||
def batch_exists(self, path_infos, callback):
|
||||
def _exists(chunk_and_channel):
|
||||
chunk, channel = chunk_and_channel
|
||||
@@ -315,14 +314,14 @@ class SSHRemote(BaseRemote):
|
||||
|
||||
return results
|
||||
|
||||
def checksums_exist(self, checksums, jobs=None, name=None):
|
||||
def hashes_exist(self, hashes, jobs=None, name=None):
|
||||
"""This is older implementation used in remote/base.py
|
||||
We are reusing it in RemoteSSH, because SSH's batch_exists proved to be
|
||||
faster than current approach (relying on exists(path_info)) applied in
|
||||
remote/base.
|
||||
"""
|
||||
if not self.CAN_TRAVERSE:
|
||||
return list(set(checksums) & set(self.all()))
|
||||
if not self.tree.CAN_TRAVERSE:
|
||||
return list(set(hashes) & set(self.tree.all()))
|
||||
|
||||
# possibly prompt for credentials before "Querying" progress output
|
||||
self.tree.ensure_credentials()
|
||||
@@ -330,21 +329,19 @@ class SSHRemote(BaseRemote):
|
||||
with Tqdm(
|
||||
desc="Querying "
|
||||
+ ("cache in " + name if name else "remote cache"),
|
||||
total=len(checksums),
|
||||
total=len(hashes),
|
||||
unit="file",
|
||||
) as pbar:
|
||||
|
||||
def exists_with_progress(chunks):
|
||||
return self.batch_exists(chunks, callback=pbar.update_msg)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=jobs or self.JOBS) as executor:
|
||||
path_infos = [self.checksum_to_path_info(x) for x in checksums]
|
||||
chunks = to_chunks(path_infos, num_chunks=self.JOBS)
|
||||
with ThreadPoolExecutor(
|
||||
max_workers=jobs or self.tree.JOBS
|
||||
) as executor:
|
||||
path_infos = [self.hash_to_path_info(x) for x in hashes]
|
||||
chunks = to_chunks(path_infos, num_chunks=self.tree.JOBS)
|
||||
results = executor.map(exists_with_progress, chunks)
|
||||
in_remote = itertools.chain.from_iterable(results)
|
||||
ret = list(itertools.compress(checksums, in_remote))
|
||||
ret = list(itertools.compress(hashes, in_remote))
|
||||
return ret
|
||||
|
||||
|
||||
class SSHCache(SSHRemote, CacheMixin):
|
||||
pass
|
||||
|
||||
@@ -153,7 +153,7 @@ class SSHConnection:
|
||||
yield directory, dirs, nondirs
|
||||
|
||||
def walk_files(self, directory):
|
||||
for root, dirs, files in self.walk(directory):
|
||||
for root, _, files in self.walk(directory):
|
||||
for fname in files:
|
||||
yield posixpath.join(root, fname)
|
||||
|
||||
|
||||
@@ -187,7 +187,7 @@ class Repo:
|
||||
return Repo(root_dir)
|
||||
|
||||
def unprotect(self, target):
|
||||
return self.cache.local.unprotect(PathInfo(target))
|
||||
return self.cache.local.tree.unprotect(PathInfo(target))
|
||||
|
||||
def _ignore(self):
|
||||
flist = [self.config.files["local"], self.tmp_dir]
|
||||
@@ -485,7 +485,7 @@ class Repo:
|
||||
continue
|
||||
|
||||
dep_key = dep.path_info.parts
|
||||
overlapping = list(n.value for n in outs.prefixes(dep_key))
|
||||
overlapping = [n.value for n in outs.prefixes(dep_key)]
|
||||
if outs.has_subtrie(dep_key):
|
||||
overlapping.extend(outs.values(prefix=dep_key))
|
||||
|
||||
|
||||
+1
-1
@@ -37,7 +37,7 @@ def diff(self, a_rev="HEAD", b_rev=None):
|
||||
|
||||
def _to_checksum(output):
|
||||
if on_working_tree:
|
||||
return self.cache.local.get_checksum(output.path_info)
|
||||
return self.cache.local.get_hash(output.path_info)
|
||||
return output.checksum
|
||||
|
||||
def _exists(output):
|
||||
|
||||
+12
-9
@@ -8,8 +8,10 @@ from . import locked
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _do_gc(typ, func, clist, jobs=None):
|
||||
removed = func(clist, jobs=jobs)
|
||||
def _do_gc(typ, remote, clist, jobs=None):
|
||||
from dvc.remote.base import Remote
|
||||
|
||||
removed = Remote.gc(clist, remote, jobs=jobs)
|
||||
if not removed:
|
||||
logger.info(f"No unused '{typ}' cache to remove.")
|
||||
|
||||
@@ -74,22 +76,23 @@ def gc(
|
||||
)
|
||||
)
|
||||
|
||||
_do_gc("local", self.cache.local.gc, used, jobs)
|
||||
# treat caches as remotes for garbage collection
|
||||
_do_gc("local", self.cache.local, used, jobs)
|
||||
|
||||
if self.cache.s3:
|
||||
_do_gc("s3", self.cache.s3.gc, used, jobs)
|
||||
_do_gc("s3", self.cache.s3, used, jobs)
|
||||
|
||||
if self.cache.gs:
|
||||
_do_gc("gs", self.cache.gs.gc, used, jobs)
|
||||
_do_gc("gs", self.cache.gs, used, jobs)
|
||||
|
||||
if self.cache.ssh:
|
||||
_do_gc("ssh", self.cache.ssh.gc, used, jobs)
|
||||
_do_gc("ssh", self.cache.ssh, used, jobs)
|
||||
|
||||
if self.cache.hdfs:
|
||||
_do_gc("hdfs", self.cache.hdfs.gc, used, jobs)
|
||||
_do_gc("hdfs", self.cache.hdfs, used, jobs)
|
||||
|
||||
if self.cache.azure:
|
||||
_do_gc("azure", self.cache.azure.gc, used, jobs)
|
||||
_do_gc("azure", self.cache.azure, used, jobs)
|
||||
|
||||
if cloud:
|
||||
_do_gc("remote", self.cloud.get_remote(remote, "gc -c").gc, used, jobs)
|
||||
_do_gc("remote", self.cloud.get_remote(remote, "gc -c"), used, jobs)
|
||||
|
||||
@@ -16,4 +16,6 @@ def get_url(url, out=None):
|
||||
|
||||
(dep,) = dependency.loads_from(None, [url])
|
||||
(out,) = output.loads_from(None, [out], use_cache=False)
|
||||
dep.save()
|
||||
dep.download(out)
|
||||
out.save()
|
||||
|
||||
+6
-1
@@ -50,13 +50,18 @@ def ls(
|
||||
def _ls(repo, path_info, recursive=None, dvc_only=False):
|
||||
from dvc.repo.tree import RepoTree
|
||||
|
||||
def onerror(exc):
|
||||
raise exc
|
||||
|
||||
# use our own RepoTree instance instead of repo.repo_tree since we do not
|
||||
# want fetch/stream enabled for ls
|
||||
tree = RepoTree(repo)
|
||||
|
||||
ret = {}
|
||||
try:
|
||||
for root, dirs, files in tree.walk(path_info.fspath, dvcfiles=True):
|
||||
for root, dirs, files in tree.walk(
|
||||
path_info.fspath, onerror=onerror, dvcfiles=True
|
||||
):
|
||||
for fname in files:
|
||||
info = PathInfo(root) / fname
|
||||
dvc = tree.isdvc(info)
|
||||
|
||||
@@ -17,6 +17,10 @@ class NotAPlotError(DvcException):
|
||||
)
|
||||
|
||||
|
||||
class PropsNotFoundError(DvcException):
|
||||
pass
|
||||
|
||||
|
||||
class Plots:
|
||||
def __init__(self, repo):
|
||||
self.repo = repo
|
||||
@@ -51,9 +55,8 @@ class Plots:
|
||||
try:
|
||||
with tree.open(path_info) as fd:
|
||||
data[rev][datafile]["data"] = fd.read()
|
||||
except FileNotFoundError as e:
|
||||
except FileNotFoundError:
|
||||
# This might happen simply because cache is absent
|
||||
print(e)
|
||||
pass
|
||||
|
||||
return data
|
||||
@@ -93,6 +96,17 @@ class Plots:
|
||||
|
||||
return diff(self.repo, *args, **kwargs)
|
||||
|
||||
@staticmethod
|
||||
def _unset(out, props):
|
||||
missing = list(set(props) - set(out.plot.keys()))
|
||||
if missing:
|
||||
raise PropsNotFoundError(
|
||||
f"display properties {missing} not found in plot '{out}'"
|
||||
)
|
||||
|
||||
for prop in props:
|
||||
out.plot.pop(prop)
|
||||
|
||||
def modify(self, path, props=None, unset=None):
|
||||
from dvc.dvcfile import Dvcfile
|
||||
|
||||
@@ -109,8 +123,9 @@ class Plots:
|
||||
if not isinstance(out.plot, dict):
|
||||
out.plot = {}
|
||||
|
||||
for field in unset or ():
|
||||
out.plot.pop(field, None)
|
||||
if unset:
|
||||
self._unset(out, unset)
|
||||
|
||||
out.plot.update(props)
|
||||
|
||||
# Empty dict will move it to non-plots
|
||||
|
||||
@@ -196,7 +196,7 @@ class CSVPlotData(PlotData):
|
||||
)
|
||||
|
||||
fieldnames = reader.fieldnames
|
||||
data = [row for row in reader]
|
||||
data = list(reader)
|
||||
|
||||
return [
|
||||
OrderedDict([(field, data_point[field]) for field in fieldnames])
|
||||
|
||||
+71
-29
@@ -12,6 +12,10 @@ class TemplateNotFoundError(DvcException):
|
||||
super().__init__(f"Template '{path}' not found.")
|
||||
|
||||
|
||||
class BadTemplateError(DvcException):
|
||||
pass
|
||||
|
||||
|
||||
class NoFieldInDataError(DvcException):
|
||||
def __init__(self, field_name):
|
||||
super().__init__(
|
||||
@@ -23,13 +27,7 @@ class Template:
|
||||
INDENT = 4
|
||||
SEPARATORS = (",", ": ")
|
||||
EXTENSION = ".json"
|
||||
ANCHOR = '"<DVC_METRIC_{}>"'
|
||||
METRIC_DATA_ANCHOR = "<DVC_METRIC_DATA>"
|
||||
X_ANCHOR = "<DVC_METRIC_X>"
|
||||
Y_ANCHOR = "<DVC_METRIC_Y>"
|
||||
TITLE_ANCHOR = "<DVC_METRIC_TITLE>"
|
||||
X_LABEL_ANCHOR = "<DVC_METRIC_X_LABEL>"
|
||||
Y_LABEL_ANCHOR = "<DVC_METRIC_Y_LABEL>"
|
||||
ANCHOR = "<DVC_METRIC_{}>"
|
||||
|
||||
def __init__(self, content=None, name=None):
|
||||
self.content = self.DEFAULT_CONTENT if content is None else content
|
||||
@@ -39,6 +37,12 @@ class Template:
|
||||
def render(self, data, props=None):
|
||||
props = props or {}
|
||||
|
||||
if self._anchor_str("data") not in self.content:
|
||||
anchor = self.anchor("data")
|
||||
raise BadTemplateError(
|
||||
f"Template '{self.filename}' is not using '{anchor}' anchor"
|
||||
)
|
||||
|
||||
if props.get("x"):
|
||||
Template._check_field_exists(data, props.get("x"))
|
||||
if props.get("y"):
|
||||
@@ -49,19 +53,23 @@ class Template:
|
||||
|
||||
return content
|
||||
|
||||
@classmethod
|
||||
def anchor(cls, name):
|
||||
return cls.ANCHOR.format(name.upper())
|
||||
|
||||
def has_anchor(self, name):
|
||||
return self._anchor(name) in self.content
|
||||
return self._anchor_str(name) in self.content
|
||||
|
||||
@classmethod
|
||||
def _fill_anchor(cls, content, name, value):
|
||||
value_str = json.dumps(
|
||||
value, indent=cls.INDENT, separators=cls.SEPARATORS, sort_keys=True
|
||||
)
|
||||
return content.replace(cls._anchor(name), value_str)
|
||||
return content.replace(cls._anchor_str(name), value_str)
|
||||
|
||||
@classmethod
|
||||
def _anchor(cls, name):
|
||||
return cls.ANCHOR.format(name.upper())
|
||||
def _anchor_str(cls, name):
|
||||
return '"{}"'.format(cls.anchor(name))
|
||||
|
||||
@classmethod
|
||||
def _fill_metadata(cls, content, props):
|
||||
@@ -88,19 +96,19 @@ class DefaultLinearTemplate(Template):
|
||||
|
||||
DEFAULT_CONTENT = {
|
||||
"$schema": "https://vega.github.io/schema/vega-lite/v4.json",
|
||||
"data": {"values": Template.METRIC_DATA_ANCHOR},
|
||||
"title": Template.TITLE_ANCHOR,
|
||||
"data": {"values": Template.anchor("data")},
|
||||
"title": Template.anchor("title"),
|
||||
"mark": {"type": "line"},
|
||||
"encoding": {
|
||||
"x": {
|
||||
"field": Template.X_ANCHOR,
|
||||
"field": Template.anchor("x"),
|
||||
"type": "quantitative",
|
||||
"title": Template.X_LABEL_ANCHOR,
|
||||
"title": Template.anchor("x_label"),
|
||||
},
|
||||
"y": {
|
||||
"field": Template.Y_ANCHOR,
|
||||
"field": Template.anchor("y"),
|
||||
"type": "quantitative",
|
||||
"title": Template.Y_LABEL_ANCHOR,
|
||||
"title": Template.anchor("y_label"),
|
||||
"scale": {"zero": False},
|
||||
},
|
||||
"color": {"field": "rev", "type": "nominal"},
|
||||
@@ -112,21 +120,21 @@ class DefaultConfusionTemplate(Template):
|
||||
DEFAULT_NAME = "confusion"
|
||||
DEFAULT_CONTENT = {
|
||||
"$schema": "https://vega.github.io/schema/vega-lite/v4.json",
|
||||
"data": {"values": Template.METRIC_DATA_ANCHOR},
|
||||
"title": Template.TITLE_ANCHOR,
|
||||
"data": {"values": Template.anchor("data")},
|
||||
"title": Template.anchor("title"),
|
||||
"mark": "rect",
|
||||
"encoding": {
|
||||
"x": {
|
||||
"field": Template.X_ANCHOR,
|
||||
"field": Template.anchor("x"),
|
||||
"type": "nominal",
|
||||
"sort": "ascending",
|
||||
"title": Template.X_LABEL_ANCHOR,
|
||||
"title": Template.anchor("x_label"),
|
||||
},
|
||||
"y": {
|
||||
"field": Template.Y_ANCHOR,
|
||||
"field": Template.anchor("y"),
|
||||
"type": "nominal",
|
||||
"sort": "ascending",
|
||||
"title": Template.Y_LABEL_ANCHOR,
|
||||
"title": Template.anchor("y_label"),
|
||||
},
|
||||
"color": {"aggregate": "count", "type": "quantitative"},
|
||||
"facet": {"field": "rev", "type": "nominal"},
|
||||
@@ -138,19 +146,19 @@ class DefaultScatterTemplate(Template):
|
||||
DEFAULT_NAME = "scatter"
|
||||
DEFAULT_CONTENT = {
|
||||
"$schema": "https://vega.github.io/schema/vega-lite/v4.json",
|
||||
"data": {"values": Template.METRIC_DATA_ANCHOR},
|
||||
"title": Template.TITLE_ANCHOR,
|
||||
"data": {"values": Template.anchor("data")},
|
||||
"title": Template.anchor("title"),
|
||||
"mark": "point",
|
||||
"encoding": {
|
||||
"x": {
|
||||
"field": Template.X_ANCHOR,
|
||||
"field": Template.anchor("x"),
|
||||
"type": "quantitative",
|
||||
"title": Template.X_LABEL_ANCHOR,
|
||||
"title": Template.anchor("x_label"),
|
||||
},
|
||||
"y": {
|
||||
"field": Template.Y_ANCHOR,
|
||||
"field": Template.anchor("y"),
|
||||
"type": "quantitative",
|
||||
"title": Template.Y_LABEL_ANCHOR,
|
||||
"title": Template.anchor("y_label"),
|
||||
"scale": {"zero": False},
|
||||
},
|
||||
"color": {"field": "rev", "type": "nominal"},
|
||||
@@ -158,12 +166,46 @@ class DefaultScatterTemplate(Template):
|
||||
}
|
||||
|
||||
|
||||
class SmoothLinearTemplate(Template):
|
||||
DEFAULT_NAME = "smooth"
|
||||
|
||||
DEFAULT_CONTENT = {
|
||||
"$schema": "https://vega.github.io/schema/vega-lite/v4.json",
|
||||
"data": {"values": Template.anchor("data")},
|
||||
"title": Template.anchor("title"),
|
||||
"mark": {"type": "line"},
|
||||
"encoding": {
|
||||
"x": {
|
||||
"field": Template.anchor("x"),
|
||||
"type": "quantitative",
|
||||
"title": Template.anchor("x_label"),
|
||||
},
|
||||
"y": {
|
||||
"field": Template.anchor("y"),
|
||||
"type": "quantitative",
|
||||
"title": Template.anchor("y_label"),
|
||||
"scale": {"zero": False},
|
||||
},
|
||||
"color": {"field": "rev", "type": "nominal"},
|
||||
},
|
||||
"transform": [
|
||||
{
|
||||
"loess": Template.anchor("y"),
|
||||
"on": Template.anchor("x"),
|
||||
"groupby": ["rev"],
|
||||
"bandwidth": 0.3,
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
class PlotTemplates:
|
||||
TEMPLATES_DIR = "plots"
|
||||
TEMPLATES = [
|
||||
DefaultLinearTemplate,
|
||||
DefaultConfusionTemplate,
|
||||
DefaultScatterTemplate,
|
||||
SmoothLinearTemplate,
|
||||
]
|
||||
|
||||
@cached_property
|
||||
|
||||
+38
-26
@@ -48,11 +48,11 @@ class DvcTree(BaseTree):
|
||||
raise FileNotFoundError
|
||||
dir_cache = out.get_dir_cache(remote=remote)
|
||||
for entry in dir_cache:
|
||||
entry_relpath = entry[out.remote.PARAM_RELPATH]
|
||||
entry_relpath = entry[out.remote.tree.PARAM_RELPATH]
|
||||
if os.name == "nt":
|
||||
entry_relpath = entry_relpath.replace("/", os.sep)
|
||||
if path == out.path_info / entry_relpath:
|
||||
return entry[out.remote.PARAM_CHECKSUM]
|
||||
return entry[out.remote.tree.PARAM_CHECKSUM]
|
||||
raise FileNotFoundError
|
||||
|
||||
def open(self, path, mode="r", encoding="utf-8", remote=None):
|
||||
@@ -82,7 +82,7 @@ class DvcTree(BaseTree):
|
||||
else:
|
||||
checksum = out.checksum
|
||||
try:
|
||||
remote_info = remote_obj.checksum_to_path_info(checksum)
|
||||
remote_info = remote_obj.hash_to_path_info(checksum)
|
||||
return remote_obj.open(
|
||||
remote_info, mode=mode, encoding=encoding
|
||||
)
|
||||
@@ -93,7 +93,7 @@ class DvcTree(BaseTree):
|
||||
|
||||
if out.is_dir_checksum:
|
||||
checksum = self._get_granular_checksum(path, out)
|
||||
cache_path = out.cache.checksum_to_path_info(checksum).url
|
||||
cache_path = out.cache.hash_to_path_info(checksum).url
|
||||
else:
|
||||
cache_path = out.cache_path
|
||||
return open(cache_path, mode=mode, encoding=encoding)
|
||||
@@ -142,7 +142,7 @@ class DvcTree(BaseTree):
|
||||
files = []
|
||||
|
||||
root_len = len(root.parts)
|
||||
for key, out in trie.iteritems(prefix=root.parts):
|
||||
for key, out in trie.iteritems(prefix=root.parts): # noqa: B301
|
||||
if key == root.parts:
|
||||
continue
|
||||
|
||||
@@ -153,25 +153,29 @@ class DvcTree(BaseTree):
|
||||
|
||||
files.append(name)
|
||||
|
||||
if topdown:
|
||||
dirs = list(dirs)
|
||||
yield root.fspath, dirs, files
|
||||
assert topdown
|
||||
dirs = list(dirs)
|
||||
yield root.fspath, dirs, files
|
||||
|
||||
for dname in dirs:
|
||||
yield from self._walk(root / dname, trie)
|
||||
else:
|
||||
assert False
|
||||
for dname in dirs:
|
||||
yield from self._walk(root / dname, trie)
|
||||
|
||||
def walk(self, top, topdown=True, download_callback=None, **kwargs):
|
||||
def walk(
|
||||
self, top, topdown=True, onerror=None, download_callback=None, **kwargs
|
||||
):
|
||||
from pygtrie import Trie
|
||||
|
||||
assert topdown
|
||||
|
||||
if not self.exists(top):
|
||||
raise FileNotFoundError
|
||||
if onerror is not None:
|
||||
onerror(FileNotFoundError(top))
|
||||
return
|
||||
|
||||
if not self.isdir(top):
|
||||
raise NotADirectoryError
|
||||
if onerror is not None:
|
||||
onerror(NotADirectoryError(top))
|
||||
return
|
||||
|
||||
root = PathInfo(os.path.abspath(top))
|
||||
outs = self._find_outs(top, recursive=True, strict=False)
|
||||
@@ -194,7 +198,7 @@ class DvcTree(BaseTree):
|
||||
download_callback(downloaded)
|
||||
|
||||
for entry in dir_cache:
|
||||
entry_relpath = entry[out.remote.PARAM_RELPATH]
|
||||
entry_relpath = entry[out.remote.tree.PARAM_RELPATH]
|
||||
if os.name == "nt":
|
||||
entry_relpath = entry_relpath.replace("/", os.sep)
|
||||
path_info = out.path_info / entry_relpath
|
||||
@@ -212,7 +216,7 @@ class DvcTree(BaseTree):
|
||||
def isexec(self, path):
|
||||
return False
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
def get_file_hash(self, path_info):
|
||||
outs = self._find_outs(path_info, strict=False)
|
||||
if len(outs) != 1:
|
||||
raise OutputNotFoundError
|
||||
@@ -333,12 +337,14 @@ class RepoTree(BaseTree):
|
||||
elif dirname in repo_set:
|
||||
yield from self._walk_one(repo_walk)
|
||||
|
||||
def walk(self, top, topdown=True, dvcfiles=False, **kwargs):
|
||||
def walk(self, top, topdown=True, onerror=None, dvcfiles=False, **kwargs):
|
||||
"""Walk and merge both DVC and repo trees.
|
||||
|
||||
Args:
|
||||
top: path to walk from
|
||||
topdown: if True, tree will be walked from top down.
|
||||
onerror: if set, onerror function will be called if an error
|
||||
occurs (by default errors are ignored).
|
||||
dvcfiles: if True, dvcfiles will be included in the files list
|
||||
for walked directories.
|
||||
|
||||
@@ -348,21 +354,27 @@ class RepoTree(BaseTree):
|
||||
assert topdown
|
||||
|
||||
if not self.exists(top):
|
||||
raise FileNotFoundError
|
||||
if onerror is not None:
|
||||
onerror(FileNotFoundError(top))
|
||||
return
|
||||
|
||||
if not self.isdir(top):
|
||||
raise NotADirectoryError
|
||||
if onerror is not None:
|
||||
onerror(NotADirectoryError(top))
|
||||
return
|
||||
|
||||
dvc_exists = self.dvctree and self.dvctree.exists(top)
|
||||
repo_exists = self.repo.tree.exists(top)
|
||||
if dvc_exists and not repo_exists:
|
||||
yield from self.dvctree.walk(top, topdown=topdown, **kwargs)
|
||||
yield from self.dvctree.walk(
|
||||
top, topdown=topdown, onerror=onerror, **kwargs
|
||||
)
|
||||
return
|
||||
if repo_exists and not dvc_exists:
|
||||
yield from self.repo.tree.walk(top, topdown=topdown)
|
||||
yield from self.repo.tree.walk(
|
||||
top, topdown=topdown, onerror=onerror
|
||||
)
|
||||
return
|
||||
if not dvc_exists and not repo_exists:
|
||||
raise FileNotFoundError
|
||||
|
||||
dvc_walk = self.dvctree.walk(top, topdown=topdown, **kwargs)
|
||||
repo_walk = self.repo.tree.walk(top, topdown=topdown)
|
||||
@@ -373,7 +385,7 @@ class RepoTree(BaseTree):
|
||||
for fname in files:
|
||||
yield PathInfo(root) / fname
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
def get_file_hash(self, path_info):
|
||||
"""Return file checksum for specified path.
|
||||
|
||||
If path_info is a DVC out, the pre-computed checksum for the file
|
||||
@@ -384,7 +396,7 @@ class RepoTree(BaseTree):
|
||||
raise FileNotFoundError
|
||||
if self.dvctree and self.dvctree.exists(path_info):
|
||||
try:
|
||||
return self.dvctree.get_file_checksum(path_info)
|
||||
return self.dvctree.get_file_hash(path_info)
|
||||
except OutputNotFoundError:
|
||||
pass
|
||||
return file_md5(path_info, self)[0]
|
||||
|
||||
@@ -51,9 +51,6 @@ def _edit_rwlock(lock_dir):
|
||||
yield lock
|
||||
with open(path, "w+") as fobj:
|
||||
json.dump(lock, fobj)
|
||||
# NOTE: flush and fsync to ensure that rwlock contents are saved
|
||||
fobj.flush()
|
||||
os.fsync(fobj.fileno())
|
||||
|
||||
|
||||
def _infos_to_str(infos):
|
||||
|
||||
+16
-18
@@ -37,24 +37,22 @@ class TqdmGit(Tqdm):
|
||||
def code2desc(op_code):
|
||||
from git import RootUpdateProgress as OP
|
||||
|
||||
ops = dict(
|
||||
(
|
||||
(OP.COUNTING, "Counting"),
|
||||
(OP.COMPRESSING, "Compressing"),
|
||||
(OP.WRITING, "Writing"),
|
||||
(OP.RECEIVING, "Receiving"),
|
||||
(OP.RESOLVING, "Resolving"),
|
||||
(OP.FINDING_SOURCES, "Finding sources"),
|
||||
(OP.CHECKING_OUT, "Checking out"),
|
||||
(OP.CLONE, "Cloning"),
|
||||
(OP.FETCH, "Fetching"),
|
||||
(OP.UPDWKTREE, "Updating working tree"),
|
||||
(OP.REMOVE, "Removing"),
|
||||
(OP.PATHCHANGE, "Changing path"),
|
||||
(OP.URLCHANGE, "Changing URL"),
|
||||
(OP.BRANCHCHANGE, "Changing branch"),
|
||||
)
|
||||
)
|
||||
ops = {
|
||||
OP.COUNTING: "Counting",
|
||||
OP.COMPRESSING: "Compressing",
|
||||
OP.WRITING: "Writing",
|
||||
OP.RECEIVING: "Receiving",
|
||||
OP.RESOLVING: "Resolving",
|
||||
OP.FINDING_SOURCES: "Finding sources",
|
||||
OP.CHECKING_OUT: "Checking out",
|
||||
OP.CLONE: "Cloning",
|
||||
OP.FETCH: "Fetching",
|
||||
OP.UPDWKTREE: "Updating working tree",
|
||||
OP.REMOVE: "Removing",
|
||||
OP.PATHCHANGE: "Changing path",
|
||||
OP.URLCHANGE: "Changing URL",
|
||||
OP.BRANCHCHANGE: "Changing branch",
|
||||
}
|
||||
return ops.get(op_code & OP.OP_MASK, "")
|
||||
|
||||
|
||||
|
||||
+8
-3
@@ -128,17 +128,22 @@ class GitTree(BaseTree):
|
||||
if not topdown:
|
||||
yield os.path.normpath(tree.abspath), dirs, nondirs
|
||||
|
||||
def walk(self, top, topdown=True):
|
||||
def walk(self, top, topdown=True, onerror=None):
|
||||
"""Directory tree generator.
|
||||
|
||||
See `os.walk` for the docs. Differences:
|
||||
- no support for symlinks
|
||||
- it could raise exceptions, there is no onerror argument
|
||||
"""
|
||||
|
||||
tree = self.git_object_by_path(top)
|
||||
if tree is None:
|
||||
raise OSError(errno.ENOENT, "No such file")
|
||||
if onerror is not None:
|
||||
onerror(OSError(errno.ENOENT, "No such file", top))
|
||||
return
|
||||
if tree.mode != GIT_MODE_DIR:
|
||||
if onerror is not None:
|
||||
onerror(NotADirectoryError(top))
|
||||
return
|
||||
|
||||
yield from self._walk(tree, topdown=topdown)
|
||||
|
||||
|
||||
+4
-9
@@ -21,12 +21,11 @@ class BaseTree:
|
||||
def isfile(self, path):
|
||||
"""Test whether a path is a regular file"""
|
||||
|
||||
def walk(self, top, topdown=True):
|
||||
def walk(self, top, topdown=True, onerror=None):
|
||||
"""Directory tree generator.
|
||||
|
||||
See `os.walk` for the docs. Differences:
|
||||
- no support for symlinks
|
||||
- it could raise exceptions, there is no onerror argument
|
||||
"""
|
||||
|
||||
def walk_files(self, top):
|
||||
@@ -39,7 +38,8 @@ class BaseTree:
|
||||
class WorkingTree(BaseTree):
|
||||
"""Proxies the repo file access methods to working tree files"""
|
||||
|
||||
def __init__(self, repo_root=os.getcwd()):
|
||||
def __init__(self, repo_root=None):
|
||||
repo_root = repo_root or os.getcwd()
|
||||
self.repo_root = repo_root
|
||||
|
||||
@property
|
||||
@@ -64,17 +64,12 @@ class WorkingTree(BaseTree):
|
||||
"""Test whether a path is a regular file"""
|
||||
return os.path.isfile(path)
|
||||
|
||||
def walk(self, top, topdown=True):
|
||||
def walk(self, top, topdown=True, onerror=None):
|
||||
"""Directory tree generator.
|
||||
|
||||
See `os.walk` for the docs. Differences:
|
||||
- no support for symlinks
|
||||
- it could raise exceptions, there is no onerror argument
|
||||
"""
|
||||
|
||||
def onerror(e):
|
||||
raise e
|
||||
|
||||
for root, dirs, files in os.walk(
|
||||
top, topdown=topdown, onerror=onerror
|
||||
):
|
||||
|
||||
+10
-3
@@ -3,7 +3,7 @@ import os
|
||||
import string
|
||||
from collections import defaultdict
|
||||
|
||||
from funcy import project
|
||||
from funcy import cached_property, project
|
||||
|
||||
import dvc.dependency as dependency
|
||||
import dvc.prompt as prompt
|
||||
@@ -127,6 +127,8 @@ class Stage(params.StageParams):
|
||||
@path.setter
|
||||
def path(self, path):
|
||||
self._path = path
|
||||
self.__dict__.pop("path_in_repo", None)
|
||||
self.__dict__.pop("relpath", None)
|
||||
|
||||
@property
|
||||
def dvcfile(self):
|
||||
@@ -172,11 +174,11 @@ class Stage(params.StageParams):
|
||||
and self.path_in_repo == other.path_in_repo
|
||||
)
|
||||
|
||||
@property
|
||||
@cached_property
|
||||
def path_in_repo(self):
|
||||
return relpath(self.path, self.repo.root_dir)
|
||||
|
||||
@property
|
||||
@cached_property
|
||||
def relpath(self):
|
||||
return relpath(self.path)
|
||||
|
||||
@@ -284,12 +286,17 @@ class Stage(params.StageParams):
|
||||
for out in self.outs:
|
||||
out.unprotect()
|
||||
|
||||
def ignore_remove_outs(self):
|
||||
for out in self.outs:
|
||||
out.ignore_remove()
|
||||
|
||||
@rwlocked(write=["outs"])
|
||||
def remove(self, force=False, remove_outs=True, purge=True):
|
||||
if remove_outs:
|
||||
self.remove_outs(ignore_remove=True, force=force)
|
||||
else:
|
||||
self.unprotect_outs()
|
||||
self.ignore_remove_outs()
|
||||
if purge:
|
||||
self.dvcfile.remove_stage(self)
|
||||
|
||||
|
||||
@@ -26,4 +26,5 @@ def sync_import(stage, dry=False, force=False):
|
||||
if not force and stage.already_cached():
|
||||
stage.outs[0].checkout()
|
||||
else:
|
||||
stage.save_deps()
|
||||
stage.deps[0].download(stage.outs[0])
|
||||
|
||||
@@ -111,7 +111,7 @@ def _serialize_params_values(params: List[ParamsDependency]):
|
||||
def to_pipeline_file(stage: "PipelineStage"):
|
||||
wdir = resolve_wdir(stage.wdir, stage.path)
|
||||
params, deps = split_params_deps(stage)
|
||||
deps = sorted([d.def_path for d in deps])
|
||||
deps = sorted(d.def_path for d in deps)
|
||||
params = _serialize_params_keys(params)
|
||||
|
||||
outs, metrics, plots = _serialize_outs(stage.outs)
|
||||
|
||||
+4
-3
@@ -8,7 +8,8 @@ from dvc import dependency, output
|
||||
from dvc.utils.fs import path_isin
|
||||
|
||||
from ..dependency import ParamsDependency
|
||||
from ..remote import LocalRemote, S3Remote
|
||||
from ..remote.local import LocalRemoteTree
|
||||
from ..remote.s3 import S3RemoteTree
|
||||
from ..utils import dict_md5, format_link, relpath
|
||||
from .exceptions import (
|
||||
MissingDataSource,
|
||||
@@ -132,8 +133,8 @@ def stage_dump_eq(stage_cls, old_d, new_d):
|
||||
new_d.pop(stage_cls.PARAM_MD5, None)
|
||||
outs = old_d.get(stage_cls.PARAM_OUTS, [])
|
||||
for out in outs:
|
||||
out.pop(LocalRemote.PARAM_CHECKSUM, None)
|
||||
out.pop(S3Remote.PARAM_CHECKSUM, None)
|
||||
out.pop(LocalRemoteTree.PARAM_CHECKSUM, None)
|
||||
out.pop(S3RemoteTree.PARAM_CHECKSUM, None)
|
||||
|
||||
# outs and deps are lists of dicts. To check equality, we need to make
|
||||
# them independent of the order, so, we convert them to dicts.
|
||||
|
||||
+1
-1
@@ -6,7 +6,7 @@
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
_BASE_VERSION = "1.0.0b1"
|
||||
_BASE_VERSION = "1.0.0b6"
|
||||
|
||||
|
||||
def _generate_version(base_version):
|
||||
|
||||
@@ -115,10 +115,10 @@ build_dvc() {
|
||||
# [1] https://github.com/iterative/dvc/issues/2585
|
||||
if [[ "$(uname)" == 'Linux' ]]; then
|
||||
mkdir -p $BUILD_DIR/$BASH_CMPLT_DIR
|
||||
cp scripts/completion/dvc.bash $BUILD_DIR/$BASH_CMPLT_DIR/dvc
|
||||
$LIB_DIR/dvc/dvc completion -s bash > $BUILD_DIR/$BASH_CMPLT_DIR/dvc
|
||||
|
||||
mkdir -p $BUILD_DIR/$ZSH_CMPLT_DIR
|
||||
cp scripts/completion/dvc.zsh $BUILD_DIR/$ZSH_CMPLT_DIR
|
||||
$LIB_DIR/dvc/dvc completion -s zsh > $BUILD_DIR/$ZSH_CMPLT_DIR/_dvc
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
@@ -1,146 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# References:
|
||||
# - https://www.gnu.org/software/bash/manual/html_node/Programmable-Completion.html
|
||||
# - https://opensource.com/article/18/3/creating-bash-completion-script
|
||||
# - https://stackoverflow.com/questions/12933362
|
||||
|
||||
_dvc_commands='add cache checkout commit config dag destroy diff fetch freeze \
|
||||
get-url get gc import-url import init install list metrics move params \
|
||||
plots pull push remote remove repro root run status unfreeze unprotect \
|
||||
update version'
|
||||
|
||||
_dvc_options='-h --help -V --version'
|
||||
_dvc_global_options='-h --help -q --quiet -v --verbose'
|
||||
|
||||
_dvc_add='-R --recursive -f --file --no-commit --external'
|
||||
_dvc_add_COMPGEN=_dvc_compgen_files
|
||||
_dvc_cache='dir'
|
||||
_dvc_cache_dir=' --global --system --local -u --unset'
|
||||
_dvc_checkout='-d --with-deps -R --recursive -f --force --relink --summary'
|
||||
_dvc_checkout_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_commit='-f --force -d --with-deps -R --recursive'
|
||||
_dvc_commit_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_config='-u --unset --local --system --global'
|
||||
_dvc_dag='--dot --full'
|
||||
_dvc_dag_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_destroy='-f --force'
|
||||
_dvc_diff='-t --show-json --show-hash --show-md'
|
||||
_dvc_fetch='-j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -R --recursive'
|
||||
_dvc_fetch_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_gc='-a --all-branches --all-commits -T --all-tags -w --workspace -c --cloud -r --remote -f --force -p --projects -j --jobs'
|
||||
_dvc_get='-o --out --rev --show-url'
|
||||
_dvc_get_url=''
|
||||
_dvc_import='-o --out --rev'
|
||||
_dvc_import_url='-f --file'
|
||||
_dvc_init='--no-scm -f --force'
|
||||
_dvc_install=''
|
||||
_dvc_list='-R --recursive --dvc-only --rev'
|
||||
_dvc_list_COMPGEN=_dvc_compgen_files
|
||||
_dvc_freeze=''
|
||||
_dvc_freeze_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_metrics='diff show'
|
||||
_dvc_metrics_diff='--targets -t --type -x --xpath -R --show-json --show-md --no-path --old'
|
||||
_dvc_metrics_show='-t --type -x --xpath -a --all-branches -T --all-tags -R --recursive'
|
||||
_dvc_metrics_show_COMPGEN=_dvc_compgen_files
|
||||
_dvc_move=''
|
||||
_dvc_move_COMPGEN=_dvc_compgen_files
|
||||
_dvc_params='diff'
|
||||
_dvc_params_diff='--all --show-json --show-md --no-path'
|
||||
_dvc_plots='show diff'
|
||||
_dvc_plots_show='-t --template -o --out -x -y --show-vega --no-csv-header --title --x-label --y-label'
|
||||
_dvc_plots_diff='-t --template --targets -o --out -x -y --show-vega --no-csv-header --title --x-label --y-label'
|
||||
_dvc_plots_modify='-t --template -x -y --no-csv-header --title --x-label --y-label'
|
||||
_dvc_pull='-j --jobs -r --remote -a --all-branches -T --all-tags -f --force -d --with-deps -R --recursive'
|
||||
_dvc_pull_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_push='-j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -R --recursive'
|
||||
_dvc_push_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_remote='add default list modify remove rename'
|
||||
_dvc_remote_add='--global --system --local -d --default -f --force'
|
||||
_dvc_remote_default='--global --system --local -u --unset'
|
||||
_dvc_remote_list='--global --system --local'
|
||||
_dvc_remote_modify='--global --system --local -u --unset'
|
||||
_dvc_remote_remove='--global --system --local'
|
||||
_dvc_remote_rename='--global --system --local'
|
||||
_dvc_remove='-o --outs -p --purge -f --force'
|
||||
_dvc_remove_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_repro='-f --force -s --single-item -c --cwd -m --metrics --dry -i --interactive -p --pipeline -P --all-pipelines --no-run-cache --force-downstream --no-commit -R --recursive --downstream'
|
||||
_dvc_repro_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_root=''
|
||||
_dvc_run='--no-exec -f --file -d --deps -o --outs -O --outs-no-cache --outs-persist --outs-persist-no-cache -m --metrics -M --metrics-no-cache --overwrite-dvcfile --no-run-cache --no-commit -w --wdir --external'
|
||||
_dvc_run_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_status='-j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -c --cloud'
|
||||
_dvc_status_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_unfreeze_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_unprotect_COMPGEN=_dvc_compgen_files
|
||||
_dvc_update='--rev'
|
||||
_dvc_update_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_version=''
|
||||
|
||||
# $1=COMP_WORDS[1]
|
||||
_dvc_compgen_DVCFiles() {
|
||||
compgen -f -X '!*?.dvc' -- $1
|
||||
compgen -d -S '/' -- $1 # recurse into subdirs
|
||||
# Note that the recurse into dirs is only for looking for DVC-files.
|
||||
# Since dirs themselves are not required, we need `-o nospace` at the bottom
|
||||
# unfortunately :(
|
||||
}
|
||||
|
||||
# $1=COMP_WORDS[1]
|
||||
_dvc_compgen_files() {
|
||||
compgen -f -- $1
|
||||
compgen -d -S '/' -- $1 # recurse into subdirs
|
||||
}
|
||||
|
||||
# $1=COMP_WORDS[1]
|
||||
_dvc_replace_hyphen() {
|
||||
echo $1 | sed 's/-/_/g'
|
||||
}
|
||||
|
||||
# $1=COMP_WORDS[1]
|
||||
_dvc_compgen_command() {
|
||||
local flags_list="_dvc_$(_dvc_replace_hyphen $1)"
|
||||
local args_gen="${flags_list}_COMPGEN"
|
||||
COMPREPLY=( $(compgen -W "$_dvc_global_options ${!flags_list}" -- "$word"; [ -n "${!args_gen}" ] && ${!args_gen} "$word") )
|
||||
}
|
||||
|
||||
# $1=COMP_WORDS[1]
|
||||
# $2=COMP_WORDS[2]
|
||||
_dvc_compgen_subcommand() {
|
||||
local flags_list="_dvc_$(_dvc_replace_hyphen $1)_$(_dvc_replace_hyphen $2)"
|
||||
local args_gen="${flags_list}_COMPGEN"
|
||||
[ -n "${!args_gen}" ] && local opts_more="$(${!args_gen} "$word")"
|
||||
local opts="${!flags_list}"
|
||||
if [ -z "$opts$opts_more" ]; then
|
||||
_dvc_compgen_command $1
|
||||
else
|
||||
COMPREPLY=( $(compgen -W "$_dvc_global_options $opts" -- "$word"; [ -n "$opts_more" ] && echo "$opts_more") )
|
||||
fi
|
||||
}
|
||||
|
||||
# Notes:
|
||||
# `COMPREPLY` contains what will be rendered after completion is triggered
|
||||
# `word` refers to the current typed word
|
||||
# `${!var}` is to evaluate the content of `var` and expand its content as a variable
|
||||
# hello="world"
|
||||
# x="hello"
|
||||
# ${!x} -> ${hello} -> "world"
|
||||
_dvc() {
|
||||
local word="${COMP_WORDS[COMP_CWORD]}"
|
||||
|
||||
COMPREPLY=()
|
||||
|
||||
if [ "${COMP_CWORD}" -eq 1 ]; then
|
||||
case "$word" in
|
||||
-*) COMPREPLY=($(compgen -W "$_dvc_options" -- "$word")) ;;
|
||||
*) COMPREPLY=($(compgen -W "$_dvc_commands" -- "$word")) ;;
|
||||
esac
|
||||
elif [ "${COMP_CWORD}" -eq 2 ]; then
|
||||
_dvc_compgen_command ${COMP_WORDS[1]}
|
||||
elif [ "${COMP_CWORD}" -ge 3 ]; then
|
||||
_dvc_compgen_subcommand ${COMP_WORDS[1]} ${COMP_WORDS[2]}
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
complete -o nospace -F _dvc dvc
|
||||
@@ -1,340 +0,0 @@
|
||||
#compdef dvc
|
||||
|
||||
#----------------------------------------------------------
|
||||
# Repository: https://github.com/iterative/dvc
|
||||
#
|
||||
# References:
|
||||
# - https://github.com/zsh-users/zsh-completions
|
||||
# - http://zsh.sourceforge.net/Doc/Release/Completion-System.html
|
||||
# - https://mads-hartmann.com/2017/08/06/writing-zsh-completion-scripts.html
|
||||
# - http://www.linux-mag.com/id/1106/
|
||||
#----------------------------------------------------------
|
||||
|
||||
_dvc_commands() {
|
||||
local _commands=(
|
||||
"add:Track data files or directories with DVC."
|
||||
"cache:Manage cache settings."
|
||||
"checkout:Checkout data files from cache."
|
||||
"commit:Save changed data to cache and update DVC-files."
|
||||
"config:Get or set config settings."
|
||||
"dag:Visualize DVC project DAG."
|
||||
"destroy:Remove DVC-files, local DVC config and data cache."
|
||||
"diff:Show added, modified, or deleted data between commits in the DVC repository, or between a commit and the workspace."
|
||||
"fetch:Get files or directories tracked by DVC from remote storage into the cache."
|
||||
"get-url:Download or copy files from URL."
|
||||
"get:Download data from DVC repository."
|
||||
"gc:Garbage collect unused objects from cache or remote storage."
|
||||
"import-url:Download or copy file from URL and take it under DVC control."
|
||||
"import:Download data from DVC repository and take it under DVC control."
|
||||
"init:Initialize DVC in the current directory."
|
||||
"install:Install DVC git hooks into the repository."
|
||||
"list:List repository contents, including files and directories tracked by DVC and by Git."
|
||||
"lock:Lock DVC-file."
|
||||
"metrics:Commands to display and compare metrics."
|
||||
"move:Rename or move a DVC controlled data file or a directory."
|
||||
"params:Commands to display params."
|
||||
"pull:Pull data files from a DVC remote storage."
|
||||
"push:Push data files to a DVC remote storage."
|
||||
"plots:Generate plot for metrics structured as JSON, CSV or TSV."
|
||||
"remote:Manage remote storage configuration."
|
||||
"remove:Remove outputs of DVC-file."
|
||||
"repro:Check for changes and reproduce DVC-file and dependencies."
|
||||
"root:Relative path to project's directory."
|
||||
"run:Generate a stage file from a command and execute the command."
|
||||
"status:Show changed stages, compare local cache and a remote storage."
|
||||
"unlock:Unlock DVC-file."
|
||||
"unprotect:Unprotect data file/directory."
|
||||
"update:Update data artifacts imported from other DVC repositories."
|
||||
"version:Show DVC version and system/environment information."
|
||||
)
|
||||
|
||||
_describe 'dvc commands' _commands
|
||||
}
|
||||
|
||||
_dvc_options=(
|
||||
"(-)"{-h,--help}"[Show help message.]"
|
||||
"(-)"{-V,--version}"[Show program's version]"
|
||||
)
|
||||
|
||||
_dvc_global_options=(
|
||||
"(-)"{-h,--help}"[Show help message related to the command.]"
|
||||
"(-)"{-q,--quiet}"[Be quiet.]"
|
||||
"(-)"{-v,--verbose}"[Be verbose.]"
|
||||
)
|
||||
|
||||
_dvc_add=(
|
||||
{-R,--recursive}"[Recursively add each file under the directory.]"
|
||||
"--no-commit[Don't put files/directories into cache.]"
|
||||
{-f,--file}"[Specify name of the DVC-file it generates.]:File:_files"
|
||||
"--external[Allow targets that are outside of the DVC project.]"
|
||||
"1:File:_files"
|
||||
)
|
||||
|
||||
_dvc_cache=(
|
||||
"1:Sub command:(dir)"
|
||||
)
|
||||
|
||||
_dvc_checkout=(
|
||||
{-d,--with-deps}"[Checkout all dependencies of the specified target.]"
|
||||
{-R,--recursive}"[Checkout all subdirectories of the specified directory.]"
|
||||
{-f,--force}"[Do not prompt when removing working directory files.]"
|
||||
"--relink[Recreate links or copies from cache to workspace.]"
|
||||
"--summary[Show summary of the changes.]"
|
||||
"1:Stages:_files -g '(*.dvc|Dvcfile)'"
|
||||
)
|
||||
|
||||
_dvc_commit=(
|
||||
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
|
||||
{-f,--force}"[Commit even if checksums for dependencies/outputs changed.]"
|
||||
{-d,--with-deps}"[Commit all dependencies of the specified target.]"
|
||||
{-R,--recursive}"[Commit cache for subdirectories of the specified directory.]"
|
||||
)
|
||||
|
||||
_dvc_config=(
|
||||
"--global[Use global config.]"
|
||||
"--system[Use system config.]"
|
||||
"--local[Use local config.]"
|
||||
{-u,--unset}"[Unset option.]"
|
||||
)
|
||||
|
||||
_dvc_dag=(
|
||||
"--dot[Print DAG in DOT format.]"
|
||||
"--full[Show full DAG that the target belongs too, instead of showing DAG consisting only of ancestors.]"
|
||||
"1:Stage:"
|
||||
)
|
||||
|
||||
_dvc_destroy=(
|
||||
{-f,--force}"[Force destruction.]"
|
||||
)
|
||||
|
||||
_dvc_diff=(
|
||||
"--show-json[Format the output into a JSON]"
|
||||
"--show-hash[Display hash value for each entry]"
|
||||
"--show-md[Format the output into a Markdown table]"
|
||||
"1:Old Git commit to compare (defaults to HEAD):"
|
||||
"2:New Git commit to compare (defaults to the current workspace):"
|
||||
)
|
||||
|
||||
_dvc_fetch=(
|
||||
{-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:"
|
||||
{-r,--remote}"[Remote repository to fetch from.]:Remote repository:"
|
||||
{-a,--all-branches}"[Fetch cache for all branches.]"
|
||||
{-T,--all-tags}"[Fetch cache for all tags.]"
|
||||
{-d,--with-deps}"[Fetch cache for all dependencies of the specified target.]"
|
||||
{-R,--recursive}"[Fetch cache for subdirectories of specified directory.]"
|
||||
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
|
||||
)
|
||||
|
||||
_dvc_geturl=(
|
||||
"1:URL:"
|
||||
"2:Output:"
|
||||
)
|
||||
|
||||
_dvc_get=(
|
||||
{-o,--out}"[Destination path to put data to.]:OUT:_files -/"
|
||||
"--rev[Git revision (e.g. SHA, branch, tag)]:Revision:"
|
||||
"--show-url[Returns path/url to the location in remote for given path]"
|
||||
"1:URL:"
|
||||
"2:Path:"
|
||||
)
|
||||
|
||||
_dvc_gc=(
|
||||
{-w,--workspace}"[Keep data files used in the current workspace.]"
|
||||
{-a,--all-branches}"[Keep data files for the tips of all Git branches.]"
|
||||
"--all-commits[Keep data files for all Git commits.]"
|
||||
{-T,--all-tags}"[Keep data files for all Git tags.]"
|
||||
{-c,--cloud}"[Collect garbage in remote repository.]"
|
||||
{-r,--remote}"[Remote storage to collect garbage in.]:Remote repository:"
|
||||
{-f,--force}"[Force garbage collection - automatically agree to all prompts.]:Repos:_files"
|
||||
{-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:"
|
||||
{-p,--projects}"[Keep data files required by these projects in addition to the current one.]:Repos:_files"
|
||||
)
|
||||
|
||||
_dvc_importurl=(
|
||||
{-f,--file}"[Specify name of the DVC-file it generates.]:File:_files"
|
||||
"1:URL:"
|
||||
"2:Output:"
|
||||
)
|
||||
|
||||
_dvc_import=(
|
||||
{-o,--out}"[Destination path to put data to.]:OUT:_files -/"
|
||||
"--rev[Git revision (e.g. SHA, branch, tag)]:Commit hash:"
|
||||
"1:URL:"
|
||||
"2:Path:"
|
||||
)
|
||||
|
||||
_dvc_init=(
|
||||
"--no-scm[Initiate dvc in directory that is not tracked by any scm tool.]"
|
||||
{-f,--force}"[Overwrite existing '.dvc' directory. This operation removes local cache.]"
|
||||
)
|
||||
|
||||
_dvc_install=()
|
||||
|
||||
_dvc_freeze=(
|
||||
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
|
||||
)
|
||||
|
||||
_dvc_list=(
|
||||
"--rev[Git revision (e.g. branch, tag, SHA)]:Revision:"
|
||||
{-R,--recursive}"[Recursively add each file under the directory.]"
|
||||
"--dvc-only[Only outputs DVC-outs.]"
|
||||
"1:URL:"
|
||||
"2:Path:"
|
||||
)
|
||||
|
||||
_dvc_metrics=(
|
||||
"1:Sub command:(show diff)"
|
||||
)
|
||||
|
||||
_dvc_move=(
|
||||
"1:Source:_files"
|
||||
"2:Destination:"
|
||||
)
|
||||
|
||||
_dvc_params=(
|
||||
"1:Sub command:(diff)"
|
||||
)
|
||||
|
||||
_dvc_pull=(
|
||||
{-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:"
|
||||
{-r,--remote}"[Remote repository to pull from.]:Remote repository:"
|
||||
{-a,--all-branches}"[Fetch cache for all branches.]"
|
||||
{-T,--all-tags}"[Fetch cache for all tags.]"
|
||||
{-d,--with-deps}"[Fetch cache for all dependencies of the specified target.]"
|
||||
{-f,--force}"[Do not prompt when removing working directory files.]"
|
||||
{-R,--recursive}"[Pull cache for subdirectories of the specified directory.]"
|
||||
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
|
||||
)
|
||||
|
||||
_dvc_push=(
|
||||
{-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:"
|
||||
{-r,--remote}"[Remote repository to push to.]:Remote repository:"
|
||||
{-a,--all-branches}"[Push cache for all branches.]"
|
||||
{-T,--all-tags}"[Push cache for all tags.]"
|
||||
{-d,--with-deps}"[Push cache for all dependencies of the specified target.]"
|
||||
{-R,--recursive}"[Push cache for subdirectories of specified directory.]"
|
||||
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
|
||||
)
|
||||
|
||||
_dvc_plots=(
|
||||
"1:Sub command:(show diff modify)"
|
||||
)
|
||||
|
||||
_dvc_remote=(
|
||||
"1:Sub command:(add default remove modify list rename)"
|
||||
)
|
||||
|
||||
_dvc_remove=(
|
||||
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
|
||||
"--dry[Only print the commands that would be executed without actually executing]"
|
||||
{-o,--outs}"[Only remove DVC-file outputs. (Default)]"
|
||||
{-p,--purge}"[Remove DVC-file and all its outputs.]"
|
||||
{-f,--force}"[Force purge.]"
|
||||
)
|
||||
|
||||
_dvc_repro=(
|
||||
{-f,--force}"[Reproduce even if dependencies were not changed.]"
|
||||
{-s,--single-item}"[Reproduce only single data item without recursive dependencies check.]"
|
||||
{-c,--cwd}"[Directory within your repo to reproduce from.]:CWD:_files -/"
|
||||
{-m,--metrics}"[Show metrics after reproduction.]"
|
||||
"--dry[Only print the commands that would be executed without actually executing]"
|
||||
{-i,--interactive}"[Ask for confirmation before reproducing each stage.]"
|
||||
{-p,--pipeline}"[Reproduce the whole pipeline that the specified stage file belongs to.]"
|
||||
{-P,--all-pipelines}"[Reproduce all pipelines in the repo.]"
|
||||
{-R,--recursive}"[Reproduce all stages in the specified directory.]"
|
||||
"--force-downstream[Reproduce all descendants of a changed stage even if their direct dependencies didn't change.]"
|
||||
"--no-run-cache[Run changed stage even if it has been already ran with the same command/dependencies/outputs/etc before.]"
|
||||
"--no-commit[Don't put files/directories into cache.]"
|
||||
"--downstream[Start from the specified stages when reproducing pipelines.]"
|
||||
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
|
||||
)
|
||||
|
||||
_dvc_root=()
|
||||
|
||||
_dvc_run=(
|
||||
"*"{-d,--deps}"[Declare dependencies for reproducible cmd.]:Dependency:_files"
|
||||
"*"{-o,--outs}"[Declare output file or directory.]:Output data:_files"
|
||||
"*"{-O,--outs-no-cache}"[Declare output file or directory (do not put into DVC cache).]:Output regular:_files"
|
||||
"*"{-m,--metrics}"[Declare output metric file or directory.]:Metrics:_files"
|
||||
"*"{-M,--metrics-no-cache}"[Declare output metric file or directory (do not put into DVC cache).]:Metrics (no cache):_files"
|
||||
{-f,--file}"[Specify name of the DVC-file it generates.]:File:_files"
|
||||
{-c,--cwd}"[Deprecated, use -w and -f instead.]:CWD:_files -/"
|
||||
{-w,--wdir}"[Directory within your repo to run your command in.]:WDIR:_files -/"
|
||||
"--no-exec[Only create stage file without actually running it.]"
|
||||
{-y,--yes}"[Deprecated, use --overwrite-dvcfile instead]"
|
||||
"--overwrite-dvcfile[Overwrite existing DVC-file without asking for confirmation.]"
|
||||
"--no-run-cache[Run this stage even if it has been already ran with the same command/dependencies/outputs/etc before.]"
|
||||
"--remove-outs[Deprecated, this is now the default behavior]"
|
||||
"--no-commit[Don't put files/directories into cache.]"
|
||||
"--outs-persist[Declare output file or directory that will not be removed upon repro.]:Output persistent:_files"
|
||||
"--outs-persist-no-cache[Declare output file or directory that will not be removed upon repro (do not put into DVC cache).]:Output persistent regular:_files"
|
||||
"--external[Allow outputs that are outside of the DVC project.]"
|
||||
)
|
||||
|
||||
_dvc_status=(
|
||||
{-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:"
|
||||
{-q,--quiet}"[Suppresses all output. Exit with 0 if pipelines are up to date, otherwise 1.]"
|
||||
{-c,--cloud}"[Show status of a local cache compared to a remote repository.]"
|
||||
{-r,--remote}"[Remote repository to compare local cache to.]:Remote repository:"
|
||||
{-a,--all-branches}"[Show status of a local cache compared to a remote repository for all branches.]"
|
||||
{-T,--all-tags}"[Show status of a local cache compared to a remote repository for all tags.]"
|
||||
{-d,--with-deps}"[Show status for all dependencies of the specified target.]"
|
||||
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
|
||||
)
|
||||
|
||||
_dvc_unfreeze=(
|
||||
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
|
||||
)
|
||||
|
||||
_dvc_unprotect=(
|
||||
"*:Data files:_files"
|
||||
)
|
||||
|
||||
_dvc_update=(
|
||||
"--rev[Git revision (e.g. SHA, branch, tag)]:Revision:"
|
||||
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
|
||||
)
|
||||
|
||||
typeset -A opt_args
|
||||
local context state line curcontext="$curcontext"
|
||||
|
||||
_arguments \
|
||||
$_dvc_options \
|
||||
'1: :_dvc_commands' \
|
||||
'*::args:->args'
|
||||
|
||||
case $words[1] in
|
||||
add) _arguments $_dvc_global_options $_dvc_add ;;
|
||||
cache) _arguments $_dvc_global_options $_dvc_cache ;;
|
||||
checkout) _arguments $_dvc_global_options $_dvc_checkout ;;
|
||||
commit) _arguments $_dvc_global_options $_dvc_commit ;;
|
||||
config) _arguments $_dvc_global_options $_dvc_config ;;
|
||||
dag) _arguments $_dvc_global_options $_dvc_dag ;;
|
||||
destroy) _arguments $_dvc_global_options $_dvc_destroy ;;
|
||||
diff) _arguments $_dvc_global_options $_dvc_diff ;;
|
||||
fetch) _arguments $_dvc_global_options $_dvc_fetch ;;
|
||||
freeze) _arguments $_dvc_global_options $_dvc_freeze ;;
|
||||
get-url) _arguments $_dvc_global_options $_dvc_geturl ;;
|
||||
get) _arguments $_dvc_global_options $_dvc_get ;;
|
||||
gc) _arguments $_dvc_global_options $_dvc_gc ;;
|
||||
import-url) _arguments $_dvc_global_options $_dvc_importurl ;;
|
||||
import) _arguments $_dvc_global_options $_dvc_import ;;
|
||||
init) _arguments $_dvc_global_options $_dvc_init ;;
|
||||
install) _arguments $_dvc_global_options $_dvc_install ;;
|
||||
list) _arguments $_dvc_global_options $_dvc_list ;;
|
||||
metrics) _arguments $_dvc_global_options $_dvc_metrics ;;
|
||||
move) _arguments $_dvc_global_options $_dvc_move ;;
|
||||
params) _arguments $_dvc_global_options $_dvc_params ;;
|
||||
pull) _arguments $_dvc_global_options $_dvc_pull ;;
|
||||
push) _arguments $_dvc_global_options $_dvc_push ;;
|
||||
plots) _arguments $_dvc_global_options $_dvc_plots ;;
|
||||
remote) _arguments $_dvc_global_options $_dvc_remote ;;
|
||||
remove) _arguments $_dvc_global_options $_dvc_remove ;;
|
||||
repro) _arguments $_dvc_global_options $_dvc_repro ;;
|
||||
root) _arguments $_dvc_global_options $_dvc_root ;;
|
||||
run) _arguments $_dvc_global_options $_dvc_run ;;
|
||||
status) _arguments $_dvc_global_options $_dvc_status ;;
|
||||
unfreeze) _arguments $_dvc_global_options $_dvc_unfreeze ;;
|
||||
unprotect) _arguments $_dvc_global_options $_dvc_unprotect ;;
|
||||
update) _arguments $_dvc_global_options $_dvc_update ;;
|
||||
esac
|
||||
@@ -1,6 +1,3 @@
|
||||
md5: 3cc286c534a71504476da009ed174423
|
||||
outs:
|
||||
- cache: true
|
||||
md5: 90104d9e83cfb825cf45507e90aadd27
|
||||
metric: false
|
||||
- md5: 90104d9e83cfb825cf45507e90aadd27
|
||||
path: dvc.ico
|
||||
|
||||
@@ -1,6 +1,3 @@
|
||||
md5: c7c2944695a3eb7f04a0a7ae20c9dc0c
|
||||
outs:
|
||||
- cache: true
|
||||
md5: 9106cda08aa427e73492389a0f17c72d
|
||||
metric: false
|
||||
- md5: 9106cda08aa427e73492389a0f17c72d
|
||||
path: dvc_left.bmp
|
||||
|
||||
@@ -1,6 +1,3 @@
|
||||
md5: cde0fe37927a5771b1ba8d1c909e120c
|
||||
outs:
|
||||
- cache: true
|
||||
md5: 94614d6650e062655f9f77507dc9c1f2
|
||||
metric: false
|
||||
- md5: 94614d6650e062655f9f77507dc9c1f2
|
||||
path: dvc_up.bmp
|
||||
|
||||
@@ -6,13 +6,18 @@ ignore=
|
||||
E203, # Whitespace before ':'
|
||||
E266, # Too many leading '#' for block comment
|
||||
W503, # Line break occurred before a binary operator
|
||||
P1, # unindexed parameters in the str.format, see:
|
||||
# https://pypi.org/project/flake8-string-format/
|
||||
max_line_length=79
|
||||
select=B,C,E,F,W,T4,B9
|
||||
max-complexity = 15
|
||||
select=B,C,E,F,W,T4,B902,T,P
|
||||
show_source=true
|
||||
count=true
|
||||
|
||||
[isort]
|
||||
include_trailing_comma=true
|
||||
known_first_party=dvc,tests
|
||||
known_third_party=PyInstaller,RangeHTTPServer,boto3,colorama,configobj,distro,dpath,flaky,flufl,funcy,git,google,grandalf,mock,mockssh,moto,nanotime,networkx,packaging,paramiko,pathspec,pytest,requests,ruamel,setuptools,shortuuid,tqdm,voluptuous,yaml,zc
|
||||
known_third_party=PyInstaller,RangeHTTPServer,boto3,colorama,configobj,distro,dpath,flaky,flufl,funcy,git,google,grandalf,mock,moto,nanotime,networkx,packaging,paramiko,pathspec,pytest,requests,ruamel,setuptools,shortuuid,shtab,tqdm,voluptuous,yaml,zc
|
||||
line_length=79
|
||||
force_grid_wrap=0
|
||||
use_parentheses=True
|
||||
|
||||
@@ -78,6 +78,7 @@ install_requires = [
|
||||
"tabulate>=0.8.7",
|
||||
"pygtrie==2.3.2",
|
||||
"dpath>=2.0.1,<3",
|
||||
"shtab>=0.0.2",
|
||||
]
|
||||
|
||||
|
||||
@@ -106,6 +107,7 @@ tests_requirements = [
|
||||
"pytest-cov>=2.6.1",
|
||||
"pytest-xdist>=1.26.1",
|
||||
"pytest-mock==1.11.2",
|
||||
"pytest-lazy-fixture",
|
||||
"flaky>=3.5.3",
|
||||
"mock>=3.0.0",
|
||||
"xmltodict>=0.11.0",
|
||||
@@ -122,6 +124,9 @@ tests_requirements = [
|
||||
"moto==1.3.14.dev464",
|
||||
"rangehttpserver==1.2.0",
|
||||
"beautifulsoup4==4.4.0",
|
||||
"flake8-bugbear",
|
||||
"flake8-comprehensions",
|
||||
"flake8-string-format",
|
||||
]
|
||||
|
||||
if (sys.version_info) >= (3, 6):
|
||||
|
||||
+8
-4
@@ -31,17 +31,21 @@ parts:
|
||||
snapcraftctl pull
|
||||
snapcraftctl set-version $(cd $SNAPCRAFT_PART_SRC && git describe --tags)
|
||||
git diff --quiet || error_dirty_build
|
||||
echo 'PKG = "snap"' > $SNAPCRAFT_PART_SRC/dvc/utils/build.py
|
||||
echo 'PKG = "snap"' > dvc/utils/build.py
|
||||
# install all optional extras
|
||||
sed -ri 's/(=install_requires)/\1+all_remotes+hdfs/' $SNAPCRAFT_PART_SRC/setup.py
|
||||
sed -ri 's/(=install_requires)/\1+all_remotes+hdfs/' setup.py
|
||||
# ensure dvc knows the state isn't really dirty
|
||||
sed -rin 's/.*git.*diff.*--quiet.*//' $SNAPCRAFT_PART_SRC/dvc/version.py
|
||||
sed -rin 's/.*git.*diff.*--quiet.*//' dvc/version.py
|
||||
override-build: |
|
||||
snapcraftctl build
|
||||
# prevent user site packages interfering with this snap - reference:
|
||||
# https://github.com/snapcore/snapcraft/blob/19393ef36cd773a28131cec10cc0bfb3bf9c7e77/tools/snapcraft-override-build.sh#L18
|
||||
sed -ri 's/^(ENABLE_USER_SITE = )None$/\1False/' $SNAPCRAFT_PART_INSTALL/usr/lib/python*/site.py
|
||||
cp $SNAPCRAFT_PART_BUILD/scripts/completion/dvc.bash $SNAPCRAFT_PART_INSTALL/completion.sh
|
||||
# make plugin aware that a file (generated later) needs to be staged
|
||||
touch $SNAPCRAFT_PART_INSTALL/completion.sh
|
||||
override-stage: |
|
||||
snapcraftctl stage
|
||||
usr/bin/python3 -m dvc completion -s bash > completion.sh
|
||||
apps:
|
||||
dvc:
|
||||
command: bin/dvc
|
||||
|
||||
+1
-34
@@ -1,12 +1,9 @@
|
||||
import os
|
||||
|
||||
import mockssh
|
||||
import pytest
|
||||
|
||||
from dvc.remote.ssh.connection import SSHConnection
|
||||
from tests.utils.httpd import PushRequestHandler, StaticFileServer
|
||||
|
||||
from .dir_helpers import * # noqa
|
||||
from .remotes import * # noqa
|
||||
|
||||
# Prevent updater and analytics from running their processes
|
||||
os.environ["DVC_TEST"] = "true"
|
||||
@@ -28,39 +25,9 @@ def reset_loglevel(request, caplog):
|
||||
yield
|
||||
|
||||
|
||||
here = os.path.abspath(os.path.dirname(__file__))
|
||||
|
||||
user = "user"
|
||||
key_path = os.path.join(here, f"{user}.key")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def ssh_server():
|
||||
users = {user: key_path}
|
||||
with mockssh.Server(users) as s:
|
||||
s.test_creds = {
|
||||
"host": s.host,
|
||||
"port": s.port,
|
||||
"username": user,
|
||||
"key_filename": key_path,
|
||||
}
|
||||
yield s
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def ssh(ssh_server):
|
||||
yield SSHConnection(**ssh_server.test_creds)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
def _close_pools():
|
||||
from dvc.remote.pool import close_pools
|
||||
|
||||
yield
|
||||
close_pools()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def http_server(tmp_dir):
|
||||
with StaticFileServer(handler_class=PushRequestHandler) as httpd:
|
||||
yield httpd
|
||||
|
||||
+21
-18
@@ -63,7 +63,6 @@ __all__ = [
|
||||
"run_head",
|
||||
"erepo_dir",
|
||||
"git_dir",
|
||||
"setup_remote",
|
||||
"git_init",
|
||||
]
|
||||
|
||||
@@ -177,6 +176,27 @@ class TmpDir(pathlib.Path):
|
||||
if commit:
|
||||
self.scm.commit(commit)
|
||||
|
||||
def add_remote(
|
||||
self, *, url=None, config=None, name="upstream", default=True
|
||||
):
|
||||
self._require("dvc")
|
||||
|
||||
assert bool(url) ^ bool(config)
|
||||
|
||||
if url:
|
||||
config = {"url": url}
|
||||
|
||||
with self.dvc.config.edit() as conf:
|
||||
conf["remote"][name] = config
|
||||
if default:
|
||||
conf["core"]["remote"] = name
|
||||
|
||||
if hasattr(self, "scm"):
|
||||
self.scm.add(self.dvc.config.files["repo"])
|
||||
self.scm.commit(f"add '{name}' remote")
|
||||
|
||||
return url or config["url"]
|
||||
|
||||
# contexts
|
||||
@contextmanager
|
||||
def chdir(self):
|
||||
@@ -315,20 +335,3 @@ def git_dir(make_tmp_dir):
|
||||
path = make_tmp_dir("git-erepo", scm=True)
|
||||
path.scm.commit("init repo")
|
||||
return path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def setup_remote(make_tmp_dir):
|
||||
def create(repo, url=None, name="upstream", default=True):
|
||||
if not url:
|
||||
url = os.fspath(make_tmp_dir("local_remote"))
|
||||
with repo.config.edit() as conf:
|
||||
conf["remote"][name] = {"url": url}
|
||||
if default:
|
||||
conf["core"]["remote"] = name
|
||||
|
||||
repo.scm.add(repo.config.files["repo"])
|
||||
repo.scm.commit(f"add '{name}' remote")
|
||||
return url
|
||||
|
||||
return create
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def run_copy_metrics(tmp_dir, run_copy):
|
||||
def run(file1, file2, commit=None, tag=None, single_stage=True, **kwargs):
|
||||
stage = tmp_dir.dvc.run(
|
||||
cmd=f"python copy.py {file1} {file2}",
|
||||
deps=[file1],
|
||||
single_stage=single_stage,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
if hasattr(tmp_dir.dvc, "scm"):
|
||||
files = [stage.path]
|
||||
files += [
|
||||
os.fspath(out.path_info)
|
||||
for out in stage.outs
|
||||
if not out.use_cache
|
||||
]
|
||||
tmp_dir.dvc.scm.add(files)
|
||||
if commit:
|
||||
tmp_dir.dvc.scm.commit(commit)
|
||||
if tag:
|
||||
tmp_dir.dvc.scm.tag(tag)
|
||||
return stage
|
||||
|
||||
return run
|
||||
@@ -1 +0,0 @@
|
||||
from tests.func.plots import run_copy_metrics # noqa: F401
|
||||
@@ -1,30 +0,0 @@
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def run_copy_metrics(tmp_dir, run_copy):
|
||||
def run(file1, file2, commit=None, tag=None, single_stage=True, **kwargs):
|
||||
stage = tmp_dir.dvc.run(
|
||||
cmd=f"python copy.py {file1} {file2}",
|
||||
deps=[file1],
|
||||
single_stage=single_stage,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
if hasattr(tmp_dir.dvc, "scm"):
|
||||
files = [stage.path]
|
||||
files += [
|
||||
os.fspath(out.path_info)
|
||||
for out in stage.outs
|
||||
if not out.use_cache
|
||||
]
|
||||
tmp_dir.dvc.scm.add(files)
|
||||
if commit:
|
||||
tmp_dir.dvc.scm.commit(commit)
|
||||
if tag:
|
||||
tmp_dir.dvc.scm.tag(tag)
|
||||
return stage
|
||||
|
||||
return run
|
||||
|
||||
@@ -1 +1,12 @@
|
||||
from . import run_copy_metrics # noqa: F401
|
||||
import shutil
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def custom_template(tmp_dir, dvc):
|
||||
custom_template = tmp_dir / "custom_template.json"
|
||||
shutil.copy(
|
||||
tmp_dir / ".dvc" / "plots" / "default.json", custom_template,
|
||||
)
|
||||
return custom_template
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
import pytest
|
||||
|
||||
from dvc.dvcfile import PIPELINE_LOCK
|
||||
from dvc.repo.plots import PropsNotFoundError
|
||||
from dvc.repo.plots.template import TemplateNotFoundError
|
||||
from dvc.utils import relpath
|
||||
|
||||
from .test_plots import _write_json
|
||||
|
||||
|
||||
def test_plots_modify_existing_template(
|
||||
tmp_dir, dvc, run_copy_metrics, custom_template
|
||||
):
|
||||
metric = [{"a": 1, "b": 2}, {"a": 2, "b": 3}]
|
||||
_write_json(tmp_dir, metric, "metric_t.json")
|
||||
stage = run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
plots_no_cache=["metric.json"],
|
||||
name="copy-metrics",
|
||||
single_stage=False,
|
||||
)
|
||||
dvc.plots.modify(
|
||||
"metric.json", props={"template": relpath(custom_template)}
|
||||
)
|
||||
stage = stage.reload()
|
||||
assert stage.outs[0].plot == {"template": relpath(custom_template)}
|
||||
|
||||
|
||||
def test_plots_modify_should_not_change_lockfile(
|
||||
tmp_dir, dvc, run_copy_metrics, custom_template
|
||||
):
|
||||
_write_json(tmp_dir, [{"a": 1, "b": 2}], "metric_t.json")
|
||||
run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
plots_no_cache=["metric.json"],
|
||||
name="copy-metrics",
|
||||
single_stage=False,
|
||||
)
|
||||
|
||||
(tmp_dir / PIPELINE_LOCK).unlink()
|
||||
dvc.plots.modify(
|
||||
"metric.json", props={"template": relpath(custom_template)}
|
||||
)
|
||||
assert not (tmp_dir / PIPELINE_LOCK).exists()
|
||||
|
||||
|
||||
def test_plots_modify_not_existing_template(dvc):
|
||||
with pytest.raises(TemplateNotFoundError):
|
||||
dvc.plots.modify(
|
||||
"metric.json", props={"template": "not-existing-template.json"}
|
||||
)
|
||||
|
||||
|
||||
def test_unset_nonexistent(tmp_dir, dvc, run_copy_metrics, custom_template):
|
||||
metric = [{"a": 1, "b": 2}, {"a": 2, "b": 3}]
|
||||
_write_json(tmp_dir, metric, "metric_t.json")
|
||||
run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
plots_no_cache=["metric.json"],
|
||||
name="copy-metrics",
|
||||
single_stage=False,
|
||||
)
|
||||
|
||||
with pytest.raises(PropsNotFoundError):
|
||||
dvc.plots.modify(
|
||||
"metric.json", unset=["nonexistent"],
|
||||
)
|
||||
@@ -9,7 +9,6 @@ import pytest
|
||||
import yaml
|
||||
from funcy import first
|
||||
|
||||
from dvc.dvcfile import PIPELINE_LOCK
|
||||
from dvc.repo.plots.data import (
|
||||
JSONPlotData,
|
||||
NoMetricInHistoryError,
|
||||
@@ -17,8 +16,11 @@ from dvc.repo.plots.data import (
|
||||
PlotMetricTypeError,
|
||||
YAMLPlotData,
|
||||
)
|
||||
from dvc.repo.plots.template import NoFieldInDataError, TemplateNotFoundError
|
||||
from dvc.utils import relpath
|
||||
from dvc.repo.plots.template import (
|
||||
BadTemplateError,
|
||||
NoFieldInDataError,
|
||||
TemplateNotFoundError,
|
||||
)
|
||||
|
||||
|
||||
def _write_csv(metric, filename, header=True):
|
||||
@@ -358,15 +360,6 @@ def test_throw_on_no_metric_at_all(tmp_dir, scm, dvc, caplog):
|
||||
assert str(error.value) == "Could not find 'metric.json'."
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def custom_template(tmp_dir, dvc):
|
||||
custom_template = tmp_dir / "custom_template.json"
|
||||
shutil.copy(
|
||||
tmp_dir / ".dvc" / "plots" / "default.json", custom_template,
|
||||
)
|
||||
return custom_template
|
||||
|
||||
|
||||
def test_custom_template(tmp_dir, scm, dvc, custom_template, run_copy_metrics):
|
||||
metric = [{"a": 1, "b": 2}, {"a": 2, "b": 3}]
|
||||
_write_json(tmp_dir, metric, "metric_t.json")
|
||||
@@ -416,6 +409,23 @@ def test_should_raise_on_no_template(tmp_dir, dvc, run_copy_metrics):
|
||||
dvc.plots.show("metric.json", props=props)
|
||||
|
||||
|
||||
def test_bad_template(tmp_dir, dvc, run_copy_metrics):
|
||||
metric = [{"val": 2}, {"val": 3}]
|
||||
_write_json(tmp_dir, metric, "metric_t.json")
|
||||
run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
plots_no_cache=["metric.json"],
|
||||
commit="first run",
|
||||
)
|
||||
|
||||
tmp_dir.gen("template.json", json.dumps({"a": "b", "c": "d"}))
|
||||
|
||||
with pytest.raises(BadTemplateError):
|
||||
props = {"template": "template.json"}
|
||||
dvc.plots.show("metric.json", props=props)
|
||||
|
||||
|
||||
def test_plot_wrong_metric_type(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
tmp_dir.gen("metric_t.txt", "some text")
|
||||
run_copy_metrics(
|
||||
@@ -542,51 +552,6 @@ def test_load_metric_from_dict_yaml(tmp_dir):
|
||||
assert list(map(dict, plot_data.to_datapoints())) == expected
|
||||
|
||||
|
||||
def test_plots_modify_existing_template(
|
||||
tmp_dir, dvc, run_copy_metrics, custom_template
|
||||
):
|
||||
metric = [{"a": 1, "b": 2}, {"a": 2, "b": 3}]
|
||||
_write_json(tmp_dir, metric, "metric_t.json")
|
||||
stage = run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
plots_no_cache=["metric.json"],
|
||||
name="copy-metrics",
|
||||
single_stage=False,
|
||||
)
|
||||
dvc.plots.modify(
|
||||
"metric.json", props={"template": relpath(custom_template)}
|
||||
)
|
||||
stage = stage.reload()
|
||||
assert stage.outs[0].plot == {"template": relpath(custom_template)}
|
||||
|
||||
|
||||
def test_plots_modify_should_not_change_lockfile(
|
||||
tmp_dir, dvc, run_copy_metrics, custom_template
|
||||
):
|
||||
_write_json(tmp_dir, [{"a": 1, "b": 2}], "metric_t.json")
|
||||
run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
plots_no_cache=["metric.json"],
|
||||
name="copy-metrics",
|
||||
single_stage=False,
|
||||
)
|
||||
|
||||
(tmp_dir / PIPELINE_LOCK).unlink()
|
||||
dvc.plots.modify(
|
||||
"metric.json", props={"template": relpath(custom_template)}
|
||||
)
|
||||
assert not (tmp_dir / PIPELINE_LOCK).exists()
|
||||
|
||||
|
||||
def test_plots_modify_not_existing_template(dvc):
|
||||
with pytest.raises(TemplateNotFoundError):
|
||||
dvc.plots.modify(
|
||||
"metric.json", props={"template": "not-existing-template.json"}
|
||||
)
|
||||
|
||||
|
||||
def test_multiple_plots(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
metric1 = [
|
||||
OrderedDict([("first_val", 100), ("second_val", 100), ("val", 2)]),
|
||||
|
||||
@@ -3,7 +3,7 @@ import os
|
||||
import pytest
|
||||
|
||||
from dvc.exceptions import DownloadError, UploadError
|
||||
from dvc.remote.base import BaseRemote
|
||||
from dvc.remote.base import Remote
|
||||
from dvc.remote.index import RemoteIndex
|
||||
from dvc.remote.local import LocalRemote, LocalRemoteTree
|
||||
from dvc.utils.fs import remove
|
||||
@@ -15,12 +15,12 @@ def remote(tmp_dir, dvc, tmp_path_factory, mocker):
|
||||
dvc.config["remote"]["upstream"] = {"url": url}
|
||||
dvc.config["core"]["remote"] = "upstream"
|
||||
|
||||
# patch checksums_exist since the LocalRemote normally overrides
|
||||
# BaseRemote.checksums_exist.
|
||||
def checksums_exist(self, *args, **kwargs):
|
||||
return BaseRemote.checksums_exist(self, *args, **kwargs)
|
||||
# patch hashes_exist since the LocalRemote normally overrides
|
||||
# BaseRemoteTree.hashes_exist.
|
||||
def hashes_exist(self, *args, **kwargs):
|
||||
return Remote.hashes_exist(self, *args, **kwargs)
|
||||
|
||||
mocker.patch.object(LocalRemote, "checksums_exist", checksums_exist)
|
||||
mocker.patch.object(LocalRemote, "hashes_exist", hashes_exist)
|
||||
|
||||
# patch index class since LocalRemote normally overrides index class
|
||||
mocker.patch.object(LocalRemote, "INDEX_CLS", RemoteIndex)
|
||||
@@ -38,9 +38,9 @@ def test_indexed_on_status(tmp_dir, dvc, tmp_path_factory, remote):
|
||||
|
||||
dvc.status(cloud=True)
|
||||
with remote.index:
|
||||
assert {bar.checksum, baz["md5"]} == set(remote.index.checksums())
|
||||
assert [bar.checksum] == list(remote.index.dir_checksums())
|
||||
assert foo.checksum not in remote.index.checksums()
|
||||
assert {bar.checksum, baz["md5"]} == set(remote.index.hashes())
|
||||
assert [bar.checksum] == list(remote.index.dir_hashes())
|
||||
assert foo.checksum not in remote.index.hashes()
|
||||
|
||||
|
||||
def test_indexed_on_push(tmp_dir, dvc, tmp_path_factory, remote):
|
||||
@@ -50,9 +50,9 @@ def test_indexed_on_push(tmp_dir, dvc, tmp_path_factory, remote):
|
||||
|
||||
dvc.push()
|
||||
with remote.index:
|
||||
assert {bar.checksum, baz["md5"]} == set(remote.index.checksums())
|
||||
assert [bar.checksum] == list(remote.index.dir_checksums())
|
||||
assert foo.checksum not in remote.index.checksums()
|
||||
assert {bar.checksum, baz["md5"]} == set(remote.index.hashes())
|
||||
assert [bar.checksum] == list(remote.index.dir_hashes())
|
||||
assert foo.checksum not in remote.index.hashes()
|
||||
|
||||
|
||||
def test_indexed_dir_missing(tmp_dir, dvc, tmp_path_factory, remote):
|
||||
@@ -61,7 +61,7 @@ def test_indexed_dir_missing(tmp_dir, dvc, tmp_path_factory, remote):
|
||||
remote.index.update([bar.checksum], [])
|
||||
dvc.status(cloud=True)
|
||||
with remote.index:
|
||||
assert not list(remote.index.checksums())
|
||||
assert not list(remote.index.hashes())
|
||||
|
||||
|
||||
def test_clear_on_gc(tmp_dir, dvc, tmp_path_factory, remote, mocker):
|
||||
@@ -101,4 +101,4 @@ def test_partial_upload(tmp_dir, dvc, tmp_path_factory, remote, mocker):
|
||||
with pytest.raises(UploadError):
|
||||
dvc.push()
|
||||
with remote.index:
|
||||
assert not list(remote.index.checksums())
|
||||
assert not list(remote.index.hashes())
|
||||
|
||||
@@ -21,7 +21,7 @@ from dvc.exceptions import (
|
||||
)
|
||||
from dvc.main import main
|
||||
from dvc.output.base import OutputAlreadyTrackedError, OutputIsStageFileError
|
||||
from dvc.remote.local import LocalRemote, LocalRemoteTree
|
||||
from dvc.remote.local import LocalRemoteTree
|
||||
from dvc.repo import Repo as DvcRepo
|
||||
from dvc.stage import Stage
|
||||
from dvc.system import System
|
||||
@@ -340,7 +340,7 @@ class TestAddCommit(TestDvc):
|
||||
|
||||
def test_should_collect_dir_cache_only_once(mocker, tmp_dir, dvc):
|
||||
tmp_dir.gen({"data/data": "foo"})
|
||||
get_dir_checksum_counter = mocker.spy(LocalRemoteTree, "get_dir_checksum")
|
||||
get_dir_hash_counter = mocker.spy(LocalRemoteTree, "get_dir_hash")
|
||||
ret = main(["add", "data"])
|
||||
assert ret == 0
|
||||
|
||||
@@ -349,7 +349,7 @@ def test_should_collect_dir_cache_only_once(mocker, tmp_dir, dvc):
|
||||
|
||||
ret = main(["status"])
|
||||
assert ret == 0
|
||||
assert get_dir_checksum_counter.mock.call_count == 1
|
||||
assert get_dir_hash_counter.mock.call_count == 1
|
||||
|
||||
|
||||
class SymlinkAddTestBase(TestDvc):
|
||||
@@ -583,7 +583,7 @@ def test_readding_dir_should_not_unprotect_all(tmp_dir, dvc, mocker):
|
||||
dvc.add("dir")
|
||||
tmp_dir.gen("dir/new_file", "new_file_content")
|
||||
|
||||
unprotect_spy = mocker.spy(LocalRemote, "unprotect")
|
||||
unprotect_spy = mocker.spy(LocalRemoteTree, "unprotect")
|
||||
dvc.add("dir")
|
||||
|
||||
assert not unprotect_spy.mock.called
|
||||
|
||||
+32
-90
@@ -5,93 +5,42 @@ import pytest
|
||||
from dvc import api
|
||||
from dvc.api import UrlNotDvcRepoError
|
||||
from dvc.exceptions import FileMissingError
|
||||
from dvc.main import main
|
||||
from dvc.path_info import URLInfo
|
||||
from dvc.utils.fs import remove
|
||||
from tests.remotes import (
|
||||
GCP,
|
||||
HDFS,
|
||||
OSS,
|
||||
S3,
|
||||
SSH,
|
||||
TEST_REMOTE,
|
||||
Azure,
|
||||
GDrive,
|
||||
Local,
|
||||
)
|
||||
|
||||
remote_params = [S3, GCP, Azure, GDrive, OSS, SSH, HDFS]
|
||||
all_remote_params = [Local] + remote_params
|
||||
cloud_names = [
|
||||
"s3",
|
||||
"gs",
|
||||
"azure",
|
||||
"gdrive",
|
||||
"oss",
|
||||
"ssh",
|
||||
"hdfs",
|
||||
"http",
|
||||
]
|
||||
clouds = [pytest.lazy_fixture(cloud) for cloud in cloud_names]
|
||||
all_clouds = [pytest.lazy_fixture("local")] + clouds
|
||||
remotes = [pytest.lazy_fixture(f"{cloud}_remote") for cloud in cloud_names]
|
||||
all_remotes = [pytest.lazy_fixture("local_remote")] + remotes
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def remote_url(request):
|
||||
if not request.param.should_test():
|
||||
raise pytest.skip()
|
||||
return request.param.get_url()
|
||||
|
||||
|
||||
def run_dvc(*argv):
|
||||
assert main(argv) == 0
|
||||
|
||||
|
||||
def ensure_dir(dvc, url):
|
||||
if url.startswith("gdrive://"):
|
||||
GDrive.create_dir(dvc, url)
|
||||
run_dvc(
|
||||
"remote",
|
||||
"modify",
|
||||
TEST_REMOTE,
|
||||
"gdrive_service_account_email",
|
||||
"test",
|
||||
)
|
||||
run_dvc(
|
||||
"remote",
|
||||
"modify",
|
||||
TEST_REMOTE,
|
||||
"gdrive_service_account_p12_file_path",
|
||||
"test.p12",
|
||||
)
|
||||
run_dvc(
|
||||
"remote",
|
||||
"modify",
|
||||
TEST_REMOTE,
|
||||
"gdrive_use_service_account",
|
||||
"True",
|
||||
)
|
||||
|
||||
|
||||
def ensure_dir_scm(dvc, url):
|
||||
if url.startswith("gdrive://"):
|
||||
GDrive.create_dir(dvc, url)
|
||||
with dvc.config.edit() as conf:
|
||||
conf["remote"][TEST_REMOTE].update(
|
||||
gdrive_service_account_email="test",
|
||||
gdrive_service_account_p12_file_path="test.p12",
|
||||
gdrive_use_service_account=True,
|
||||
)
|
||||
dvc.scm.add(dvc.config.files["repo"])
|
||||
dvc.scm.commit(f"modify '{TEST_REMOTE}' remote")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("remote_url", remote_params, indirect=True)
|
||||
def test_get_url(tmp_dir, dvc, remote_url):
|
||||
run_dvc("remote", "add", "-d", TEST_REMOTE, remote_url)
|
||||
@pytest.mark.parametrize("remote", remotes)
|
||||
def test_get_url(tmp_dir, dvc, request, remote):
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
|
||||
expected_url = URLInfo(remote_url) / "ac/bd18db4cc2f85cedef654fccc4a4d8"
|
||||
expected_url = URLInfo(remote.url) / "ac/bd18db4cc2f85cedef654fccc4a4d8"
|
||||
assert api.get_url("foo") == expected_url
|
||||
|
||||
|
||||
@pytest.mark.parametrize("remote_url", remote_params, indirect=True)
|
||||
def test_get_url_external(erepo_dir, remote_url, setup_remote):
|
||||
setup_remote(erepo_dir.dvc, url=remote_url)
|
||||
@pytest.mark.parametrize("cloud", clouds)
|
||||
def test_get_url_external(erepo_dir, cloud):
|
||||
erepo_dir.add_remote(config=cloud.config)
|
||||
with erepo_dir.chdir():
|
||||
erepo_dir.dvc_gen("foo", "foo", commit="add foo")
|
||||
|
||||
# Using file url to force clone to tmp repo
|
||||
repo_url = f"file://{erepo_dir}"
|
||||
expected_url = URLInfo(remote_url) / "ac/bd18db4cc2f85cedef654fccc4a4d8"
|
||||
expected_url = URLInfo(cloud.url) / "ac/bd18db4cc2f85cedef654fccc4a4d8"
|
||||
assert api.get_url("foo", repo=repo_url) == expected_url
|
||||
|
||||
|
||||
@@ -105,12 +54,10 @@ def test_get_url_requires_dvc(tmp_dir, scm):
|
||||
api.get_url("foo", repo=f"file://{tmp_dir}")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("remote_url", all_remote_params, indirect=True)
|
||||
def test_open(remote_url, tmp_dir, dvc):
|
||||
run_dvc("remote", "add", "-d", TEST_REMOTE, remote_url)
|
||||
ensure_dir(dvc, remote_url)
|
||||
@pytest.mark.parametrize("remote", all_remotes)
|
||||
def test_open(tmp_dir, dvc, remote):
|
||||
tmp_dir.dvc_gen("foo", "foo-text")
|
||||
run_dvc("push")
|
||||
dvc.push()
|
||||
|
||||
# Remove cache to force download
|
||||
remove(dvc.cache.local.cache_dir)
|
||||
@@ -119,10 +66,9 @@ def test_open(remote_url, tmp_dir, dvc):
|
||||
assert fd.read() == "foo-text"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("remote_url", all_remote_params, indirect=True)
|
||||
def test_open_external(remote_url, erepo_dir, setup_remote):
|
||||
setup_remote(erepo_dir.dvc, url=remote_url)
|
||||
ensure_dir_scm(erepo_dir.dvc, remote_url)
|
||||
@pytest.mark.parametrize("cloud", clouds)
|
||||
def test_open_external(erepo_dir, cloud):
|
||||
erepo_dir.add_remote(config=cloud.config)
|
||||
|
||||
with erepo_dir.chdir():
|
||||
erepo_dir.dvc_gen("version", "master", commit="add version")
|
||||
@@ -144,12 +90,10 @@ def test_open_external(remote_url, erepo_dir, setup_remote):
|
||||
assert api.read("version", repo=repo_url, rev="branch") == "branchver"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("remote_url", all_remote_params, indirect=True)
|
||||
def test_open_granular(remote_url, tmp_dir, dvc):
|
||||
run_dvc("remote", "add", "-d", TEST_REMOTE, remote_url)
|
||||
ensure_dir(dvc, remote_url)
|
||||
@pytest.mark.parametrize("remote", all_remotes)
|
||||
def test_open_granular(tmp_dir, dvc, remote):
|
||||
tmp_dir.dvc_gen({"dir": {"foo": "foo-text"}})
|
||||
run_dvc("push")
|
||||
dvc.push()
|
||||
|
||||
# Remove cache to force download
|
||||
remove(dvc.cache.local.cache_dir)
|
||||
@@ -158,11 +102,9 @@ def test_open_granular(remote_url, tmp_dir, dvc):
|
||||
assert fd.read() == "foo-text"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("remote_url", all_remote_params, indirect=True)
|
||||
def test_missing(remote_url, tmp_dir, dvc):
|
||||
@pytest.mark.parametrize("remote", all_remotes)
|
||||
def test_missing(tmp_dir, dvc, remote):
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
run_dvc("remote", "add", "-d", TEST_REMOTE, remote_url)
|
||||
ensure_dir(dvc, remote_url)
|
||||
|
||||
# Remove cache to make foo missing
|
||||
remove(dvc.cache.local.cache_dir)
|
||||
|
||||
@@ -30,14 +30,14 @@ class TestCache(TestDvc):
|
||||
self.create(self.cache2, "2")
|
||||
|
||||
def test_all(self):
|
||||
md5_list = list(Cache(self.dvc).local.all())
|
||||
md5_list = list(Cache(self.dvc).local.tree.all())
|
||||
self.assertEqual(len(md5_list), 2)
|
||||
self.assertIn(self.cache1_md5, md5_list)
|
||||
self.assertIn(self.cache2_md5, md5_list)
|
||||
|
||||
def test_get(self):
|
||||
cache = Cache(self.dvc).local.get(self.cache1_md5)
|
||||
self.assertEqual(cache, self.cache1)
|
||||
cache = Cache(self.dvc).local.hash_to_path_info(self.cache1_md5)
|
||||
self.assertEqual(os.fspath(cache), self.cache1)
|
||||
|
||||
|
||||
class TestCacheLoadBadDirCache(TestDvc):
|
||||
@@ -46,16 +46,16 @@ class TestCacheLoadBadDirCache(TestDvc):
|
||||
self.assertEqual(len(ret), 0)
|
||||
|
||||
def test(self):
|
||||
checksum = "123.dir"
|
||||
fname = self.dvc.cache.local.get(checksum)
|
||||
dir_hash = "123.dir"
|
||||
fname = os.fspath(self.dvc.cache.local.hash_to_path_info(dir_hash))
|
||||
self.create(fname, "<clearly>not,json")
|
||||
with pytest.raises(DirCacheError):
|
||||
self.dvc.cache.local.load_dir_cache(checksum)
|
||||
self.dvc.cache.local.load_dir_cache(dir_hash)
|
||||
|
||||
checksum = "234.dir"
|
||||
fname = self.dvc.cache.local.get(checksum)
|
||||
dir_hash = "234.dir"
|
||||
fname = os.fspath(self.dvc.cache.local.hash_to_path_info(dir_hash))
|
||||
self.create(fname, '{"a": "b"}')
|
||||
self._do_test(self.dvc.cache.local.load_dir_cache(checksum))
|
||||
self._do_test(self.dvc.cache.local.load_dir_cache(dir_hash))
|
||||
|
||||
|
||||
class TestExternalCacheDir(TestDvc):
|
||||
|
||||
@@ -16,8 +16,9 @@ from dvc.exceptions import (
|
||||
NoOutputOrStageError,
|
||||
)
|
||||
from dvc.main import main
|
||||
from dvc.remote import S3Cache, S3Remote
|
||||
from dvc.remote.local import LocalRemote
|
||||
from dvc.remote.base import CloudCache, Remote
|
||||
from dvc.remote.local import LocalRemoteTree
|
||||
from dvc.remote.s3 import S3RemoteTree
|
||||
from dvc.repo import Repo as DvcRepo
|
||||
from dvc.stage import Stage
|
||||
from dvc.stage.exceptions import StageFileDoesNotExistError
|
||||
@@ -99,8 +100,8 @@ class TestCheckoutCorruptedCacheDir(TestDvc):
|
||||
# NOTE: modifying cache file for one of the files inside the directory
|
||||
# to check if dvc will detect that the cache is corrupted.
|
||||
entry = self.dvc.cache.local.load_dir_cache(out.checksum)[0]
|
||||
checksum = entry[self.dvc.cache.local.PARAM_CHECKSUM]
|
||||
cache = self.dvc.cache.local.get(checksum)
|
||||
entry_hash = entry[self.dvc.cache.local.tree.PARAM_CHECKSUM]
|
||||
cache = os.fspath(self.dvc.cache.local.hash_to_path_info(entry_hash))
|
||||
|
||||
os.chmod(cache, 0o644)
|
||||
with open(cache, "w+") as fobj:
|
||||
@@ -305,8 +306,8 @@ class TestGitIgnoreWhenCheckout(CheckoutBase):
|
||||
class TestCheckoutMissingMd5InStageFile(TestRepro):
|
||||
def test(self):
|
||||
d = load_yaml(self.file1_stage)
|
||||
del d[Stage.PARAM_OUTS][0][LocalRemote.PARAM_CHECKSUM]
|
||||
del d[Stage.PARAM_DEPS][0][LocalRemote.PARAM_CHECKSUM]
|
||||
del d[Stage.PARAM_OUTS][0][LocalRemoteTree.PARAM_CHECKSUM]
|
||||
del d[Stage.PARAM_DEPS][0][LocalRemoteTree.PARAM_CHECKSUM]
|
||||
dump_yaml(self.file1_stage, d)
|
||||
|
||||
with pytest.raises(CheckoutError):
|
||||
@@ -755,9 +756,9 @@ def test_checkout_recursive(tmp_dir, dvc):
|
||||
not S3.should_test(), reason="Only run with S3 credentials"
|
||||
)
|
||||
def test_checkout_for_external_outputs(tmp_dir, dvc):
|
||||
dvc.cache.s3 = S3Cache(dvc, {"url": S3.get_url()})
|
||||
dvc.cache.s3 = CloudCache(S3RemoteTree(dvc, {"url": S3.get_url()}))
|
||||
|
||||
remote = S3Remote(dvc, {"url": S3.get_url()})
|
||||
remote = Remote(S3RemoteTree(dvc, {"url": S3.get_url()}))
|
||||
file_path = remote.path_info / "foo"
|
||||
remote.tree.s3.put_object(
|
||||
Bucket=remote.path_info.bucket, Key=file_path.path, Body="foo"
|
||||
|
||||
+148
-580
@@ -1,28 +1,13 @@
|
||||
import copy
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import uuid
|
||||
from unittest import SkipTest
|
||||
|
||||
import pytest
|
||||
from flaky.flaky_decorator import flaky
|
||||
|
||||
from dvc.cache import NamedCache
|
||||
from dvc.data_cloud import DataCloud
|
||||
from dvc.external_repo import clean_repos
|
||||
from dvc.main import main
|
||||
from dvc.remote import (
|
||||
AzureRemote,
|
||||
GDriveRemote,
|
||||
GSRemote,
|
||||
HDFSRemote,
|
||||
HTTPRemote,
|
||||
LocalRemote,
|
||||
OSSRemote,
|
||||
S3Remote,
|
||||
SSHRemote,
|
||||
)
|
||||
from dvc.remote.base import STATUS_DELETED, STATUS_NEW, STATUS_OK
|
||||
from dvc.remote.local import LocalRemoteTree
|
||||
from dvc.stage.exceptions import StageNotFound
|
||||
@@ -30,465 +15,149 @@ from dvc.utils import file_md5
|
||||
from dvc.utils.fs import remove
|
||||
from dvc.utils.yaml import dump_yaml, load_yaml
|
||||
from tests.basic_env import TestDvc
|
||||
from tests.remotes import (
|
||||
GCP,
|
||||
HDFS,
|
||||
HTTP,
|
||||
OSS,
|
||||
S3,
|
||||
TEST_CONFIG,
|
||||
TEST_GCP_CREDS_FILE,
|
||||
TEST_REMOTE,
|
||||
Azure,
|
||||
GDrive,
|
||||
Local,
|
||||
SSHMocked,
|
||||
)
|
||||
|
||||
from .test_api import all_remotes
|
||||
|
||||
|
||||
class TestDataCloud(TestDvc):
|
||||
def _test_cloud(self, config, cl):
|
||||
self.dvc.config = config
|
||||
cloud = DataCloud(self.dvc)
|
||||
self.assertIsInstance(cloud.get_remote(), cl)
|
||||
@pytest.mark.parametrize("remote", all_remotes)
|
||||
def test_cloud(tmp_dir, dvc, remote):
|
||||
(stage,) = tmp_dir.dvc_gen("foo", "foo")
|
||||
out = stage.outs[0]
|
||||
cache = out.cache_path
|
||||
md5 = out.checksum
|
||||
info = out.get_used_cache()
|
||||
|
||||
def test(self):
|
||||
config = copy.deepcopy(TEST_CONFIG)
|
||||
|
||||
clist = [
|
||||
("s3://mybucket/", S3Remote),
|
||||
("gs://mybucket/", GSRemote),
|
||||
("ssh://user@localhost:/", SSHRemote),
|
||||
("http://localhost:8000/", HTTPRemote),
|
||||
("azure://ContainerName=mybucket;conn_string;", AzureRemote),
|
||||
("oss://mybucket/", OSSRemote),
|
||||
(TestDvc.mkdtemp(), LocalRemote),
|
||||
]
|
||||
|
||||
for scheme, cl in clist:
|
||||
remote_url = scheme + str(uuid.uuid4())
|
||||
config["remote"][TEST_REMOTE]["url"] = remote_url
|
||||
self._test_cloud(config, cl)
|
||||
|
||||
|
||||
class TestDataCloudBase(TestDvc):
|
||||
def _get_cloud_class(self):
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def should_test():
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def get_url():
|
||||
return NotImplementedError
|
||||
|
||||
def _get_keyfile(self):
|
||||
return None
|
||||
|
||||
def _ensure_should_run(self):
|
||||
if not self.should_test():
|
||||
raise SkipTest(f"Test {self.__class__.__name__} is disabled")
|
||||
|
||||
def _setup_cloud(self):
|
||||
self._ensure_should_run()
|
||||
|
||||
repo = self.get_url()
|
||||
keyfile = self._get_keyfile()
|
||||
|
||||
config = copy.deepcopy(TEST_CONFIG)
|
||||
config["remote"][TEST_REMOTE] = {"url": repo, "keyfile": keyfile}
|
||||
self.dvc.config = config
|
||||
self.cloud = DataCloud(self.dvc)
|
||||
|
||||
self.assertIsInstance(self.cloud.get_remote(), self._get_cloud_class())
|
||||
|
||||
def _test_cloud(self):
|
||||
self._setup_cloud()
|
||||
|
||||
stages = self.dvc.add(self.FOO)
|
||||
self.assertEqual(len(stages), 1)
|
||||
stage = stages[0]
|
||||
self.assertTrue(stage is not None)
|
||||
out = stage.outs[0]
|
||||
cache = out.cache_path
|
||||
md5 = out.checksum
|
||||
info = out.get_used_cache()
|
||||
|
||||
stages = self.dvc.add(self.DATA_DIR)
|
||||
self.assertEqual(len(stages), 1)
|
||||
stage_dir = stages[0]
|
||||
self.assertTrue(stage_dir is not None)
|
||||
out_dir = stage_dir.outs[0]
|
||||
cache_dir = out_dir.cache_path
|
||||
name_dir = str(out_dir)
|
||||
md5_dir = out_dir.checksum
|
||||
info_dir = NamedCache.make(out_dir.scheme, md5_dir, name_dir)
|
||||
|
||||
with self.cloud.repo.state:
|
||||
# Check status
|
||||
status = self.cloud.status(info, show_checksums=True)
|
||||
expected = {md5: {"name": md5, "status": STATUS_NEW}}
|
||||
self.assertEqual(status, expected)
|
||||
|
||||
status_dir = self.cloud.status(info_dir, show_checksums=True)
|
||||
expected = {md5_dir: {"name": md5_dir, "status": STATUS_NEW}}
|
||||
self.assertEqual(status_dir, expected)
|
||||
|
||||
# Push and check status
|
||||
self.cloud.push(info)
|
||||
self.assertTrue(os.path.exists(cache))
|
||||
self.assertTrue(os.path.isfile(cache))
|
||||
|
||||
self.cloud.push(info_dir)
|
||||
self.assertTrue(os.path.isfile(cache_dir))
|
||||
|
||||
status = self.cloud.status(info, show_checksums=True)
|
||||
expected = {md5: {"name": md5, "status": STATUS_OK}}
|
||||
self.assertEqual(status, expected)
|
||||
|
||||
status_dir = self.cloud.status(info_dir, show_checksums=True)
|
||||
expected = {md5_dir: {"name": md5_dir, "status": STATUS_OK}}
|
||||
self.assertEqual(status_dir, expected)
|
||||
|
||||
# Remove and check status
|
||||
remove(self.dvc.cache.local.cache_dir)
|
||||
|
||||
status = self.cloud.status(info, show_checksums=True)
|
||||
expected = {md5: {"name": md5, "status": STATUS_DELETED}}
|
||||
self.assertEqual(status, expected)
|
||||
|
||||
status_dir = self.cloud.status(info_dir, show_checksums=True)
|
||||
expected = {md5_dir: {"name": md5_dir, "status": STATUS_DELETED}}
|
||||
self.assertEqual(status_dir, expected)
|
||||
|
||||
# Pull and check status
|
||||
self.cloud.pull(info)
|
||||
self.assertTrue(os.path.exists(cache))
|
||||
self.assertTrue(os.path.isfile(cache))
|
||||
with open(cache) as fd:
|
||||
self.assertEqual(fd.read(), self.FOO_CONTENTS)
|
||||
|
||||
self.cloud.pull(info_dir)
|
||||
self.assertTrue(os.path.isfile(cache_dir))
|
||||
|
||||
status = self.cloud.status(info, show_checksums=True)
|
||||
expected = {md5: {"name": md5, "status": STATUS_OK}}
|
||||
self.assertEqual(status, expected)
|
||||
|
||||
status_dir = self.cloud.status(info_dir, show_checksums=True)
|
||||
expected = {md5_dir: {"name": md5_dir, "status": STATUS_OK}}
|
||||
self.assertTrue(status_dir, expected)
|
||||
|
||||
def test(self):
|
||||
self._ensure_should_run()
|
||||
self._test_cloud()
|
||||
|
||||
|
||||
class TestS3Remote(S3, TestDataCloudBase):
|
||||
def _get_cloud_class(self):
|
||||
return S3Remote
|
||||
|
||||
|
||||
class TestGDriveRemote(GDrive, TestDataCloudBase):
|
||||
def _setup_cloud(self):
|
||||
self._ensure_should_run()
|
||||
|
||||
url = self.get_url()
|
||||
self.create_dir(self.dvc, url)
|
||||
|
||||
config = copy.deepcopy(TEST_CONFIG)
|
||||
config["remote"][TEST_REMOTE] = {
|
||||
"url": url,
|
||||
"gdrive_service_account_email": "test",
|
||||
"gdrive_service_account_p12_file_path": "test.p12",
|
||||
"gdrive_use_service_account": True,
|
||||
(stage_dir,) = tmp_dir.dvc_gen(
|
||||
{
|
||||
"data_dir": {
|
||||
"data_sub_dir": {"data_sub": "data_sub"},
|
||||
"data": "data",
|
||||
}
|
||||
}
|
||||
)
|
||||
out_dir = stage_dir.outs[0]
|
||||
cache_dir = out_dir.cache_path
|
||||
name_dir = str(out_dir)
|
||||
md5_dir = out_dir.checksum
|
||||
info_dir = NamedCache.make(out_dir.scheme, md5_dir, name_dir)
|
||||
|
||||
self.dvc.config = config
|
||||
self.cloud = DataCloud(self.dvc)
|
||||
remote = self.cloud.get_remote()
|
||||
self.assertIsInstance(remote, self._get_cloud_class())
|
||||
with dvc.state:
|
||||
# Check status
|
||||
status = dvc.cloud.status(info, show_checksums=True)
|
||||
expected = {md5: {"name": md5, "status": STATUS_NEW}}
|
||||
assert status == expected
|
||||
|
||||
def _get_cloud_class(self):
|
||||
return GDriveRemote
|
||||
status_dir = dvc.cloud.status(info_dir, show_checksums=True)
|
||||
expected = {md5_dir: {"name": md5_dir, "status": STATUS_NEW}}
|
||||
assert status_dir == expected
|
||||
|
||||
# Push and check status
|
||||
dvc.cloud.push(info)
|
||||
assert os.path.exists(cache)
|
||||
assert os.path.isfile(cache)
|
||||
|
||||
class TestGSRemote(GCP, TestDataCloudBase):
|
||||
def _setup_cloud(self):
|
||||
self._ensure_should_run()
|
||||
dvc.cloud.push(info_dir)
|
||||
assert os.path.isfile(cache_dir)
|
||||
|
||||
repo = self.get_url()
|
||||
status = dvc.cloud.status(info, show_checksums=True)
|
||||
expected = {md5: {"name": md5, "status": STATUS_OK}}
|
||||
assert status == expected
|
||||
|
||||
config = copy.deepcopy(TEST_CONFIG)
|
||||
config["remote"][TEST_REMOTE] = {
|
||||
"url": repo,
|
||||
"credentialpath": TEST_GCP_CREDS_FILE,
|
||||
}
|
||||
self.dvc.config = config
|
||||
self.cloud = DataCloud(self.dvc)
|
||||
status_dir = dvc.cloud.status(info_dir, show_checksums=True)
|
||||
expected = {md5_dir: {"name": md5_dir, "status": STATUS_OK}}
|
||||
assert status_dir == expected
|
||||
|
||||
self.assertIsInstance(self.cloud.get_remote(), self._get_cloud_class())
|
||||
# Remove and check status
|
||||
remove(dvc.cache.local.cache_dir)
|
||||
|
||||
def _get_cloud_class(self):
|
||||
return GSRemote
|
||||
status = dvc.cloud.status(info, show_checksums=True)
|
||||
expected = {md5: {"name": md5, "status": STATUS_DELETED}}
|
||||
assert status == expected
|
||||
|
||||
status_dir = dvc.cloud.status(info_dir, show_checksums=True)
|
||||
expected = {md5_dir: {"name": md5_dir, "status": STATUS_DELETED}}
|
||||
assert status_dir == expected
|
||||
|
||||
class TestAzureRemote(Azure, TestDataCloudBase):
|
||||
def _get_cloud_class(self):
|
||||
return AzureRemote
|
||||
|
||||
|
||||
class TestOSSRemote(OSS, TestDataCloudBase):
|
||||
def _get_cloud_class(self):
|
||||
return OSSRemote
|
||||
|
||||
|
||||
class TestLocalRemote(Local, TestDataCloudBase):
|
||||
def _get_cloud_class(self):
|
||||
return LocalRemote
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("ssh_server")
|
||||
class TestSSHRemoteMocked(SSHMocked, TestDataCloudBase):
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup_method_fixture(self, request, ssh_server):
|
||||
self.ssh_server = ssh_server
|
||||
self.method_name = request.function.__name__
|
||||
|
||||
def _setup_cloud(self):
|
||||
self._ensure_should_run()
|
||||
|
||||
repo = self.get_url()
|
||||
keyfile = self._get_keyfile()
|
||||
|
||||
self._get_cloud_class().CAN_TRAVERSE = False
|
||||
config = copy.deepcopy(TEST_CONFIG)
|
||||
config["remote"][TEST_REMOTE] = {
|
||||
"url": repo,
|
||||
"keyfile": keyfile,
|
||||
}
|
||||
self.dvc.config = config
|
||||
self.cloud = DataCloud(self.dvc)
|
||||
|
||||
self.assertIsInstance(self.cloud.get_remote(), self._get_cloud_class())
|
||||
|
||||
def get_url(self):
|
||||
user = self.ssh_server.test_creds["username"]
|
||||
return super().get_url(user, self.ssh_server.port)
|
||||
|
||||
def _get_keyfile(self):
|
||||
return self.ssh_server.test_creds["key_filename"]
|
||||
|
||||
def _get_cloud_class(self):
|
||||
return SSHRemote
|
||||
|
||||
|
||||
class TestHDFSRemote(HDFS, TestDataCloudBase):
|
||||
def _get_cloud_class(self):
|
||||
return HDFSRemote
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("http_server")
|
||||
class TestHTTPRemote(HTTP, TestDataCloudBase):
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup_method_fixture(self, request, http_server):
|
||||
self.http_server = http_server
|
||||
self.method_name = request.function.__name__
|
||||
|
||||
def get_url(self):
|
||||
return super().get_url(self.http_server.server_port)
|
||||
|
||||
def _get_cloud_class(self):
|
||||
return HTTPRemote
|
||||
|
||||
|
||||
class TestDataCloudCLIBase(TestDvc):
|
||||
def main(self, args):
|
||||
ret = main(args)
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
@staticmethod
|
||||
def should_test():
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def get_url():
|
||||
raise NotImplementedError
|
||||
|
||||
def _setup_cloud(self):
|
||||
pass
|
||||
|
||||
def _test_cloud(self, remote=None):
|
||||
self._setup_cloud()
|
||||
|
||||
args = ["-v", "-j", "2"]
|
||||
if remote:
|
||||
args += ["-r", remote]
|
||||
else:
|
||||
args += []
|
||||
|
||||
stages = self.dvc.add(self.FOO)
|
||||
self.assertEqual(len(stages), 1)
|
||||
stage = stages[0]
|
||||
self.assertTrue(stage is not None)
|
||||
cache = stage.outs[0].cache_path
|
||||
|
||||
stages = self.dvc.add(self.DATA_DIR)
|
||||
self.assertEqual(len(stages), 1)
|
||||
stage_dir = stages[0]
|
||||
self.assertTrue(stage_dir is not None)
|
||||
cache_dir = stage_dir.outs[0].cache_path
|
||||
|
||||
# FIXME check status output
|
||||
|
||||
self.main(["push"] + args)
|
||||
self.assertTrue(os.path.exists(cache))
|
||||
self.assertTrue(os.path.isfile(cache))
|
||||
self.assertTrue(os.path.isfile(cache_dir))
|
||||
|
||||
remove(self.dvc.cache.local.cache_dir)
|
||||
|
||||
self.main(["fetch"] + args)
|
||||
self.assertTrue(os.path.exists(cache))
|
||||
self.assertTrue(os.path.isfile(cache))
|
||||
self.assertTrue(os.path.isfile(cache_dir))
|
||||
|
||||
self.main(["pull"] + args)
|
||||
self.assertTrue(os.path.exists(cache))
|
||||
self.assertTrue(os.path.isfile(cache))
|
||||
self.assertTrue(os.path.isfile(cache_dir))
|
||||
self.assertTrue(os.path.isfile(self.FOO))
|
||||
self.assertTrue(os.path.isdir(self.DATA_DIR))
|
||||
|
||||
# Pull and check status
|
||||
dvc.cloud.pull(info)
|
||||
assert os.path.exists(cache)
|
||||
assert os.path.isfile(cache)
|
||||
with open(cache) as fd:
|
||||
self.assertEqual(fd.read(), self.FOO_CONTENTS)
|
||||
self.assertTrue(os.path.isfile(cache_dir))
|
||||
assert fd.read() == "foo"
|
||||
|
||||
# NOTE: check if remote gc works correctly on directories
|
||||
self.main(["gc", "-cw", "-f"] + args)
|
||||
shutil.move(
|
||||
self.dvc.cache.local.cache_dir,
|
||||
self.dvc.cache.local.cache_dir + ".back",
|
||||
)
|
||||
dvc.cloud.pull(info_dir)
|
||||
assert os.path.isfile(cache_dir)
|
||||
|
||||
self.main(["fetch"] + args)
|
||||
status = dvc.cloud.status(info, show_checksums=True)
|
||||
expected = {md5: {"name": md5, "status": STATUS_OK}}
|
||||
assert status == expected
|
||||
|
||||
self.main(["pull", "-f"] + args)
|
||||
self.assertTrue(os.path.exists(cache))
|
||||
self.assertTrue(os.path.isfile(cache))
|
||||
self.assertTrue(os.path.isfile(cache_dir))
|
||||
self.assertTrue(os.path.isfile(self.FOO))
|
||||
self.assertTrue(os.path.isdir(self.DATA_DIR))
|
||||
|
||||
def _test(self):
|
||||
pass
|
||||
|
||||
def test(self):
|
||||
if not self.should_test():
|
||||
raise SkipTest(f"Test {self.__class__.__name__} is disabled")
|
||||
self._test()
|
||||
status_dir = dvc.cloud.status(info_dir, show_checksums=True)
|
||||
expected = {md5_dir: {"name": md5_dir, "status": STATUS_OK}}
|
||||
assert status_dir == expected
|
||||
|
||||
|
||||
class TestLocalRemoteCLI(Local, TestDataCloudCLIBase):
|
||||
def _test(self):
|
||||
url = self.get_url()
|
||||
@pytest.mark.parametrize("remote", all_remotes)
|
||||
def test_cloud_cli(tmp_dir, dvc, remote):
|
||||
args = ["-v", "-j", "2"]
|
||||
|
||||
self.main(["remote", "add", TEST_REMOTE, url])
|
||||
(stage,) = tmp_dir.dvc_gen("foo", "foo")
|
||||
cache = stage.outs[0].cache_path
|
||||
|
||||
self._test_cloud(TEST_REMOTE)
|
||||
(stage_dir,) = tmp_dir.dvc_gen(
|
||||
{
|
||||
"data_dir": {
|
||||
"data_sub_dir": {"data_sub": "data_sub"},
|
||||
"data": "data",
|
||||
}
|
||||
}
|
||||
)
|
||||
assert stage_dir is not None
|
||||
cache_dir = stage_dir.outs[0].cache_path
|
||||
|
||||
# FIXME check status output
|
||||
|
||||
class TestRemoteHDFSCLI(HDFS, TestDataCloudCLIBase):
|
||||
def _test(self):
|
||||
url = self.get_url()
|
||||
assert main(["push"] + args) == 0
|
||||
assert os.path.exists(cache)
|
||||
assert os.path.isfile(cache)
|
||||
assert os.path.isfile(cache_dir)
|
||||
|
||||
self.main(["remote", "add", TEST_REMOTE, url])
|
||||
remove(dvc.cache.local.cache_dir)
|
||||
|
||||
self._test_cloud(TEST_REMOTE)
|
||||
assert main(["fetch"] + args) == 0
|
||||
assert os.path.exists(cache)
|
||||
assert os.path.isfile(cache)
|
||||
assert os.path.isfile(cache_dir)
|
||||
|
||||
assert main(["pull"] + args) == 0
|
||||
assert os.path.exists(cache)
|
||||
assert os.path.isfile(cache)
|
||||
assert os.path.isfile(cache_dir)
|
||||
assert os.path.isfile("foo")
|
||||
assert os.path.isdir("data_dir")
|
||||
|
||||
class TestS3RemoteCLI(S3, TestDataCloudCLIBase):
|
||||
def _test(self):
|
||||
url = self.get_url()
|
||||
with open(cache) as fd:
|
||||
assert fd.read() == "foo"
|
||||
assert os.path.isfile(cache_dir)
|
||||
|
||||
self.main(["remote", "add", TEST_REMOTE, url])
|
||||
# NOTE: http doesn't support gc yet
|
||||
if remote.url.startswith("http"):
|
||||
return
|
||||
|
||||
self._test_cloud(TEST_REMOTE)
|
||||
# NOTE: check if remote gc works correctly on directories
|
||||
assert main(["gc", "-cw", "-f"] + args) == 0
|
||||
shutil.move(
|
||||
dvc.cache.local.cache_dir, dvc.cache.local.cache_dir + ".back",
|
||||
)
|
||||
|
||||
assert main(["fetch"] + args) == 0
|
||||
|
||||
class TestGDriveRemoteCLI(GDrive, TestDataCloudCLIBase):
|
||||
def _test(self):
|
||||
url = self.get_url()
|
||||
|
||||
self.create_dir(self.dvc, url)
|
||||
|
||||
self.main(["remote", "add", TEST_REMOTE, url])
|
||||
self.main(
|
||||
[
|
||||
"remote",
|
||||
"modify",
|
||||
TEST_REMOTE,
|
||||
"gdrive_service_account_email",
|
||||
"test",
|
||||
]
|
||||
)
|
||||
self.main(
|
||||
[
|
||||
"remote",
|
||||
"modify",
|
||||
TEST_REMOTE,
|
||||
"gdrive_service_account_p12_file_path",
|
||||
"test.p12",
|
||||
]
|
||||
)
|
||||
self.main(
|
||||
[
|
||||
"remote",
|
||||
"modify",
|
||||
TEST_REMOTE,
|
||||
"gdrive_use_service_account",
|
||||
"True",
|
||||
]
|
||||
)
|
||||
|
||||
self._test_cloud(TEST_REMOTE)
|
||||
|
||||
|
||||
class TestGSRemoteCLI(GCP, TestDataCloudCLIBase):
|
||||
def _test(self):
|
||||
url = self.get_url()
|
||||
|
||||
self.main(["remote", "add", TEST_REMOTE, url])
|
||||
self.main(
|
||||
[
|
||||
"remote",
|
||||
"modify",
|
||||
TEST_REMOTE,
|
||||
"credentialpath",
|
||||
TEST_GCP_CREDS_FILE,
|
||||
]
|
||||
)
|
||||
|
||||
self._test_cloud(TEST_REMOTE)
|
||||
|
||||
|
||||
class TestAzureRemoteCLI(Azure, TestDataCloudCLIBase):
|
||||
def _test(self):
|
||||
url = self.get_url()
|
||||
|
||||
self.main(["remote", "add", TEST_REMOTE, url])
|
||||
|
||||
self._test_cloud(TEST_REMOTE)
|
||||
|
||||
|
||||
class TestOSSRemoteCLI(OSS, TestDataCloudCLIBase):
|
||||
def _test(self):
|
||||
url = self.get_url()
|
||||
|
||||
self.main(["remote", "add", TEST_REMOTE, url])
|
||||
|
||||
self._test_cloud(TEST_REMOTE)
|
||||
assert main(["pull", "-f"] + args) == 0
|
||||
assert os.path.exists(cache)
|
||||
assert os.path.isfile(cache)
|
||||
assert os.path.isfile(cache_dir)
|
||||
assert os.path.isfile("foo")
|
||||
assert os.path.isdir("data_dir")
|
||||
|
||||
|
||||
class TestDataCloudErrorCLI(TestDvc):
|
||||
@@ -504,123 +173,30 @@ class TestDataCloudErrorCLI(TestDvc):
|
||||
self.main_fail(["fetch", f])
|
||||
|
||||
|
||||
class TestWarnOnOutdatedStage(TestDvc):
|
||||
def main(self, args):
|
||||
ret = main(args)
|
||||
self.assertEqual(ret, 0)
|
||||
def test_warn_on_outdated_stage(tmp_dir, dvc, local_remote, caplog):
|
||||
stage = dvc.run(outs=["bar"], cmd="echo bar > bar", single_stage=True)
|
||||
assert main(["push"]) == 0
|
||||
|
||||
def _test(self):
|
||||
url = Local.get_url()
|
||||
self.main(["remote", "add", "-d", TEST_REMOTE, url])
|
||||
stage_file_path = stage.relpath
|
||||
content = load_yaml(stage_file_path)
|
||||
del content["outs"][0]["md5"]
|
||||
dump_yaml(stage_file_path, content)
|
||||
|
||||
stage = self.dvc.run(
|
||||
outs=["bar"], cmd="echo bar > bar", single_stage=True
|
||||
)
|
||||
self.main(["push"])
|
||||
|
||||
stage_file_path = stage.relpath
|
||||
content = load_yaml(stage_file_path)
|
||||
del content["outs"][0]["md5"]
|
||||
dump_yaml(stage_file_path, content)
|
||||
|
||||
with self._caplog.at_level(logging.WARNING, logger="dvc"):
|
||||
self._caplog.clear()
|
||||
self.main(["status", "-c"])
|
||||
expected_warning = (
|
||||
"Output 'bar'(stage: 'bar.dvc') is missing version info. "
|
||||
"Cache for it will not be collected. "
|
||||
"Use `dvc repro` to get your pipeline up to date."
|
||||
)
|
||||
|
||||
assert expected_warning in self._caplog.text
|
||||
|
||||
def test(self):
|
||||
self._test()
|
||||
|
||||
|
||||
class TestRecursiveSyncOperations(Local, TestDataCloudBase):
|
||||
def main(self, args):
|
||||
ret = main(args)
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
def _get_cloud_class(self):
|
||||
return LocalRemote
|
||||
|
||||
def _prepare_repo(self):
|
||||
remote = self.cloud.get_remote()
|
||||
self.main(
|
||||
["remote", "add", "-d", TEST_REMOTE, remote.path_info.fspath]
|
||||
with caplog.at_level(logging.WARNING, logger="dvc"):
|
||||
caplog.clear()
|
||||
assert main(["status", "-c"]) == 0
|
||||
expected_warning = (
|
||||
"Output 'bar'(stage: 'bar.dvc') is missing version info. "
|
||||
"Cache for it will not be collected. "
|
||||
"Use `dvc repro` to get your pipeline up to date."
|
||||
)
|
||||
|
||||
self.dvc.add(self.DATA)
|
||||
self.dvc.add(self.DATA_SUB)
|
||||
|
||||
def _remove_local_data_files(self):
|
||||
os.remove(self.DATA)
|
||||
os.remove(self.DATA_SUB)
|
||||
|
||||
def _test_recursive_pull(self):
|
||||
self._remove_local_data_files()
|
||||
self._clear_local_cache()
|
||||
|
||||
self.assertFalse(os.path.exists(self.DATA))
|
||||
self.assertFalse(os.path.exists(self.DATA_SUB))
|
||||
|
||||
self.main(["pull", "-R", self.DATA_DIR])
|
||||
|
||||
self.assertTrue(os.path.exists(self.DATA))
|
||||
self.assertTrue(os.path.exists(self.DATA_SUB))
|
||||
|
||||
def _clear_local_cache(self):
|
||||
remove(self.dvc.cache.local.cache_dir)
|
||||
|
||||
def _test_recursive_fetch(self, data_md5, data_sub_md5):
|
||||
self._clear_local_cache()
|
||||
|
||||
local_cache_data_path = self.dvc.cache.local.get(data_md5)
|
||||
local_cache_data_sub_path = self.dvc.cache.local.get(data_sub_md5)
|
||||
|
||||
self.assertFalse(os.path.exists(local_cache_data_path))
|
||||
self.assertFalse(os.path.exists(local_cache_data_sub_path))
|
||||
|
||||
self.main(["fetch", "-R", self.DATA_DIR])
|
||||
|
||||
self.assertTrue(os.path.exists(local_cache_data_path))
|
||||
self.assertTrue(os.path.exists(local_cache_data_sub_path))
|
||||
|
||||
def _test_recursive_push(self, data_md5, data_sub_md5):
|
||||
remote = self.cloud.get_remote()
|
||||
cloud_data_path = remote.get(data_md5)
|
||||
cloud_data_sub_path = remote.get(data_sub_md5)
|
||||
|
||||
self.assertFalse(os.path.exists(cloud_data_path))
|
||||
self.assertFalse(os.path.exists(cloud_data_sub_path))
|
||||
|
||||
self.main(["push", "-R", self.DATA_DIR])
|
||||
|
||||
self.assertTrue(os.path.exists(cloud_data_path))
|
||||
self.assertTrue(os.path.exists(cloud_data_sub_path))
|
||||
|
||||
def test(self):
|
||||
self._setup_cloud()
|
||||
self._prepare_repo()
|
||||
|
||||
data_md5 = file_md5(self.DATA)[0]
|
||||
data_sub_md5 = file_md5(self.DATA_SUB)[0]
|
||||
|
||||
self._test_recursive_push(data_md5, data_sub_md5)
|
||||
|
||||
self._test_recursive_fetch(data_md5, data_sub_md5)
|
||||
|
||||
self._test_recursive_pull()
|
||||
assert expected_warning in caplog.text
|
||||
|
||||
|
||||
def test_checksum_recalculation(mocker, dvc, tmp_dir):
|
||||
def test_hash_recalculation(mocker, dvc, tmp_dir, local_remote):
|
||||
tmp_dir.gen({"foo": "foo"})
|
||||
test_get_file_checksum = mocker.spy(LocalRemoteTree, "get_file_checksum")
|
||||
url = Local.get_url()
|
||||
ret = main(["remote", "add", "-d", TEST_REMOTE, url])
|
||||
assert ret == 0
|
||||
test_get_file_hash = mocker.spy(LocalRemoteTree, "get_file_hash")
|
||||
ret = main(["config", "cache.type", "hardlink"])
|
||||
assert ret == 0
|
||||
ret = main(["add", "foo"])
|
||||
@@ -629,7 +205,7 @@ def test_checksum_recalculation(mocker, dvc, tmp_dir):
|
||||
assert ret == 0
|
||||
ret = main(["run", "--single-stage", "-d", "foo", "echo foo"])
|
||||
assert ret == 0
|
||||
assert test_get_file_checksum.mock.call_count == 1
|
||||
assert test_get_file_hash.mock.call_count == 1
|
||||
|
||||
|
||||
class TestShouldWarnOnNoChecksumInLocalAndRemoteCache(TestDvc):
|
||||
@@ -682,11 +258,9 @@ class TestShouldWarnOnNoChecksumInLocalAndRemoteCache(TestDvc):
|
||||
assert self.message_bar_part in self._caplog.text
|
||||
|
||||
|
||||
def test_verify_checksums(
|
||||
tmp_dir, scm, dvc, mocker, tmp_path_factory, setup_remote
|
||||
def test_verify_hashes(
|
||||
tmp_dir, scm, dvc, mocker, tmp_path_factory, local_remote
|
||||
):
|
||||
|
||||
setup_remote(dvc, name="upstream")
|
||||
tmp_dir.dvc_gen({"file": "file1 content"}, commit="add file")
|
||||
tmp_dir.dvc_gen({"dir": {"subfile": "file2 content"}}, commit="add dir")
|
||||
dvc.push()
|
||||
@@ -696,10 +270,10 @@ def test_verify_checksums(
|
||||
remove("dir")
|
||||
remove(dvc.cache.local.cache_dir)
|
||||
|
||||
checksum_spy = mocker.spy(dvc.cache.local.tree, "get_file_checksum")
|
||||
hash_spy = mocker.spy(dvc.cache.local.tree, "get_file_hash")
|
||||
|
||||
dvc.pull()
|
||||
assert checksum_spy.call_count == 0
|
||||
assert hash_spy.call_count == 0
|
||||
|
||||
# Removing cache will invalidate existing state entries
|
||||
remove(dvc.cache.local.cache_dir)
|
||||
@@ -707,7 +281,7 @@ def test_verify_checksums(
|
||||
dvc.config["remote"]["upstream"]["verify"] = True
|
||||
|
||||
dvc.pull()
|
||||
assert checksum_spy.call_count == 3
|
||||
assert hash_spy.call_count == 3
|
||||
|
||||
|
||||
@flaky(max_runs=3, min_passes=1)
|
||||
@@ -778,8 +352,7 @@ def recurse_list_dir(d):
|
||||
]
|
||||
|
||||
|
||||
def test_dvc_pull_pipeline_stages(tmp_dir, dvc, run_copy, setup_remote):
|
||||
setup_remote(dvc)
|
||||
def test_dvc_pull_pipeline_stages(tmp_dir, dvc, run_copy, local_remote):
|
||||
(stage0,) = tmp_dir.dvc_gen("foo", "foo")
|
||||
stage1 = run_copy("foo", "bar", single_stage=True)
|
||||
stage2 = run_copy("bar", "foobar", name="copy-bar-foobar")
|
||||
@@ -810,8 +383,8 @@ def test_dvc_pull_pipeline_stages(tmp_dir, dvc, run_copy, setup_remote):
|
||||
assert set(stats["added"]) == set(outs)
|
||||
|
||||
|
||||
def test_pipeline_file_target_ops(tmp_dir, dvc, run_copy, setup_remote):
|
||||
remote_path = setup_remote(dvc)
|
||||
def test_pipeline_file_target_ops(tmp_dir, dvc, run_copy, local_remote):
|
||||
path = local_remote.url
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
run_copy("foo", "bar", single_stage=True)
|
||||
|
||||
@@ -830,7 +403,7 @@ def test_pipeline_file_target_ops(tmp_dir, dvc, run_copy, setup_remote):
|
||||
outs = ["foo", "bar", "lorem", "ipsum", "baz", "lorem2"]
|
||||
|
||||
# each one's a copy of other, hence 3
|
||||
assert len(recurse_list_dir(remote_path)) == 3
|
||||
assert len(recurse_list_dir(path)) == 3
|
||||
|
||||
clean(outs, dvc)
|
||||
assert set(dvc.pull(["dvc.yaml"])["added"]) == {"lorem2", "baz"}
|
||||
@@ -841,13 +414,13 @@ def test_pipeline_file_target_ops(tmp_dir, dvc, run_copy, setup_remote):
|
||||
# clean everything in remote and push
|
||||
from tests.dir_helpers import TmpDir
|
||||
|
||||
clean(TmpDir(remote_path).iterdir())
|
||||
clean(TmpDir(path).iterdir())
|
||||
dvc.push(["dvc.yaml:copy-ipsum-baz"])
|
||||
assert len(recurse_list_dir(remote_path)) == 1
|
||||
assert len(recurse_list_dir(path)) == 1
|
||||
|
||||
clean(TmpDir(remote_path).iterdir())
|
||||
clean(TmpDir(path).iterdir())
|
||||
dvc.push(["dvc.yaml"])
|
||||
assert len(recurse_list_dir(remote_path)) == 2
|
||||
assert len(recurse_list_dir(path)) == 2
|
||||
|
||||
with pytest.raises(StageNotFound):
|
||||
dvc.push(["dvc.yaml:StageThatDoesNotExist"])
|
||||
@@ -864,8 +437,7 @@ def test_pipeline_file_target_ops(tmp_dir, dvc, run_copy, setup_remote):
|
||||
({}, "Everything is up to date"),
|
||||
],
|
||||
)
|
||||
def test_push_stats(tmp_dir, dvc, fs, msg, caplog, setup_remote):
|
||||
setup_remote(dvc)
|
||||
def test_push_stats(tmp_dir, dvc, fs, msg, caplog, local_remote):
|
||||
tmp_dir.dvc_gen(fs)
|
||||
|
||||
caplog.clear()
|
||||
@@ -882,8 +454,7 @@ def test_push_stats(tmp_dir, dvc, fs, msg, caplog, setup_remote):
|
||||
({}, "Everything is up to date."),
|
||||
],
|
||||
)
|
||||
def test_fetch_stats(tmp_dir, dvc, fs, msg, caplog, setup_remote):
|
||||
setup_remote(dvc)
|
||||
def test_fetch_stats(tmp_dir, dvc, fs, msg, caplog, local_remote):
|
||||
tmp_dir.dvc_gen(fs)
|
||||
dvc.push()
|
||||
clean(list(fs.keys()), dvc)
|
||||
@@ -893,8 +464,7 @@ def test_fetch_stats(tmp_dir, dvc, fs, msg, caplog, setup_remote):
|
||||
assert msg in caplog.text
|
||||
|
||||
|
||||
def test_pull_stats(tmp_dir, dvc, caplog, setup_remote):
|
||||
setup_remote(dvc)
|
||||
def test_pull_stats(tmp_dir, dvc, caplog, local_remote):
|
||||
tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
|
||||
dvc.push()
|
||||
clean(["foo", "bar"], dvc)
|
||||
@@ -915,8 +485,7 @@ def test_pull_stats(tmp_dir, dvc, caplog, setup_remote):
|
||||
@pytest.mark.parametrize(
|
||||
"key,expected", [("all_tags", 2), ("all_branches", 3), ("all_commits", 3)]
|
||||
)
|
||||
def test_push_pull_all(tmp_dir, scm, dvc, setup_remote, key, expected):
|
||||
setup_remote(dvc)
|
||||
def test_push_pull_all(tmp_dir, scm, dvc, local_remote, key, expected):
|
||||
tmp_dir.dvc_gen({"foo": "foo"}, commit="first")
|
||||
scm.tag("v1")
|
||||
dvc.remove("foo.dvc")
|
||||
@@ -932,13 +501,12 @@ def test_push_pull_all(tmp_dir, scm, dvc, setup_remote, key, expected):
|
||||
assert dvc.pull(**{key: True})["fetched"] == expected
|
||||
|
||||
|
||||
def test_push_pull_fetch_pipeline_stages(tmp_dir, dvc, run_copy, setup_remote):
|
||||
remote_path = setup_remote(dvc)
|
||||
def test_push_pull_fetch_pipeline_stages(tmp_dir, dvc, run_copy, local_remote):
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
run_copy("foo", "bar", no_commit=True, name="copy-foo-bar")
|
||||
|
||||
dvc.push("copy-foo-bar")
|
||||
assert len(recurse_list_dir(remote_path)) == 1
|
||||
assert len(recurse_list_dir(local_remote.url)) == 1
|
||||
# pushing everything so as we can check pull/fetch only downloads
|
||||
# from specified targets
|
||||
dvc.push()
|
||||
|
||||
@@ -157,7 +157,7 @@ def test_stage_collection(tmp_dir, dvc):
|
||||
always_changed=True,
|
||||
single_stage=True,
|
||||
)
|
||||
assert {s for s in dvc.stages} == {stage1, stage3, stage2}
|
||||
assert set(dvc.stages) == {stage1, stage3, stage2}
|
||||
|
||||
|
||||
def test_remove_stage(tmp_dir, dvc, run_copy):
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user