mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 07:18:27 +01:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a2f1367a9a | ||
|
|
5d97f92bdd | ||
|
|
02b147b6f4 | ||
|
|
fa0f182028 | ||
|
|
15c787f2ac | ||
|
|
4af3919062 | ||
|
|
3a469f2e93 | ||
|
|
51a8c782ae | ||
|
|
b579d06c8e | ||
|
|
039cd49259 | ||
|
|
f516dd252a | ||
|
|
706dd05e8d | ||
|
|
5d957b1830 | ||
|
|
387867fbbe | ||
|
|
7c6351d039 | ||
|
|
613ca0bdb5 | ||
|
|
0fb9870de6 | ||
|
|
73e60a8c13 | ||
|
|
9d1dde4051 | ||
|
|
38dc0f85d4 | ||
|
|
d2dbe9649c |
+19
-5
@@ -1,5 +1,19 @@
|
||||
languages:
|
||||
Python: true
|
||||
|
||||
exclude_paths:
|
||||
- tests/
|
||||
version: "2"
|
||||
checks:
|
||||
argument-count:
|
||||
config:
|
||||
threshold: 8
|
||||
complex-logic:
|
||||
config:
|
||||
threshold: 6
|
||||
file-lines:
|
||||
config:
|
||||
threshold: 350
|
||||
method-complexity:
|
||||
config:
|
||||
threshold: 15
|
||||
method-count:
|
||||
config:
|
||||
threshold: 30
|
||||
exclude_patterns:
|
||||
- tests
|
||||
|
||||
@@ -4,10 +4,6 @@ test_patterns = [
|
||||
'tests/**'
|
||||
]
|
||||
|
||||
exclude_patterns = [
|
||||
'dvc/utils/compat.py'
|
||||
]
|
||||
|
||||
[[analyzers]]
|
||||
name = 'python'
|
||||
enabled = true
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
* [ ] ❗ I have followed the [Contributing to DVC](https://dvc.org/doc/user-guide/contributing/core) checklist.
|
||||
|
||||
* [ ] 📖 If this PR requires [documentation](https://dvc.org/doc) updates, I have created a separate PR (or issue, at least) in [dvc.org](https://github.com/iterative/dvc.org) and linked it here. If the CLI API is changed, I have updated [tab completion scripts](https://github.com/iterative/dvc/tree/master/scripts/completion).
|
||||
* [ ] 📖 If this PR requires [documentation](https://dvc.org/doc) updates, I have created a separate PR (or issue, at least) in [dvc.org](https://github.com/iterative/dvc.org) and linked it here.
|
||||
|
||||
* [ ] ❌ I will check DeepSource, CodeClimate, and other sanity checks below. (We consider them recommendatory and don't expect everything to be addressed. Please fix things that actually improve code or fix bugs.)
|
||||
|
||||
|
||||
+65
-60
@@ -1,63 +1,68 @@
|
||||
repos:
|
||||
- hooks:
|
||||
- id: black
|
||||
language_version: python3
|
||||
repo: https://github.com/ambv/black
|
||||
rev: 19.10b0
|
||||
- repo: https://github.com/asottile/seed-isort-config
|
||||
- hooks:
|
||||
- id: black
|
||||
language_version: python3
|
||||
repo: https://github.com/ambv/black
|
||||
rev: 19.10b0
|
||||
- repo: https://github.com/asottile/seed-isort-config
|
||||
rev: v2.1.1
|
||||
hooks:
|
||||
- id: seed-isort-config
|
||||
- hooks:
|
||||
- id: isort
|
||||
language_version: python3
|
||||
repo: https://github.com/timothycrosley/isort
|
||||
rev: 4.3.21
|
||||
- hooks:
|
||||
- id: flake8
|
||||
language_version: python3
|
||||
repo: https://gitlab.com/pycqa/flake8
|
||||
rev: 3.7.9
|
||||
- hooks:
|
||||
- args:
|
||||
- -i
|
||||
- '2'
|
||||
id: beautysh
|
||||
language_version: python3
|
||||
repo: https://github.com/lovesegfault/beautysh
|
||||
rev: master
|
||||
- hooks:
|
||||
- args:
|
||||
- git-hook
|
||||
- pre-commit
|
||||
entry: dvc
|
||||
id: dvc-pre-commit
|
||||
language: system
|
||||
name: DVC pre-commit
|
||||
stages:
|
||||
- commit
|
||||
verbose: true
|
||||
require_serial: true
|
||||
- args:
|
||||
- git-hook
|
||||
- pre-push
|
||||
entry: dvc
|
||||
id: dvc-pre-push
|
||||
language: system
|
||||
name: DVC pre-push
|
||||
stages:
|
||||
- push
|
||||
require_serial: true
|
||||
- always_run: true
|
||||
args:
|
||||
- git-hook
|
||||
- post-checkout
|
||||
entry: dvc
|
||||
id: dvc-post-checkout
|
||||
language: system
|
||||
minimum_pre_commit_version: 2.2.0
|
||||
name: DVC post-checkout
|
||||
stages:
|
||||
- post-checkout
|
||||
require_serial: true
|
||||
repo: local
|
||||
- id: seed-isort-config
|
||||
- hooks:
|
||||
- id: isort
|
||||
language_version: python3
|
||||
repo: https://github.com/timothycrosley/isort
|
||||
rev: 4.3.21
|
||||
- hooks:
|
||||
- id: flake8
|
||||
language_version: python3
|
||||
additional_dependencies:
|
||||
- flake8-bugbear
|
||||
- flake8-comprehensions
|
||||
- flake8-debugger
|
||||
- flake8-string-format
|
||||
repo: https://gitlab.com/pycqa/flake8
|
||||
rev: 3.7.9
|
||||
- hooks:
|
||||
- args:
|
||||
- -i
|
||||
- "2"
|
||||
id: beautysh
|
||||
language_version: python3
|
||||
repo: https://github.com/lovesegfault/beautysh
|
||||
rev: master
|
||||
- hooks:
|
||||
- args:
|
||||
- git-hook
|
||||
- pre-commit
|
||||
entry: dvc
|
||||
id: dvc-pre-commit
|
||||
language: system
|
||||
name: DVC pre-commit
|
||||
stages:
|
||||
- commit
|
||||
verbose: true
|
||||
require_serial: true
|
||||
- args:
|
||||
- git-hook
|
||||
- pre-push
|
||||
entry: dvc
|
||||
id: dvc-pre-push
|
||||
language: system
|
||||
name: DVC pre-push
|
||||
stages:
|
||||
- push
|
||||
require_serial: true
|
||||
- always_run: true
|
||||
args:
|
||||
- git-hook
|
||||
- post-checkout
|
||||
entry: dvc
|
||||
id: dvc-post-checkout
|
||||
language: system
|
||||
minimum_pre_commit_version: 2.2.0
|
||||
name: DVC post-checkout
|
||||
stages:
|
||||
- post-checkout
|
||||
require_serial: true
|
||||
repo: local
|
||||
|
||||
+1
-1
@@ -30,7 +30,7 @@ def get_url(path, repo=None, rev=None, remote=None):
|
||||
raise UrlNotDvcRepoError(_repo.url)
|
||||
out = _repo.find_out_by_relpath(path)
|
||||
remote_obj = _repo.cloud.get_remote(remote)
|
||||
return str(remote_obj.checksum_to_path_info(out.checksum))
|
||||
return str(remote_obj.hash_to_path_info(out.checksum))
|
||||
|
||||
|
||||
def open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
|
||||
|
||||
+16
-10
@@ -8,6 +8,7 @@ from .command import (
|
||||
cache,
|
||||
checkout,
|
||||
commit,
|
||||
completion,
|
||||
config,
|
||||
daemon,
|
||||
dag,
|
||||
@@ -70,6 +71,7 @@ COMMANDS = [
|
||||
dag,
|
||||
daemon,
|
||||
commit,
|
||||
completion,
|
||||
diff,
|
||||
version,
|
||||
update,
|
||||
@@ -140,15 +142,7 @@ def get_parent_parser():
|
||||
return parent_parser
|
||||
|
||||
|
||||
def parse_args(argv=None):
|
||||
"""Parses CLI arguments.
|
||||
|
||||
Args:
|
||||
argv: optional list of arguments to parse. sys.argv is used by default.
|
||||
|
||||
Raises:
|
||||
dvc.exceptions.DvcParserError: raised for argument parsing errors.
|
||||
"""
|
||||
def get_main_parser():
|
||||
parent_parser = get_parent_parser()
|
||||
|
||||
# Main parser
|
||||
@@ -196,6 +190,18 @@ def parse_args(argv=None):
|
||||
for cmd in COMMANDS:
|
||||
cmd.add_parser(subparsers, parent_parser)
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
return parser
|
||||
|
||||
|
||||
def parse_args(argv=None):
|
||||
"""Parses CLI arguments.
|
||||
|
||||
Args:
|
||||
argv: optional list of arguments to parse. sys.argv is used by default.
|
||||
|
||||
Raises:
|
||||
dvc.exceptions.DvcParserError: raised for argument parsing errors.
|
||||
"""
|
||||
parser = get_main_parser()
|
||||
args = parser.parse_args(argv)
|
||||
return args
|
||||
|
||||
+5
-1
@@ -1,6 +1,7 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException, RecursiveAddingWhileUsingFilename
|
||||
|
||||
@@ -62,6 +63,9 @@ def add_parser(subparsers, parent_parser):
|
||||
metavar="<filename>",
|
||||
)
|
||||
parser.add_argument(
|
||||
"targets", nargs="+", help="Input files/directories to add."
|
||||
"targets",
|
||||
nargs="+",
|
||||
help="Input files/directories to add.",
|
||||
choices=completion.Required.FILE,
|
||||
)
|
||||
parser.set_defaults(func=CmdAdd)
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import argparse
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import append_doc_link, fix_subparsers
|
||||
from dvc.command.config import CmdConfig
|
||||
|
||||
@@ -55,5 +56,6 @@ def add_parser(subparsers, parent_parser):
|
||||
help="Path to cache directory. Relative paths are resolved relative "
|
||||
"to the current directory and saved to config relative to the "
|
||||
"config file location.",
|
||||
choices=completion.Required.DIR,
|
||||
)
|
||||
cache_dir_parser.set_defaults(func=CmdCacheDir)
|
||||
|
||||
@@ -4,6 +4,7 @@ import operator
|
||||
|
||||
import colorama
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import CheckoutError
|
||||
from dvc.utils.humanize import get_summary
|
||||
@@ -113,5 +114,6 @@ def add_parser(subparsers, parent_parser):
|
||||
nargs="*",
|
||||
help="DVC-files to checkout. Optional. "
|
||||
"(Finds all DVC-files in the workspace by default.)",
|
||||
choices=completion.Optional.DVC_FILE,
|
||||
)
|
||||
checkout_parser.set_defaults(func=CmdCheckout)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
@@ -66,5 +67,6 @@ def add_parser(subparsers, parent_parser):
|
||||
nargs="*",
|
||||
help="DVC-files to commit. Optional. "
|
||||
"(Finds all DVC-files in the workspace by default.)",
|
||||
choices=completion.Optional.DVC_FILE,
|
||||
)
|
||||
commit_parser.set_defaults(func=CmdCommit)
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
import shtab
|
||||
|
||||
from dvc.command.base import CmdBaseNoRepo, append_doc_link
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
CHOICE_FUNCTIONS = {
|
||||
"bash": {"DVCFile": "_dvc_compgen_DVCFiles"},
|
||||
"zsh": {"DVCFile": "_files -g '(*?.dvc|Dvcfile|dvc.yaml)'"},
|
||||
}
|
||||
PREAMBLE = {
|
||||
"bash": """
|
||||
# $1=COMP_WORDS[1]
|
||||
_dvc_compgen_DVCFiles() {
|
||||
compgen -d -S '/' -- $1 # recurse into subdirs
|
||||
compgen -f -X '!*?.dvc' -- $1
|
||||
compgen -f -X '!*Dvcfile' -- $1
|
||||
compgen -f -X '!*dvc.yaml' -- $1
|
||||
}
|
||||
""",
|
||||
"zsh": "",
|
||||
}
|
||||
|
||||
|
||||
class Optional(shtab.Optional):
|
||||
DVC_FILE = [shtab.Choice("DVCFile", required=False)]
|
||||
|
||||
|
||||
class Required(shtab.Required):
|
||||
DVC_FILE = [shtab.Choice("DVCFile", required=True)]
|
||||
|
||||
|
||||
class CmdCompletion(CmdBaseNoRepo):
|
||||
def run(self):
|
||||
from dvc.cli import get_main_parser
|
||||
|
||||
parser = get_main_parser()
|
||||
shell = self.args.shell
|
||||
script = shtab.complete(
|
||||
parser,
|
||||
shell=shell,
|
||||
preamble=PREAMBLE[shell],
|
||||
choice_functions=CHOICE_FUNCTIONS[shell],
|
||||
)
|
||||
print(script)
|
||||
return 0
|
||||
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
COMPLETION_HELP = "Generate shell tab completion."
|
||||
COMPLETION_DESCRIPTION = "Prints out shell tab completion scripts."
|
||||
completion_parser = subparsers.add_parser(
|
||||
"completion",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(COMPLETION_DESCRIPTION, "completion"),
|
||||
help=COMPLETION_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
completion_parser.add_argument(
|
||||
"-s",
|
||||
"--shell",
|
||||
help="Shell syntax for completions.",
|
||||
default="bash",
|
||||
choices=["bash", "zsh"],
|
||||
)
|
||||
completion_parser.set_defaults(func=CmdCompletion)
|
||||
@@ -1,3 +1,4 @@
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBaseNoRepo, fix_subparsers
|
||||
|
||||
|
||||
@@ -64,5 +65,7 @@ def add_parser(subparsers, parent_parser):
|
||||
description=DAEMON_ANALYTICS_HELP,
|
||||
help=DAEMON_ANALYTICS_HELP,
|
||||
)
|
||||
daemon_analytics_parser.add_argument("target", help="Analytics file.")
|
||||
daemon_analytics_parser.add_argument(
|
||||
"target", help="Analytics file.", choices=completion.Required.FILE
|
||||
)
|
||||
daemon_analytics_parser.set_defaults(func=CmdDaemonAnalytics)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.command.checkout import log_changes
|
||||
from dvc.exceptions import CheckoutError, DvcException
|
||||
@@ -104,6 +105,7 @@ def shared_parent_parser():
|
||||
nargs="*",
|
||||
help="Limit command scope to these DVC-files. "
|
||||
"Using -R, directories to search DVC-files in can also be given.",
|
||||
choices=completion.Optional.DVC_FILE,
|
||||
)
|
||||
|
||||
return parent_parser
|
||||
|
||||
+1
-1
@@ -19,7 +19,7 @@ def _show_md(diff):
|
||||
entries = diff.get(status, [])
|
||||
if not entries:
|
||||
continue
|
||||
paths = sorted([entry["path"] for entry in entries])
|
||||
paths = sorted(entry["path"] for entry in entries)
|
||||
for path in paths:
|
||||
rows.append([status, path])
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
@@ -39,7 +40,10 @@ def add_parser(subparsers, parent_parser):
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
freeze_parser.add_argument(
|
||||
"targets", nargs="+", help="DVC-files to freeze."
|
||||
"targets",
|
||||
nargs="+",
|
||||
help="DVC-files to freeze.",
|
||||
choices=completion.Required.DVC_FILE,
|
||||
)
|
||||
freeze_parser.set_defaults(func=CmdFreeze)
|
||||
|
||||
@@ -52,6 +56,9 @@ def add_parser(subparsers, parent_parser):
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
unfreeze_parser.add_argument(
|
||||
"targets", nargs="+", help="DVC-files to unfreeze."
|
||||
"targets",
|
||||
nargs="+",
|
||||
help="DVC-files to unfreeze.",
|
||||
choices=completion.Required.DVC_FILE,
|
||||
)
|
||||
unfreeze_parser.set_defaults(func=CmdUnfreeze)
|
||||
|
||||
+5
-1
@@ -3,6 +3,7 @@ import logging
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
from . import completion
|
||||
from .base import CmdBaseNoRepo, append_doc_link
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -62,7 +63,9 @@ def add_parser(subparsers, parent_parser):
|
||||
"url", help="Location of DVC or Git repository to download from"
|
||||
)
|
||||
get_parser.add_argument(
|
||||
"path", help="Path to a file or directory within the repository"
|
||||
"path",
|
||||
help="Path to a file or directory within the repository",
|
||||
choices=completion.Required.FILE,
|
||||
)
|
||||
get_parser.add_argument(
|
||||
"-o",
|
||||
@@ -70,6 +73,7 @@ def add_parser(subparsers, parent_parser):
|
||||
nargs="?",
|
||||
help="Destination path to download files to",
|
||||
metavar="<path>",
|
||||
choices=completion.Optional.DIR,
|
||||
)
|
||||
get_parser.add_argument(
|
||||
"--rev",
|
||||
|
||||
@@ -3,6 +3,7 @@ import logging
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
from . import completion
|
||||
from .base import CmdBaseNoRepo, append_doc_link
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -33,6 +34,9 @@ def add_parser(subparsers, parent_parser):
|
||||
"url", help="See `dvc import-url -h` for full list of supported URLs."
|
||||
)
|
||||
get_parser.add_argument(
|
||||
"out", nargs="?", help="Destination path to put data to."
|
||||
"out",
|
||||
nargs="?",
|
||||
help="Destination path to put data to.",
|
||||
choices=completion.Optional.DIR,
|
||||
)
|
||||
get_parser.set_defaults(func=CmdGetUrl)
|
||||
|
||||
+5
-1
@@ -1,6 +1,7 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
@@ -43,7 +44,9 @@ def add_parser(subparsers, parent_parser):
|
||||
"url", help="Location of DVC or Git repository to download from"
|
||||
)
|
||||
import_parser.add_argument(
|
||||
"path", help="Path to a file or directory within the repository"
|
||||
"path",
|
||||
help="Path to a file or directory within the repository",
|
||||
choices=completion.Required.FILE,
|
||||
)
|
||||
import_parser.add_argument(
|
||||
"-o",
|
||||
@@ -51,6 +54,7 @@ def add_parser(subparsers, parent_parser):
|
||||
nargs="?",
|
||||
help="Destination path to download files to",
|
||||
metavar="<path>",
|
||||
choices=completion.Optional.DIR,
|
||||
)
|
||||
import_parser.add_argument(
|
||||
"--rev",
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
@@ -54,11 +55,15 @@ def add_parser(subparsers, parent_parser):
|
||||
"remote://myremote/path/to/file (see `dvc remote`)",
|
||||
)
|
||||
import_parser.add_argument(
|
||||
"out", nargs="?", help="Destination path to put files to."
|
||||
"out",
|
||||
nargs="?",
|
||||
help="Destination path to put files to.",
|
||||
choices=completion.Optional.DIR,
|
||||
)
|
||||
import_parser.add_argument(
|
||||
"--file",
|
||||
help="Specify name of the DVC-file this command will generate.",
|
||||
metavar="<filename>",
|
||||
choices=completion.Optional.DIR,
|
||||
)
|
||||
import_parser.set_defaults(func=CmdImportUrl)
|
||||
|
||||
@@ -2,6 +2,7 @@ import argparse
|
||||
import logging
|
||||
import sys
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBaseNoRepo, append_doc_link
|
||||
from dvc.command.ls.ls_colors import LsColors
|
||||
from dvc.exceptions import DvcException
|
||||
@@ -74,5 +75,6 @@ def add_parser(subparsers, parent_parser):
|
||||
"path",
|
||||
nargs="?",
|
||||
help="Path to directory within the repository to list outputs for",
|
||||
choices=completion.Optional.DIR,
|
||||
)
|
||||
list_parser.set_defaults(func=CmdList)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
|
||||
from dvc.exceptions import BadMetricError, DvcException
|
||||
|
||||
@@ -169,6 +170,7 @@ def add_parser(subparsers, parent_parser):
|
||||
"Limit command scope to these metric files. Using -R, "
|
||||
"directories to search metric files in can also be given."
|
||||
),
|
||||
choices=completion.Optional.FILE,
|
||||
)
|
||||
metrics_show_parser.add_argument(
|
||||
"-a",
|
||||
@@ -235,6 +237,7 @@ def add_parser(subparsers, parent_parser):
|
||||
"directories to search metric files in can also be given."
|
||||
),
|
||||
metavar="<paths>",
|
||||
choices=completion.Optional.FILE,
|
||||
)
|
||||
metrics_diff_parser.add_argument(
|
||||
"-R",
|
||||
|
||||
+7
-2
@@ -1,6 +1,7 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
@@ -36,7 +37,11 @@ def add_parser(subparsers, parent_parser):
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
move_parser.add_argument(
|
||||
"src", help="Source path to a data file or directory."
|
||||
"src",
|
||||
help="Source path to a data file or directory.",
|
||||
choices=completion.Required.FILE,
|
||||
)
|
||||
move_parser.add_argument(
|
||||
"dst", help="Destination path.", choices=completion.Required.FILE,
|
||||
)
|
||||
move_parser.add_argument("dst", help="Destination path.")
|
||||
move_parser.set_defaults(func=CmdMove)
|
||||
|
||||
@@ -2,6 +2,7 @@ import argparse
|
||||
import logging
|
||||
import os
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.schema import PLOT_PROPS
|
||||
@@ -128,6 +129,7 @@ def add_parser(subparsers, parent_parser):
|
||||
"targets",
|
||||
nargs="*",
|
||||
help="Plots files to visualize. Shows all plots by default.",
|
||||
choices=completion.Optional.FILE,
|
||||
)
|
||||
_add_props_arguments(plots_show_parser)
|
||||
_add_output_arguments(plots_show_parser)
|
||||
@@ -149,6 +151,7 @@ def add_parser(subparsers, parent_parser):
|
||||
nargs="*",
|
||||
help="Plots file to visualize. Shows all plots by default.",
|
||||
metavar="<path>",
|
||||
choices=completion.Optional.FILE,
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"revisions", nargs="*", default=None, help="Git commits to plot from",
|
||||
@@ -166,7 +169,9 @@ def add_parser(subparsers, parent_parser):
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
plots_modify_parser.add_argument(
|
||||
"target", help="Metric file to set properties to",
|
||||
"target",
|
||||
help="Metric file to set properties to",
|
||||
choices=completion.Required.FILE,
|
||||
)
|
||||
_add_props_arguments(plots_modify_parser)
|
||||
plots_modify_parser.add_argument(
|
||||
@@ -191,6 +196,7 @@ def _add_props_arguments(parser):
|
||||
)
|
||||
),
|
||||
metavar="<path>",
|
||||
choices=completion.Optional.FILE,
|
||||
)
|
||||
parser.add_argument(
|
||||
"-x", default=None, help="Field name for X axis.", metavar="<field>"
|
||||
@@ -223,6 +229,7 @@ def _add_output_arguments(parser):
|
||||
default=None,
|
||||
help="Destination path to save plots to",
|
||||
metavar="<path>",
|
||||
choices=completion.Optional.DIR,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--show-vega",
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
@@ -34,6 +35,9 @@ def add_parser(subparsers, parent_parser):
|
||||
help="Remove outputs as well.",
|
||||
)
|
||||
remove_parser.add_argument(
|
||||
"targets", nargs="+", help="DVC-files to remove."
|
||||
"targets",
|
||||
nargs="+",
|
||||
help="DVC-files to remove.",
|
||||
choices=completion.Required.DVC_FILE,
|
||||
)
|
||||
remove_parser.set_defaults(func=CmdRemove)
|
||||
|
||||
@@ -2,6 +2,7 @@ import argparse
|
||||
import logging
|
||||
import os
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.command.metrics import show_metrics
|
||||
from dvc.command.status import CmdDataStatus
|
||||
@@ -71,6 +72,7 @@ def add_parser(subparsers, parent_parser):
|
||||
"targets",
|
||||
nargs="*",
|
||||
help=f"Stages to reproduce. '{PIPELINE_FILE}' by default.",
|
||||
choices=completion.Optional.DVC_FILE,
|
||||
)
|
||||
repro_parser.add_argument(
|
||||
"-f",
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
@@ -32,6 +33,9 @@ def add_parser(subparsers, parent_parser):
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
unprotect_parser.add_argument(
|
||||
"targets", nargs="+", help="Data files/directories to unprotect."
|
||||
"targets",
|
||||
nargs="+",
|
||||
help="Data files/directories to unprotect.",
|
||||
choices=completion.Required.FILE,
|
||||
)
|
||||
unprotect_parser.set_defaults(func=CmdUnprotect)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
@@ -32,7 +33,10 @@ def add_parser(subparsers, parent_parser):
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
update_parser.add_argument(
|
||||
"targets", nargs="+", help="DVC-files to update."
|
||||
"targets",
|
||||
nargs="+",
|
||||
help="DVC-files to update.",
|
||||
choices=completion.Required.DVC_FILE,
|
||||
)
|
||||
update_parser.add_argument(
|
||||
"--rev",
|
||||
|
||||
+1
-1
@@ -222,7 +222,7 @@ def _build_sugiyama_layout(vertexes, edges):
|
||||
vertex.view = VertexViewer(vertex.data)
|
||||
|
||||
# NOTE: determine min box length to create the best layout
|
||||
minw = min([v.view.w for v in vertexes])
|
||||
minw = min(v.view.w for v in vertexes)
|
||||
|
||||
for edge in edges:
|
||||
edge.view = EdgeViewer()
|
||||
|
||||
+1
-1
@@ -92,7 +92,7 @@ class DataCloud:
|
||||
|
||||
def _save_pulled_checksums(self, cache):
|
||||
for checksum in cache.scheme_keys("local"):
|
||||
cache_file = self.repo.cache.local.checksum_to_path_info(checksum)
|
||||
cache_file = self.repo.cache.local.hash_to_path_info(checksum)
|
||||
if self.repo.cache.local.tree.exists(cache_file):
|
||||
# We can safely save here, as existing corrupted files will
|
||||
# be removed upon status, while files corrupted during
|
||||
|
||||
@@ -64,8 +64,8 @@ class RepoDependency(LocalDependency):
|
||||
|
||||
# We are polluting our repo cache with some dir listing here
|
||||
if tree.isdir(path):
|
||||
return self.repo.cache.local.get_checksum(path, tree)
|
||||
return tree.get_file_checksum(path)
|
||||
return self.repo.cache.local.get_hash(path, tree)
|
||||
return tree.get_file_hash(path)
|
||||
|
||||
def status(self):
|
||||
current_checksum = self._get_checksum(locked=True)
|
||||
|
||||
+5
-2
@@ -39,6 +39,7 @@ class DvcIgnorePatterns(DvcIgnore):
|
||||
for ignore, group in groupby(
|
||||
regex_pattern_list, lambda x: x[1]
|
||||
)
|
||||
if ignore is not None
|
||||
]
|
||||
|
||||
def __call__(self, root, dirs, files):
|
||||
@@ -229,8 +230,10 @@ class CleanTree(BaseTree):
|
||||
return False
|
||||
return True
|
||||
|
||||
def walk(self, top, topdown=True):
|
||||
for root, dirs, files in self.tree.walk(top, topdown):
|
||||
def walk(self, top, topdown=True, onerror=None):
|
||||
for root, dirs, files in self.tree.walk(
|
||||
top, topdown=topdown, onerror=onerror
|
||||
):
|
||||
dirs[:], files[:] = self.dvcignore(
|
||||
os.path.abspath(root), dirs, files
|
||||
)
|
||||
|
||||
+11
-5
@@ -163,7 +163,7 @@ class BaseOutput:
|
||||
|
||||
@property
|
||||
def cache_path(self):
|
||||
return self.cache.checksum_to_path_info(self.checksum).url
|
||||
return self.cache.hash_to_path_info(self.checksum).url
|
||||
|
||||
@property
|
||||
def checksum_type(self):
|
||||
@@ -178,11 +178,11 @@ class BaseOutput:
|
||||
self.info[self.remote.tree.PARAM_CHECKSUM] = checksum
|
||||
|
||||
def get_checksum(self):
|
||||
return self.remote.get_checksum(self.path_info)
|
||||
return self.remote.get_hash(self.path_info)
|
||||
|
||||
@property
|
||||
def is_dir_checksum(self):
|
||||
return self.remote.is_dir_checksum(self.checksum)
|
||||
return self.remote.is_dir_hash(self.checksum)
|
||||
|
||||
@property
|
||||
def exists(self):
|
||||
@@ -239,6 +239,12 @@ class BaseOutput:
|
||||
|
||||
self.repo.scm.ignore(self.fspath)
|
||||
|
||||
def ignore_remove(self):
|
||||
if not self.use_scm_ignore:
|
||||
return
|
||||
|
||||
self.repo.scm.ignore_remove(self.fspath)
|
||||
|
||||
def save(self):
|
||||
if not self.exists:
|
||||
raise self.DoesNotExistError(self)
|
||||
@@ -336,8 +342,8 @@ class BaseOutput:
|
||||
if self.scheme != "local":
|
||||
return
|
||||
|
||||
if ignore_remove and self.use_scm_ignore:
|
||||
self.repo.scm.ignore_remove(self.fspath)
|
||||
if ignore_remove:
|
||||
self.ignore_remove()
|
||||
|
||||
def move(self, out):
|
||||
if self.scheme == "local" and self.use_scm_ignore:
|
||||
|
||||
+2
-2
@@ -35,14 +35,14 @@ class LocalOutput(BaseOutput):
|
||||
# FIXME: if we have Windows path containing / or posix one with \
|
||||
# then we have #2059 bug and can't really handle that.
|
||||
p = self.TREE_CLS.PATH_CLS(path)
|
||||
if not p.is_absolute():
|
||||
if self.stage and not p.is_absolute():
|
||||
p = self.stage.wdir / p
|
||||
|
||||
abs_p = os.path.abspath(os.path.normpath(p))
|
||||
return self.TREE_CLS.PATH_CLS(abs_p)
|
||||
|
||||
def __str__(self):
|
||||
if not self.is_in_repo:
|
||||
if not self.repo or not self.is_in_repo:
|
||||
return str(self.def_path)
|
||||
|
||||
cur_dir = os.getcwd()
|
||||
|
||||
+1
-1
@@ -113,7 +113,7 @@ class AzureRemoteTree(BaseRemoteTree):
|
||||
logger.debug(f"Removing {path_info}")
|
||||
self.blob_service.delete_blob(path_info.bucket, path_info.path)
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
def get_file_hash(self, path_info):
|
||||
return self.get_etag(path_info)
|
||||
|
||||
def _upload(
|
||||
|
||||
+249
-274
File diff suppressed because it is too large
Load Diff
@@ -564,7 +564,7 @@ class GDriveRemoteTree(BaseRemoteTree):
|
||||
item_id = self._get_item_id(path_info)
|
||||
self.gdrive_delete_file(item_id)
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
def get_file_hash(self, path_info):
|
||||
raise NotImplementedError
|
||||
|
||||
def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
|
||||
|
||||
+19
-2
@@ -91,13 +91,30 @@ class GSRemoteTree(BaseRemoteTree):
|
||||
)
|
||||
|
||||
def _generate_download_url(self, path_info, expires=3600):
|
||||
import google.auth
|
||||
from google.auth import compute_engine
|
||||
|
||||
expiration = timedelta(seconds=int(expires))
|
||||
|
||||
bucket = self.gs.bucket(path_info.bucket)
|
||||
blob = bucket.get_blob(path_info.path)
|
||||
if blob is None:
|
||||
raise FileNotFoundError
|
||||
return blob.generate_signed_url(expiration=expiration)
|
||||
|
||||
if isinstance(
|
||||
blob.client._credentials, google.auth.credentials.Signing
|
||||
):
|
||||
# sign if we're able to sign with credentials.
|
||||
return blob.generate_signed_url(expiration=expiration)
|
||||
|
||||
auth_request = google.auth.transport.requests.Request()
|
||||
# create signing credentials with the default credentials
|
||||
# for use with Compute Engine and other environments where
|
||||
# Client credentials cannot sign.
|
||||
signing_credentials = compute_engine.IDTokenCredentials(
|
||||
auth_request, ""
|
||||
)
|
||||
return signing_credentials.signer.sign(blob)
|
||||
|
||||
def exists(self, path_info):
|
||||
"""Check if the blob exists. If it does not exist,
|
||||
@@ -160,7 +177,7 @@ class GSRemoteTree(BaseRemoteTree):
|
||||
to_bucket = self.gs.bucket(to_info.bucket)
|
||||
from_bucket.copy_blob(blob, to_bucket, new_name=to_info.path)
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
def get_file_hash(self, path_info):
|
||||
import base64
|
||||
import codecs
|
||||
|
||||
|
||||
+1
-1
@@ -159,7 +159,7 @@ class HDFSRemoteTree(BaseRemoteTree):
|
||||
assert match is not None
|
||||
return match.group(gname)
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
def get_file_hash(self, path_info):
|
||||
# NOTE: pyarrow doesn't support checksum, so we need to use hadoop
|
||||
regex = r".*\t.*\t(?P<checksum>.*)"
|
||||
stdout = self.hadoop_fs(
|
||||
|
||||
+1
-1
@@ -124,7 +124,7 @@ class HTTPRemoteTree(BaseRemoteTree):
|
||||
def exists(self, path_info):
|
||||
return bool(self.request("HEAD", path_info.url))
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
def get_file_hash(self, path_info):
|
||||
url = path_info.url
|
||||
headers = self.request("HEAD", url).headers
|
||||
etag = headers.get("ETag") or headers.get("Content-MD5")
|
||||
|
||||
+27
-31
@@ -25,15 +25,15 @@ class RemoteIndexNoop:
|
||||
def __iter__(self):
|
||||
return iter([])
|
||||
|
||||
def __contains__(self, checksum):
|
||||
def __contains__(self, hash_):
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def checksums():
|
||||
def hashes():
|
||||
return []
|
||||
|
||||
@staticmethod
|
||||
def dir_checksums():
|
||||
def dir_hashes():
|
||||
return []
|
||||
|
||||
def load(self):
|
||||
@@ -54,13 +54,13 @@ class RemoteIndexNoop:
|
||||
|
||||
|
||||
class RemoteIndex:
|
||||
"""Class for indexing remote checksums in a sqlite3 database.
|
||||
"""Class for indexing remote hashes in a sqlite3 database.
|
||||
|
||||
Args:
|
||||
repo: repo for this remote index.
|
||||
name: name for this index. Index db will be loaded from and saved to
|
||||
``.dvc/tmp/index/{name}.idx``.
|
||||
dir_suffix: suffix used for naming directory checksums
|
||||
dir_suffix: suffix used for naming directory hashes
|
||||
"""
|
||||
|
||||
INDEX_SUFFIX = ".idx"
|
||||
@@ -79,8 +79,8 @@ class RemoteIndex:
|
||||
|
||||
def __iter__(self):
|
||||
cmd = f"SELECT checksum FROM {self.INDEX_TABLE}"
|
||||
for (checksum,) in self._execute(cmd):
|
||||
yield checksum
|
||||
for (hash_,) in self._execute(cmd):
|
||||
yield hash_
|
||||
|
||||
def __enter__(self):
|
||||
self.lock.acquire()
|
||||
@@ -90,25 +90,25 @@ class RemoteIndex:
|
||||
self.dump()
|
||||
self.lock.release()
|
||||
|
||||
def __contains__(self, checksum):
|
||||
def __contains__(self, hash_):
|
||||
cmd = "SELECT checksum FROM {} WHERE checksum = (?)".format(
|
||||
self.INDEX_TABLE
|
||||
)
|
||||
self._execute(cmd, (checksum,))
|
||||
self._execute(cmd, (hash_,))
|
||||
return self.cursor.fetchone() is not None
|
||||
|
||||
def checksums(self):
|
||||
"""Iterate over checksums stored in the index."""
|
||||
def hashes(self):
|
||||
"""Iterate over hashes stored in the index."""
|
||||
return iter(self)
|
||||
|
||||
def dir_checksums(self):
|
||||
"""Iterate over .dir checksums stored in the index."""
|
||||
def dir_hashes(self):
|
||||
"""Iterate over .dir hashes stored in the index."""
|
||||
cmd = f"SELECT checksum FROM {self.INDEX_TABLE} WHERE dir = 1"
|
||||
for (checksum,) in self._execute(cmd):
|
||||
yield checksum
|
||||
for (hash_,) in self._execute(cmd):
|
||||
yield hash_
|
||||
|
||||
def is_dir_checksum(self, checksum):
|
||||
return checksum.endswith(self.dir_suffix)
|
||||
def is_dir_hash(self, hash_):
|
||||
return hash_.endswith(self.dir_suffix)
|
||||
|
||||
def _execute(self, cmd, parameters=()):
|
||||
return self.cursor.execute(cmd, parameters)
|
||||
@@ -185,28 +185,24 @@ class RemoteIndex:
|
||||
cmd = f"DELETE FROM {self.INDEX_TABLE}"
|
||||
self._execute(cmd)
|
||||
|
||||
def update(self, dir_checksums, file_checksums):
|
||||
"""Update this index, adding the specified checksums.
|
||||
def update(self, dir_hashes, file_hashes):
|
||||
"""Update this index, adding the specified hashes.
|
||||
|
||||
Changes to the index will not committed until dump() is called.
|
||||
"""
|
||||
cmd = "INSERT OR IGNORE INTO {} (checksum, dir) VALUES (?, ?)".format(
|
||||
self.INDEX_TABLE
|
||||
)
|
||||
self._executemany(
|
||||
cmd, ((checksum, True) for checksum in dir_checksums)
|
||||
)
|
||||
self._executemany(
|
||||
cmd, ((checksum, False) for checksum in file_checksums)
|
||||
)
|
||||
self._executemany(cmd, ((hash_, True) for hash_ in dir_hashes))
|
||||
self._executemany(cmd, ((hash_, False) for hash_ in file_hashes))
|
||||
|
||||
def intersection(self, checksums):
|
||||
"""Iterate over values from `checksums` which exist in the index."""
|
||||
def intersection(self, hashes):
|
||||
"""Iterate over values from `hashes` which exist in the index."""
|
||||
# sqlite has a compile time limit of 999, see:
|
||||
# https://www.sqlite.org/c3ref/c_limit_attached.html#sqlitelimitvariablenumber
|
||||
for chunk in lchunks(999, checksums):
|
||||
for chunk in lchunks(999, hashes):
|
||||
cmd = "SELECT checksum FROM {} WHERE checksum IN ({})".format(
|
||||
self.INDEX_TABLE, ",".join("?" for checksum in chunk)
|
||||
self.INDEX_TABLE, ",".join("?" for hash_ in chunk)
|
||||
)
|
||||
for (checksum,) in self._execute(cmd, chunk):
|
||||
yield checksum
|
||||
for (hash_,) in self._execute(cmd, chunk):
|
||||
yield hash_
|
||||
|
||||
+63
-69
@@ -56,7 +56,9 @@ class LocalRemoteTree(BaseRemoteTree):
|
||||
|
||||
@property
|
||||
def state(self):
|
||||
return self.repo.state
|
||||
from dvc.state import StateNoop
|
||||
|
||||
return self.repo.state if self.repo else StateNoop()
|
||||
|
||||
@cached_property
|
||||
def work_tree(self):
|
||||
@@ -169,7 +171,7 @@ class LocalRemoteTree(BaseRemoteTree):
|
||||
# and the cache type is `hardlink`, we might reach link limits and
|
||||
# will get something like: `too many links error`
|
||||
#
|
||||
# This is because all those empty files will have the same checksum
|
||||
# This is because all those empty files will have the same hash
|
||||
# (i.e. 68b329da9893e34099c7d8ad5cb9c940), therefore, they will be
|
||||
# linked to the same file in the cache.
|
||||
#
|
||||
@@ -268,7 +270,7 @@ class LocalRemoteTree(BaseRemoteTree):
|
||||
|
||||
return stat.S_IMODE(mode) == self.CACHE_MODE
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
def get_file_hash(self, path_info):
|
||||
return file_md5(path_info)[0]
|
||||
|
||||
@staticmethod
|
||||
@@ -313,8 +315,8 @@ class LocalRemoteTree(BaseRemoteTree):
|
||||
else:
|
||||
yield from walk_files(path_info)
|
||||
|
||||
def _remove_unpacked_dir(self, checksum):
|
||||
info = self.checksum_to_path_info(checksum)
|
||||
def _remove_unpacked_dir(self, hash_):
|
||||
info = self.hash_to_path_info(hash_)
|
||||
path_info = info.with_name(info.name + self.UNPACKED_DIR_SUFFIX)
|
||||
self.remove(path_info)
|
||||
|
||||
@@ -367,30 +369,28 @@ class LocalCache(CloudCache):
|
||||
def cache_path(self):
|
||||
return os.path.abspath(self.cache_dir)
|
||||
|
||||
def checksum_to_path(self, checksum):
|
||||
def hash_to_path(self, hash_):
|
||||
# NOTE: `self.cache_path` is already normalized so we can simply use
|
||||
# `os.sep` instead of `os.path.join`. This results in this helper
|
||||
# being ~5.5 times faster.
|
||||
return (
|
||||
f"{self.cache_path}{os.sep}{checksum[0:2]}{os.sep}{checksum[2:]}"
|
||||
)
|
||||
return f"{self.cache_path}{os.sep}{hash_[0:2]}{os.sep}{hash_[2:]}"
|
||||
|
||||
def checksums_exist(self, checksums, jobs=None, name=None):
|
||||
def hashes_exist(self, hashes, jobs=None, name=None):
|
||||
return [
|
||||
checksum
|
||||
for checksum in Tqdm(
|
||||
checksums,
|
||||
hash_
|
||||
for hash_ in Tqdm(
|
||||
hashes,
|
||||
unit="file",
|
||||
desc="Querying "
|
||||
+ ("cache in " + name if name else "local cache"),
|
||||
)
|
||||
if not self.changed_cache_file(checksum)
|
||||
if not self.changed_cache_file(hash_)
|
||||
]
|
||||
|
||||
def already_cached(self, path_info):
|
||||
assert path_info.scheme in ["", "local"]
|
||||
|
||||
current_md5 = self.get_checksum(path_info)
|
||||
current_md5 = self.get_hash(path_info)
|
||||
|
||||
if not current_md5:
|
||||
return False
|
||||
@@ -434,7 +434,7 @@ class LocalCache(CloudCache):
|
||||
|
||||
dir_status_info contains status for .dir files, file_status_info
|
||||
contains status for all other files, and dir_contents is a dict of
|
||||
{dir_checksum: set(file_checksum, ...)} which can be used to map
|
||||
{dir_hash: set(file_hash, ...)} which can be used to map
|
||||
a .dir file to its file contents.
|
||||
"""
|
||||
logger.debug(f"Preparing to collect status from {remote.path_info}")
|
||||
@@ -442,7 +442,7 @@ class LocalCache(CloudCache):
|
||||
|
||||
logger.debug("Collecting information from local cache...")
|
||||
local_exists = frozenset(
|
||||
self.checksums_exist(md5s, jobs=jobs, name=self.cache_dir)
|
||||
self.hashes_exist(md5s, jobs=jobs, name=self.cache_dir)
|
||||
)
|
||||
|
||||
# This is a performance optimization. We can safely assume that,
|
||||
@@ -457,12 +457,12 @@ class LocalCache(CloudCache):
|
||||
dir_md5s = set(named_cache.dir_keys(self.scheme))
|
||||
if dir_md5s:
|
||||
remote_exists.update(
|
||||
self._indexed_dir_checksums(named_cache, remote, dir_md5s)
|
||||
self._indexed_dir_hashes(named_cache, remote, dir_md5s)
|
||||
)
|
||||
md5s.difference_update(remote_exists)
|
||||
if md5s:
|
||||
remote_exists.update(
|
||||
remote.checksums_exist(
|
||||
remote.hashes_exist(
|
||||
md5s, jobs=jobs, name=str(remote.path_info)
|
||||
)
|
||||
)
|
||||
@@ -473,23 +473,23 @@ class LocalCache(CloudCache):
|
||||
def _make_status(
|
||||
self, named_cache, show_checksums, local_exists, remote_exists
|
||||
):
|
||||
def make_names(checksum, names):
|
||||
return {"name": checksum if show_checksums else " ".join(names)}
|
||||
def make_names(hash_, names):
|
||||
return {"name": hash_ if show_checksums else " ".join(names)}
|
||||
|
||||
dir_status = {}
|
||||
file_status = {}
|
||||
dir_contents = {}
|
||||
for checksum, item in named_cache[self.scheme].items():
|
||||
for hash_, item in named_cache[self.scheme].items():
|
||||
if item.children:
|
||||
dir_status[checksum] = make_names(checksum, item.names)
|
||||
dir_contents[checksum] = set()
|
||||
for child_checksum, child in item.children.items():
|
||||
file_status[child_checksum] = make_names(
|
||||
child_checksum, child.names
|
||||
dir_status[hash_] = make_names(hash_, item.names)
|
||||
dir_contents[hash_] = set()
|
||||
for child_hash, child in item.children.items():
|
||||
file_status[child_hash] = make_names(
|
||||
child_hash, child.names
|
||||
)
|
||||
dir_contents[checksum].add(child_checksum)
|
||||
dir_contents[hash_].add(child_hash)
|
||||
else:
|
||||
file_status[checksum] = make_names(checksum, item.names)
|
||||
file_status[hash_] = make_names(hash_, item.names)
|
||||
|
||||
self._fill_statuses(dir_status, local_exists, remote_exists)
|
||||
self._fill_statuses(file_status, local_exists, remote_exists)
|
||||
@@ -498,52 +498,50 @@ class LocalCache(CloudCache):
|
||||
|
||||
return dir_status, file_status, dir_contents
|
||||
|
||||
def _indexed_dir_checksums(self, named_cache, remote, dir_md5s):
|
||||
# Validate our index by verifying all indexed .dir checksums
|
||||
def _indexed_dir_hashes(self, named_cache, remote, dir_md5s):
|
||||
# Validate our index by verifying all indexed .dir hashes
|
||||
# still exist on the remote
|
||||
indexed_dirs = set(remote.index.dir_checksums())
|
||||
indexed_dirs = set(remote.index.dir_hashes())
|
||||
indexed_dir_exists = set()
|
||||
if indexed_dirs:
|
||||
indexed_dir_exists.update(
|
||||
remote.tree.list_checksums_exists(indexed_dirs)
|
||||
remote.tree.list_hashes_exists(indexed_dirs)
|
||||
)
|
||||
missing_dirs = indexed_dirs.difference(indexed_dir_exists)
|
||||
if missing_dirs:
|
||||
logger.debug(
|
||||
"Remote cache missing indexed .dir checksums '{}', "
|
||||
"Remote cache missing indexed .dir hashes '{}', "
|
||||
"clearing remote index".format(", ".join(missing_dirs))
|
||||
)
|
||||
remote.index.clear()
|
||||
|
||||
# Check if non-indexed (new) dir checksums exist on remote
|
||||
# Check if non-indexed (new) dir hashes exist on remote
|
||||
dir_exists = dir_md5s.intersection(indexed_dir_exists)
|
||||
dir_exists.update(
|
||||
remote.tree.list_checksums_exists(dir_md5s - dir_exists)
|
||||
remote.tree.list_hashes_exists(dir_md5s - dir_exists)
|
||||
)
|
||||
|
||||
# If .dir checksum exists on the remote, assume directory contents
|
||||
# If .dir hash exists on the remote, assume directory contents
|
||||
# still exists on the remote
|
||||
for dir_checksum in dir_exists:
|
||||
file_checksums = list(
|
||||
named_cache.child_keys(self.scheme, dir_checksum)
|
||||
)
|
||||
if dir_checksum not in remote.index:
|
||||
for dir_hash in dir_exists:
|
||||
file_hashes = list(named_cache.child_keys(self.scheme, dir_hash))
|
||||
if dir_hash not in remote.index:
|
||||
logger.debug(
|
||||
"Indexing new .dir '{}' with '{}' nested files".format(
|
||||
dir_checksum, len(file_checksums)
|
||||
dir_hash, len(file_hashes)
|
||||
)
|
||||
)
|
||||
remote.index.update([dir_checksum], file_checksums)
|
||||
yield dir_checksum
|
||||
yield from file_checksums
|
||||
remote.index.update([dir_hash], file_hashes)
|
||||
yield dir_hash
|
||||
yield from file_hashes
|
||||
|
||||
@staticmethod
|
||||
def _fill_statuses(checksum_info_dir, local_exists, remote_exists):
|
||||
def _fill_statuses(hash_info_dir, local_exists, remote_exists):
|
||||
# Using sets because they are way faster for lookups
|
||||
local = set(local_exists)
|
||||
remote = set(remote_exists)
|
||||
|
||||
for md5, info in checksum_info_dir.items():
|
||||
for md5, info in hash_info_dir.items():
|
||||
status = STATUS_MAP[(md5 in local, md5 in remote)]
|
||||
info["status"] = status
|
||||
|
||||
@@ -551,15 +549,15 @@ class LocalCache(CloudCache):
|
||||
cache = []
|
||||
path_infos = []
|
||||
names = []
|
||||
checksums = []
|
||||
hashes = []
|
||||
for md5, info in Tqdm(
|
||||
status_info.items(), desc="Analysing status", unit="file"
|
||||
):
|
||||
if info["status"] == status:
|
||||
cache.append(self.checksum_to_path_info(md5))
|
||||
path_infos.append(remote.checksum_to_path_info(md5))
|
||||
cache.append(self.hash_to_path_info(md5))
|
||||
path_infos.append(remote.hash_to_path_info(md5))
|
||||
names.append(info["name"])
|
||||
checksums.append(md5)
|
||||
hashes.append(md5)
|
||||
|
||||
if download:
|
||||
to_infos = cache
|
||||
@@ -568,7 +566,7 @@ class LocalCache(CloudCache):
|
||||
to_infos = path_infos
|
||||
from_infos = cache
|
||||
|
||||
return from_infos, to_infos, names, checksums
|
||||
return from_infos, to_infos, names, hashes
|
||||
|
||||
def _process(
|
||||
self,
|
||||
@@ -630,20 +628,18 @@ class LocalCache(CloudCache):
|
||||
# for uploads, push files first, and any .dir files last
|
||||
|
||||
file_futures = {}
|
||||
for from_info, to_info, name, checksum in zip(*file_plans):
|
||||
file_futures[checksum] = executor.submit(
|
||||
for from_info, to_info, name, hash_ in zip(*file_plans):
|
||||
file_futures[hash_] = executor.submit(
|
||||
func, from_info, to_info, name
|
||||
)
|
||||
dir_futures = {}
|
||||
for from_info, to_info, name, dir_checksum in zip(
|
||||
*dir_plans
|
||||
):
|
||||
for from_info, to_info, name, dir_hash in zip(*dir_plans):
|
||||
wait_futures = {
|
||||
future
|
||||
for file_checksum, future in file_futures.items()
|
||||
if file_checksum in dir_contents[dir_checksum]
|
||||
for file_hash, future in file_futures.items()
|
||||
if file_hash in dir_contents[dir_hash]
|
||||
}
|
||||
dir_futures[dir_checksum] = executor.submit(
|
||||
dir_futures[dir_hash] = executor.submit(
|
||||
self._dir_upload,
|
||||
func,
|
||||
wait_futures,
|
||||
@@ -666,16 +662,14 @@ class LocalCache(CloudCache):
|
||||
|
||||
if not download:
|
||||
# index successfully pushed dirs
|
||||
for dir_checksum, future in dir_futures.items():
|
||||
for dir_hash, future in dir_futures.items():
|
||||
if future.result() == 0:
|
||||
file_checksums = dir_contents[dir_checksum]
|
||||
file_hashes = dir_contents[dir_hash]
|
||||
logger.debug(
|
||||
"Indexing pushed dir '{}' with "
|
||||
"'{}' nested files".format(
|
||||
dir_checksum, len(file_checksums)
|
||||
)
|
||||
"'{}' nested files".format(dir_hash, len(file_hashes))
|
||||
)
|
||||
remote.index.update([dir_checksum], file_checksums)
|
||||
remote.index.update([dir_hash], file_hashes)
|
||||
|
||||
return len(dir_plans[0]) + len(file_plans[0])
|
||||
|
||||
@@ -714,10 +708,10 @@ class LocalCache(CloudCache):
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _log_missing_caches(checksum_info_dict):
|
||||
def _log_missing_caches(hash_info_dict):
|
||||
missing_caches = [
|
||||
(md5, info)
|
||||
for md5, info in checksum_info_dict.items()
|
||||
for md5, info in hash_info_dict.items()
|
||||
if info["status"] == STATUS_MISSING
|
||||
]
|
||||
if missing_caches:
|
||||
|
||||
+1
-1
@@ -308,7 +308,7 @@ class S3RemoteTree(BaseRemoteTree):
|
||||
if etag != cached_etag:
|
||||
raise ETagMismatchError(etag, cached_etag)
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
def get_file_hash(self, path_info):
|
||||
return self.get_etag(self.s3, path_info.bucket, path_info.path)
|
||||
|
||||
def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
|
||||
|
||||
@@ -237,7 +237,7 @@ class SSHRemoteTree(BaseRemoteTree):
|
||||
with self.ssh(from_info) as ssh:
|
||||
ssh.reflink(from_info.path, to_info.path)
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
def get_file_hash(self, path_info):
|
||||
if path_info.scheme != self.scheme:
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -314,14 +314,14 @@ class SSHRemote(Remote):
|
||||
|
||||
return results
|
||||
|
||||
def checksums_exist(self, checksums, jobs=None, name=None):
|
||||
def hashes_exist(self, hashes, jobs=None, name=None):
|
||||
"""This is older implementation used in remote/base.py
|
||||
We are reusing it in RemoteSSH, because SSH's batch_exists proved to be
|
||||
faster than current approach (relying on exists(path_info)) applied in
|
||||
remote/base.
|
||||
"""
|
||||
if not self.tree.CAN_TRAVERSE:
|
||||
return list(set(checksums) & set(self.tree.all()))
|
||||
return list(set(hashes) & set(self.tree.all()))
|
||||
|
||||
# possibly prompt for credentials before "Querying" progress output
|
||||
self.tree.ensure_credentials()
|
||||
@@ -329,7 +329,7 @@ class SSHRemote(Remote):
|
||||
with Tqdm(
|
||||
desc="Querying "
|
||||
+ ("cache in " + name if name else "remote cache"),
|
||||
total=len(checksums),
|
||||
total=len(hashes),
|
||||
unit="file",
|
||||
) as pbar:
|
||||
|
||||
@@ -339,9 +339,9 @@ class SSHRemote(Remote):
|
||||
with ThreadPoolExecutor(
|
||||
max_workers=jobs or self.tree.JOBS
|
||||
) as executor:
|
||||
path_infos = [self.checksum_to_path_info(x) for x in checksums]
|
||||
path_infos = [self.hash_to_path_info(x) for x in hashes]
|
||||
chunks = to_chunks(path_infos, num_chunks=self.tree.JOBS)
|
||||
results = executor.map(exists_with_progress, chunks)
|
||||
in_remote = itertools.chain.from_iterable(results)
|
||||
ret = list(itertools.compress(checksums, in_remote))
|
||||
ret = list(itertools.compress(hashes, in_remote))
|
||||
return ret
|
||||
|
||||
@@ -153,7 +153,7 @@ class SSHConnection:
|
||||
yield directory, dirs, nondirs
|
||||
|
||||
def walk_files(self, directory):
|
||||
for root, dirs, files in self.walk(directory):
|
||||
for root, _, files in self.walk(directory):
|
||||
for fname in files:
|
||||
yield posixpath.join(root, fname)
|
||||
|
||||
|
||||
@@ -485,7 +485,7 @@ class Repo:
|
||||
continue
|
||||
|
||||
dep_key = dep.path_info.parts
|
||||
overlapping = list(n.value for n in outs.prefixes(dep_key))
|
||||
overlapping = [n.value for n in outs.prefixes(dep_key)]
|
||||
if outs.has_subtrie(dep_key):
|
||||
overlapping.extend(outs.values(prefix=dep_key))
|
||||
|
||||
|
||||
+1
-1
@@ -37,7 +37,7 @@ def diff(self, a_rev="HEAD", b_rev=None):
|
||||
|
||||
def _to_checksum(output):
|
||||
if on_working_tree:
|
||||
return self.cache.local.get_checksum(output.path_info)
|
||||
return self.cache.local.get_hash(output.path_info)
|
||||
return output.checksum
|
||||
|
||||
def _exists(output):
|
||||
|
||||
@@ -16,4 +16,6 @@ def get_url(url, out=None):
|
||||
|
||||
(dep,) = dependency.loads_from(None, [url])
|
||||
(out,) = output.loads_from(None, [out], use_cache=False)
|
||||
dep.save()
|
||||
dep.download(out)
|
||||
out.save()
|
||||
|
||||
+6
-1
@@ -50,13 +50,18 @@ def ls(
|
||||
def _ls(repo, path_info, recursive=None, dvc_only=False):
|
||||
from dvc.repo.tree import RepoTree
|
||||
|
||||
def onerror(exc):
|
||||
raise exc
|
||||
|
||||
# use our own RepoTree instance instead of repo.repo_tree since we do not
|
||||
# want fetch/stream enabled for ls
|
||||
tree = RepoTree(repo)
|
||||
|
||||
ret = {}
|
||||
try:
|
||||
for root, dirs, files in tree.walk(path_info.fspath, dvcfiles=True):
|
||||
for root, dirs, files in tree.walk(
|
||||
path_info.fspath, onerror=onerror, dvcfiles=True
|
||||
):
|
||||
for fname in files:
|
||||
info = PathInfo(root) / fname
|
||||
dvc = tree.isdvc(info)
|
||||
|
||||
@@ -55,9 +55,8 @@ class Plots:
|
||||
try:
|
||||
with tree.open(path_info) as fd:
|
||||
data[rev][datafile]["data"] = fd.read()
|
||||
except FileNotFoundError as e:
|
||||
except FileNotFoundError:
|
||||
# This might happen simply because cache is absent
|
||||
print(e)
|
||||
pass
|
||||
|
||||
return data
|
||||
|
||||
@@ -196,7 +196,7 @@ class CSVPlotData(PlotData):
|
||||
)
|
||||
|
||||
fieldnames = reader.fieldnames
|
||||
data = [row for row in reader]
|
||||
data = list(reader)
|
||||
|
||||
return [
|
||||
OrderedDict([(field, data_point[field]) for field in fieldnames])
|
||||
|
||||
+64
-32
@@ -27,13 +27,7 @@ class Template:
|
||||
INDENT = 4
|
||||
SEPARATORS = (",", ": ")
|
||||
EXTENSION = ".json"
|
||||
ANCHOR = '"<DVC_METRIC_{}>"'
|
||||
METRIC_DATA_ANCHOR = "<DVC_METRIC_DATA>"
|
||||
X_ANCHOR = "<DVC_METRIC_X>"
|
||||
Y_ANCHOR = "<DVC_METRIC_Y>"
|
||||
TITLE_ANCHOR = "<DVC_METRIC_TITLE>"
|
||||
X_LABEL_ANCHOR = "<DVC_METRIC_X_LABEL>"
|
||||
Y_LABEL_ANCHOR = "<DVC_METRIC_Y_LABEL>"
|
||||
ANCHOR = "<DVC_METRIC_{}>"
|
||||
|
||||
def __init__(self, content=None, name=None):
|
||||
self.content = self.DEFAULT_CONTENT if content is None else content
|
||||
@@ -43,10 +37,10 @@ class Template:
|
||||
def render(self, data, props=None):
|
||||
props = props or {}
|
||||
|
||||
if self.METRIC_DATA_ANCHOR not in self.content:
|
||||
if self._anchor_str("data") not in self.content:
|
||||
anchor = self.anchor("data")
|
||||
raise BadTemplateError(
|
||||
f"Template '{self.filename}' is not using "
|
||||
f"'{self.METRIC_DATA_ANCHOR}' anchor"
|
||||
f"Template '{self.filename}' is not using '{anchor}' anchor"
|
||||
)
|
||||
|
||||
if props.get("x"):
|
||||
@@ -59,19 +53,23 @@ class Template:
|
||||
|
||||
return content
|
||||
|
||||
@classmethod
|
||||
def anchor(cls, name):
|
||||
return cls.ANCHOR.format(name.upper())
|
||||
|
||||
def has_anchor(self, name):
|
||||
return self._anchor(name) in self.content
|
||||
return self._anchor_str(name) in self.content
|
||||
|
||||
@classmethod
|
||||
def _fill_anchor(cls, content, name, value):
|
||||
value_str = json.dumps(
|
||||
value, indent=cls.INDENT, separators=cls.SEPARATORS, sort_keys=True
|
||||
)
|
||||
return content.replace(cls._anchor(name), value_str)
|
||||
return content.replace(cls._anchor_str(name), value_str)
|
||||
|
||||
@classmethod
|
||||
def _anchor(cls, name):
|
||||
return cls.ANCHOR.format(name.upper())
|
||||
def _anchor_str(cls, name):
|
||||
return '"{}"'.format(cls.anchor(name))
|
||||
|
||||
@classmethod
|
||||
def _fill_metadata(cls, content, props):
|
||||
@@ -98,19 +96,19 @@ class DefaultLinearTemplate(Template):
|
||||
|
||||
DEFAULT_CONTENT = {
|
||||
"$schema": "https://vega.github.io/schema/vega-lite/v4.json",
|
||||
"data": {"values": Template.METRIC_DATA_ANCHOR},
|
||||
"title": Template.TITLE_ANCHOR,
|
||||
"data": {"values": Template.anchor("data")},
|
||||
"title": Template.anchor("title"),
|
||||
"mark": {"type": "line"},
|
||||
"encoding": {
|
||||
"x": {
|
||||
"field": Template.X_ANCHOR,
|
||||
"field": Template.anchor("x"),
|
||||
"type": "quantitative",
|
||||
"title": Template.X_LABEL_ANCHOR,
|
||||
"title": Template.anchor("x_label"),
|
||||
},
|
||||
"y": {
|
||||
"field": Template.Y_ANCHOR,
|
||||
"field": Template.anchor("y"),
|
||||
"type": "quantitative",
|
||||
"title": Template.Y_LABEL_ANCHOR,
|
||||
"title": Template.anchor("y_label"),
|
||||
"scale": {"zero": False},
|
||||
},
|
||||
"color": {"field": "rev", "type": "nominal"},
|
||||
@@ -122,21 +120,21 @@ class DefaultConfusionTemplate(Template):
|
||||
DEFAULT_NAME = "confusion"
|
||||
DEFAULT_CONTENT = {
|
||||
"$schema": "https://vega.github.io/schema/vega-lite/v4.json",
|
||||
"data": {"values": Template.METRIC_DATA_ANCHOR},
|
||||
"title": Template.TITLE_ANCHOR,
|
||||
"data": {"values": Template.anchor("data")},
|
||||
"title": Template.anchor("title"),
|
||||
"mark": "rect",
|
||||
"encoding": {
|
||||
"x": {
|
||||
"field": Template.X_ANCHOR,
|
||||
"field": Template.anchor("x"),
|
||||
"type": "nominal",
|
||||
"sort": "ascending",
|
||||
"title": Template.X_LABEL_ANCHOR,
|
||||
"title": Template.anchor("x_label"),
|
||||
},
|
||||
"y": {
|
||||
"field": Template.Y_ANCHOR,
|
||||
"field": Template.anchor("y"),
|
||||
"type": "nominal",
|
||||
"sort": "ascending",
|
||||
"title": Template.Y_LABEL_ANCHOR,
|
||||
"title": Template.anchor("y_label"),
|
||||
},
|
||||
"color": {"aggregate": "count", "type": "quantitative"},
|
||||
"facet": {"field": "rev", "type": "nominal"},
|
||||
@@ -148,19 +146,19 @@ class DefaultScatterTemplate(Template):
|
||||
DEFAULT_NAME = "scatter"
|
||||
DEFAULT_CONTENT = {
|
||||
"$schema": "https://vega.github.io/schema/vega-lite/v4.json",
|
||||
"data": {"values": Template.METRIC_DATA_ANCHOR},
|
||||
"title": Template.TITLE_ANCHOR,
|
||||
"data": {"values": Template.anchor("data")},
|
||||
"title": Template.anchor("title"),
|
||||
"mark": "point",
|
||||
"encoding": {
|
||||
"x": {
|
||||
"field": Template.X_ANCHOR,
|
||||
"field": Template.anchor("x"),
|
||||
"type": "quantitative",
|
||||
"title": Template.X_LABEL_ANCHOR,
|
||||
"title": Template.anchor("x_label"),
|
||||
},
|
||||
"y": {
|
||||
"field": Template.Y_ANCHOR,
|
||||
"field": Template.anchor("y"),
|
||||
"type": "quantitative",
|
||||
"title": Template.Y_LABEL_ANCHOR,
|
||||
"title": Template.anchor("y_label"),
|
||||
"scale": {"zero": False},
|
||||
},
|
||||
"color": {"field": "rev", "type": "nominal"},
|
||||
@@ -168,12 +166,46 @@ class DefaultScatterTemplate(Template):
|
||||
}
|
||||
|
||||
|
||||
class SmoothLinearTemplate(Template):
|
||||
DEFAULT_NAME = "smooth"
|
||||
|
||||
DEFAULT_CONTENT = {
|
||||
"$schema": "https://vega.github.io/schema/vega-lite/v4.json",
|
||||
"data": {"values": Template.anchor("data")},
|
||||
"title": Template.anchor("title"),
|
||||
"mark": {"type": "line"},
|
||||
"encoding": {
|
||||
"x": {
|
||||
"field": Template.anchor("x"),
|
||||
"type": "quantitative",
|
||||
"title": Template.anchor("x_label"),
|
||||
},
|
||||
"y": {
|
||||
"field": Template.anchor("y"),
|
||||
"type": "quantitative",
|
||||
"title": Template.anchor("y_label"),
|
||||
"scale": {"zero": False},
|
||||
},
|
||||
"color": {"field": "rev", "type": "nominal"},
|
||||
},
|
||||
"transform": [
|
||||
{
|
||||
"loess": Template.anchor("y"),
|
||||
"on": Template.anchor("x"),
|
||||
"groupby": ["rev"],
|
||||
"bandwidth": 0.3,
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
class PlotTemplates:
|
||||
TEMPLATES_DIR = "plots"
|
||||
TEMPLATES = [
|
||||
DefaultLinearTemplate,
|
||||
DefaultConfusionTemplate,
|
||||
DefaultScatterTemplate,
|
||||
SmoothLinearTemplate,
|
||||
]
|
||||
|
||||
@cached_property
|
||||
|
||||
+35
-23
@@ -82,7 +82,7 @@ class DvcTree(BaseTree):
|
||||
else:
|
||||
checksum = out.checksum
|
||||
try:
|
||||
remote_info = remote_obj.checksum_to_path_info(checksum)
|
||||
remote_info = remote_obj.hash_to_path_info(checksum)
|
||||
return remote_obj.open(
|
||||
remote_info, mode=mode, encoding=encoding
|
||||
)
|
||||
@@ -93,7 +93,7 @@ class DvcTree(BaseTree):
|
||||
|
||||
if out.is_dir_checksum:
|
||||
checksum = self._get_granular_checksum(path, out)
|
||||
cache_path = out.cache.checksum_to_path_info(checksum).url
|
||||
cache_path = out.cache.hash_to_path_info(checksum).url
|
||||
else:
|
||||
cache_path = out.cache_path
|
||||
return open(cache_path, mode=mode, encoding=encoding)
|
||||
@@ -142,7 +142,7 @@ class DvcTree(BaseTree):
|
||||
files = []
|
||||
|
||||
root_len = len(root.parts)
|
||||
for key, out in trie.iteritems(prefix=root.parts):
|
||||
for key, out in trie.iteritems(prefix=root.parts): # noqa: B301
|
||||
if key == root.parts:
|
||||
continue
|
||||
|
||||
@@ -153,25 +153,29 @@ class DvcTree(BaseTree):
|
||||
|
||||
files.append(name)
|
||||
|
||||
if topdown:
|
||||
dirs = list(dirs)
|
||||
yield root.fspath, dirs, files
|
||||
assert topdown
|
||||
dirs = list(dirs)
|
||||
yield root.fspath, dirs, files
|
||||
|
||||
for dname in dirs:
|
||||
yield from self._walk(root / dname, trie)
|
||||
else:
|
||||
assert False
|
||||
for dname in dirs:
|
||||
yield from self._walk(root / dname, trie)
|
||||
|
||||
def walk(self, top, topdown=True, download_callback=None, **kwargs):
|
||||
def walk(
|
||||
self, top, topdown=True, onerror=None, download_callback=None, **kwargs
|
||||
):
|
||||
from pygtrie import Trie
|
||||
|
||||
assert topdown
|
||||
|
||||
if not self.exists(top):
|
||||
raise FileNotFoundError
|
||||
if onerror is not None:
|
||||
onerror(FileNotFoundError(top))
|
||||
return
|
||||
|
||||
if not self.isdir(top):
|
||||
raise NotADirectoryError
|
||||
if onerror is not None:
|
||||
onerror(NotADirectoryError(top))
|
||||
return
|
||||
|
||||
root = PathInfo(os.path.abspath(top))
|
||||
outs = self._find_outs(top, recursive=True, strict=False)
|
||||
@@ -212,7 +216,7 @@ class DvcTree(BaseTree):
|
||||
def isexec(self, path):
|
||||
return False
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
def get_file_hash(self, path_info):
|
||||
outs = self._find_outs(path_info, strict=False)
|
||||
if len(outs) != 1:
|
||||
raise OutputNotFoundError
|
||||
@@ -333,12 +337,14 @@ class RepoTree(BaseTree):
|
||||
elif dirname in repo_set:
|
||||
yield from self._walk_one(repo_walk)
|
||||
|
||||
def walk(self, top, topdown=True, dvcfiles=False, **kwargs):
|
||||
def walk(self, top, topdown=True, onerror=None, dvcfiles=False, **kwargs):
|
||||
"""Walk and merge both DVC and repo trees.
|
||||
|
||||
Args:
|
||||
top: path to walk from
|
||||
topdown: if True, tree will be walked from top down.
|
||||
onerror: if set, onerror function will be called if an error
|
||||
occurs (by default errors are ignored).
|
||||
dvcfiles: if True, dvcfiles will be included in the files list
|
||||
for walked directories.
|
||||
|
||||
@@ -348,21 +354,27 @@ class RepoTree(BaseTree):
|
||||
assert topdown
|
||||
|
||||
if not self.exists(top):
|
||||
raise FileNotFoundError
|
||||
if onerror is not None:
|
||||
onerror(FileNotFoundError(top))
|
||||
return
|
||||
|
||||
if not self.isdir(top):
|
||||
raise NotADirectoryError
|
||||
if onerror is not None:
|
||||
onerror(NotADirectoryError(top))
|
||||
return
|
||||
|
||||
dvc_exists = self.dvctree and self.dvctree.exists(top)
|
||||
repo_exists = self.repo.tree.exists(top)
|
||||
if dvc_exists and not repo_exists:
|
||||
yield from self.dvctree.walk(top, topdown=topdown, **kwargs)
|
||||
yield from self.dvctree.walk(
|
||||
top, topdown=topdown, onerror=onerror, **kwargs
|
||||
)
|
||||
return
|
||||
if repo_exists and not dvc_exists:
|
||||
yield from self.repo.tree.walk(top, topdown=topdown)
|
||||
yield from self.repo.tree.walk(
|
||||
top, topdown=topdown, onerror=onerror
|
||||
)
|
||||
return
|
||||
if not dvc_exists and not repo_exists:
|
||||
raise FileNotFoundError
|
||||
|
||||
dvc_walk = self.dvctree.walk(top, topdown=topdown, **kwargs)
|
||||
repo_walk = self.repo.tree.walk(top, topdown=topdown)
|
||||
@@ -373,7 +385,7 @@ class RepoTree(BaseTree):
|
||||
for fname in files:
|
||||
yield PathInfo(root) / fname
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
def get_file_hash(self, path_info):
|
||||
"""Return file checksum for specified path.
|
||||
|
||||
If path_info is a DVC out, the pre-computed checksum for the file
|
||||
@@ -384,7 +396,7 @@ class RepoTree(BaseTree):
|
||||
raise FileNotFoundError
|
||||
if self.dvctree and self.dvctree.exists(path_info):
|
||||
try:
|
||||
return self.dvctree.get_file_checksum(path_info)
|
||||
return self.dvctree.get_file_hash(path_info)
|
||||
except OutputNotFoundError:
|
||||
pass
|
||||
return file_md5(path_info, self)[0]
|
||||
|
||||
@@ -51,9 +51,6 @@ def _edit_rwlock(lock_dir):
|
||||
yield lock
|
||||
with open(path, "w+") as fobj:
|
||||
json.dump(lock, fobj)
|
||||
# NOTE: flush and fsync to ensure that rwlock contents are saved
|
||||
fobj.flush()
|
||||
os.fsync(fobj.fileno())
|
||||
|
||||
|
||||
def _infos_to_str(infos):
|
||||
|
||||
+16
-18
@@ -37,24 +37,22 @@ class TqdmGit(Tqdm):
|
||||
def code2desc(op_code):
|
||||
from git import RootUpdateProgress as OP
|
||||
|
||||
ops = dict(
|
||||
(
|
||||
(OP.COUNTING, "Counting"),
|
||||
(OP.COMPRESSING, "Compressing"),
|
||||
(OP.WRITING, "Writing"),
|
||||
(OP.RECEIVING, "Receiving"),
|
||||
(OP.RESOLVING, "Resolving"),
|
||||
(OP.FINDING_SOURCES, "Finding sources"),
|
||||
(OP.CHECKING_OUT, "Checking out"),
|
||||
(OP.CLONE, "Cloning"),
|
||||
(OP.FETCH, "Fetching"),
|
||||
(OP.UPDWKTREE, "Updating working tree"),
|
||||
(OP.REMOVE, "Removing"),
|
||||
(OP.PATHCHANGE, "Changing path"),
|
||||
(OP.URLCHANGE, "Changing URL"),
|
||||
(OP.BRANCHCHANGE, "Changing branch"),
|
||||
)
|
||||
)
|
||||
ops = {
|
||||
OP.COUNTING: "Counting",
|
||||
OP.COMPRESSING: "Compressing",
|
||||
OP.WRITING: "Writing",
|
||||
OP.RECEIVING: "Receiving",
|
||||
OP.RESOLVING: "Resolving",
|
||||
OP.FINDING_SOURCES: "Finding sources",
|
||||
OP.CHECKING_OUT: "Checking out",
|
||||
OP.CLONE: "Cloning",
|
||||
OP.FETCH: "Fetching",
|
||||
OP.UPDWKTREE: "Updating working tree",
|
||||
OP.REMOVE: "Removing",
|
||||
OP.PATHCHANGE: "Changing path",
|
||||
OP.URLCHANGE: "Changing URL",
|
||||
OP.BRANCHCHANGE: "Changing branch",
|
||||
}
|
||||
return ops.get(op_code & OP.OP_MASK, "")
|
||||
|
||||
|
||||
|
||||
+8
-3
@@ -128,17 +128,22 @@ class GitTree(BaseTree):
|
||||
if not topdown:
|
||||
yield os.path.normpath(tree.abspath), dirs, nondirs
|
||||
|
||||
def walk(self, top, topdown=True):
|
||||
def walk(self, top, topdown=True, onerror=None):
|
||||
"""Directory tree generator.
|
||||
|
||||
See `os.walk` for the docs. Differences:
|
||||
- no support for symlinks
|
||||
- it could raise exceptions, there is no onerror argument
|
||||
"""
|
||||
|
||||
tree = self.git_object_by_path(top)
|
||||
if tree is None:
|
||||
raise OSError(errno.ENOENT, "No such file")
|
||||
if onerror is not None:
|
||||
onerror(OSError(errno.ENOENT, "No such file", top))
|
||||
return
|
||||
if tree.mode != GIT_MODE_DIR:
|
||||
if onerror is not None:
|
||||
onerror(NotADirectoryError(top))
|
||||
return
|
||||
|
||||
yield from self._walk(tree, topdown=topdown)
|
||||
|
||||
|
||||
+4
-9
@@ -21,12 +21,11 @@ class BaseTree:
|
||||
def isfile(self, path):
|
||||
"""Test whether a path is a regular file"""
|
||||
|
||||
def walk(self, top, topdown=True):
|
||||
def walk(self, top, topdown=True, onerror=None):
|
||||
"""Directory tree generator.
|
||||
|
||||
See `os.walk` for the docs. Differences:
|
||||
- no support for symlinks
|
||||
- it could raise exceptions, there is no onerror argument
|
||||
"""
|
||||
|
||||
def walk_files(self, top):
|
||||
@@ -39,7 +38,8 @@ class BaseTree:
|
||||
class WorkingTree(BaseTree):
|
||||
"""Proxies the repo file access methods to working tree files"""
|
||||
|
||||
def __init__(self, repo_root=os.getcwd()):
|
||||
def __init__(self, repo_root=None):
|
||||
repo_root = repo_root or os.getcwd()
|
||||
self.repo_root = repo_root
|
||||
|
||||
@property
|
||||
@@ -64,17 +64,12 @@ class WorkingTree(BaseTree):
|
||||
"""Test whether a path is a regular file"""
|
||||
return os.path.isfile(path)
|
||||
|
||||
def walk(self, top, topdown=True):
|
||||
def walk(self, top, topdown=True, onerror=None):
|
||||
"""Directory tree generator.
|
||||
|
||||
See `os.walk` for the docs. Differences:
|
||||
- no support for symlinks
|
||||
- it could raise exceptions, there is no onerror argument
|
||||
"""
|
||||
|
||||
def onerror(e):
|
||||
raise e
|
||||
|
||||
for root, dirs, files in os.walk(
|
||||
top, topdown=topdown, onerror=onerror
|
||||
):
|
||||
|
||||
+10
-3
@@ -3,7 +3,7 @@ import os
|
||||
import string
|
||||
from collections import defaultdict
|
||||
|
||||
from funcy import project
|
||||
from funcy import cached_property, project
|
||||
|
||||
import dvc.dependency as dependency
|
||||
import dvc.prompt as prompt
|
||||
@@ -127,6 +127,8 @@ class Stage(params.StageParams):
|
||||
@path.setter
|
||||
def path(self, path):
|
||||
self._path = path
|
||||
self.__dict__.pop("path_in_repo", None)
|
||||
self.__dict__.pop("relpath", None)
|
||||
|
||||
@property
|
||||
def dvcfile(self):
|
||||
@@ -172,11 +174,11 @@ class Stage(params.StageParams):
|
||||
and self.path_in_repo == other.path_in_repo
|
||||
)
|
||||
|
||||
@property
|
||||
@cached_property
|
||||
def path_in_repo(self):
|
||||
return relpath(self.path, self.repo.root_dir)
|
||||
|
||||
@property
|
||||
@cached_property
|
||||
def relpath(self):
|
||||
return relpath(self.path)
|
||||
|
||||
@@ -284,12 +286,17 @@ class Stage(params.StageParams):
|
||||
for out in self.outs:
|
||||
out.unprotect()
|
||||
|
||||
def ignore_remove_outs(self):
|
||||
for out in self.outs:
|
||||
out.ignore_remove()
|
||||
|
||||
@rwlocked(write=["outs"])
|
||||
def remove(self, force=False, remove_outs=True, purge=True):
|
||||
if remove_outs:
|
||||
self.remove_outs(ignore_remove=True, force=force)
|
||||
else:
|
||||
self.unprotect_outs()
|
||||
self.ignore_remove_outs()
|
||||
if purge:
|
||||
self.dvcfile.remove_stage(self)
|
||||
|
||||
|
||||
@@ -26,4 +26,5 @@ def sync_import(stage, dry=False, force=False):
|
||||
if not force and stage.already_cached():
|
||||
stage.outs[0].checkout()
|
||||
else:
|
||||
stage.save_deps()
|
||||
stage.deps[0].download(stage.outs[0])
|
||||
|
||||
@@ -111,7 +111,7 @@ def _serialize_params_values(params: List[ParamsDependency]):
|
||||
def to_pipeline_file(stage: "PipelineStage"):
|
||||
wdir = resolve_wdir(stage.wdir, stage.path)
|
||||
params, deps = split_params_deps(stage)
|
||||
deps = sorted([d.def_path for d in deps])
|
||||
deps = sorted(d.def_path for d in deps)
|
||||
params = _serialize_params_keys(params)
|
||||
|
||||
outs, metrics, plots = _serialize_outs(stage.outs)
|
||||
|
||||
+1
-1
@@ -6,7 +6,7 @@
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
_BASE_VERSION = "1.0.0b2"
|
||||
_BASE_VERSION = "1.0.0b6"
|
||||
|
||||
|
||||
def _generate_version(base_version):
|
||||
|
||||
@@ -115,10 +115,10 @@ build_dvc() {
|
||||
# [1] https://github.com/iterative/dvc/issues/2585
|
||||
if [[ "$(uname)" == 'Linux' ]]; then
|
||||
mkdir -p $BUILD_DIR/$BASH_CMPLT_DIR
|
||||
cp scripts/completion/dvc.bash $BUILD_DIR/$BASH_CMPLT_DIR/dvc
|
||||
$LIB_DIR/dvc/dvc completion -s bash > $BUILD_DIR/$BASH_CMPLT_DIR/dvc
|
||||
|
||||
mkdir -p $BUILD_DIR/$ZSH_CMPLT_DIR
|
||||
cp scripts/completion/dvc.zsh $BUILD_DIR/$ZSH_CMPLT_DIR
|
||||
$LIB_DIR/dvc/dvc completion -s zsh > $BUILD_DIR/$ZSH_CMPLT_DIR/_dvc
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
@@ -1,146 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# References:
|
||||
# - https://www.gnu.org/software/bash/manual/html_node/Programmable-Completion.html
|
||||
# - https://opensource.com/article/18/3/creating-bash-completion-script
|
||||
# - https://stackoverflow.com/questions/12933362
|
||||
|
||||
_dvc_commands='add cache checkout commit config dag destroy diff fetch freeze \
|
||||
get-url get gc import-url import init install list metrics move params \
|
||||
plots pull push remote remove repro root run status unfreeze unprotect \
|
||||
update version'
|
||||
|
||||
_dvc_options='-h --help -V --version'
|
||||
_dvc_global_options='-h --help -q --quiet -v --verbose'
|
||||
|
||||
_dvc_add='-R --recursive -f --file --no-commit --external'
|
||||
_dvc_add_COMPGEN=_dvc_compgen_files
|
||||
_dvc_cache='dir'
|
||||
_dvc_cache_dir=' --global --system --local -u --unset'
|
||||
_dvc_checkout='-d --with-deps -R --recursive -f --force --relink --summary'
|
||||
_dvc_checkout_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_commit='-f --force -d --with-deps -R --recursive'
|
||||
_dvc_commit_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_config='-u --unset --local --system --global'
|
||||
_dvc_dag='--dot --full'
|
||||
_dvc_dag_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_destroy='-f --force'
|
||||
_dvc_diff='-t --show-json --show-hash --show-md'
|
||||
_dvc_fetch='-j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -R --recursive'
|
||||
_dvc_fetch_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_gc='-a --all-branches --all-commits -T --all-tags -w --workspace -c --cloud -r --remote -f --force -p --projects -j --jobs'
|
||||
_dvc_get='-o --out --rev --show-url'
|
||||
_dvc_get_url=''
|
||||
_dvc_import='-o --out --rev'
|
||||
_dvc_import_url='-f --file'
|
||||
_dvc_init='--no-scm -f --force'
|
||||
_dvc_install=''
|
||||
_dvc_list='-R --recursive --dvc-only --rev'
|
||||
_dvc_list_COMPGEN=_dvc_compgen_files
|
||||
_dvc_freeze=''
|
||||
_dvc_freeze_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_metrics='diff show'
|
||||
_dvc_metrics_diff='--targets -t --type -x --xpath -R --show-json --show-md --no-path --old'
|
||||
_dvc_metrics_show='-t --type -x --xpath -a --all-branches -T --all-tags -R --recursive'
|
||||
_dvc_metrics_show_COMPGEN=_dvc_compgen_files
|
||||
_dvc_move=''
|
||||
_dvc_move_COMPGEN=_dvc_compgen_files
|
||||
_dvc_params='diff'
|
||||
_dvc_params_diff='--all --show-json --show-md --no-path'
|
||||
_dvc_plots='show diff'
|
||||
_dvc_plots_show='-t --template -o --out -x -y --show-vega --no-csv-header --title --x-label --y-label'
|
||||
_dvc_plots_diff='-t --template --targets -o --out -x -y --show-vega --no-csv-header --title --x-label --y-label'
|
||||
_dvc_plots_modify='-t --template -x -y --no-csv-header --title --x-label --y-label'
|
||||
_dvc_pull='-j --jobs -r --remote -a --all-branches -T --all-tags -f --force -d --with-deps -R --recursive'
|
||||
_dvc_pull_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_push='-j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -R --recursive'
|
||||
_dvc_push_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_remote='add default list modify remove rename'
|
||||
_dvc_remote_add='--global --system --local -d --default -f --force'
|
||||
_dvc_remote_default='--global --system --local -u --unset'
|
||||
_dvc_remote_list='--global --system --local'
|
||||
_dvc_remote_modify='--global --system --local -u --unset'
|
||||
_dvc_remote_remove='--global --system --local'
|
||||
_dvc_remote_rename='--global --system --local'
|
||||
_dvc_remove='-o --outs -p --purge -f --force'
|
||||
_dvc_remove_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_repro='-f --force -s --single-item -c --cwd -m --metrics --dry -i --interactive -p --pipeline -P --all-pipelines --no-run-cache --force-downstream --no-commit -R --recursive --downstream'
|
||||
_dvc_repro_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_root=''
|
||||
_dvc_run='--no-exec -f --file -d --deps -o --outs -O --outs-no-cache --outs-persist --outs-persist-no-cache -m --metrics -M --metrics-no-cache --overwrite-dvcfile --no-run-cache --no-commit -w --wdir --external'
|
||||
_dvc_run_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_status='-j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -c --cloud'
|
||||
_dvc_status_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_unfreeze_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_unprotect_COMPGEN=_dvc_compgen_files
|
||||
_dvc_update='--rev'
|
||||
_dvc_update_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_version=''
|
||||
|
||||
# $1=COMP_WORDS[1]
|
||||
_dvc_compgen_DVCFiles() {
|
||||
compgen -f -X '!*?.dvc' -- $1
|
||||
compgen -d -S '/' -- $1 # recurse into subdirs
|
||||
# Note that the recurse into dirs is only for looking for DVC-files.
|
||||
# Since dirs themselves are not required, we need `-o nospace` at the bottom
|
||||
# unfortunately :(
|
||||
}
|
||||
|
||||
# $1=COMP_WORDS[1]
|
||||
_dvc_compgen_files() {
|
||||
compgen -f -- $1
|
||||
compgen -d -S '/' -- $1 # recurse into subdirs
|
||||
}
|
||||
|
||||
# $1=COMP_WORDS[1]
|
||||
_dvc_replace_hyphen() {
|
||||
echo $1 | sed 's/-/_/g'
|
||||
}
|
||||
|
||||
# $1=COMP_WORDS[1]
|
||||
_dvc_compgen_command() {
|
||||
local flags_list="_dvc_$(_dvc_replace_hyphen $1)"
|
||||
local args_gen="${flags_list}_COMPGEN"
|
||||
COMPREPLY=( $(compgen -W "$_dvc_global_options ${!flags_list}" -- "$word"; [ -n "${!args_gen}" ] && ${!args_gen} "$word") )
|
||||
}
|
||||
|
||||
# $1=COMP_WORDS[1]
|
||||
# $2=COMP_WORDS[2]
|
||||
_dvc_compgen_subcommand() {
|
||||
local flags_list="_dvc_$(_dvc_replace_hyphen $1)_$(_dvc_replace_hyphen $2)"
|
||||
local args_gen="${flags_list}_COMPGEN"
|
||||
[ -n "${!args_gen}" ] && local opts_more="$(${!args_gen} "$word")"
|
||||
local opts="${!flags_list}"
|
||||
if [ -z "$opts$opts_more" ]; then
|
||||
_dvc_compgen_command $1
|
||||
else
|
||||
COMPREPLY=( $(compgen -W "$_dvc_global_options $opts" -- "$word"; [ -n "$opts_more" ] && echo "$opts_more") )
|
||||
fi
|
||||
}
|
||||
|
||||
# Notes:
|
||||
# `COMPREPLY` contains what will be rendered after completion is triggered
|
||||
# `word` refers to the current typed word
|
||||
# `${!var}` is to evaluate the content of `var` and expand its content as a variable
|
||||
# hello="world"
|
||||
# x="hello"
|
||||
# ${!x} -> ${hello} -> "world"
|
||||
_dvc() {
|
||||
local word="${COMP_WORDS[COMP_CWORD]}"
|
||||
|
||||
COMPREPLY=()
|
||||
|
||||
if [ "${COMP_CWORD}" -eq 1 ]; then
|
||||
case "$word" in
|
||||
-*) COMPREPLY=($(compgen -W "$_dvc_options" -- "$word")) ;;
|
||||
*) COMPREPLY=($(compgen -W "$_dvc_commands" -- "$word")) ;;
|
||||
esac
|
||||
elif [ "${COMP_CWORD}" -eq 2 ]; then
|
||||
_dvc_compgen_command ${COMP_WORDS[1]}
|
||||
elif [ "${COMP_CWORD}" -ge 3 ]; then
|
||||
_dvc_compgen_subcommand ${COMP_WORDS[1]} ${COMP_WORDS[2]}
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
complete -o nospace -F _dvc dvc
|
||||
@@ -1,340 +0,0 @@
|
||||
#compdef dvc
|
||||
|
||||
#----------------------------------------------------------
|
||||
# Repository: https://github.com/iterative/dvc
|
||||
#
|
||||
# References:
|
||||
# - https://github.com/zsh-users/zsh-completions
|
||||
# - http://zsh.sourceforge.net/Doc/Release/Completion-System.html
|
||||
# - https://mads-hartmann.com/2017/08/06/writing-zsh-completion-scripts.html
|
||||
# - http://www.linux-mag.com/id/1106/
|
||||
#----------------------------------------------------------
|
||||
|
||||
_dvc_commands() {
|
||||
local _commands=(
|
||||
"add:Track data files or directories with DVC."
|
||||
"cache:Manage cache settings."
|
||||
"checkout:Checkout data files from cache."
|
||||
"commit:Save changed data to cache and update DVC-files."
|
||||
"config:Get or set config settings."
|
||||
"dag:Visualize DVC project DAG."
|
||||
"destroy:Remove DVC-files, local DVC config and data cache."
|
||||
"diff:Show added, modified, or deleted data between commits in the DVC repository, or between a commit and the workspace."
|
||||
"fetch:Get files or directories tracked by DVC from remote storage into the cache."
|
||||
"get-url:Download or copy files from URL."
|
||||
"get:Download data from DVC repository."
|
||||
"gc:Garbage collect unused objects from cache or remote storage."
|
||||
"import-url:Download or copy file from URL and take it under DVC control."
|
||||
"import:Download data from DVC repository and take it under DVC control."
|
||||
"init:Initialize DVC in the current directory."
|
||||
"install:Install DVC git hooks into the repository."
|
||||
"list:List repository contents, including files and directories tracked by DVC and by Git."
|
||||
"lock:Lock DVC-file."
|
||||
"metrics:Commands to display and compare metrics."
|
||||
"move:Rename or move a DVC controlled data file or a directory."
|
||||
"params:Commands to display params."
|
||||
"pull:Pull data files from a DVC remote storage."
|
||||
"push:Push data files to a DVC remote storage."
|
||||
"plots:Generate plot for metrics structured as JSON, CSV or TSV."
|
||||
"remote:Manage remote storage configuration."
|
||||
"remove:Remove outputs of DVC-file."
|
||||
"repro:Check for changes and reproduce DVC-file and dependencies."
|
||||
"root:Relative path to project's directory."
|
||||
"run:Generate a stage file from a command and execute the command."
|
||||
"status:Show changed stages, compare local cache and a remote storage."
|
||||
"unlock:Unlock DVC-file."
|
||||
"unprotect:Unprotect data file/directory."
|
||||
"update:Update data artifacts imported from other DVC repositories."
|
||||
"version:Show DVC version and system/environment information."
|
||||
)
|
||||
|
||||
_describe 'dvc commands' _commands
|
||||
}
|
||||
|
||||
_dvc_options=(
|
||||
"(-)"{-h,--help}"[Show help message.]"
|
||||
"(-)"{-V,--version}"[Show program's version]"
|
||||
)
|
||||
|
||||
_dvc_global_options=(
|
||||
"(-)"{-h,--help}"[Show help message related to the command.]"
|
||||
"(-)"{-q,--quiet}"[Be quiet.]"
|
||||
"(-)"{-v,--verbose}"[Be verbose.]"
|
||||
)
|
||||
|
||||
_dvc_add=(
|
||||
{-R,--recursive}"[Recursively add each file under the directory.]"
|
||||
"--no-commit[Don't put files/directories into cache.]"
|
||||
{-f,--file}"[Specify name of the DVC-file it generates.]:File:_files"
|
||||
"--external[Allow targets that are outside of the DVC project.]"
|
||||
"1:File:_files"
|
||||
)
|
||||
|
||||
_dvc_cache=(
|
||||
"1:Sub command:(dir)"
|
||||
)
|
||||
|
||||
_dvc_checkout=(
|
||||
{-d,--with-deps}"[Checkout all dependencies of the specified target.]"
|
||||
{-R,--recursive}"[Checkout all subdirectories of the specified directory.]"
|
||||
{-f,--force}"[Do not prompt when removing working directory files.]"
|
||||
"--relink[Recreate links or copies from cache to workspace.]"
|
||||
"--summary[Show summary of the changes.]"
|
||||
"1:Stages:_files -g '(*.dvc|Dvcfile)'"
|
||||
)
|
||||
|
||||
_dvc_commit=(
|
||||
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
|
||||
{-f,--force}"[Commit even if checksums for dependencies/outputs changed.]"
|
||||
{-d,--with-deps}"[Commit all dependencies of the specified target.]"
|
||||
{-R,--recursive}"[Commit cache for subdirectories of the specified directory.]"
|
||||
)
|
||||
|
||||
_dvc_config=(
|
||||
"--global[Use global config.]"
|
||||
"--system[Use system config.]"
|
||||
"--local[Use local config.]"
|
||||
{-u,--unset}"[Unset option.]"
|
||||
)
|
||||
|
||||
_dvc_dag=(
|
||||
"--dot[Print DAG in DOT format.]"
|
||||
"--full[Show full DAG that the target belongs too, instead of showing DAG consisting only of ancestors.]"
|
||||
"1:Stage:"
|
||||
)
|
||||
|
||||
_dvc_destroy=(
|
||||
{-f,--force}"[Force destruction.]"
|
||||
)
|
||||
|
||||
_dvc_diff=(
|
||||
"--show-json[Format the output into a JSON]"
|
||||
"--show-hash[Display hash value for each entry]"
|
||||
"--show-md[Format the output into a Markdown table]"
|
||||
"1:Old Git commit to compare (defaults to HEAD):"
|
||||
"2:New Git commit to compare (defaults to the current workspace):"
|
||||
)
|
||||
|
||||
_dvc_fetch=(
|
||||
{-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:"
|
||||
{-r,--remote}"[Remote repository to fetch from.]:Remote repository:"
|
||||
{-a,--all-branches}"[Fetch cache for all branches.]"
|
||||
{-T,--all-tags}"[Fetch cache for all tags.]"
|
||||
{-d,--with-deps}"[Fetch cache for all dependencies of the specified target.]"
|
||||
{-R,--recursive}"[Fetch cache for subdirectories of specified directory.]"
|
||||
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
|
||||
)
|
||||
|
||||
_dvc_geturl=(
|
||||
"1:URL:"
|
||||
"2:Output:"
|
||||
)
|
||||
|
||||
_dvc_get=(
|
||||
{-o,--out}"[Destination path to put data to.]:OUT:_files -/"
|
||||
"--rev[Git revision (e.g. SHA, branch, tag)]:Revision:"
|
||||
"--show-url[Returns path/url to the location in remote for given path]"
|
||||
"1:URL:"
|
||||
"2:Path:"
|
||||
)
|
||||
|
||||
_dvc_gc=(
|
||||
{-w,--workspace}"[Keep data files used in the current workspace.]"
|
||||
{-a,--all-branches}"[Keep data files for the tips of all Git branches.]"
|
||||
"--all-commits[Keep data files for all Git commits.]"
|
||||
{-T,--all-tags}"[Keep data files for all Git tags.]"
|
||||
{-c,--cloud}"[Collect garbage in remote repository.]"
|
||||
{-r,--remote}"[Remote storage to collect garbage in.]:Remote repository:"
|
||||
{-f,--force}"[Force garbage collection - automatically agree to all prompts.]:Repos:_files"
|
||||
{-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:"
|
||||
{-p,--projects}"[Keep data files required by these projects in addition to the current one.]:Repos:_files"
|
||||
)
|
||||
|
||||
_dvc_importurl=(
|
||||
{-f,--file}"[Specify name of the DVC-file it generates.]:File:_files"
|
||||
"1:URL:"
|
||||
"2:Output:"
|
||||
)
|
||||
|
||||
_dvc_import=(
|
||||
{-o,--out}"[Destination path to put data to.]:OUT:_files -/"
|
||||
"--rev[Git revision (e.g. SHA, branch, tag)]:Commit hash:"
|
||||
"1:URL:"
|
||||
"2:Path:"
|
||||
)
|
||||
|
||||
_dvc_init=(
|
||||
"--no-scm[Initiate dvc in directory that is not tracked by any scm tool.]"
|
||||
{-f,--force}"[Overwrite existing '.dvc' directory. This operation removes local cache.]"
|
||||
)
|
||||
|
||||
_dvc_install=()
|
||||
|
||||
_dvc_freeze=(
|
||||
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
|
||||
)
|
||||
|
||||
_dvc_list=(
|
||||
"--rev[Git revision (e.g. branch, tag, SHA)]:Revision:"
|
||||
{-R,--recursive}"[Recursively add each file under the directory.]"
|
||||
"--dvc-only[Only outputs DVC-outs.]"
|
||||
"1:URL:"
|
||||
"2:Path:"
|
||||
)
|
||||
|
||||
_dvc_metrics=(
|
||||
"1:Sub command:(show diff)"
|
||||
)
|
||||
|
||||
_dvc_move=(
|
||||
"1:Source:_files"
|
||||
"2:Destination:"
|
||||
)
|
||||
|
||||
_dvc_params=(
|
||||
"1:Sub command:(diff)"
|
||||
)
|
||||
|
||||
_dvc_pull=(
|
||||
{-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:"
|
||||
{-r,--remote}"[Remote repository to pull from.]:Remote repository:"
|
||||
{-a,--all-branches}"[Fetch cache for all branches.]"
|
||||
{-T,--all-tags}"[Fetch cache for all tags.]"
|
||||
{-d,--with-deps}"[Fetch cache for all dependencies of the specified target.]"
|
||||
{-f,--force}"[Do not prompt when removing working directory files.]"
|
||||
{-R,--recursive}"[Pull cache for subdirectories of the specified directory.]"
|
||||
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
|
||||
)
|
||||
|
||||
_dvc_push=(
|
||||
{-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:"
|
||||
{-r,--remote}"[Remote repository to push to.]:Remote repository:"
|
||||
{-a,--all-branches}"[Push cache for all branches.]"
|
||||
{-T,--all-tags}"[Push cache for all tags.]"
|
||||
{-d,--with-deps}"[Push cache for all dependencies of the specified target.]"
|
||||
{-R,--recursive}"[Push cache for subdirectories of specified directory.]"
|
||||
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
|
||||
)
|
||||
|
||||
_dvc_plots=(
|
||||
"1:Sub command:(show diff modify)"
|
||||
)
|
||||
|
||||
_dvc_remote=(
|
||||
"1:Sub command:(add default remove modify list rename)"
|
||||
)
|
||||
|
||||
_dvc_remove=(
|
||||
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
|
||||
"--dry[Only print the commands that would be executed without actually executing]"
|
||||
{-o,--outs}"[Only remove DVC-file outputs. (Default)]"
|
||||
{-p,--purge}"[Remove DVC-file and all its outputs.]"
|
||||
{-f,--force}"[Force purge.]"
|
||||
)
|
||||
|
||||
_dvc_repro=(
|
||||
{-f,--force}"[Reproduce even if dependencies were not changed.]"
|
||||
{-s,--single-item}"[Reproduce only single data item without recursive dependencies check.]"
|
||||
{-c,--cwd}"[Directory within your repo to reproduce from.]:CWD:_files -/"
|
||||
{-m,--metrics}"[Show metrics after reproduction.]"
|
||||
"--dry[Only print the commands that would be executed without actually executing]"
|
||||
{-i,--interactive}"[Ask for confirmation before reproducing each stage.]"
|
||||
{-p,--pipeline}"[Reproduce the whole pipeline that the specified stage file belongs to.]"
|
||||
{-P,--all-pipelines}"[Reproduce all pipelines in the repo.]"
|
||||
{-R,--recursive}"[Reproduce all stages in the specified directory.]"
|
||||
"--force-downstream[Reproduce all descendants of a changed stage even if their direct dependencies didn't change.]"
|
||||
"--no-run-cache[Run changed stage even if it has been already ran with the same command/dependencies/outputs/etc before.]"
|
||||
"--no-commit[Don't put files/directories into cache.]"
|
||||
"--downstream[Start from the specified stages when reproducing pipelines.]"
|
||||
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
|
||||
)
|
||||
|
||||
_dvc_root=()
|
||||
|
||||
_dvc_run=(
|
||||
"*"{-d,--deps}"[Declare dependencies for reproducible cmd.]:Dependency:_files"
|
||||
"*"{-o,--outs}"[Declare output file or directory.]:Output data:_files"
|
||||
"*"{-O,--outs-no-cache}"[Declare output file or directory (do not put into DVC cache).]:Output regular:_files"
|
||||
"*"{-m,--metrics}"[Declare output metric file or directory.]:Metrics:_files"
|
||||
"*"{-M,--metrics-no-cache}"[Declare output metric file or directory (do not put into DVC cache).]:Metrics (no cache):_files"
|
||||
{-f,--file}"[Specify name of the DVC-file it generates.]:File:_files"
|
||||
{-c,--cwd}"[Deprecated, use -w and -f instead.]:CWD:_files -/"
|
||||
{-w,--wdir}"[Directory within your repo to run your command in.]:WDIR:_files -/"
|
||||
"--no-exec[Only create stage file without actually running it.]"
|
||||
{-y,--yes}"[Deprecated, use --overwrite-dvcfile instead]"
|
||||
"--overwrite-dvcfile[Overwrite existing DVC-file without asking for confirmation.]"
|
||||
"--no-run-cache[Run this stage even if it has been already ran with the same command/dependencies/outputs/etc before.]"
|
||||
"--remove-outs[Deprecated, this is now the default behavior]"
|
||||
"--no-commit[Don't put files/directories into cache.]"
|
||||
"--outs-persist[Declare output file or directory that will not be removed upon repro.]:Output persistent:_files"
|
||||
"--outs-persist-no-cache[Declare output file or directory that will not be removed upon repro (do not put into DVC cache).]:Output persistent regular:_files"
|
||||
"--external[Allow outputs that are outside of the DVC project.]"
|
||||
)
|
||||
|
||||
_dvc_status=(
|
||||
{-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:"
|
||||
{-q,--quiet}"[Suppresses all output. Exit with 0 if pipelines are up to date, otherwise 1.]"
|
||||
{-c,--cloud}"[Show status of a local cache compared to a remote repository.]"
|
||||
{-r,--remote}"[Remote repository to compare local cache to.]:Remote repository:"
|
||||
{-a,--all-branches}"[Show status of a local cache compared to a remote repository for all branches.]"
|
||||
{-T,--all-tags}"[Show status of a local cache compared to a remote repository for all tags.]"
|
||||
{-d,--with-deps}"[Show status for all dependencies of the specified target.]"
|
||||
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
|
||||
)
|
||||
|
||||
_dvc_unfreeze=(
|
||||
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
|
||||
)
|
||||
|
||||
_dvc_unprotect=(
|
||||
"*:Data files:_files"
|
||||
)
|
||||
|
||||
_dvc_update=(
|
||||
"--rev[Git revision (e.g. SHA, branch, tag)]:Revision:"
|
||||
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
|
||||
)
|
||||
|
||||
typeset -A opt_args
|
||||
local context state line curcontext="$curcontext"
|
||||
|
||||
_arguments \
|
||||
$_dvc_options \
|
||||
'1: :_dvc_commands' \
|
||||
'*::args:->args'
|
||||
|
||||
case $words[1] in
|
||||
add) _arguments $_dvc_global_options $_dvc_add ;;
|
||||
cache) _arguments $_dvc_global_options $_dvc_cache ;;
|
||||
checkout) _arguments $_dvc_global_options $_dvc_checkout ;;
|
||||
commit) _arguments $_dvc_global_options $_dvc_commit ;;
|
||||
config) _arguments $_dvc_global_options $_dvc_config ;;
|
||||
dag) _arguments $_dvc_global_options $_dvc_dag ;;
|
||||
destroy) _arguments $_dvc_global_options $_dvc_destroy ;;
|
||||
diff) _arguments $_dvc_global_options $_dvc_diff ;;
|
||||
fetch) _arguments $_dvc_global_options $_dvc_fetch ;;
|
||||
freeze) _arguments $_dvc_global_options $_dvc_freeze ;;
|
||||
get-url) _arguments $_dvc_global_options $_dvc_geturl ;;
|
||||
get) _arguments $_dvc_global_options $_dvc_get ;;
|
||||
gc) _arguments $_dvc_global_options $_dvc_gc ;;
|
||||
import-url) _arguments $_dvc_global_options $_dvc_importurl ;;
|
||||
import) _arguments $_dvc_global_options $_dvc_import ;;
|
||||
init) _arguments $_dvc_global_options $_dvc_init ;;
|
||||
install) _arguments $_dvc_global_options $_dvc_install ;;
|
||||
list) _arguments $_dvc_global_options $_dvc_list ;;
|
||||
metrics) _arguments $_dvc_global_options $_dvc_metrics ;;
|
||||
move) _arguments $_dvc_global_options $_dvc_move ;;
|
||||
params) _arguments $_dvc_global_options $_dvc_params ;;
|
||||
pull) _arguments $_dvc_global_options $_dvc_pull ;;
|
||||
push) _arguments $_dvc_global_options $_dvc_push ;;
|
||||
plots) _arguments $_dvc_global_options $_dvc_plots ;;
|
||||
remote) _arguments $_dvc_global_options $_dvc_remote ;;
|
||||
remove) _arguments $_dvc_global_options $_dvc_remove ;;
|
||||
repro) _arguments $_dvc_global_options $_dvc_repro ;;
|
||||
root) _arguments $_dvc_global_options $_dvc_root ;;
|
||||
run) _arguments $_dvc_global_options $_dvc_run ;;
|
||||
status) _arguments $_dvc_global_options $_dvc_status ;;
|
||||
unfreeze) _arguments $_dvc_global_options $_dvc_unfreeze ;;
|
||||
unprotect) _arguments $_dvc_global_options $_dvc_unprotect ;;
|
||||
update) _arguments $_dvc_global_options $_dvc_update ;;
|
||||
esac
|
||||
@@ -1,6 +1,3 @@
|
||||
md5: 3cc286c534a71504476da009ed174423
|
||||
outs:
|
||||
- cache: true
|
||||
md5: 90104d9e83cfb825cf45507e90aadd27
|
||||
metric: false
|
||||
- md5: 90104d9e83cfb825cf45507e90aadd27
|
||||
path: dvc.ico
|
||||
|
||||
@@ -1,6 +1,3 @@
|
||||
md5: c7c2944695a3eb7f04a0a7ae20c9dc0c
|
||||
outs:
|
||||
- cache: true
|
||||
md5: 9106cda08aa427e73492389a0f17c72d
|
||||
metric: false
|
||||
- md5: 9106cda08aa427e73492389a0f17c72d
|
||||
path: dvc_left.bmp
|
||||
|
||||
@@ -1,6 +1,3 @@
|
||||
md5: cde0fe37927a5771b1ba8d1c909e120c
|
||||
outs:
|
||||
- cache: true
|
||||
md5: 94614d6650e062655f9f77507dc9c1f2
|
||||
metric: false
|
||||
- md5: 94614d6650e062655f9f77507dc9c1f2
|
||||
path: dvc_up.bmp
|
||||
|
||||
@@ -6,13 +6,18 @@ ignore=
|
||||
E203, # Whitespace before ':'
|
||||
E266, # Too many leading '#' for block comment
|
||||
W503, # Line break occurred before a binary operator
|
||||
P1, # unindexed parameters in the str.format, see:
|
||||
# https://pypi.org/project/flake8-string-format/
|
||||
max_line_length=79
|
||||
select=B,C,E,F,W,T4,B9
|
||||
max-complexity = 15
|
||||
select=B,C,E,F,W,T4,B902,T,P
|
||||
show_source=true
|
||||
count=true
|
||||
|
||||
[isort]
|
||||
include_trailing_comma=true
|
||||
known_first_party=dvc,tests
|
||||
known_third_party=PyInstaller,RangeHTTPServer,boto3,colorama,configobj,distro,dpath,flaky,flufl,funcy,git,google,grandalf,mock,mockssh,moto,nanotime,networkx,packaging,paramiko,pathspec,pytest,requests,ruamel,setuptools,shortuuid,tqdm,voluptuous,yaml,zc
|
||||
known_third_party=PyInstaller,RangeHTTPServer,boto3,colorama,configobj,distro,dpath,flaky,flufl,funcy,git,google,grandalf,mock,moto,nanotime,networkx,packaging,paramiko,pathspec,pytest,requests,ruamel,setuptools,shortuuid,shtab,tqdm,voluptuous,yaml,zc
|
||||
line_length=79
|
||||
force_grid_wrap=0
|
||||
use_parentheses=True
|
||||
|
||||
@@ -78,6 +78,7 @@ install_requires = [
|
||||
"tabulate>=0.8.7",
|
||||
"pygtrie==2.3.2",
|
||||
"dpath>=2.0.1,<3",
|
||||
"shtab>=0.0.2",
|
||||
]
|
||||
|
||||
|
||||
@@ -106,6 +107,7 @@ tests_requirements = [
|
||||
"pytest-cov>=2.6.1",
|
||||
"pytest-xdist>=1.26.1",
|
||||
"pytest-mock==1.11.2",
|
||||
"pytest-lazy-fixture",
|
||||
"flaky>=3.5.3",
|
||||
"mock>=3.0.0",
|
||||
"xmltodict>=0.11.0",
|
||||
@@ -122,6 +124,9 @@ tests_requirements = [
|
||||
"moto==1.3.14.dev464",
|
||||
"rangehttpserver==1.2.0",
|
||||
"beautifulsoup4==4.4.0",
|
||||
"flake8-bugbear",
|
||||
"flake8-comprehensions",
|
||||
"flake8-string-format",
|
||||
]
|
||||
|
||||
if (sys.version_info) >= (3, 6):
|
||||
|
||||
+8
-4
@@ -31,17 +31,21 @@ parts:
|
||||
snapcraftctl pull
|
||||
snapcraftctl set-version $(cd $SNAPCRAFT_PART_SRC && git describe --tags)
|
||||
git diff --quiet || error_dirty_build
|
||||
echo 'PKG = "snap"' > $SNAPCRAFT_PART_SRC/dvc/utils/build.py
|
||||
echo 'PKG = "snap"' > dvc/utils/build.py
|
||||
# install all optional extras
|
||||
sed -ri 's/(=install_requires)/\1+all_remotes+hdfs/' $SNAPCRAFT_PART_SRC/setup.py
|
||||
sed -ri 's/(=install_requires)/\1+all_remotes+hdfs/' setup.py
|
||||
# ensure dvc knows the state isn't really dirty
|
||||
sed -rin 's/.*git.*diff.*--quiet.*//' $SNAPCRAFT_PART_SRC/dvc/version.py
|
||||
sed -rin 's/.*git.*diff.*--quiet.*//' dvc/version.py
|
||||
override-build: |
|
||||
snapcraftctl build
|
||||
# prevent user site packages interfering with this snap - reference:
|
||||
# https://github.com/snapcore/snapcraft/blob/19393ef36cd773a28131cec10cc0bfb3bf9c7e77/tools/snapcraft-override-build.sh#L18
|
||||
sed -ri 's/^(ENABLE_USER_SITE = )None$/\1False/' $SNAPCRAFT_PART_INSTALL/usr/lib/python*/site.py
|
||||
cp $SNAPCRAFT_PART_BUILD/scripts/completion/dvc.bash $SNAPCRAFT_PART_INSTALL/completion.sh
|
||||
# make plugin aware that a file (generated later) needs to be staged
|
||||
touch $SNAPCRAFT_PART_INSTALL/completion.sh
|
||||
override-stage: |
|
||||
snapcraftctl stage
|
||||
usr/bin/python3 -m dvc completion -s bash > completion.sh
|
||||
apps:
|
||||
dvc:
|
||||
command: bin/dvc
|
||||
|
||||
+1
-34
@@ -1,12 +1,9 @@
|
||||
import os
|
||||
|
||||
import mockssh
|
||||
import pytest
|
||||
|
||||
from dvc.remote.ssh.connection import SSHConnection
|
||||
from tests.utils.httpd import PushRequestHandler, StaticFileServer
|
||||
|
||||
from .dir_helpers import * # noqa
|
||||
from .remotes import * # noqa
|
||||
|
||||
# Prevent updater and analytics from running their processes
|
||||
os.environ["DVC_TEST"] = "true"
|
||||
@@ -28,39 +25,9 @@ def reset_loglevel(request, caplog):
|
||||
yield
|
||||
|
||||
|
||||
here = os.path.abspath(os.path.dirname(__file__))
|
||||
|
||||
user = "user"
|
||||
key_path = os.path.join(here, f"{user}.key")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def ssh_server():
|
||||
users = {user: key_path}
|
||||
with mockssh.Server(users) as s:
|
||||
s.test_creds = {
|
||||
"host": s.host,
|
||||
"port": s.port,
|
||||
"username": user,
|
||||
"key_filename": key_path,
|
||||
}
|
||||
yield s
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def ssh(ssh_server):
|
||||
yield SSHConnection(**ssh_server.test_creds)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
def _close_pools():
|
||||
from dvc.remote.pool import close_pools
|
||||
|
||||
yield
|
||||
close_pools()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def http_server(tmp_dir):
|
||||
with StaticFileServer(handler_class=PushRequestHandler) as httpd:
|
||||
yield httpd
|
||||
|
||||
+21
-18
@@ -63,7 +63,6 @@ __all__ = [
|
||||
"run_head",
|
||||
"erepo_dir",
|
||||
"git_dir",
|
||||
"setup_remote",
|
||||
"git_init",
|
||||
]
|
||||
|
||||
@@ -177,6 +176,27 @@ class TmpDir(pathlib.Path):
|
||||
if commit:
|
||||
self.scm.commit(commit)
|
||||
|
||||
def add_remote(
|
||||
self, *, url=None, config=None, name="upstream", default=True
|
||||
):
|
||||
self._require("dvc")
|
||||
|
||||
assert bool(url) ^ bool(config)
|
||||
|
||||
if url:
|
||||
config = {"url": url}
|
||||
|
||||
with self.dvc.config.edit() as conf:
|
||||
conf["remote"][name] = config
|
||||
if default:
|
||||
conf["core"]["remote"] = name
|
||||
|
||||
if hasattr(self, "scm"):
|
||||
self.scm.add(self.dvc.config.files["repo"])
|
||||
self.scm.commit(f"add '{name}' remote")
|
||||
|
||||
return url or config["url"]
|
||||
|
||||
# contexts
|
||||
@contextmanager
|
||||
def chdir(self):
|
||||
@@ -315,20 +335,3 @@ def git_dir(make_tmp_dir):
|
||||
path = make_tmp_dir("git-erepo", scm=True)
|
||||
path.scm.commit("init repo")
|
||||
return path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def setup_remote(make_tmp_dir):
|
||||
def create(repo, url=None, name="upstream", default=True):
|
||||
if not url:
|
||||
url = os.fspath(make_tmp_dir("local_remote"))
|
||||
with repo.config.edit() as conf:
|
||||
conf["remote"][name] = {"url": url}
|
||||
if default:
|
||||
conf["core"]["remote"] = name
|
||||
|
||||
repo.scm.add(repo.config.files["repo"])
|
||||
repo.scm.commit(f"add '{name}' remote")
|
||||
return url
|
||||
|
||||
return create
|
||||
|
||||
@@ -15,12 +15,12 @@ def remote(tmp_dir, dvc, tmp_path_factory, mocker):
|
||||
dvc.config["remote"]["upstream"] = {"url": url}
|
||||
dvc.config["core"]["remote"] = "upstream"
|
||||
|
||||
# patch checksums_exist since the LocalRemote normally overrides
|
||||
# BaseRemoteTree.checksums_exist.
|
||||
def checksums_exist(self, *args, **kwargs):
|
||||
return Remote.checksums_exist(self, *args, **kwargs)
|
||||
# patch hashes_exist since the LocalRemote normally overrides
|
||||
# BaseRemoteTree.hashes_exist.
|
||||
def hashes_exist(self, *args, **kwargs):
|
||||
return Remote.hashes_exist(self, *args, **kwargs)
|
||||
|
||||
mocker.patch.object(LocalRemote, "checksums_exist", checksums_exist)
|
||||
mocker.patch.object(LocalRemote, "hashes_exist", hashes_exist)
|
||||
|
||||
# patch index class since LocalRemote normally overrides index class
|
||||
mocker.patch.object(LocalRemote, "INDEX_CLS", RemoteIndex)
|
||||
@@ -38,9 +38,9 @@ def test_indexed_on_status(tmp_dir, dvc, tmp_path_factory, remote):
|
||||
|
||||
dvc.status(cloud=True)
|
||||
with remote.index:
|
||||
assert {bar.checksum, baz["md5"]} == set(remote.index.checksums())
|
||||
assert [bar.checksum] == list(remote.index.dir_checksums())
|
||||
assert foo.checksum not in remote.index.checksums()
|
||||
assert {bar.checksum, baz["md5"]} == set(remote.index.hashes())
|
||||
assert [bar.checksum] == list(remote.index.dir_hashes())
|
||||
assert foo.checksum not in remote.index.hashes()
|
||||
|
||||
|
||||
def test_indexed_on_push(tmp_dir, dvc, tmp_path_factory, remote):
|
||||
@@ -50,9 +50,9 @@ def test_indexed_on_push(tmp_dir, dvc, tmp_path_factory, remote):
|
||||
|
||||
dvc.push()
|
||||
with remote.index:
|
||||
assert {bar.checksum, baz["md5"]} == set(remote.index.checksums())
|
||||
assert [bar.checksum] == list(remote.index.dir_checksums())
|
||||
assert foo.checksum not in remote.index.checksums()
|
||||
assert {bar.checksum, baz["md5"]} == set(remote.index.hashes())
|
||||
assert [bar.checksum] == list(remote.index.dir_hashes())
|
||||
assert foo.checksum not in remote.index.hashes()
|
||||
|
||||
|
||||
def test_indexed_dir_missing(tmp_dir, dvc, tmp_path_factory, remote):
|
||||
@@ -61,7 +61,7 @@ def test_indexed_dir_missing(tmp_dir, dvc, tmp_path_factory, remote):
|
||||
remote.index.update([bar.checksum], [])
|
||||
dvc.status(cloud=True)
|
||||
with remote.index:
|
||||
assert not list(remote.index.checksums())
|
||||
assert not list(remote.index.hashes())
|
||||
|
||||
|
||||
def test_clear_on_gc(tmp_dir, dvc, tmp_path_factory, remote, mocker):
|
||||
@@ -101,4 +101,4 @@ def test_partial_upload(tmp_dir, dvc, tmp_path_factory, remote, mocker):
|
||||
with pytest.raises(UploadError):
|
||||
dvc.push()
|
||||
with remote.index:
|
||||
assert not list(remote.index.checksums())
|
||||
assert not list(remote.index.hashes())
|
||||
|
||||
@@ -340,7 +340,7 @@ class TestAddCommit(TestDvc):
|
||||
|
||||
def test_should_collect_dir_cache_only_once(mocker, tmp_dir, dvc):
|
||||
tmp_dir.gen({"data/data": "foo"})
|
||||
get_dir_checksum_counter = mocker.spy(LocalRemoteTree, "get_dir_checksum")
|
||||
get_dir_hash_counter = mocker.spy(LocalRemoteTree, "get_dir_hash")
|
||||
ret = main(["add", "data"])
|
||||
assert ret == 0
|
||||
|
||||
@@ -349,7 +349,7 @@ def test_should_collect_dir_cache_only_once(mocker, tmp_dir, dvc):
|
||||
|
||||
ret = main(["status"])
|
||||
assert ret == 0
|
||||
assert get_dir_checksum_counter.mock.call_count == 1
|
||||
assert get_dir_hash_counter.mock.call_count == 1
|
||||
|
||||
|
||||
class SymlinkAddTestBase(TestDvc):
|
||||
|
||||
+32
-90
@@ -5,93 +5,42 @@ import pytest
|
||||
from dvc import api
|
||||
from dvc.api import UrlNotDvcRepoError
|
||||
from dvc.exceptions import FileMissingError
|
||||
from dvc.main import main
|
||||
from dvc.path_info import URLInfo
|
||||
from dvc.utils.fs import remove
|
||||
from tests.remotes import (
|
||||
GCP,
|
||||
HDFS,
|
||||
OSS,
|
||||
S3,
|
||||
SSH,
|
||||
TEST_REMOTE,
|
||||
Azure,
|
||||
GDrive,
|
||||
Local,
|
||||
)
|
||||
|
||||
remote_params = [S3, GCP, Azure, GDrive, OSS, SSH, HDFS]
|
||||
all_remote_params = [Local] + remote_params
|
||||
cloud_names = [
|
||||
"s3",
|
||||
"gs",
|
||||
"azure",
|
||||
"gdrive",
|
||||
"oss",
|
||||
"ssh",
|
||||
"hdfs",
|
||||
"http",
|
||||
]
|
||||
clouds = [pytest.lazy_fixture(cloud) for cloud in cloud_names]
|
||||
all_clouds = [pytest.lazy_fixture("local")] + clouds
|
||||
remotes = [pytest.lazy_fixture(f"{cloud}_remote") for cloud in cloud_names]
|
||||
all_remotes = [pytest.lazy_fixture("local_remote")] + remotes
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def remote_url(request):
|
||||
if not request.param.should_test():
|
||||
raise pytest.skip()
|
||||
return request.param.get_url()
|
||||
|
||||
|
||||
def run_dvc(*argv):
|
||||
assert main(argv) == 0
|
||||
|
||||
|
||||
def ensure_dir(dvc, url):
|
||||
if url.startswith("gdrive://"):
|
||||
GDrive.create_dir(dvc, url)
|
||||
run_dvc(
|
||||
"remote",
|
||||
"modify",
|
||||
TEST_REMOTE,
|
||||
"gdrive_service_account_email",
|
||||
"test",
|
||||
)
|
||||
run_dvc(
|
||||
"remote",
|
||||
"modify",
|
||||
TEST_REMOTE,
|
||||
"gdrive_service_account_p12_file_path",
|
||||
"test.p12",
|
||||
)
|
||||
run_dvc(
|
||||
"remote",
|
||||
"modify",
|
||||
TEST_REMOTE,
|
||||
"gdrive_use_service_account",
|
||||
"True",
|
||||
)
|
||||
|
||||
|
||||
def ensure_dir_scm(dvc, url):
|
||||
if url.startswith("gdrive://"):
|
||||
GDrive.create_dir(dvc, url)
|
||||
with dvc.config.edit() as conf:
|
||||
conf["remote"][TEST_REMOTE].update(
|
||||
gdrive_service_account_email="test",
|
||||
gdrive_service_account_p12_file_path="test.p12",
|
||||
gdrive_use_service_account=True,
|
||||
)
|
||||
dvc.scm.add(dvc.config.files["repo"])
|
||||
dvc.scm.commit(f"modify '{TEST_REMOTE}' remote")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("remote_url", remote_params, indirect=True)
|
||||
def test_get_url(tmp_dir, dvc, remote_url):
|
||||
run_dvc("remote", "add", "-d", TEST_REMOTE, remote_url)
|
||||
@pytest.mark.parametrize("remote", remotes)
|
||||
def test_get_url(tmp_dir, dvc, request, remote):
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
|
||||
expected_url = URLInfo(remote_url) / "ac/bd18db4cc2f85cedef654fccc4a4d8"
|
||||
expected_url = URLInfo(remote.url) / "ac/bd18db4cc2f85cedef654fccc4a4d8"
|
||||
assert api.get_url("foo") == expected_url
|
||||
|
||||
|
||||
@pytest.mark.parametrize("remote_url", remote_params, indirect=True)
|
||||
def test_get_url_external(erepo_dir, remote_url, setup_remote):
|
||||
setup_remote(erepo_dir.dvc, url=remote_url)
|
||||
@pytest.mark.parametrize("cloud", clouds)
|
||||
def test_get_url_external(erepo_dir, cloud):
|
||||
erepo_dir.add_remote(config=cloud.config)
|
||||
with erepo_dir.chdir():
|
||||
erepo_dir.dvc_gen("foo", "foo", commit="add foo")
|
||||
|
||||
# Using file url to force clone to tmp repo
|
||||
repo_url = f"file://{erepo_dir}"
|
||||
expected_url = URLInfo(remote_url) / "ac/bd18db4cc2f85cedef654fccc4a4d8"
|
||||
expected_url = URLInfo(cloud.url) / "ac/bd18db4cc2f85cedef654fccc4a4d8"
|
||||
assert api.get_url("foo", repo=repo_url) == expected_url
|
||||
|
||||
|
||||
@@ -105,12 +54,10 @@ def test_get_url_requires_dvc(tmp_dir, scm):
|
||||
api.get_url("foo", repo=f"file://{tmp_dir}")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("remote_url", all_remote_params, indirect=True)
|
||||
def test_open(remote_url, tmp_dir, dvc):
|
||||
run_dvc("remote", "add", "-d", TEST_REMOTE, remote_url)
|
||||
ensure_dir(dvc, remote_url)
|
||||
@pytest.mark.parametrize("remote", all_remotes)
|
||||
def test_open(tmp_dir, dvc, remote):
|
||||
tmp_dir.dvc_gen("foo", "foo-text")
|
||||
run_dvc("push")
|
||||
dvc.push()
|
||||
|
||||
# Remove cache to force download
|
||||
remove(dvc.cache.local.cache_dir)
|
||||
@@ -119,10 +66,9 @@ def test_open(remote_url, tmp_dir, dvc):
|
||||
assert fd.read() == "foo-text"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("remote_url", all_remote_params, indirect=True)
|
||||
def test_open_external(remote_url, erepo_dir, setup_remote):
|
||||
setup_remote(erepo_dir.dvc, url=remote_url)
|
||||
ensure_dir_scm(erepo_dir.dvc, remote_url)
|
||||
@pytest.mark.parametrize("cloud", clouds)
|
||||
def test_open_external(erepo_dir, cloud):
|
||||
erepo_dir.add_remote(config=cloud.config)
|
||||
|
||||
with erepo_dir.chdir():
|
||||
erepo_dir.dvc_gen("version", "master", commit="add version")
|
||||
@@ -144,12 +90,10 @@ def test_open_external(remote_url, erepo_dir, setup_remote):
|
||||
assert api.read("version", repo=repo_url, rev="branch") == "branchver"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("remote_url", all_remote_params, indirect=True)
|
||||
def test_open_granular(remote_url, tmp_dir, dvc):
|
||||
run_dvc("remote", "add", "-d", TEST_REMOTE, remote_url)
|
||||
ensure_dir(dvc, remote_url)
|
||||
@pytest.mark.parametrize("remote", all_remotes)
|
||||
def test_open_granular(tmp_dir, dvc, remote):
|
||||
tmp_dir.dvc_gen({"dir": {"foo": "foo-text"}})
|
||||
run_dvc("push")
|
||||
dvc.push()
|
||||
|
||||
# Remove cache to force download
|
||||
remove(dvc.cache.local.cache_dir)
|
||||
@@ -158,11 +102,9 @@ def test_open_granular(remote_url, tmp_dir, dvc):
|
||||
assert fd.read() == "foo-text"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("remote_url", all_remote_params, indirect=True)
|
||||
def test_missing(remote_url, tmp_dir, dvc):
|
||||
@pytest.mark.parametrize("remote", all_remotes)
|
||||
def test_missing(tmp_dir, dvc, remote):
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
run_dvc("remote", "add", "-d", TEST_REMOTE, remote_url)
|
||||
ensure_dir(dvc, remote_url)
|
||||
|
||||
# Remove cache to make foo missing
|
||||
remove(dvc.cache.local.cache_dir)
|
||||
|
||||
@@ -36,7 +36,7 @@ class TestCache(TestDvc):
|
||||
self.assertIn(self.cache2_md5, md5_list)
|
||||
|
||||
def test_get(self):
|
||||
cache = Cache(self.dvc).local.checksum_to_path_info(self.cache1_md5)
|
||||
cache = Cache(self.dvc).local.hash_to_path_info(self.cache1_md5)
|
||||
self.assertEqual(os.fspath(cache), self.cache1)
|
||||
|
||||
|
||||
@@ -46,16 +46,16 @@ class TestCacheLoadBadDirCache(TestDvc):
|
||||
self.assertEqual(len(ret), 0)
|
||||
|
||||
def test(self):
|
||||
checksum = "123.dir"
|
||||
fname = os.fspath(self.dvc.cache.local.checksum_to_path_info(checksum))
|
||||
dir_hash = "123.dir"
|
||||
fname = os.fspath(self.dvc.cache.local.hash_to_path_info(dir_hash))
|
||||
self.create(fname, "<clearly>not,json")
|
||||
with pytest.raises(DirCacheError):
|
||||
self.dvc.cache.local.load_dir_cache(checksum)
|
||||
self.dvc.cache.local.load_dir_cache(dir_hash)
|
||||
|
||||
checksum = "234.dir"
|
||||
fname = os.fspath(self.dvc.cache.local.checksum_to_path_info(checksum))
|
||||
dir_hash = "234.dir"
|
||||
fname = os.fspath(self.dvc.cache.local.hash_to_path_info(dir_hash))
|
||||
self.create(fname, '{"a": "b"}')
|
||||
self._do_test(self.dvc.cache.local.load_dir_cache(checksum))
|
||||
self._do_test(self.dvc.cache.local.load_dir_cache(dir_hash))
|
||||
|
||||
|
||||
class TestExternalCacheDir(TestDvc):
|
||||
|
||||
@@ -100,8 +100,8 @@ class TestCheckoutCorruptedCacheDir(TestDvc):
|
||||
# NOTE: modifying cache file for one of the files inside the directory
|
||||
# to check if dvc will detect that the cache is corrupted.
|
||||
entry = self.dvc.cache.local.load_dir_cache(out.checksum)[0]
|
||||
checksum = entry[self.dvc.cache.local.tree.PARAM_CHECKSUM]
|
||||
cache = os.fspath(self.dvc.cache.local.checksum_to_path_info(checksum))
|
||||
entry_hash = entry[self.dvc.cache.local.tree.PARAM_CHECKSUM]
|
||||
cache = os.fspath(self.dvc.cache.local.hash_to_path_info(entry_hash))
|
||||
|
||||
os.chmod(cache, 0o644)
|
||||
with open(cache, "w+") as fobj:
|
||||
|
||||
+148
-586
@@ -1,497 +1,163 @@
|
||||
import copy
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import uuid
|
||||
from unittest import SkipTest
|
||||
|
||||
import pytest
|
||||
from flaky.flaky_decorator import flaky
|
||||
|
||||
from dvc.cache import NamedCache
|
||||
from dvc.data_cloud import DataCloud
|
||||
from dvc.external_repo import clean_repos
|
||||
from dvc.main import main
|
||||
from dvc.remote.azure import AzureRemoteTree
|
||||
from dvc.remote.base import STATUS_DELETED, STATUS_NEW, STATUS_OK
|
||||
from dvc.remote.gdrive import GDriveRemoteTree
|
||||
from dvc.remote.gs import GSRemoteTree
|
||||
from dvc.remote.hdfs import HDFSRemoteTree
|
||||
from dvc.remote.http import HTTPRemoteTree
|
||||
from dvc.remote.local import LocalRemoteTree
|
||||
from dvc.remote.oss import OSSRemoteTree
|
||||
from dvc.remote.s3 import S3RemoteTree
|
||||
from dvc.remote.ssh import SSHRemoteTree
|
||||
from dvc.stage.exceptions import StageNotFound
|
||||
from dvc.utils import file_md5
|
||||
from dvc.utils.fs import remove
|
||||
from dvc.utils.yaml import dump_yaml, load_yaml
|
||||
from tests.basic_env import TestDvc
|
||||
from tests.remotes import (
|
||||
GCP,
|
||||
HDFS,
|
||||
HTTP,
|
||||
OSS,
|
||||
S3,
|
||||
TEST_CONFIG,
|
||||
TEST_GCP_CREDS_FILE,
|
||||
TEST_REMOTE,
|
||||
Azure,
|
||||
GDrive,
|
||||
Local,
|
||||
SSHMocked,
|
||||
)
|
||||
|
||||
from .test_api import all_remotes
|
||||
|
||||
|
||||
class TestDataCloud(TestDvc):
|
||||
def _test_cloud(self, config, cl):
|
||||
self.dvc.config = config
|
||||
cloud = DataCloud(self.dvc)
|
||||
self.assertIsInstance(cloud.get_remote().tree, cl)
|
||||
@pytest.mark.parametrize("remote", all_remotes)
|
||||
def test_cloud(tmp_dir, dvc, remote):
|
||||
(stage,) = tmp_dir.dvc_gen("foo", "foo")
|
||||
out = stage.outs[0]
|
||||
cache = out.cache_path
|
||||
md5 = out.checksum
|
||||
info = out.get_used_cache()
|
||||
|
||||
def test(self):
|
||||
config = copy.deepcopy(TEST_CONFIG)
|
||||
|
||||
clist = [
|
||||
("s3://mybucket/", S3RemoteTree),
|
||||
("gs://mybucket/", GSRemoteTree),
|
||||
("ssh://user@localhost:/", SSHRemoteTree),
|
||||
("http://localhost:8000/", HTTPRemoteTree),
|
||||
("azure://ContainerName=mybucket;conn_string;", AzureRemoteTree),
|
||||
("oss://mybucket/", OSSRemoteTree),
|
||||
(TestDvc.mkdtemp(), LocalRemoteTree),
|
||||
]
|
||||
|
||||
for scheme, cl in clist:
|
||||
remote_url = scheme + str(uuid.uuid4())
|
||||
config["remote"][TEST_REMOTE]["url"] = remote_url
|
||||
self._test_cloud(config, cl)
|
||||
|
||||
|
||||
class TestDataCloudBase(TestDvc):
|
||||
def _get_cloud_class(self):
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def should_test():
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def get_url():
|
||||
return NotImplementedError
|
||||
|
||||
def _get_keyfile(self):
|
||||
return None
|
||||
|
||||
def _ensure_should_run(self):
|
||||
if not self.should_test():
|
||||
raise SkipTest(f"Test {self.__class__.__name__} is disabled")
|
||||
|
||||
def _setup_cloud(self):
|
||||
self._ensure_should_run()
|
||||
|
||||
repo = self.get_url()
|
||||
keyfile = self._get_keyfile()
|
||||
|
||||
config = copy.deepcopy(TEST_CONFIG)
|
||||
config["remote"][TEST_REMOTE] = {"url": repo, "keyfile": keyfile}
|
||||
self.dvc.config = config
|
||||
self.cloud = DataCloud(self.dvc)
|
||||
|
||||
self.assertIsInstance(
|
||||
self.cloud.get_remote().tree, self._get_cloud_class()
|
||||
)
|
||||
|
||||
def _test_cloud(self):
|
||||
self._setup_cloud()
|
||||
|
||||
stages = self.dvc.add(self.FOO)
|
||||
self.assertEqual(len(stages), 1)
|
||||
stage = stages[0]
|
||||
self.assertTrue(stage is not None)
|
||||
out = stage.outs[0]
|
||||
cache = out.cache_path
|
||||
md5 = out.checksum
|
||||
info = out.get_used_cache()
|
||||
|
||||
stages = self.dvc.add(self.DATA_DIR)
|
||||
self.assertEqual(len(stages), 1)
|
||||
stage_dir = stages[0]
|
||||
self.assertTrue(stage_dir is not None)
|
||||
out_dir = stage_dir.outs[0]
|
||||
cache_dir = out_dir.cache_path
|
||||
name_dir = str(out_dir)
|
||||
md5_dir = out_dir.checksum
|
||||
info_dir = NamedCache.make(out_dir.scheme, md5_dir, name_dir)
|
||||
|
||||
with self.cloud.repo.state:
|
||||
# Check status
|
||||
status = self.cloud.status(info, show_checksums=True)
|
||||
expected = {md5: {"name": md5, "status": STATUS_NEW}}
|
||||
self.assertEqual(status, expected)
|
||||
|
||||
status_dir = self.cloud.status(info_dir, show_checksums=True)
|
||||
expected = {md5_dir: {"name": md5_dir, "status": STATUS_NEW}}
|
||||
self.assertEqual(status_dir, expected)
|
||||
|
||||
# Push and check status
|
||||
self.cloud.push(info)
|
||||
self.assertTrue(os.path.exists(cache))
|
||||
self.assertTrue(os.path.isfile(cache))
|
||||
|
||||
self.cloud.push(info_dir)
|
||||
self.assertTrue(os.path.isfile(cache_dir))
|
||||
|
||||
status = self.cloud.status(info, show_checksums=True)
|
||||
expected = {md5: {"name": md5, "status": STATUS_OK}}
|
||||
self.assertEqual(status, expected)
|
||||
|
||||
status_dir = self.cloud.status(info_dir, show_checksums=True)
|
||||
expected = {md5_dir: {"name": md5_dir, "status": STATUS_OK}}
|
||||
self.assertEqual(status_dir, expected)
|
||||
|
||||
# Remove and check status
|
||||
remove(self.dvc.cache.local.cache_dir)
|
||||
|
||||
status = self.cloud.status(info, show_checksums=True)
|
||||
expected = {md5: {"name": md5, "status": STATUS_DELETED}}
|
||||
self.assertEqual(status, expected)
|
||||
|
||||
status_dir = self.cloud.status(info_dir, show_checksums=True)
|
||||
expected = {md5_dir: {"name": md5_dir, "status": STATUS_DELETED}}
|
||||
self.assertEqual(status_dir, expected)
|
||||
|
||||
# Pull and check status
|
||||
self.cloud.pull(info)
|
||||
self.assertTrue(os.path.exists(cache))
|
||||
self.assertTrue(os.path.isfile(cache))
|
||||
with open(cache) as fd:
|
||||
self.assertEqual(fd.read(), self.FOO_CONTENTS)
|
||||
|
||||
self.cloud.pull(info_dir)
|
||||
self.assertTrue(os.path.isfile(cache_dir))
|
||||
|
||||
status = self.cloud.status(info, show_checksums=True)
|
||||
expected = {md5: {"name": md5, "status": STATUS_OK}}
|
||||
self.assertEqual(status, expected)
|
||||
|
||||
status_dir = self.cloud.status(info_dir, show_checksums=True)
|
||||
expected = {md5_dir: {"name": md5_dir, "status": STATUS_OK}}
|
||||
self.assertTrue(status_dir, expected)
|
||||
|
||||
def test(self):
|
||||
self._ensure_should_run()
|
||||
self._test_cloud()
|
||||
|
||||
|
||||
class TestS3Remote(S3, TestDataCloudBase):
|
||||
def _get_cloud_class(self):
|
||||
return S3RemoteTree
|
||||
|
||||
|
||||
class TestGDriveRemote(GDrive, TestDataCloudBase):
|
||||
def _setup_cloud(self):
|
||||
self._ensure_should_run()
|
||||
|
||||
url = self.get_url()
|
||||
self.create_dir(self.dvc, url)
|
||||
|
||||
config = copy.deepcopy(TEST_CONFIG)
|
||||
config["remote"][TEST_REMOTE] = {
|
||||
"url": url,
|
||||
"gdrive_service_account_email": "test",
|
||||
"gdrive_service_account_p12_file_path": "test.p12",
|
||||
"gdrive_use_service_account": True,
|
||||
(stage_dir,) = tmp_dir.dvc_gen(
|
||||
{
|
||||
"data_dir": {
|
||||
"data_sub_dir": {"data_sub": "data_sub"},
|
||||
"data": "data",
|
||||
}
|
||||
}
|
||||
)
|
||||
out_dir = stage_dir.outs[0]
|
||||
cache_dir = out_dir.cache_path
|
||||
name_dir = str(out_dir)
|
||||
md5_dir = out_dir.checksum
|
||||
info_dir = NamedCache.make(out_dir.scheme, md5_dir, name_dir)
|
||||
|
||||
self.dvc.config = config
|
||||
self.cloud = DataCloud(self.dvc)
|
||||
remote = self.cloud.get_remote()
|
||||
self.assertIsInstance(remote.tree, self._get_cloud_class())
|
||||
with dvc.state:
|
||||
# Check status
|
||||
status = dvc.cloud.status(info, show_checksums=True)
|
||||
expected = {md5: {"name": md5, "status": STATUS_NEW}}
|
||||
assert status == expected
|
||||
|
||||
def _get_cloud_class(self):
|
||||
return GDriveRemoteTree
|
||||
status_dir = dvc.cloud.status(info_dir, show_checksums=True)
|
||||
expected = {md5_dir: {"name": md5_dir, "status": STATUS_NEW}}
|
||||
assert status_dir == expected
|
||||
|
||||
# Push and check status
|
||||
dvc.cloud.push(info)
|
||||
assert os.path.exists(cache)
|
||||
assert os.path.isfile(cache)
|
||||
|
||||
class TestGSRemote(GCP, TestDataCloudBase):
|
||||
def _setup_cloud(self):
|
||||
self._ensure_should_run()
|
||||
dvc.cloud.push(info_dir)
|
||||
assert os.path.isfile(cache_dir)
|
||||
|
||||
repo = self.get_url()
|
||||
status = dvc.cloud.status(info, show_checksums=True)
|
||||
expected = {md5: {"name": md5, "status": STATUS_OK}}
|
||||
assert status == expected
|
||||
|
||||
config = copy.deepcopy(TEST_CONFIG)
|
||||
config["remote"][TEST_REMOTE] = {
|
||||
"url": repo,
|
||||
"credentialpath": TEST_GCP_CREDS_FILE,
|
||||
}
|
||||
self.dvc.config = config
|
||||
self.cloud = DataCloud(self.dvc)
|
||||
status_dir = dvc.cloud.status(info_dir, show_checksums=True)
|
||||
expected = {md5_dir: {"name": md5_dir, "status": STATUS_OK}}
|
||||
assert status_dir == expected
|
||||
|
||||
self.assertIsInstance(
|
||||
self.cloud.get_remote().tree, self._get_cloud_class()
|
||||
)
|
||||
# Remove and check status
|
||||
remove(dvc.cache.local.cache_dir)
|
||||
|
||||
def _get_cloud_class(self):
|
||||
return GSRemoteTree
|
||||
status = dvc.cloud.status(info, show_checksums=True)
|
||||
expected = {md5: {"name": md5, "status": STATUS_DELETED}}
|
||||
assert status == expected
|
||||
|
||||
status_dir = dvc.cloud.status(info_dir, show_checksums=True)
|
||||
expected = {md5_dir: {"name": md5_dir, "status": STATUS_DELETED}}
|
||||
assert status_dir == expected
|
||||
|
||||
class TestAzureRemote(Azure, TestDataCloudBase):
|
||||
def _get_cloud_class(self):
|
||||
return AzureRemoteTree
|
||||
|
||||
|
||||
class TestOSSRemote(OSS, TestDataCloudBase):
|
||||
def _get_cloud_class(self):
|
||||
return OSSRemoteTree
|
||||
|
||||
|
||||
class TestLocalRemote(Local, TestDataCloudBase):
|
||||
def _get_cloud_class(self):
|
||||
return LocalRemoteTree
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("ssh_server")
|
||||
class TestSSHRemoteMocked(SSHMocked, TestDataCloudBase):
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup_method_fixture(self, request, ssh_server):
|
||||
self.ssh_server = ssh_server
|
||||
self.method_name = request.function.__name__
|
||||
|
||||
def _setup_cloud(self):
|
||||
self._ensure_should_run()
|
||||
|
||||
repo = self.get_url()
|
||||
keyfile = self._get_keyfile()
|
||||
|
||||
self._get_cloud_class().CAN_TRAVERSE = False
|
||||
config = copy.deepcopy(TEST_CONFIG)
|
||||
config["remote"][TEST_REMOTE] = {
|
||||
"url": repo,
|
||||
"keyfile": keyfile,
|
||||
}
|
||||
self.dvc.config = config
|
||||
self.cloud = DataCloud(self.dvc)
|
||||
|
||||
self.assertIsInstance(
|
||||
self.cloud.get_remote().tree, self._get_cloud_class()
|
||||
)
|
||||
|
||||
def get_url(self):
|
||||
user = self.ssh_server.test_creds["username"]
|
||||
return super().get_url(user, self.ssh_server.port)
|
||||
|
||||
def _get_keyfile(self):
|
||||
return self.ssh_server.test_creds["key_filename"]
|
||||
|
||||
def _get_cloud_class(self):
|
||||
return SSHRemoteTree
|
||||
|
||||
|
||||
class TestHDFSRemote(HDFS, TestDataCloudBase):
|
||||
def _get_cloud_class(self):
|
||||
return HDFSRemoteTree
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("http_server")
|
||||
class TestHTTPRemote(HTTP, TestDataCloudBase):
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup_method_fixture(self, request, http_server):
|
||||
self.http_server = http_server
|
||||
self.method_name = request.function.__name__
|
||||
|
||||
def get_url(self):
|
||||
return super().get_url(self.http_server.server_port)
|
||||
|
||||
def _get_cloud_class(self):
|
||||
return HTTPRemoteTree
|
||||
|
||||
|
||||
class TestDataCloudCLIBase(TestDvc):
|
||||
def main(self, args):
|
||||
ret = main(args)
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
@staticmethod
|
||||
def should_test():
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def get_url():
|
||||
raise NotImplementedError
|
||||
|
||||
def _setup_cloud(self):
|
||||
pass
|
||||
|
||||
def _test_cloud(self, remote=None):
|
||||
self._setup_cloud()
|
||||
|
||||
args = ["-v", "-j", "2"]
|
||||
if remote:
|
||||
args += ["-r", remote]
|
||||
else:
|
||||
args += []
|
||||
|
||||
stages = self.dvc.add(self.FOO)
|
||||
self.assertEqual(len(stages), 1)
|
||||
stage = stages[0]
|
||||
self.assertTrue(stage is not None)
|
||||
cache = stage.outs[0].cache_path
|
||||
|
||||
stages = self.dvc.add(self.DATA_DIR)
|
||||
self.assertEqual(len(stages), 1)
|
||||
stage_dir = stages[0]
|
||||
self.assertTrue(stage_dir is not None)
|
||||
cache_dir = stage_dir.outs[0].cache_path
|
||||
|
||||
# FIXME check status output
|
||||
|
||||
self.main(["push"] + args)
|
||||
self.assertTrue(os.path.exists(cache))
|
||||
self.assertTrue(os.path.isfile(cache))
|
||||
self.assertTrue(os.path.isfile(cache_dir))
|
||||
|
||||
remove(self.dvc.cache.local.cache_dir)
|
||||
|
||||
self.main(["fetch"] + args)
|
||||
self.assertTrue(os.path.exists(cache))
|
||||
self.assertTrue(os.path.isfile(cache))
|
||||
self.assertTrue(os.path.isfile(cache_dir))
|
||||
|
||||
self.main(["pull"] + args)
|
||||
self.assertTrue(os.path.exists(cache))
|
||||
self.assertTrue(os.path.isfile(cache))
|
||||
self.assertTrue(os.path.isfile(cache_dir))
|
||||
self.assertTrue(os.path.isfile(self.FOO))
|
||||
self.assertTrue(os.path.isdir(self.DATA_DIR))
|
||||
|
||||
# Pull and check status
|
||||
dvc.cloud.pull(info)
|
||||
assert os.path.exists(cache)
|
||||
assert os.path.isfile(cache)
|
||||
with open(cache) as fd:
|
||||
self.assertEqual(fd.read(), self.FOO_CONTENTS)
|
||||
self.assertTrue(os.path.isfile(cache_dir))
|
||||
assert fd.read() == "foo"
|
||||
|
||||
# NOTE: check if remote gc works correctly on directories
|
||||
self.main(["gc", "-cw", "-f"] + args)
|
||||
shutil.move(
|
||||
self.dvc.cache.local.cache_dir,
|
||||
self.dvc.cache.local.cache_dir + ".back",
|
||||
)
|
||||
dvc.cloud.pull(info_dir)
|
||||
assert os.path.isfile(cache_dir)
|
||||
|
||||
self.main(["fetch"] + args)
|
||||
status = dvc.cloud.status(info, show_checksums=True)
|
||||
expected = {md5: {"name": md5, "status": STATUS_OK}}
|
||||
assert status == expected
|
||||
|
||||
self.main(["pull", "-f"] + args)
|
||||
self.assertTrue(os.path.exists(cache))
|
||||
self.assertTrue(os.path.isfile(cache))
|
||||
self.assertTrue(os.path.isfile(cache_dir))
|
||||
self.assertTrue(os.path.isfile(self.FOO))
|
||||
self.assertTrue(os.path.isdir(self.DATA_DIR))
|
||||
|
||||
def _test(self):
|
||||
pass
|
||||
|
||||
def test(self):
|
||||
if not self.should_test():
|
||||
raise SkipTest(f"Test {self.__class__.__name__} is disabled")
|
||||
self._test()
|
||||
status_dir = dvc.cloud.status(info_dir, show_checksums=True)
|
||||
expected = {md5_dir: {"name": md5_dir, "status": STATUS_OK}}
|
||||
assert status_dir == expected
|
||||
|
||||
|
||||
class TestLocalRemoteCLI(Local, TestDataCloudCLIBase):
|
||||
def _test(self):
|
||||
url = self.get_url()
|
||||
@pytest.mark.parametrize("remote", all_remotes)
|
||||
def test_cloud_cli(tmp_dir, dvc, remote):
|
||||
args = ["-v", "-j", "2"]
|
||||
|
||||
self.main(["remote", "add", TEST_REMOTE, url])
|
||||
(stage,) = tmp_dir.dvc_gen("foo", "foo")
|
||||
cache = stage.outs[0].cache_path
|
||||
|
||||
self._test_cloud(TEST_REMOTE)
|
||||
(stage_dir,) = tmp_dir.dvc_gen(
|
||||
{
|
||||
"data_dir": {
|
||||
"data_sub_dir": {"data_sub": "data_sub"},
|
||||
"data": "data",
|
||||
}
|
||||
}
|
||||
)
|
||||
assert stage_dir is not None
|
||||
cache_dir = stage_dir.outs[0].cache_path
|
||||
|
||||
# FIXME check status output
|
||||
|
||||
class TestRemoteHDFSCLI(HDFS, TestDataCloudCLIBase):
|
||||
def _test(self):
|
||||
url = self.get_url()
|
||||
assert main(["push"] + args) == 0
|
||||
assert os.path.exists(cache)
|
||||
assert os.path.isfile(cache)
|
||||
assert os.path.isfile(cache_dir)
|
||||
|
||||
self.main(["remote", "add", TEST_REMOTE, url])
|
||||
remove(dvc.cache.local.cache_dir)
|
||||
|
||||
self._test_cloud(TEST_REMOTE)
|
||||
assert main(["fetch"] + args) == 0
|
||||
assert os.path.exists(cache)
|
||||
assert os.path.isfile(cache)
|
||||
assert os.path.isfile(cache_dir)
|
||||
|
||||
assert main(["pull"] + args) == 0
|
||||
assert os.path.exists(cache)
|
||||
assert os.path.isfile(cache)
|
||||
assert os.path.isfile(cache_dir)
|
||||
assert os.path.isfile("foo")
|
||||
assert os.path.isdir("data_dir")
|
||||
|
||||
class TestS3RemoteCLI(S3, TestDataCloudCLIBase):
|
||||
def _test(self):
|
||||
url = self.get_url()
|
||||
with open(cache) as fd:
|
||||
assert fd.read() == "foo"
|
||||
assert os.path.isfile(cache_dir)
|
||||
|
||||
self.main(["remote", "add", TEST_REMOTE, url])
|
||||
# NOTE: http doesn't support gc yet
|
||||
if remote.url.startswith("http"):
|
||||
return
|
||||
|
||||
self._test_cloud(TEST_REMOTE)
|
||||
# NOTE: check if remote gc works correctly on directories
|
||||
assert main(["gc", "-cw", "-f"] + args) == 0
|
||||
shutil.move(
|
||||
dvc.cache.local.cache_dir, dvc.cache.local.cache_dir + ".back",
|
||||
)
|
||||
|
||||
assert main(["fetch"] + args) == 0
|
||||
|
||||
class TestGDriveRemoteCLI(GDrive, TestDataCloudCLIBase):
|
||||
def _test(self):
|
||||
url = self.get_url()
|
||||
|
||||
self.create_dir(self.dvc, url)
|
||||
|
||||
self.main(["remote", "add", TEST_REMOTE, url])
|
||||
self.main(
|
||||
[
|
||||
"remote",
|
||||
"modify",
|
||||
TEST_REMOTE,
|
||||
"gdrive_service_account_email",
|
||||
"test",
|
||||
]
|
||||
)
|
||||
self.main(
|
||||
[
|
||||
"remote",
|
||||
"modify",
|
||||
TEST_REMOTE,
|
||||
"gdrive_service_account_p12_file_path",
|
||||
"test.p12",
|
||||
]
|
||||
)
|
||||
self.main(
|
||||
[
|
||||
"remote",
|
||||
"modify",
|
||||
TEST_REMOTE,
|
||||
"gdrive_use_service_account",
|
||||
"True",
|
||||
]
|
||||
)
|
||||
|
||||
self._test_cloud(TEST_REMOTE)
|
||||
|
||||
|
||||
class TestGSRemoteCLI(GCP, TestDataCloudCLIBase):
|
||||
def _test(self):
|
||||
url = self.get_url()
|
||||
|
||||
self.main(["remote", "add", TEST_REMOTE, url])
|
||||
self.main(
|
||||
[
|
||||
"remote",
|
||||
"modify",
|
||||
TEST_REMOTE,
|
||||
"credentialpath",
|
||||
TEST_GCP_CREDS_FILE,
|
||||
]
|
||||
)
|
||||
|
||||
self._test_cloud(TEST_REMOTE)
|
||||
|
||||
|
||||
class TestAzureRemoteCLI(Azure, TestDataCloudCLIBase):
|
||||
def _test(self):
|
||||
url = self.get_url()
|
||||
|
||||
self.main(["remote", "add", TEST_REMOTE, url])
|
||||
|
||||
self._test_cloud(TEST_REMOTE)
|
||||
|
||||
|
||||
class TestOSSRemoteCLI(OSS, TestDataCloudCLIBase):
|
||||
def _test(self):
|
||||
url = self.get_url()
|
||||
|
||||
self.main(["remote", "add", TEST_REMOTE, url])
|
||||
|
||||
self._test_cloud(TEST_REMOTE)
|
||||
assert main(["pull", "-f"] + args) == 0
|
||||
assert os.path.exists(cache)
|
||||
assert os.path.isfile(cache)
|
||||
assert os.path.isfile(cache_dir)
|
||||
assert os.path.isfile("foo")
|
||||
assert os.path.isdir("data_dir")
|
||||
|
||||
|
||||
class TestDataCloudErrorCLI(TestDvc):
|
||||
@@ -507,126 +173,30 @@ class TestDataCloudErrorCLI(TestDvc):
|
||||
self.main_fail(["fetch", f])
|
||||
|
||||
|
||||
class TestWarnOnOutdatedStage(TestDvc):
|
||||
def main(self, args):
|
||||
ret = main(args)
|
||||
self.assertEqual(ret, 0)
|
||||
def test_warn_on_outdated_stage(tmp_dir, dvc, local_remote, caplog):
|
||||
stage = dvc.run(outs=["bar"], cmd="echo bar > bar", single_stage=True)
|
||||
assert main(["push"]) == 0
|
||||
|
||||
def _test(self):
|
||||
url = Local.get_url()
|
||||
self.main(["remote", "add", "-d", TEST_REMOTE, url])
|
||||
stage_file_path = stage.relpath
|
||||
content = load_yaml(stage_file_path)
|
||||
del content["outs"][0]["md5"]
|
||||
dump_yaml(stage_file_path, content)
|
||||
|
||||
stage = self.dvc.run(
|
||||
outs=["bar"], cmd="echo bar > bar", single_stage=True
|
||||
)
|
||||
self.main(["push"])
|
||||
|
||||
stage_file_path = stage.relpath
|
||||
content = load_yaml(stage_file_path)
|
||||
del content["outs"][0]["md5"]
|
||||
dump_yaml(stage_file_path, content)
|
||||
|
||||
with self._caplog.at_level(logging.WARNING, logger="dvc"):
|
||||
self._caplog.clear()
|
||||
self.main(["status", "-c"])
|
||||
expected_warning = (
|
||||
"Output 'bar'(stage: 'bar.dvc') is missing version info. "
|
||||
"Cache for it will not be collected. "
|
||||
"Use `dvc repro` to get your pipeline up to date."
|
||||
)
|
||||
|
||||
assert expected_warning in self._caplog.text
|
||||
|
||||
def test(self):
|
||||
self._test()
|
||||
|
||||
|
||||
class TestRecursiveSyncOperations(Local, TestDataCloudBase):
|
||||
def main(self, args):
|
||||
ret = main(args)
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
def _get_cloud_class(self):
|
||||
return LocalRemoteTree
|
||||
|
||||
def _prepare_repo(self):
|
||||
remote = self.cloud.get_remote()
|
||||
self.main(
|
||||
["remote", "add", "-d", TEST_REMOTE, remote.path_info.fspath]
|
||||
with caplog.at_level(logging.WARNING, logger="dvc"):
|
||||
caplog.clear()
|
||||
assert main(["status", "-c"]) == 0
|
||||
expected_warning = (
|
||||
"Output 'bar'(stage: 'bar.dvc') is missing version info. "
|
||||
"Cache for it will not be collected. "
|
||||
"Use `dvc repro` to get your pipeline up to date."
|
||||
)
|
||||
|
||||
self.dvc.add(self.DATA)
|
||||
self.dvc.add(self.DATA_SUB)
|
||||
|
||||
def _remove_local_data_files(self):
|
||||
os.remove(self.DATA)
|
||||
os.remove(self.DATA_SUB)
|
||||
|
||||
def _test_recursive_pull(self):
|
||||
self._remove_local_data_files()
|
||||
self._clear_local_cache()
|
||||
|
||||
self.assertFalse(os.path.exists(self.DATA))
|
||||
self.assertFalse(os.path.exists(self.DATA_SUB))
|
||||
|
||||
self.main(["pull", "-R", self.DATA_DIR])
|
||||
|
||||
self.assertTrue(os.path.exists(self.DATA))
|
||||
self.assertTrue(os.path.exists(self.DATA_SUB))
|
||||
|
||||
def _clear_local_cache(self):
|
||||
remove(self.dvc.cache.local.cache_dir)
|
||||
|
||||
def _test_recursive_fetch(self, data_md5, data_sub_md5):
|
||||
self._clear_local_cache()
|
||||
|
||||
local_cache = self.dvc.cache.local
|
||||
local_cache_data_path = local_cache.checksum_to_path_info(data_md5)
|
||||
local_cache_data_sub_path = local_cache.checksum_to_path_info(
|
||||
data_sub_md5
|
||||
)
|
||||
|
||||
self.assertFalse(os.path.exists(local_cache_data_path))
|
||||
self.assertFalse(os.path.exists(local_cache_data_sub_path))
|
||||
|
||||
self.main(["fetch", "-R", self.DATA_DIR])
|
||||
|
||||
self.assertTrue(os.path.exists(local_cache_data_path))
|
||||
self.assertTrue(os.path.exists(local_cache_data_sub_path))
|
||||
|
||||
def _test_recursive_push(self, data_md5, data_sub_md5):
|
||||
remote = self.cloud.get_remote()
|
||||
cloud_data_path = remote.checksum_to_path_info(data_md5)
|
||||
cloud_data_sub_path = remote.checksum_to_path_info(data_sub_md5)
|
||||
|
||||
self.assertFalse(os.path.exists(cloud_data_path))
|
||||
self.assertFalse(os.path.exists(cloud_data_sub_path))
|
||||
|
||||
self.main(["push", "-R", self.DATA_DIR])
|
||||
|
||||
self.assertTrue(os.path.exists(cloud_data_path))
|
||||
self.assertTrue(os.path.exists(cloud_data_sub_path))
|
||||
|
||||
def test(self):
|
||||
self._setup_cloud()
|
||||
self._prepare_repo()
|
||||
|
||||
data_md5 = file_md5(self.DATA)[0]
|
||||
data_sub_md5 = file_md5(self.DATA_SUB)[0]
|
||||
|
||||
self._test_recursive_push(data_md5, data_sub_md5)
|
||||
|
||||
self._test_recursive_fetch(data_md5, data_sub_md5)
|
||||
|
||||
self._test_recursive_pull()
|
||||
assert expected_warning in caplog.text
|
||||
|
||||
|
||||
def test_checksum_recalculation(mocker, dvc, tmp_dir):
|
||||
def test_hash_recalculation(mocker, dvc, tmp_dir, local_remote):
|
||||
tmp_dir.gen({"foo": "foo"})
|
||||
test_get_file_checksum = mocker.spy(LocalRemoteTree, "get_file_checksum")
|
||||
url = Local.get_url()
|
||||
ret = main(["remote", "add", "-d", TEST_REMOTE, url])
|
||||
assert ret == 0
|
||||
test_get_file_hash = mocker.spy(LocalRemoteTree, "get_file_hash")
|
||||
ret = main(["config", "cache.type", "hardlink"])
|
||||
assert ret == 0
|
||||
ret = main(["add", "foo"])
|
||||
@@ -635,7 +205,7 @@ def test_checksum_recalculation(mocker, dvc, tmp_dir):
|
||||
assert ret == 0
|
||||
ret = main(["run", "--single-stage", "-d", "foo", "echo foo"])
|
||||
assert ret == 0
|
||||
assert test_get_file_checksum.mock.call_count == 1
|
||||
assert test_get_file_hash.mock.call_count == 1
|
||||
|
||||
|
||||
class TestShouldWarnOnNoChecksumInLocalAndRemoteCache(TestDvc):
|
||||
@@ -688,11 +258,9 @@ class TestShouldWarnOnNoChecksumInLocalAndRemoteCache(TestDvc):
|
||||
assert self.message_bar_part in self._caplog.text
|
||||
|
||||
|
||||
def test_verify_checksums(
|
||||
tmp_dir, scm, dvc, mocker, tmp_path_factory, setup_remote
|
||||
def test_verify_hashes(
|
||||
tmp_dir, scm, dvc, mocker, tmp_path_factory, local_remote
|
||||
):
|
||||
|
||||
setup_remote(dvc, name="upstream")
|
||||
tmp_dir.dvc_gen({"file": "file1 content"}, commit="add file")
|
||||
tmp_dir.dvc_gen({"dir": {"subfile": "file2 content"}}, commit="add dir")
|
||||
dvc.push()
|
||||
@@ -702,10 +270,10 @@ def test_verify_checksums(
|
||||
remove("dir")
|
||||
remove(dvc.cache.local.cache_dir)
|
||||
|
||||
checksum_spy = mocker.spy(dvc.cache.local.tree, "get_file_checksum")
|
||||
hash_spy = mocker.spy(dvc.cache.local.tree, "get_file_hash")
|
||||
|
||||
dvc.pull()
|
||||
assert checksum_spy.call_count == 0
|
||||
assert hash_spy.call_count == 0
|
||||
|
||||
# Removing cache will invalidate existing state entries
|
||||
remove(dvc.cache.local.cache_dir)
|
||||
@@ -713,7 +281,7 @@ def test_verify_checksums(
|
||||
dvc.config["remote"]["upstream"]["verify"] = True
|
||||
|
||||
dvc.pull()
|
||||
assert checksum_spy.call_count == 3
|
||||
assert hash_spy.call_count == 3
|
||||
|
||||
|
||||
@flaky(max_runs=3, min_passes=1)
|
||||
@@ -784,8 +352,7 @@ def recurse_list_dir(d):
|
||||
]
|
||||
|
||||
|
||||
def test_dvc_pull_pipeline_stages(tmp_dir, dvc, run_copy, setup_remote):
|
||||
setup_remote(dvc)
|
||||
def test_dvc_pull_pipeline_stages(tmp_dir, dvc, run_copy, local_remote):
|
||||
(stage0,) = tmp_dir.dvc_gen("foo", "foo")
|
||||
stage1 = run_copy("foo", "bar", single_stage=True)
|
||||
stage2 = run_copy("bar", "foobar", name="copy-bar-foobar")
|
||||
@@ -816,8 +383,8 @@ def test_dvc_pull_pipeline_stages(tmp_dir, dvc, run_copy, setup_remote):
|
||||
assert set(stats["added"]) == set(outs)
|
||||
|
||||
|
||||
def test_pipeline_file_target_ops(tmp_dir, dvc, run_copy, setup_remote):
|
||||
remote_path = setup_remote(dvc)
|
||||
def test_pipeline_file_target_ops(tmp_dir, dvc, run_copy, local_remote):
|
||||
path = local_remote.url
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
run_copy("foo", "bar", single_stage=True)
|
||||
|
||||
@@ -836,7 +403,7 @@ def test_pipeline_file_target_ops(tmp_dir, dvc, run_copy, setup_remote):
|
||||
outs = ["foo", "bar", "lorem", "ipsum", "baz", "lorem2"]
|
||||
|
||||
# each one's a copy of other, hence 3
|
||||
assert len(recurse_list_dir(remote_path)) == 3
|
||||
assert len(recurse_list_dir(path)) == 3
|
||||
|
||||
clean(outs, dvc)
|
||||
assert set(dvc.pull(["dvc.yaml"])["added"]) == {"lorem2", "baz"}
|
||||
@@ -847,13 +414,13 @@ def test_pipeline_file_target_ops(tmp_dir, dvc, run_copy, setup_remote):
|
||||
# clean everything in remote and push
|
||||
from tests.dir_helpers import TmpDir
|
||||
|
||||
clean(TmpDir(remote_path).iterdir())
|
||||
clean(TmpDir(path).iterdir())
|
||||
dvc.push(["dvc.yaml:copy-ipsum-baz"])
|
||||
assert len(recurse_list_dir(remote_path)) == 1
|
||||
assert len(recurse_list_dir(path)) == 1
|
||||
|
||||
clean(TmpDir(remote_path).iterdir())
|
||||
clean(TmpDir(path).iterdir())
|
||||
dvc.push(["dvc.yaml"])
|
||||
assert len(recurse_list_dir(remote_path)) == 2
|
||||
assert len(recurse_list_dir(path)) == 2
|
||||
|
||||
with pytest.raises(StageNotFound):
|
||||
dvc.push(["dvc.yaml:StageThatDoesNotExist"])
|
||||
@@ -870,8 +437,7 @@ def test_pipeline_file_target_ops(tmp_dir, dvc, run_copy, setup_remote):
|
||||
({}, "Everything is up to date"),
|
||||
],
|
||||
)
|
||||
def test_push_stats(tmp_dir, dvc, fs, msg, caplog, setup_remote):
|
||||
setup_remote(dvc)
|
||||
def test_push_stats(tmp_dir, dvc, fs, msg, caplog, local_remote):
|
||||
tmp_dir.dvc_gen(fs)
|
||||
|
||||
caplog.clear()
|
||||
@@ -888,8 +454,7 @@ def test_push_stats(tmp_dir, dvc, fs, msg, caplog, setup_remote):
|
||||
({}, "Everything is up to date."),
|
||||
],
|
||||
)
|
||||
def test_fetch_stats(tmp_dir, dvc, fs, msg, caplog, setup_remote):
|
||||
setup_remote(dvc)
|
||||
def test_fetch_stats(tmp_dir, dvc, fs, msg, caplog, local_remote):
|
||||
tmp_dir.dvc_gen(fs)
|
||||
dvc.push()
|
||||
clean(list(fs.keys()), dvc)
|
||||
@@ -899,8 +464,7 @@ def test_fetch_stats(tmp_dir, dvc, fs, msg, caplog, setup_remote):
|
||||
assert msg in caplog.text
|
||||
|
||||
|
||||
def test_pull_stats(tmp_dir, dvc, caplog, setup_remote):
|
||||
setup_remote(dvc)
|
||||
def test_pull_stats(tmp_dir, dvc, caplog, local_remote):
|
||||
tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
|
||||
dvc.push()
|
||||
clean(["foo", "bar"], dvc)
|
||||
@@ -921,8 +485,7 @@ def test_pull_stats(tmp_dir, dvc, caplog, setup_remote):
|
||||
@pytest.mark.parametrize(
|
||||
"key,expected", [("all_tags", 2), ("all_branches", 3), ("all_commits", 3)]
|
||||
)
|
||||
def test_push_pull_all(tmp_dir, scm, dvc, setup_remote, key, expected):
|
||||
setup_remote(dvc)
|
||||
def test_push_pull_all(tmp_dir, scm, dvc, local_remote, key, expected):
|
||||
tmp_dir.dvc_gen({"foo": "foo"}, commit="first")
|
||||
scm.tag("v1")
|
||||
dvc.remove("foo.dvc")
|
||||
@@ -938,13 +501,12 @@ def test_push_pull_all(tmp_dir, scm, dvc, setup_remote, key, expected):
|
||||
assert dvc.pull(**{key: True})["fetched"] == expected
|
||||
|
||||
|
||||
def test_push_pull_fetch_pipeline_stages(tmp_dir, dvc, run_copy, setup_remote):
|
||||
remote_path = setup_remote(dvc)
|
||||
def test_push_pull_fetch_pipeline_stages(tmp_dir, dvc, run_copy, local_remote):
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
run_copy("foo", "bar", no_commit=True, name="copy-foo-bar")
|
||||
|
||||
dvc.push("copy-foo-bar")
|
||||
assert len(recurse_list_dir(remote_path)) == 1
|
||||
assert len(recurse_list_dir(local_remote.url)) == 1
|
||||
# pushing everything so as we can check pull/fetch only downloads
|
||||
# from specified targets
|
||||
dvc.push()
|
||||
|
||||
@@ -157,7 +157,7 @@ def test_stage_collection(tmp_dir, dvc):
|
||||
always_changed=True,
|
||||
single_stage=True,
|
||||
)
|
||||
assert {s for s in dvc.stages} == {stage1, stage3, stage2}
|
||||
assert set(dvc.stages) == {stage1, stage3, stage2}
|
||||
|
||||
|
||||
def test_remove_stage(tmp_dir, dvc, run_copy):
|
||||
|
||||
@@ -43,8 +43,8 @@ def test_source_change(erepo_dir):
|
||||
assert old_rev != new_rev
|
||||
|
||||
|
||||
def test_cache_reused(erepo_dir, mocker, setup_remote):
|
||||
setup_remote(erepo_dir.dvc)
|
||||
def test_cache_reused(erepo_dir, mocker, local_cloud):
|
||||
erepo_dir.add_remote(config=local_cloud.config)
|
||||
with erepo_dir.chdir():
|
||||
erepo_dir.dvc_gen("file", "text", commit="add file")
|
||||
erepo_dir.dvc.push()
|
||||
@@ -99,7 +99,7 @@ def test_pull_subdir_file(tmp_dir, erepo_dir):
|
||||
assert dest.read_text() == "contents"
|
||||
|
||||
|
||||
def test_relative_remote(erepo_dir, tmp_dir, setup_remote):
|
||||
def test_relative_remote(erepo_dir, tmp_dir):
|
||||
# these steps reproduce the script on this issue:
|
||||
# https://github.com/iterative/dvc/issues/2756
|
||||
with erepo_dir.chdir():
|
||||
@@ -107,7 +107,7 @@ def test_relative_remote(erepo_dir, tmp_dir, setup_remote):
|
||||
|
||||
upstream_dir = tmp_dir
|
||||
upstream_url = relpath(upstream_dir, erepo_dir)
|
||||
setup_remote(erepo_dir.dvc, url=upstream_url, name="upstream")
|
||||
erepo_dir.add_remote(url=upstream_url)
|
||||
|
||||
erepo_dir.dvc.push()
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ class TestGC(TestDvcGit):
|
||||
self.dvc.add(self.FOO)
|
||||
self.dvc.add(self.DATA_DIR)
|
||||
self.good_cache = [
|
||||
self.dvc.cache.local.checksum_to_path_info(md5)
|
||||
self.dvc.cache.local.hash_to_path_info(md5)
|
||||
for md5 in self.dvc.cache.local.tree.all()
|
||||
]
|
||||
|
||||
@@ -52,7 +52,7 @@ class TestGC(TestDvcGit):
|
||||
class TestGCBranchesTags(TestDvcGit):
|
||||
def _check_cache(self, num):
|
||||
total = 0
|
||||
for root, dirs, files in os.walk(os.path.join(".dvc", "cache")):
|
||||
for _, _, files in os.walk(os.path.join(".dvc", "cache")):
|
||||
total += len(files)
|
||||
self.assertEqual(total, num)
|
||||
|
||||
@@ -112,7 +112,7 @@ class TestGCBranchesTags(TestDvcGit):
|
||||
class TestGCMultipleDvcRepos(TestDvcGit):
|
||||
def _check_cache(self, num):
|
||||
total = 0
|
||||
for root, dirs, files in os.walk(os.path.join(".dvc", "cache")):
|
||||
for _, _, files in os.walk(os.path.join(".dvc", "cache")):
|
||||
total += len(files)
|
||||
self.assertEqual(total, num)
|
||||
|
||||
@@ -240,9 +240,9 @@ def test_gc_without_workspace_raises_error(tmp_dir, dvc):
|
||||
dvc.gc(force=True, workspace=False)
|
||||
|
||||
|
||||
def test_gc_cloud_with_or_without_specifier(tmp_dir, erepo_dir, setup_remote):
|
||||
def test_gc_cloud_with_or_without_specifier(tmp_dir, erepo_dir, local_cloud):
|
||||
erepo_dir.add_remote(config=local_cloud.config)
|
||||
dvc = erepo_dir.dvc
|
||||
setup_remote(dvc)
|
||||
from dvc.exceptions import InvalidArgumentError
|
||||
|
||||
with pytest.raises(InvalidArgumentError):
|
||||
@@ -297,9 +297,7 @@ def test_gc_with_possible_args_positive(tmp_dir, dvc):
|
||||
assert main(["gc", "-vf", flag]) == 0
|
||||
|
||||
|
||||
def test_gc_cloud_positive(tmp_dir, dvc, tmp_path_factory, setup_remote):
|
||||
setup_remote(dvc)
|
||||
|
||||
def test_gc_cloud_positive(tmp_dir, dvc, tmp_path_factory, local_remote):
|
||||
for flag in ["-cw", "-ca", "-cT", "-caT", "-cwT"]:
|
||||
assert main(["gc", "-vf", flag]) == 0
|
||||
|
||||
|
||||
@@ -201,8 +201,8 @@ def test_get_file_from_dir(tmp_dir, erepo_dir):
|
||||
assert (tmp_dir / "X").read_text() == "foo"
|
||||
|
||||
|
||||
def test_get_url_positive(tmp_dir, erepo_dir, caplog, setup_remote):
|
||||
setup_remote(erepo_dir.dvc)
|
||||
def test_get_url_positive(tmp_dir, erepo_dir, caplog, local_cloud):
|
||||
erepo_dir.add_remote(config=local_cloud.config)
|
||||
with erepo_dir.chdir():
|
||||
erepo_dir.dvc_gen("foo", "foo")
|
||||
erepo_dir.dvc.push()
|
||||
@@ -238,11 +238,10 @@ def test_get_url_git_only_repo(tmp_dir, scm, caplog):
|
||||
|
||||
|
||||
def test_get_pipeline_tracked_outs(
|
||||
tmp_dir, dvc, scm, git_dir, run_copy, setup_remote
|
||||
tmp_dir, dvc, scm, git_dir, run_copy, local_remote
|
||||
):
|
||||
from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
|
||||
|
||||
setup_remote(dvc)
|
||||
tmp_dir.gen("foo", "foo")
|
||||
run_copy("foo", "bar", name="copy-foo-bar")
|
||||
dvc.push()
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import pytest
|
||||
|
||||
from dvc.dependency.base import DependencyDoesNotExistError
|
||||
from dvc.repo import Repo
|
||||
|
||||
|
||||
@@ -20,3 +21,8 @@ def test_get_url_to_dir(tmp_dir, dname):
|
||||
|
||||
assert (tmp_dir / dname).is_dir()
|
||||
assert (tmp_dir / dname / "foo").read_text() == "foo contents"
|
||||
|
||||
|
||||
def test_get_url_nonexistent(tmp_dir):
|
||||
with pytest.raises(DependencyDoesNotExistError):
|
||||
Repo.get_url("nonexistent")
|
||||
|
||||
@@ -166,3 +166,10 @@ def test_ignore_subrepo(tmp_dir, scm, dvc):
|
||||
|
||||
for _ in subrepo.brancher(all_commits=True):
|
||||
assert subrepo.tree.exists(subrepo_dir / "foo")
|
||||
|
||||
|
||||
def test_ignore_blank_line(tmp_dir, dvc):
|
||||
tmp_dir.gen({"dir": {"ignored": "text", "other": "text2"}})
|
||||
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "foo\n\ndir/ignored")
|
||||
|
||||
assert _files_set("dir", dvc.tree) == {"dir/other"}
|
||||
|
||||
@@ -307,13 +307,13 @@ def test_pull_no_rev_lock(erepo_dir, tmp_dir, dvc):
|
||||
|
||||
|
||||
def test_import_from_bare_git_repo(
|
||||
tmp_dir, make_tmp_dir, erepo_dir, setup_remote
|
||||
tmp_dir, make_tmp_dir, erepo_dir, local_cloud
|
||||
):
|
||||
import git
|
||||
|
||||
git.Repo.init(os.fspath(tmp_dir), bare=True)
|
||||
|
||||
setup_remote(erepo_dir.dvc)
|
||||
erepo_dir.add_remote(config=local_cloud.config)
|
||||
with erepo_dir.chdir():
|
||||
erepo_dir.dvc_gen({"foo": "foo"}, commit="initial")
|
||||
erepo_dir.dvc.push()
|
||||
@@ -327,11 +327,10 @@ def test_import_from_bare_git_repo(
|
||||
|
||||
|
||||
def test_import_pipeline_tracked_outs(
|
||||
tmp_dir, dvc, scm, erepo_dir, run_copy, setup_remote
|
||||
tmp_dir, dvc, scm, erepo_dir, run_copy, local_remote
|
||||
):
|
||||
from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
|
||||
|
||||
setup_remote(dvc)
|
||||
tmp_dir.gen("foo", "foo")
|
||||
run_copy("foo", "bar", name="copy-foo-bar")
|
||||
dvc.push()
|
||||
|
||||
@@ -3,6 +3,7 @@ from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from dvc.dependency.base import DependencyDoesNotExistError
|
||||
from dvc.main import main
|
||||
from dvc.stage import Stage
|
||||
from dvc.utils.fs import makedirs
|
||||
@@ -97,3 +98,8 @@ def test_import_stage_accompanies_target(tmp_dir, dvc, erepo_dir):
|
||||
|
||||
assert (tmp_dir / "dir" / "imported_file").exists()
|
||||
assert (tmp_dir / "dir" / "imported_file.dvc").exists()
|
||||
|
||||
|
||||
def test_import_url_nonexistent(dvc, erepo_dir):
|
||||
with pytest.raises(DependencyDoesNotExistError):
|
||||
dvc.imp_url(os.fspath(erepo_dir / "non-existent"))
|
||||
|
||||
+12
-21
@@ -145,7 +145,7 @@ class TestRemoteShouldHandleUppercaseRemoteName(TestDvc):
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
|
||||
def test_dir_checksum_should_be_key_order_agnostic(tmp_dir, dvc):
|
||||
def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc):
|
||||
tmp_dir.gen({"data": {"1": "1 content", "2": "2 content"}})
|
||||
|
||||
path_info = PathInfo("data")
|
||||
@@ -158,7 +158,7 @@ def test_dir_checksum_should_be_key_order_agnostic(tmp_dir, dvc):
|
||||
{"relpath": "2", "md5": "2"},
|
||||
],
|
||||
):
|
||||
checksum1 = dvc.cache.local.get_checksum(path_info)
|
||||
hash1 = dvc.cache.local.get_hash(path_info)
|
||||
|
||||
with patch.object(
|
||||
BaseRemoteTree,
|
||||
@@ -168,14 +168,12 @@ def test_dir_checksum_should_be_key_order_agnostic(tmp_dir, dvc):
|
||||
{"md5": "2", "relpath": "2"},
|
||||
],
|
||||
):
|
||||
checksum2 = dvc.cache.local.get_checksum(path_info)
|
||||
hash2 = dvc.cache.local.get_hash(path_info)
|
||||
|
||||
assert checksum1 == checksum2
|
||||
assert hash1 == hash2
|
||||
|
||||
|
||||
def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory, setup_remote):
|
||||
setup_remote(dvc, name="upstream")
|
||||
|
||||
def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory, local_remote):
|
||||
foo = tmp_dir.dvc_gen({"foo": "foo content"})[0].outs[0]
|
||||
bar = tmp_dir.dvc_gen({"bar": "bar content"})[0].outs[0]
|
||||
baz = tmp_dir.dvc_gen({"baz": {"foo": "baz content"}})[0].outs[0]
|
||||
@@ -194,13 +192,9 @@ def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory, setup_remote):
|
||||
assert upload_error_info.value.amount == 3
|
||||
|
||||
remote = dvc.cloud.get_remote("upstream")
|
||||
assert not remote.tree.exists(
|
||||
remote.checksum_to_path_info(foo.checksum)
|
||||
)
|
||||
assert remote.tree.exists(remote.checksum_to_path_info(bar.checksum))
|
||||
assert not remote.tree.exists(
|
||||
remote.checksum_to_path_info(baz.checksum)
|
||||
)
|
||||
assert not remote.tree.exists(remote.hash_to_path_info(foo.checksum))
|
||||
assert remote.tree.exists(remote.hash_to_path_info(bar.checksum))
|
||||
assert not remote.tree.exists(remote.hash_to_path_info(baz.checksum))
|
||||
|
||||
# Push everything and delete local cache
|
||||
dvc.push()
|
||||
@@ -215,9 +209,8 @@ def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory, setup_remote):
|
||||
|
||||
|
||||
def test_raise_on_too_many_open_files(
|
||||
tmp_dir, dvc, tmp_path_factory, mocker, setup_remote
|
||||
tmp_dir, dvc, tmp_path_factory, mocker, local_remote
|
||||
):
|
||||
setup_remote(dvc)
|
||||
tmp_dir.dvc_gen({"file": "file content"})
|
||||
|
||||
mocker.patch.object(
|
||||
@@ -250,8 +243,7 @@ def test_external_dir_resource_on_no_cache(tmp_dir, dvc, tmp_path_factory):
|
||||
)
|
||||
|
||||
|
||||
def test_push_order(tmp_dir, dvc, tmp_path_factory, mocker, setup_remote):
|
||||
setup_remote(dvc)
|
||||
def test_push_order(tmp_dir, dvc, tmp_path_factory, mocker, local_remote):
|
||||
tmp_dir.dvc_gen({"foo": {"bar": "bar content"}})
|
||||
tmp_dir.dvc_gen({"baz": "baz content"})
|
||||
|
||||
@@ -391,13 +383,12 @@ def test_remote_default(dvc):
|
||||
assert local_config["core"]["remote"] == new_name
|
||||
|
||||
|
||||
def test_protect_local_remote(tmp_dir, dvc, setup_remote):
|
||||
setup_remote(dvc, name="upstream")
|
||||
def test_protect_local_remote(tmp_dir, dvc, local_remote):
|
||||
(stage,) = tmp_dir.dvc_gen("file", "file content")
|
||||
|
||||
dvc.push()
|
||||
remote = dvc.cloud.get_remote("upstream")
|
||||
remote_cache_file = remote.checksum_to_path_info(stage.outs[0].checksum)
|
||||
remote_cache_file = remote.hash_to_path_info(stage.outs[0].checksum)
|
||||
|
||||
assert os.path.exists(remote_cache_file)
|
||||
assert stat.S_IMODE(os.stat(remote_cache_file).st_mode) == 0o444
|
||||
|
||||
@@ -7,14 +7,19 @@ from dvc.main import main
|
||||
from dvc.stage.exceptions import StageFileDoesNotExistError
|
||||
from dvc.system import System
|
||||
from dvc.utils.fs import remove
|
||||
from tests.utils import get_gitignore_content
|
||||
|
||||
|
||||
@pytest.mark.parametrize("remove_outs", [True, False])
|
||||
def test_remove(tmp_dir, dvc, run_copy, remove_outs):
|
||||
def test_remove(tmp_dir, scm, dvc, run_copy, remove_outs):
|
||||
(stage1,) = tmp_dir.dvc_gen("foo", "foo")
|
||||
stage2 = run_copy("foo", "bar", single_stage=True)
|
||||
stage3 = run_copy("bar", "foobar", name="copy-bar-foobar")
|
||||
|
||||
assert "/foo" in get_gitignore_content()
|
||||
assert "/bar" in get_gitignore_content()
|
||||
assert "/foobar" in get_gitignore_content()
|
||||
|
||||
for stage in [stage1, stage2, stage3]:
|
||||
dvc.remove(stage.addressing, outs=remove_outs)
|
||||
out_exists = (out.exists for out in stage.outs)
|
||||
@@ -24,6 +29,8 @@ def test_remove(tmp_dir, dvc, run_copy, remove_outs):
|
||||
else:
|
||||
assert all(out_exists)
|
||||
|
||||
assert not any(out in get_gitignore_content() for out in stage.outs)
|
||||
|
||||
|
||||
def test_remove_non_existent_file(tmp_dir, dvc):
|
||||
with pytest.raises(StageFileDoesNotExistError):
|
||||
|
||||
@@ -1737,12 +1737,14 @@ def test_downstream(dvc):
|
||||
reason="external output scenario is not supported on Windows",
|
||||
)
|
||||
def test_ssh_dir_out(tmp_dir, dvc, ssh_server):
|
||||
from tests.remotes.ssh import TEST_SSH_USER, TEST_SSH_KEY_PATH
|
||||
|
||||
tmp_dir.gen({"foo": "foo content"})
|
||||
|
||||
# Set up remote and cache
|
||||
user = ssh_server.test_creds["username"]
|
||||
user = TEST_SSH_USER
|
||||
port = ssh_server.port
|
||||
keyfile = ssh_server.test_creds["key_filename"]
|
||||
keyfile = TEST_SSH_KEY_PATH
|
||||
|
||||
remote_url = SSHMocked.get_url(user, port)
|
||||
assert main(["remote", "add", "upstream", remote_url]) == 0
|
||||
|
||||
@@ -449,7 +449,7 @@ def test_repro_when_new_outs_added_does_not_exist(tmp_dir, dvc):
|
||||
{
|
||||
"stages": {
|
||||
"run-copy": {
|
||||
"cmd": "python copy {} {}".format("foo", "foobar"),
|
||||
"cmd": "python copy.py {} {}".format("foo", "foobar"),
|
||||
"deps": ["foo"],
|
||||
"outs": ["foobar", "bar"],
|
||||
}
|
||||
|
||||
@@ -8,12 +8,11 @@ def _recurse_count_files(path):
|
||||
return len([os.path.join(r, f) for r, _, fs in os.walk(path) for f in fs])
|
||||
|
||||
|
||||
def test_push_pull(tmp_dir, dvc, erepo_dir, run_copy, setup_remote):
|
||||
def test_push_pull(tmp_dir, dvc, erepo_dir, run_copy, local_remote):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
run_copy("foo", "bar", name="copy-foo-bar")
|
||||
url = setup_remote(dvc)
|
||||
assert dvc.push(run_cache=True) == 2
|
||||
setup_remote(erepo_dir.dvc, url)
|
||||
erepo_dir.add_remote(config=local_remote.config)
|
||||
with erepo_dir.chdir():
|
||||
assert not os.path.exists(erepo_dir.dvc.stage_cache.cache_dir)
|
||||
assert erepo_dir.dvc.pull(run_cache=True)["fetched"] == 2
|
||||
|
||||
@@ -220,6 +220,10 @@ def test_stage_strings_representation(tmp_dir, dvc, run_copy):
|
||||
folder = tmp_dir / "dir"
|
||||
folder.mkdir()
|
||||
with folder.chdir():
|
||||
# `Stage` caches `relpath` results, forcing it to reset
|
||||
stage1.path = stage1.path
|
||||
stage2.path = stage2.path
|
||||
|
||||
rel_path = os.path.relpath(stage1.path)
|
||||
assert stage1.addressing == rel_path
|
||||
assert repr(stage1) == f"Stage: '{rel_path}'"
|
||||
|
||||
@@ -179,8 +179,7 @@ class TestWalkInGit(AssertWalkEqualMixin, TestGit):
|
||||
)
|
||||
|
||||
|
||||
def test_repotree_walk_fetch(tmp_dir, dvc, scm, setup_remote):
|
||||
setup_remote(dvc)
|
||||
def test_repotree_walk_fetch(tmp_dir, dvc, scm, local_remote):
|
||||
out = tmp_dir.dvc_gen({"dir": {"foo": "foo"}}, commit="init")[0].outs[0]
|
||||
dvc.push()
|
||||
remove(dvc.cache.local.cache_dir)
|
||||
@@ -192,16 +191,16 @@ def test_repotree_walk_fetch(tmp_dir, dvc, scm, setup_remote):
|
||||
|
||||
assert os.path.exists(out.cache_path)
|
||||
for entry in out.dir_cache:
|
||||
checksum = entry[out.remote.tree.PARAM_CHECKSUM]
|
||||
assert os.path.exists(dvc.cache.local.checksum_to_path_info(checksum))
|
||||
hash_ = entry[out.remote.tree.PARAM_CHECKSUM]
|
||||
assert os.path.exists(dvc.cache.local.hash_to_path_info(hash_))
|
||||
|
||||
|
||||
def test_repotree_cache_save(tmp_dir, dvc, scm, erepo_dir, setup_remote):
|
||||
def test_repotree_cache_save(tmp_dir, dvc, scm, erepo_dir, local_cloud):
|
||||
with erepo_dir.chdir():
|
||||
erepo_dir.gen({"dir": {"subdir": {"foo": "foo"}, "bar": "bar"}})
|
||||
erepo_dir.dvc_add("dir/subdir", commit="subdir")
|
||||
erepo_dir.scm_add("dir", commit="dir")
|
||||
setup_remote(erepo_dir.dvc)
|
||||
erepo_dir.add_remote(config=local_cloud.config)
|
||||
erepo_dir.dvc.push()
|
||||
|
||||
# test only cares that either fetch or stream are set so that DVC dirs are
|
||||
@@ -211,7 +210,7 @@ def test_repotree_cache_save(tmp_dir, dvc, scm, erepo_dir, setup_remote):
|
||||
# into dvc.cache, not fetched or streamed from a remote
|
||||
tree = RepoTree(erepo_dir.dvc, stream=True)
|
||||
expected = [
|
||||
tree.get_file_checksum(erepo_dir / path)
|
||||
tree.get_file_hash(erepo_dir / path)
|
||||
for path in ("dir/bar", "dir/subdir/foo")
|
||||
]
|
||||
|
||||
@@ -219,8 +218,8 @@ def test_repotree_cache_save(tmp_dir, dvc, scm, erepo_dir, setup_remote):
|
||||
cache = dvc.cache.local
|
||||
with cache.state:
|
||||
cache.save(PathInfo(erepo_dir / "dir"), tree, None)
|
||||
for checksum in expected:
|
||||
assert os.path.exists(cache.checksum_to_path_info(checksum))
|
||||
for hash_ in expected:
|
||||
assert os.path.exists(cache.hash_to_path_info(hash_))
|
||||
|
||||
|
||||
def test_cleantree_subrepo(tmp_dir, dvc, scm, monkeypatch):
|
||||
|
||||
@@ -1,295 +0,0 @@
|
||||
import getpass
|
||||
import os
|
||||
import platform
|
||||
import uuid
|
||||
from contextlib import contextmanager
|
||||
from subprocess import CalledProcessError, Popen, check_output
|
||||
|
||||
from moto.s3 import mock_s3
|
||||
|
||||
from dvc.remote.base import Remote
|
||||
from dvc.remote.gdrive import GDriveRemoteTree
|
||||
from dvc.remote.gs import GSRemoteTree
|
||||
from dvc.remote.s3 import S3RemoteTree
|
||||
from dvc.utils import env2bool
|
||||
from tests.basic_env import TestDvc
|
||||
|
||||
TEST_REMOTE = "upstream"
|
||||
TEST_CONFIG = {
|
||||
"cache": {},
|
||||
"core": {"remote": TEST_REMOTE},
|
||||
"remote": {TEST_REMOTE: {"url": ""}},
|
||||
}
|
||||
|
||||
TEST_AWS_REPO_BUCKET = os.environ.get("DVC_TEST_AWS_REPO_BUCKET", "dvc-temp")
|
||||
TEST_GCP_REPO_BUCKET = os.environ.get("DVC_TEST_GCP_REPO_BUCKET", "dvc-test")
|
||||
TEST_GDRIVE_REPO_BUCKET = "root"
|
||||
TEST_OSS_REPO_BUCKET = "dvc-test"
|
||||
|
||||
TEST_GCP_CREDS_FILE = os.path.abspath(
|
||||
os.environ.get(
|
||||
"GOOGLE_APPLICATION_CREDENTIALS",
|
||||
os.path.join("scripts", "ci", "gcp-creds.json"),
|
||||
)
|
||||
)
|
||||
# Ensure that absolute path is used
|
||||
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = TEST_GCP_CREDS_FILE
|
||||
|
||||
always_test = staticmethod(lambda: True)
|
||||
|
||||
|
||||
class Local:
|
||||
should_test = always_test
|
||||
|
||||
@staticmethod
|
||||
def get_storagepath():
|
||||
return TestDvc.mkdtemp()
|
||||
|
||||
@staticmethod
|
||||
def get_url():
|
||||
return Local.get_storagepath()
|
||||
|
||||
|
||||
class S3:
|
||||
@staticmethod
|
||||
def should_test():
|
||||
do_test = env2bool("DVC_TEST_AWS", undefined=None)
|
||||
if do_test is not None:
|
||||
return do_test
|
||||
|
||||
if os.getenv("AWS_ACCESS_KEY_ID") and os.getenv(
|
||||
"AWS_SECRET_ACCESS_KEY"
|
||||
):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def get_storagepath():
|
||||
return TEST_AWS_REPO_BUCKET + "/" + str(uuid.uuid4())
|
||||
|
||||
@staticmethod
|
||||
def get_url():
|
||||
return "s3://" + S3.get_storagepath()
|
||||
|
||||
|
||||
class S3Mocked(S3):
|
||||
should_test = always_test
|
||||
|
||||
@classmethod
|
||||
@contextmanager
|
||||
def remote(cls, repo):
|
||||
with mock_s3():
|
||||
yield Remote(S3RemoteTree(repo, {"url": cls.get_url()}))
|
||||
|
||||
@staticmethod
|
||||
def put_objects(remote, objects):
|
||||
s3 = remote.tree.s3
|
||||
bucket = remote.path_info.bucket
|
||||
s3.create_bucket(Bucket=bucket)
|
||||
for key, body in objects.items():
|
||||
s3.put_object(
|
||||
Bucket=bucket, Key=(remote.path_info / key).path, Body=body
|
||||
)
|
||||
|
||||
|
||||
class GCP:
|
||||
@staticmethod
|
||||
def should_test():
|
||||
do_test = env2bool("DVC_TEST_GCP", undefined=None)
|
||||
if do_test is not None:
|
||||
return do_test
|
||||
|
||||
if not os.path.exists(TEST_GCP_CREDS_FILE):
|
||||
return False
|
||||
|
||||
try:
|
||||
check_output(
|
||||
[
|
||||
"gcloud",
|
||||
"auth",
|
||||
"activate-service-account",
|
||||
"--key-file",
|
||||
TEST_GCP_CREDS_FILE,
|
||||
]
|
||||
)
|
||||
except (CalledProcessError, OSError):
|
||||
return False
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def get_storagepath():
|
||||
return TEST_GCP_REPO_BUCKET + "/" + str(uuid.uuid4())
|
||||
|
||||
@staticmethod
|
||||
def get_url():
|
||||
return "gs://" + GCP.get_storagepath()
|
||||
|
||||
@classmethod
|
||||
@contextmanager
|
||||
def remote(cls, repo):
|
||||
yield Remote(GSRemoteTree(repo, {"url": cls.get_url()}))
|
||||
|
||||
@staticmethod
|
||||
def put_objects(remote, objects):
|
||||
client = remote.tree.gs
|
||||
bucket = client.get_bucket(remote.path_info.bucket)
|
||||
for key, body in objects.items():
|
||||
bucket.blob((remote.path_info / key).path).upload_from_string(body)
|
||||
|
||||
|
||||
class GDrive:
|
||||
@staticmethod
|
||||
def should_test():
|
||||
return os.getenv(GDriveRemoteTree.GDRIVE_CREDENTIALS_DATA) is not None
|
||||
|
||||
@staticmethod
|
||||
def create_dir(dvc, url):
|
||||
config = {
|
||||
"url": url,
|
||||
"gdrive_service_account_email": "test",
|
||||
"gdrive_service_account_p12_file_path": "test.p12",
|
||||
"gdrive_use_service_account": True,
|
||||
}
|
||||
tree = GDriveRemoteTree(dvc, config)
|
||||
tree._gdrive_create_dir("root", tree.path_info.path)
|
||||
|
||||
@staticmethod
|
||||
def get_storagepath():
|
||||
return TEST_GDRIVE_REPO_BUCKET + "/" + str(uuid.uuid4())
|
||||
|
||||
@staticmethod
|
||||
def get_url():
|
||||
# NOTE: `get_url` should always return new random url
|
||||
return "gdrive://" + GDrive.get_storagepath()
|
||||
|
||||
|
||||
class Azure:
|
||||
@staticmethod
|
||||
def should_test():
|
||||
do_test = env2bool("DVC_TEST_AZURE", undefined=None)
|
||||
if do_test is not None:
|
||||
return do_test
|
||||
|
||||
return os.getenv("AZURE_STORAGE_CONTAINER_NAME") and os.getenv(
|
||||
"AZURE_STORAGE_CONNECTION_STRING"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_url():
|
||||
container_name = os.getenv("AZURE_STORAGE_CONTAINER_NAME")
|
||||
assert container_name is not None
|
||||
return "azure://{}/{}".format(container_name, str(uuid.uuid4()))
|
||||
|
||||
|
||||
class OSS:
|
||||
@staticmethod
|
||||
def should_test():
|
||||
do_test = env2bool("DVC_TEST_OSS", undefined=None)
|
||||
if do_test is not None:
|
||||
return do_test
|
||||
|
||||
return (
|
||||
os.getenv("OSS_ENDPOINT")
|
||||
and os.getenv("OSS_ACCESS_KEY_ID")
|
||||
and os.getenv("OSS_ACCESS_KEY_SECRET")
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_storagepath():
|
||||
return f"{TEST_OSS_REPO_BUCKET}/{uuid.uuid4()}"
|
||||
|
||||
@staticmethod
|
||||
def get_url():
|
||||
return f"oss://{OSS.get_storagepath()}"
|
||||
|
||||
|
||||
class SSH:
|
||||
@staticmethod
|
||||
def should_test():
|
||||
do_test = env2bool("DVC_TEST_SSH", undefined=None)
|
||||
if do_test is not None:
|
||||
return do_test
|
||||
|
||||
# FIXME: enable on windows
|
||||
if os.name == "nt":
|
||||
return False
|
||||
|
||||
try:
|
||||
check_output(["ssh", "-o", "BatchMode=yes", "127.0.0.1", "ls"])
|
||||
except (CalledProcessError, OSError):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def get_url():
|
||||
return "ssh://{}@127.0.0.1:22{}".format(
|
||||
getpass.getuser(), Local.get_storagepath()
|
||||
)
|
||||
|
||||
|
||||
class SSHMocked:
|
||||
should_test = always_test
|
||||
|
||||
@staticmethod
|
||||
def get_url(user, port):
|
||||
path = Local.get_storagepath()
|
||||
if os.name == "nt":
|
||||
# NOTE: On Windows Local.get_storagepath() will return an
|
||||
# ntpath that looks something like `C:\some\path`, which is not
|
||||
# compatible with SFTP paths [1], so we need to convert it to
|
||||
# a proper posixpath.
|
||||
# To do that, we should construct a posixpath that would be
|
||||
# relative to the server's root.
|
||||
# In our case our ssh server is running with `c:/` as a root,
|
||||
# and our URL format requires absolute paths, so the
|
||||
# resulting path would look like `/some/path`.
|
||||
#
|
||||
# [1]https://tools.ietf.org/html/draft-ietf-secsh-filexfer-13#section-6
|
||||
drive, path = os.path.splitdrive(path)
|
||||
assert drive.lower() == "c:"
|
||||
path = path.replace("\\", "/")
|
||||
url = f"ssh://{user}@127.0.0.1:{port}{path}"
|
||||
return url
|
||||
|
||||
|
||||
class HDFS:
|
||||
@staticmethod
|
||||
def should_test():
|
||||
if platform.system() != "Linux":
|
||||
return False
|
||||
|
||||
try:
|
||||
check_output(
|
||||
["hadoop", "version"],
|
||||
shell=True,
|
||||
executable=os.getenv("SHELL"),
|
||||
)
|
||||
except (CalledProcessError, OSError):
|
||||
return False
|
||||
|
||||
p = Popen(
|
||||
"hadoop fs -ls hdfs://127.0.0.1/",
|
||||
shell=True,
|
||||
executable=os.getenv("SHELL"),
|
||||
)
|
||||
p.communicate()
|
||||
if p.returncode != 0:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def get_url():
|
||||
return "hdfs://{}@127.0.0.1{}".format(
|
||||
getpass.getuser(), Local.get_storagepath()
|
||||
)
|
||||
|
||||
|
||||
class HTTP:
|
||||
should_test = always_test
|
||||
|
||||
@staticmethod
|
||||
def get_url(port):
|
||||
return f"http://127.0.0.1:{port}"
|
||||
@@ -0,0 +1,35 @@
|
||||
TEST_REMOTE = "upstream"
|
||||
TEST_CONFIG = {
|
||||
"cache": {},
|
||||
"core": {"remote": TEST_REMOTE},
|
||||
"remote": {TEST_REMOTE: {"url": ""}},
|
||||
}
|
||||
|
||||
from .azure import Azure, azure, azure_remote # noqa: F401
|
||||
from .hdfs import HDFS, hdfs, hdfs_remote # noqa: F401
|
||||
from .http import HTTP, http, http_remote, http_server # noqa: F401
|
||||
from .local import Local, local_cloud, local_remote # noqa: F401
|
||||
from .oss import OSS, TEST_OSS_REPO_BUCKET, oss, oss_remote # noqa: F401
|
||||
from .s3 import S3, TEST_AWS_REPO_BUCKET, S3Mocked, s3, s3_remote # noqa: F401
|
||||
|
||||
from .gdrive import ( # noqa: F401; noqa: F401
|
||||
TEST_GDRIVE_REPO_BUCKET,
|
||||
GDrive,
|
||||
gdrive,
|
||||
gdrive_remote,
|
||||
)
|
||||
from .gs import ( # noqa: F401; noqa: F401
|
||||
GCP,
|
||||
TEST_GCP_CREDS_FILE,
|
||||
TEST_GCP_REPO_BUCKET,
|
||||
gs,
|
||||
gs_remote,
|
||||
)
|
||||
from .ssh import ( # noqa: F401; noqa: F401
|
||||
SSH,
|
||||
SSHMocked,
|
||||
ssh,
|
||||
ssh_connection,
|
||||
ssh_remote,
|
||||
ssh_server,
|
||||
)
|
||||
@@ -0,0 +1,39 @@
|
||||
import os
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
|
||||
from dvc.utils import env2bool
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class Azure(Base):
|
||||
@staticmethod
|
||||
def should_test():
|
||||
do_test = env2bool("DVC_TEST_AZURE", undefined=None)
|
||||
if do_test is not None:
|
||||
return do_test
|
||||
|
||||
return os.getenv("AZURE_STORAGE_CONTAINER_NAME") and os.getenv(
|
||||
"AZURE_STORAGE_CONNECTION_STRING"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_url():
|
||||
container_name = os.getenv("AZURE_STORAGE_CONTAINER_NAME")
|
||||
assert container_name is not None
|
||||
return "azure://{}/{}".format(container_name, str(uuid.uuid4()))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def azure():
|
||||
if not Azure.should_test():
|
||||
pytest.skip("no azure running")
|
||||
yield Azure()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def azure_remote(tmp_dir, dvc, azure):
|
||||
tmp_dir.add_remote(config=azure.config)
|
||||
yield azure
|
||||
@@ -0,0 +1,23 @@
|
||||
from funcy import cached_property
|
||||
|
||||
|
||||
class Base:
|
||||
@staticmethod
|
||||
def should_test():
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def get_storagepath():
|
||||
raise NotImplementedError
|
||||
|
||||
@staticmethod
|
||||
def get_url():
|
||||
raise NotImplementedError
|
||||
|
||||
@cached_property
|
||||
def url(self):
|
||||
return self.get_url()
|
||||
|
||||
@cached_property
|
||||
def config(self):
|
||||
return {"url": self.url}
|
||||
@@ -0,0 +1,55 @@
|
||||
import os
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
from funcy import cached_property
|
||||
|
||||
from dvc.remote.gdrive import GDriveRemoteTree
|
||||
|
||||
from .base import Base
|
||||
|
||||
TEST_GDRIVE_REPO_BUCKET = "root"
|
||||
|
||||
|
||||
class GDrive(Base):
|
||||
@staticmethod
|
||||
def should_test():
|
||||
return os.getenv(GDriveRemoteTree.GDRIVE_CREDENTIALS_DATA) is not None
|
||||
|
||||
@cached_property
|
||||
def config(self):
|
||||
return {
|
||||
"url": self.url,
|
||||
"gdrive_service_account_email": "test",
|
||||
"gdrive_service_account_p12_file_path": "test.p12",
|
||||
"gdrive_use_service_account": True,
|
||||
}
|
||||
|
||||
def __init__(self, dvc):
|
||||
tree = GDriveRemoteTree(dvc, self.config)
|
||||
tree._gdrive_create_dir("root", tree.path_info.path)
|
||||
|
||||
@staticmethod
|
||||
def get_storagepath():
|
||||
return TEST_GDRIVE_REPO_BUCKET + "/" + str(uuid.uuid4())
|
||||
|
||||
@staticmethod
|
||||
def get_url():
|
||||
# NOTE: `get_url` should always return new random url
|
||||
return "gdrive://" + GDrive.get_storagepath()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def gdrive(make_tmp_dir):
|
||||
if not GDrive.should_test():
|
||||
pytest.skip("no gdrive")
|
||||
|
||||
# NOTE: temporary workaround
|
||||
tmp_dir = make_tmp_dir("gdrive", dvc=True)
|
||||
return GDrive(tmp_dir.dvc)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def gdrive_remote(tmp_dir, dvc, gdrive):
|
||||
tmp_dir.add_remote(config=gdrive.config)
|
||||
return gdrive
|
||||
@@ -0,0 +1,90 @@
|
||||
import os
|
||||
import uuid
|
||||
from contextlib import contextmanager
|
||||
|
||||
import pytest
|
||||
from funcy import cached_property
|
||||
|
||||
from dvc.remote.base import Remote
|
||||
from dvc.remote.gs import GSRemoteTree
|
||||
from dvc.utils import env2bool
|
||||
|
||||
from .base import Base
|
||||
|
||||
TEST_GCP_REPO_BUCKET = os.environ.get("DVC_TEST_GCP_REPO_BUCKET", "dvc-test")
|
||||
|
||||
TEST_GCP_CREDS_FILE = os.path.abspath(
|
||||
os.environ.get(
|
||||
"GOOGLE_APPLICATION_CREDENTIALS",
|
||||
os.path.join("scripts", "ci", "gcp-creds.json"),
|
||||
)
|
||||
)
|
||||
# Ensure that absolute path is used
|
||||
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = TEST_GCP_CREDS_FILE
|
||||
|
||||
|
||||
class GCP(Base):
|
||||
@staticmethod
|
||||
def should_test():
|
||||
from subprocess import CalledProcessError, check_output
|
||||
|
||||
do_test = env2bool("DVC_TEST_GCP", undefined=None)
|
||||
if do_test is not None:
|
||||
return do_test
|
||||
|
||||
if not os.path.exists(TEST_GCP_CREDS_FILE):
|
||||
return False
|
||||
|
||||
try:
|
||||
check_output(
|
||||
[
|
||||
"gcloud",
|
||||
"auth",
|
||||
"activate-service-account",
|
||||
"--key-file",
|
||||
TEST_GCP_CREDS_FILE,
|
||||
]
|
||||
)
|
||||
except (CalledProcessError, OSError):
|
||||
return False
|
||||
return True
|
||||
|
||||
@cached_property
|
||||
def config(self):
|
||||
return {
|
||||
"url": self.url,
|
||||
"credentialpath": TEST_GCP_CREDS_FILE,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def get_storagepath():
|
||||
return TEST_GCP_REPO_BUCKET + "/" + str(uuid.uuid4())
|
||||
|
||||
@staticmethod
|
||||
def get_url():
|
||||
return "gs://" + GCP.get_storagepath()
|
||||
|
||||
@classmethod
|
||||
@contextmanager
|
||||
def remote(cls, repo):
|
||||
yield Remote(GSRemoteTree(repo, {"url": cls.get_url()}))
|
||||
|
||||
@staticmethod
|
||||
def put_objects(remote, objects):
|
||||
client = remote.tree.gs
|
||||
bucket = client.get_bucket(remote.path_info.bucket)
|
||||
for key, body in objects.items():
|
||||
bucket.blob((remote.path_info / key).path).upload_from_string(body)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def gs():
|
||||
if not GCP.should_test():
|
||||
pytest.skip("no gs")
|
||||
yield GCP()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def gs_remote(tmp_dir, dvc, gs):
|
||||
tmp_dir.add_remote(config=gs.config)
|
||||
yield gs
|
||||
@@ -0,0 +1,55 @@
|
||||
import getpass
|
||||
import os
|
||||
import platform
|
||||
from subprocess import CalledProcessError, Popen, check_output
|
||||
|
||||
import pytest
|
||||
|
||||
from .base import Base
|
||||
from .local import Local
|
||||
|
||||
|
||||
class HDFS(Base):
|
||||
@staticmethod
|
||||
def should_test():
|
||||
if platform.system() != "Linux":
|
||||
return False
|
||||
|
||||
try:
|
||||
check_output(
|
||||
["hadoop", "version"],
|
||||
shell=True,
|
||||
executable=os.getenv("SHELL"),
|
||||
)
|
||||
except (CalledProcessError, OSError):
|
||||
return False
|
||||
|
||||
p = Popen(
|
||||
"hadoop fs -ls hdfs://127.0.0.1/",
|
||||
shell=True,
|
||||
executable=os.getenv("SHELL"),
|
||||
)
|
||||
p.communicate()
|
||||
if p.returncode != 0:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def get_url():
|
||||
return "hdfs://{}@127.0.0.1{}".format(
|
||||
getpass.getuser(), Local.get_storagepath()
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def hdfs():
|
||||
if not HDFS.should_test():
|
||||
pytest.skip("no hadoop running")
|
||||
yield HDFS()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def hdfs_remote(tmp_dir, dvc, hdfs):
|
||||
tmp_dir.add_remote(config=hdfs.config)
|
||||
yield hdfs
|
||||
@@ -0,0 +1,31 @@
|
||||
import pytest
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class HTTP(Base):
|
||||
@staticmethod
|
||||
def get_url(port):
|
||||
return f"http://127.0.0.1:{port}"
|
||||
|
||||
def __init__(self, server):
|
||||
self.url = self.get_url(server.server_port)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def http_server(tmp_dir):
|
||||
from tests.utils.httpd import PushRequestHandler, StaticFileServer
|
||||
|
||||
with StaticFileServer(handler_class=PushRequestHandler) as httpd:
|
||||
yield httpd
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def http(http_server):
|
||||
yield HTTP(http_server)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def http_remote(tmp_dir, dvc, http):
|
||||
tmp_dir.add_remote(config=http.config)
|
||||
yield http
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user