mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-18 23:10:14 +01:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a2f1367a9a | ||
|
|
5d97f92bdd | ||
|
|
02b147b6f4 | ||
|
|
fa0f182028 | ||
|
|
15c787f2ac | ||
|
|
4af3919062 | ||
|
|
3a469f2e93 | ||
|
|
51a8c782ae | ||
|
|
b579d06c8e | ||
|
|
039cd49259 | ||
|
|
f516dd252a | ||
|
|
706dd05e8d | ||
|
|
5d957b1830 | ||
|
|
387867fbbe |
+19
-5
@@ -1,5 +1,19 @@
|
||||
languages:
|
||||
Python: true
|
||||
|
||||
exclude_paths:
|
||||
- tests/
|
||||
version: "2"
|
||||
checks:
|
||||
argument-count:
|
||||
config:
|
||||
threshold: 8
|
||||
complex-logic:
|
||||
config:
|
||||
threshold: 6
|
||||
file-lines:
|
||||
config:
|
||||
threshold: 350
|
||||
method-complexity:
|
||||
config:
|
||||
threshold: 15
|
||||
method-count:
|
||||
config:
|
||||
threshold: 30
|
||||
exclude_patterns:
|
||||
- tests
|
||||
|
||||
@@ -4,10 +4,6 @@ test_patterns = [
|
||||
'tests/**'
|
||||
]
|
||||
|
||||
exclude_patterns = [
|
||||
'dvc/utils/compat.py'
|
||||
]
|
||||
|
||||
[[analyzers]]
|
||||
name = 'python'
|
||||
enabled = true
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
* [ ] ❗ I have followed the [Contributing to DVC](https://dvc.org/doc/user-guide/contributing/core) checklist.
|
||||
|
||||
* [ ] 📖 If this PR requires [documentation](https://dvc.org/doc) updates, I have created a separate PR (or issue, at least) in [dvc.org](https://github.com/iterative/dvc.org) and linked it here. If the CLI API is changed, I have updated [tab completion scripts](https://github.com/iterative/dvc/tree/master/scripts/completion).
|
||||
* [ ] 📖 If this PR requires [documentation](https://dvc.org/doc) updates, I have created a separate PR (or issue, at least) in [dvc.org](https://github.com/iterative/dvc.org) and linked it here.
|
||||
|
||||
* [ ] ❌ I will check DeepSource, CodeClimate, and other sanity checks below. (We consider them recommendatory and don't expect everything to be addressed. Please fix things that actually improve code or fix bugs.)
|
||||
|
||||
|
||||
+65
-60
@@ -1,63 +1,68 @@
|
||||
repos:
|
||||
- hooks:
|
||||
- id: black
|
||||
language_version: python3
|
||||
repo: https://github.com/ambv/black
|
||||
rev: 19.10b0
|
||||
- repo: https://github.com/asottile/seed-isort-config
|
||||
- hooks:
|
||||
- id: black
|
||||
language_version: python3
|
||||
repo: https://github.com/ambv/black
|
||||
rev: 19.10b0
|
||||
- repo: https://github.com/asottile/seed-isort-config
|
||||
rev: v2.1.1
|
||||
hooks:
|
||||
- id: seed-isort-config
|
||||
- hooks:
|
||||
- id: isort
|
||||
language_version: python3
|
||||
repo: https://github.com/timothycrosley/isort
|
||||
rev: 4.3.21
|
||||
- hooks:
|
||||
- id: flake8
|
||||
language_version: python3
|
||||
repo: https://gitlab.com/pycqa/flake8
|
||||
rev: 3.7.9
|
||||
- hooks:
|
||||
- args:
|
||||
- -i
|
||||
- '2'
|
||||
id: beautysh
|
||||
language_version: python3
|
||||
repo: https://github.com/lovesegfault/beautysh
|
||||
rev: master
|
||||
- hooks:
|
||||
- args:
|
||||
- git-hook
|
||||
- pre-commit
|
||||
entry: dvc
|
||||
id: dvc-pre-commit
|
||||
language: system
|
||||
name: DVC pre-commit
|
||||
stages:
|
||||
- commit
|
||||
verbose: true
|
||||
require_serial: true
|
||||
- args:
|
||||
- git-hook
|
||||
- pre-push
|
||||
entry: dvc
|
||||
id: dvc-pre-push
|
||||
language: system
|
||||
name: DVC pre-push
|
||||
stages:
|
||||
- push
|
||||
require_serial: true
|
||||
- always_run: true
|
||||
args:
|
||||
- git-hook
|
||||
- post-checkout
|
||||
entry: dvc
|
||||
id: dvc-post-checkout
|
||||
language: system
|
||||
minimum_pre_commit_version: 2.2.0
|
||||
name: DVC post-checkout
|
||||
stages:
|
||||
- post-checkout
|
||||
require_serial: true
|
||||
repo: local
|
||||
- id: seed-isort-config
|
||||
- hooks:
|
||||
- id: isort
|
||||
language_version: python3
|
||||
repo: https://github.com/timothycrosley/isort
|
||||
rev: 4.3.21
|
||||
- hooks:
|
||||
- id: flake8
|
||||
language_version: python3
|
||||
additional_dependencies:
|
||||
- flake8-bugbear
|
||||
- flake8-comprehensions
|
||||
- flake8-debugger
|
||||
- flake8-string-format
|
||||
repo: https://gitlab.com/pycqa/flake8
|
||||
rev: 3.7.9
|
||||
- hooks:
|
||||
- args:
|
||||
- -i
|
||||
- "2"
|
||||
id: beautysh
|
||||
language_version: python3
|
||||
repo: https://github.com/lovesegfault/beautysh
|
||||
rev: master
|
||||
- hooks:
|
||||
- args:
|
||||
- git-hook
|
||||
- pre-commit
|
||||
entry: dvc
|
||||
id: dvc-pre-commit
|
||||
language: system
|
||||
name: DVC pre-commit
|
||||
stages:
|
||||
- commit
|
||||
verbose: true
|
||||
require_serial: true
|
||||
- args:
|
||||
- git-hook
|
||||
- pre-push
|
||||
entry: dvc
|
||||
id: dvc-pre-push
|
||||
language: system
|
||||
name: DVC pre-push
|
||||
stages:
|
||||
- push
|
||||
require_serial: true
|
||||
- always_run: true
|
||||
args:
|
||||
- git-hook
|
||||
- post-checkout
|
||||
entry: dvc
|
||||
id: dvc-post-checkout
|
||||
language: system
|
||||
minimum_pre_commit_version: 2.2.0
|
||||
name: DVC post-checkout
|
||||
stages:
|
||||
- post-checkout
|
||||
require_serial: true
|
||||
repo: local
|
||||
|
||||
+16
-10
@@ -8,6 +8,7 @@ from .command import (
|
||||
cache,
|
||||
checkout,
|
||||
commit,
|
||||
completion,
|
||||
config,
|
||||
daemon,
|
||||
dag,
|
||||
@@ -70,6 +71,7 @@ COMMANDS = [
|
||||
dag,
|
||||
daemon,
|
||||
commit,
|
||||
completion,
|
||||
diff,
|
||||
version,
|
||||
update,
|
||||
@@ -140,15 +142,7 @@ def get_parent_parser():
|
||||
return parent_parser
|
||||
|
||||
|
||||
def parse_args(argv=None):
|
||||
"""Parses CLI arguments.
|
||||
|
||||
Args:
|
||||
argv: optional list of arguments to parse. sys.argv is used by default.
|
||||
|
||||
Raises:
|
||||
dvc.exceptions.DvcParserError: raised for argument parsing errors.
|
||||
"""
|
||||
def get_main_parser():
|
||||
parent_parser = get_parent_parser()
|
||||
|
||||
# Main parser
|
||||
@@ -196,6 +190,18 @@ def parse_args(argv=None):
|
||||
for cmd in COMMANDS:
|
||||
cmd.add_parser(subparsers, parent_parser)
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
return parser
|
||||
|
||||
|
||||
def parse_args(argv=None):
|
||||
"""Parses CLI arguments.
|
||||
|
||||
Args:
|
||||
argv: optional list of arguments to parse. sys.argv is used by default.
|
||||
|
||||
Raises:
|
||||
dvc.exceptions.DvcParserError: raised for argument parsing errors.
|
||||
"""
|
||||
parser = get_main_parser()
|
||||
args = parser.parse_args(argv)
|
||||
return args
|
||||
|
||||
+5
-1
@@ -1,6 +1,7 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException, RecursiveAddingWhileUsingFilename
|
||||
|
||||
@@ -62,6 +63,9 @@ def add_parser(subparsers, parent_parser):
|
||||
metavar="<filename>",
|
||||
)
|
||||
parser.add_argument(
|
||||
"targets", nargs="+", help="Input files/directories to add."
|
||||
"targets",
|
||||
nargs="+",
|
||||
help="Input files/directories to add.",
|
||||
choices=completion.Required.FILE,
|
||||
)
|
||||
parser.set_defaults(func=CmdAdd)
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import argparse
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import append_doc_link, fix_subparsers
|
||||
from dvc.command.config import CmdConfig
|
||||
|
||||
@@ -55,5 +56,6 @@ def add_parser(subparsers, parent_parser):
|
||||
help="Path to cache directory. Relative paths are resolved relative "
|
||||
"to the current directory and saved to config relative to the "
|
||||
"config file location.",
|
||||
choices=completion.Required.DIR,
|
||||
)
|
||||
cache_dir_parser.set_defaults(func=CmdCacheDir)
|
||||
|
||||
@@ -4,6 +4,7 @@ import operator
|
||||
|
||||
import colorama
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import CheckoutError
|
||||
from dvc.utils.humanize import get_summary
|
||||
@@ -113,5 +114,6 @@ def add_parser(subparsers, parent_parser):
|
||||
nargs="*",
|
||||
help="DVC-files to checkout. Optional. "
|
||||
"(Finds all DVC-files in the workspace by default.)",
|
||||
choices=completion.Optional.DVC_FILE,
|
||||
)
|
||||
checkout_parser.set_defaults(func=CmdCheckout)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
@@ -66,5 +67,6 @@ def add_parser(subparsers, parent_parser):
|
||||
nargs="*",
|
||||
help="DVC-files to commit. Optional. "
|
||||
"(Finds all DVC-files in the workspace by default.)",
|
||||
choices=completion.Optional.DVC_FILE,
|
||||
)
|
||||
commit_parser.set_defaults(func=CmdCommit)
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
import shtab
|
||||
|
||||
from dvc.command.base import CmdBaseNoRepo, append_doc_link
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
CHOICE_FUNCTIONS = {
|
||||
"bash": {"DVCFile": "_dvc_compgen_DVCFiles"},
|
||||
"zsh": {"DVCFile": "_files -g '(*?.dvc|Dvcfile|dvc.yaml)'"},
|
||||
}
|
||||
PREAMBLE = {
|
||||
"bash": """
|
||||
# $1=COMP_WORDS[1]
|
||||
_dvc_compgen_DVCFiles() {
|
||||
compgen -d -S '/' -- $1 # recurse into subdirs
|
||||
compgen -f -X '!*?.dvc' -- $1
|
||||
compgen -f -X '!*Dvcfile' -- $1
|
||||
compgen -f -X '!*dvc.yaml' -- $1
|
||||
}
|
||||
""",
|
||||
"zsh": "",
|
||||
}
|
||||
|
||||
|
||||
class Optional(shtab.Optional):
|
||||
DVC_FILE = [shtab.Choice("DVCFile", required=False)]
|
||||
|
||||
|
||||
class Required(shtab.Required):
|
||||
DVC_FILE = [shtab.Choice("DVCFile", required=True)]
|
||||
|
||||
|
||||
class CmdCompletion(CmdBaseNoRepo):
|
||||
def run(self):
|
||||
from dvc.cli import get_main_parser
|
||||
|
||||
parser = get_main_parser()
|
||||
shell = self.args.shell
|
||||
script = shtab.complete(
|
||||
parser,
|
||||
shell=shell,
|
||||
preamble=PREAMBLE[shell],
|
||||
choice_functions=CHOICE_FUNCTIONS[shell],
|
||||
)
|
||||
print(script)
|
||||
return 0
|
||||
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
COMPLETION_HELP = "Generate shell tab completion."
|
||||
COMPLETION_DESCRIPTION = "Prints out shell tab completion scripts."
|
||||
completion_parser = subparsers.add_parser(
|
||||
"completion",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(COMPLETION_DESCRIPTION, "completion"),
|
||||
help=COMPLETION_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
completion_parser.add_argument(
|
||||
"-s",
|
||||
"--shell",
|
||||
help="Shell syntax for completions.",
|
||||
default="bash",
|
||||
choices=["bash", "zsh"],
|
||||
)
|
||||
completion_parser.set_defaults(func=CmdCompletion)
|
||||
@@ -1,3 +1,4 @@
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBaseNoRepo, fix_subparsers
|
||||
|
||||
|
||||
@@ -64,5 +65,7 @@ def add_parser(subparsers, parent_parser):
|
||||
description=DAEMON_ANALYTICS_HELP,
|
||||
help=DAEMON_ANALYTICS_HELP,
|
||||
)
|
||||
daemon_analytics_parser.add_argument("target", help="Analytics file.")
|
||||
daemon_analytics_parser.add_argument(
|
||||
"target", help="Analytics file.", choices=completion.Required.FILE
|
||||
)
|
||||
daemon_analytics_parser.set_defaults(func=CmdDaemonAnalytics)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.command.checkout import log_changes
|
||||
from dvc.exceptions import CheckoutError, DvcException
|
||||
@@ -104,6 +105,7 @@ def shared_parent_parser():
|
||||
nargs="*",
|
||||
help="Limit command scope to these DVC-files. "
|
||||
"Using -R, directories to search DVC-files in can also be given.",
|
||||
choices=completion.Optional.DVC_FILE,
|
||||
)
|
||||
|
||||
return parent_parser
|
||||
|
||||
+1
-1
@@ -19,7 +19,7 @@ def _show_md(diff):
|
||||
entries = diff.get(status, [])
|
||||
if not entries:
|
||||
continue
|
||||
paths = sorted([entry["path"] for entry in entries])
|
||||
paths = sorted(entry["path"] for entry in entries)
|
||||
for path in paths:
|
||||
rows.append([status, path])
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
@@ -39,7 +40,10 @@ def add_parser(subparsers, parent_parser):
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
freeze_parser.add_argument(
|
||||
"targets", nargs="+", help="DVC-files to freeze."
|
||||
"targets",
|
||||
nargs="+",
|
||||
help="DVC-files to freeze.",
|
||||
choices=completion.Required.DVC_FILE,
|
||||
)
|
||||
freeze_parser.set_defaults(func=CmdFreeze)
|
||||
|
||||
@@ -52,6 +56,9 @@ def add_parser(subparsers, parent_parser):
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
unfreeze_parser.add_argument(
|
||||
"targets", nargs="+", help="DVC-files to unfreeze."
|
||||
"targets",
|
||||
nargs="+",
|
||||
help="DVC-files to unfreeze.",
|
||||
choices=completion.Required.DVC_FILE,
|
||||
)
|
||||
unfreeze_parser.set_defaults(func=CmdUnfreeze)
|
||||
|
||||
+5
-1
@@ -3,6 +3,7 @@ import logging
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
from . import completion
|
||||
from .base import CmdBaseNoRepo, append_doc_link
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -62,7 +63,9 @@ def add_parser(subparsers, parent_parser):
|
||||
"url", help="Location of DVC or Git repository to download from"
|
||||
)
|
||||
get_parser.add_argument(
|
||||
"path", help="Path to a file or directory within the repository"
|
||||
"path",
|
||||
help="Path to a file or directory within the repository",
|
||||
choices=completion.Required.FILE,
|
||||
)
|
||||
get_parser.add_argument(
|
||||
"-o",
|
||||
@@ -70,6 +73,7 @@ def add_parser(subparsers, parent_parser):
|
||||
nargs="?",
|
||||
help="Destination path to download files to",
|
||||
metavar="<path>",
|
||||
choices=completion.Optional.DIR,
|
||||
)
|
||||
get_parser.add_argument(
|
||||
"--rev",
|
||||
|
||||
@@ -3,6 +3,7 @@ import logging
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
from . import completion
|
||||
from .base import CmdBaseNoRepo, append_doc_link
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -33,6 +34,9 @@ def add_parser(subparsers, parent_parser):
|
||||
"url", help="See `dvc import-url -h` for full list of supported URLs."
|
||||
)
|
||||
get_parser.add_argument(
|
||||
"out", nargs="?", help="Destination path to put data to."
|
||||
"out",
|
||||
nargs="?",
|
||||
help="Destination path to put data to.",
|
||||
choices=completion.Optional.DIR,
|
||||
)
|
||||
get_parser.set_defaults(func=CmdGetUrl)
|
||||
|
||||
+5
-1
@@ -1,6 +1,7 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
@@ -43,7 +44,9 @@ def add_parser(subparsers, parent_parser):
|
||||
"url", help="Location of DVC or Git repository to download from"
|
||||
)
|
||||
import_parser.add_argument(
|
||||
"path", help="Path to a file or directory within the repository"
|
||||
"path",
|
||||
help="Path to a file or directory within the repository",
|
||||
choices=completion.Required.FILE,
|
||||
)
|
||||
import_parser.add_argument(
|
||||
"-o",
|
||||
@@ -51,6 +54,7 @@ def add_parser(subparsers, parent_parser):
|
||||
nargs="?",
|
||||
help="Destination path to download files to",
|
||||
metavar="<path>",
|
||||
choices=completion.Optional.DIR,
|
||||
)
|
||||
import_parser.add_argument(
|
||||
"--rev",
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
@@ -54,11 +55,15 @@ def add_parser(subparsers, parent_parser):
|
||||
"remote://myremote/path/to/file (see `dvc remote`)",
|
||||
)
|
||||
import_parser.add_argument(
|
||||
"out", nargs="?", help="Destination path to put files to."
|
||||
"out",
|
||||
nargs="?",
|
||||
help="Destination path to put files to.",
|
||||
choices=completion.Optional.DIR,
|
||||
)
|
||||
import_parser.add_argument(
|
||||
"--file",
|
||||
help="Specify name of the DVC-file this command will generate.",
|
||||
metavar="<filename>",
|
||||
choices=completion.Optional.DIR,
|
||||
)
|
||||
import_parser.set_defaults(func=CmdImportUrl)
|
||||
|
||||
@@ -2,6 +2,7 @@ import argparse
|
||||
import logging
|
||||
import sys
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBaseNoRepo, append_doc_link
|
||||
from dvc.command.ls.ls_colors import LsColors
|
||||
from dvc.exceptions import DvcException
|
||||
@@ -74,5 +75,6 @@ def add_parser(subparsers, parent_parser):
|
||||
"path",
|
||||
nargs="?",
|
||||
help="Path to directory within the repository to list outputs for",
|
||||
choices=completion.Optional.DIR,
|
||||
)
|
||||
list_parser.set_defaults(func=CmdList)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
|
||||
from dvc.exceptions import BadMetricError, DvcException
|
||||
|
||||
@@ -169,6 +170,7 @@ def add_parser(subparsers, parent_parser):
|
||||
"Limit command scope to these metric files. Using -R, "
|
||||
"directories to search metric files in can also be given."
|
||||
),
|
||||
choices=completion.Optional.FILE,
|
||||
)
|
||||
metrics_show_parser.add_argument(
|
||||
"-a",
|
||||
@@ -235,6 +237,7 @@ def add_parser(subparsers, parent_parser):
|
||||
"directories to search metric files in can also be given."
|
||||
),
|
||||
metavar="<paths>",
|
||||
choices=completion.Optional.FILE,
|
||||
)
|
||||
metrics_diff_parser.add_argument(
|
||||
"-R",
|
||||
|
||||
+7
-2
@@ -1,6 +1,7 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
@@ -36,7 +37,11 @@ def add_parser(subparsers, parent_parser):
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
move_parser.add_argument(
|
||||
"src", help="Source path to a data file or directory."
|
||||
"src",
|
||||
help="Source path to a data file or directory.",
|
||||
choices=completion.Required.FILE,
|
||||
)
|
||||
move_parser.add_argument(
|
||||
"dst", help="Destination path.", choices=completion.Required.FILE,
|
||||
)
|
||||
move_parser.add_argument("dst", help="Destination path.")
|
||||
move_parser.set_defaults(func=CmdMove)
|
||||
|
||||
@@ -2,6 +2,7 @@ import argparse
|
||||
import logging
|
||||
import os
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.schema import PLOT_PROPS
|
||||
@@ -128,6 +129,7 @@ def add_parser(subparsers, parent_parser):
|
||||
"targets",
|
||||
nargs="*",
|
||||
help="Plots files to visualize. Shows all plots by default.",
|
||||
choices=completion.Optional.FILE,
|
||||
)
|
||||
_add_props_arguments(plots_show_parser)
|
||||
_add_output_arguments(plots_show_parser)
|
||||
@@ -149,6 +151,7 @@ def add_parser(subparsers, parent_parser):
|
||||
nargs="*",
|
||||
help="Plots file to visualize. Shows all plots by default.",
|
||||
metavar="<path>",
|
||||
choices=completion.Optional.FILE,
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"revisions", nargs="*", default=None, help="Git commits to plot from",
|
||||
@@ -166,7 +169,9 @@ def add_parser(subparsers, parent_parser):
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
plots_modify_parser.add_argument(
|
||||
"target", help="Metric file to set properties to",
|
||||
"target",
|
||||
help="Metric file to set properties to",
|
||||
choices=completion.Required.FILE,
|
||||
)
|
||||
_add_props_arguments(plots_modify_parser)
|
||||
plots_modify_parser.add_argument(
|
||||
@@ -191,6 +196,7 @@ def _add_props_arguments(parser):
|
||||
)
|
||||
),
|
||||
metavar="<path>",
|
||||
choices=completion.Optional.FILE,
|
||||
)
|
||||
parser.add_argument(
|
||||
"-x", default=None, help="Field name for X axis.", metavar="<field>"
|
||||
@@ -223,6 +229,7 @@ def _add_output_arguments(parser):
|
||||
default=None,
|
||||
help="Destination path to save plots to",
|
||||
metavar="<path>",
|
||||
choices=completion.Optional.DIR,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--show-vega",
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
@@ -34,6 +35,9 @@ def add_parser(subparsers, parent_parser):
|
||||
help="Remove outputs as well.",
|
||||
)
|
||||
remove_parser.add_argument(
|
||||
"targets", nargs="+", help="DVC-files to remove."
|
||||
"targets",
|
||||
nargs="+",
|
||||
help="DVC-files to remove.",
|
||||
choices=completion.Required.DVC_FILE,
|
||||
)
|
||||
remove_parser.set_defaults(func=CmdRemove)
|
||||
|
||||
@@ -2,6 +2,7 @@ import argparse
|
||||
import logging
|
||||
import os
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.command.metrics import show_metrics
|
||||
from dvc.command.status import CmdDataStatus
|
||||
@@ -71,6 +72,7 @@ def add_parser(subparsers, parent_parser):
|
||||
"targets",
|
||||
nargs="*",
|
||||
help=f"Stages to reproduce. '{PIPELINE_FILE}' by default.",
|
||||
choices=completion.Optional.DVC_FILE,
|
||||
)
|
||||
repro_parser.add_argument(
|
||||
"-f",
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
@@ -32,6 +33,9 @@ def add_parser(subparsers, parent_parser):
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
unprotect_parser.add_argument(
|
||||
"targets", nargs="+", help="Data files/directories to unprotect."
|
||||
"targets",
|
||||
nargs="+",
|
||||
help="Data files/directories to unprotect.",
|
||||
choices=completion.Required.FILE,
|
||||
)
|
||||
unprotect_parser.set_defaults(func=CmdUnprotect)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
@@ -32,7 +33,10 @@ def add_parser(subparsers, parent_parser):
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
update_parser.add_argument(
|
||||
"targets", nargs="+", help="DVC-files to update."
|
||||
"targets",
|
||||
nargs="+",
|
||||
help="DVC-files to update.",
|
||||
choices=completion.Required.DVC_FILE,
|
||||
)
|
||||
update_parser.add_argument(
|
||||
"--rev",
|
||||
|
||||
+1
-1
@@ -222,7 +222,7 @@ def _build_sugiyama_layout(vertexes, edges):
|
||||
vertex.view = VertexViewer(vertex.data)
|
||||
|
||||
# NOTE: determine min box length to create the best layout
|
||||
minw = min([v.view.w for v in vertexes])
|
||||
minw = min(v.view.w for v in vertexes)
|
||||
|
||||
for edge in edges:
|
||||
edge.view = EdgeViewer()
|
||||
|
||||
+5
-2
@@ -39,6 +39,7 @@ class DvcIgnorePatterns(DvcIgnore):
|
||||
for ignore, group in groupby(
|
||||
regex_pattern_list, lambda x: x[1]
|
||||
)
|
||||
if ignore is not None
|
||||
]
|
||||
|
||||
def __call__(self, root, dirs, files):
|
||||
@@ -229,8 +230,10 @@ class CleanTree(BaseTree):
|
||||
return False
|
||||
return True
|
||||
|
||||
def walk(self, top, topdown=True):
|
||||
for root, dirs, files in self.tree.walk(top, topdown):
|
||||
def walk(self, top, topdown=True, onerror=None):
|
||||
for root, dirs, files in self.tree.walk(
|
||||
top, topdown=topdown, onerror=onerror
|
||||
):
|
||||
dirs[:], files[:] = self.dvcignore(
|
||||
os.path.abspath(root), dirs, files
|
||||
)
|
||||
|
||||
+8
-2
@@ -239,6 +239,12 @@ class BaseOutput:
|
||||
|
||||
self.repo.scm.ignore(self.fspath)
|
||||
|
||||
def ignore_remove(self):
|
||||
if not self.use_scm_ignore:
|
||||
return
|
||||
|
||||
self.repo.scm.ignore_remove(self.fspath)
|
||||
|
||||
def save(self):
|
||||
if not self.exists:
|
||||
raise self.DoesNotExistError(self)
|
||||
@@ -336,8 +342,8 @@ class BaseOutput:
|
||||
if self.scheme != "local":
|
||||
return
|
||||
|
||||
if ignore_remove and self.use_scm_ignore:
|
||||
self.repo.scm.ignore_remove(self.fspath)
|
||||
if ignore_remove:
|
||||
self.ignore_remove()
|
||||
|
||||
def move(self, out):
|
||||
if self.scheme == "local" and self.use_scm_ignore:
|
||||
|
||||
+18
-1
@@ -91,13 +91,30 @@ class GSRemoteTree(BaseRemoteTree):
|
||||
)
|
||||
|
||||
def _generate_download_url(self, path_info, expires=3600):
|
||||
import google.auth
|
||||
from google.auth import compute_engine
|
||||
|
||||
expiration = timedelta(seconds=int(expires))
|
||||
|
||||
bucket = self.gs.bucket(path_info.bucket)
|
||||
blob = bucket.get_blob(path_info.path)
|
||||
if blob is None:
|
||||
raise FileNotFoundError
|
||||
return blob.generate_signed_url(expiration=expiration)
|
||||
|
||||
if isinstance(
|
||||
blob.client._credentials, google.auth.credentials.Signing
|
||||
):
|
||||
# sign if we're able to sign with credentials.
|
||||
return blob.generate_signed_url(expiration=expiration)
|
||||
|
||||
auth_request = google.auth.transport.requests.Request()
|
||||
# create signing credentials with the default credentials
|
||||
# for use with Compute Engine and other environments where
|
||||
# Client credentials cannot sign.
|
||||
signing_credentials = compute_engine.IDTokenCredentials(
|
||||
auth_request, ""
|
||||
)
|
||||
return signing_credentials.signer.sign(blob)
|
||||
|
||||
def exists(self, path_info):
|
||||
"""Check if the blob exists. If it does not exist,
|
||||
|
||||
@@ -153,7 +153,7 @@ class SSHConnection:
|
||||
yield directory, dirs, nondirs
|
||||
|
||||
def walk_files(self, directory):
|
||||
for root, dirs, files in self.walk(directory):
|
||||
for root, _, files in self.walk(directory):
|
||||
for fname in files:
|
||||
yield posixpath.join(root, fname)
|
||||
|
||||
|
||||
@@ -485,7 +485,7 @@ class Repo:
|
||||
continue
|
||||
|
||||
dep_key = dep.path_info.parts
|
||||
overlapping = list(n.value for n in outs.prefixes(dep_key))
|
||||
overlapping = [n.value for n in outs.prefixes(dep_key)]
|
||||
if outs.has_subtrie(dep_key):
|
||||
overlapping.extend(outs.values(prefix=dep_key))
|
||||
|
||||
|
||||
+6
-1
@@ -50,13 +50,18 @@ def ls(
|
||||
def _ls(repo, path_info, recursive=None, dvc_only=False):
|
||||
from dvc.repo.tree import RepoTree
|
||||
|
||||
def onerror(exc):
|
||||
raise exc
|
||||
|
||||
# use our own RepoTree instance instead of repo.repo_tree since we do not
|
||||
# want fetch/stream enabled for ls
|
||||
tree = RepoTree(repo)
|
||||
|
||||
ret = {}
|
||||
try:
|
||||
for root, dirs, files in tree.walk(path_info.fspath, dvcfiles=True):
|
||||
for root, dirs, files in tree.walk(
|
||||
path_info.fspath, onerror=onerror, dvcfiles=True
|
||||
):
|
||||
for fname in files:
|
||||
info = PathInfo(root) / fname
|
||||
dvc = tree.isdvc(info)
|
||||
|
||||
@@ -55,9 +55,8 @@ class Plots:
|
||||
try:
|
||||
with tree.open(path_info) as fd:
|
||||
data[rev][datafile]["data"] = fd.read()
|
||||
except FileNotFoundError as e:
|
||||
except FileNotFoundError:
|
||||
# This might happen simply because cache is absent
|
||||
print(e)
|
||||
pass
|
||||
|
||||
return data
|
||||
|
||||
@@ -196,7 +196,7 @@ class CSVPlotData(PlotData):
|
||||
)
|
||||
|
||||
fieldnames = reader.fieldnames
|
||||
data = [row for row in reader]
|
||||
data = list(reader)
|
||||
|
||||
return [
|
||||
OrderedDict([(field, data_point[field]) for field in fieldnames])
|
||||
|
||||
+38
-40
@@ -27,13 +27,7 @@ class Template:
|
||||
INDENT = 4
|
||||
SEPARATORS = (",", ": ")
|
||||
EXTENSION = ".json"
|
||||
ANCHOR = '"<DVC_METRIC_{}>"'
|
||||
METRIC_DATA_ANCHOR = "<DVC_METRIC_DATA>"
|
||||
X_ANCHOR = "<DVC_METRIC_X>"
|
||||
Y_ANCHOR = "<DVC_METRIC_Y>"
|
||||
TITLE_ANCHOR = "<DVC_METRIC_TITLE>"
|
||||
X_LABEL_ANCHOR = "<DVC_METRIC_X_LABEL>"
|
||||
Y_LABEL_ANCHOR = "<DVC_METRIC_Y_LABEL>"
|
||||
ANCHOR = "<DVC_METRIC_{}>"
|
||||
|
||||
def __init__(self, content=None, name=None):
|
||||
self.content = self.DEFAULT_CONTENT if content is None else content
|
||||
@@ -43,10 +37,10 @@ class Template:
|
||||
def render(self, data, props=None):
|
||||
props = props or {}
|
||||
|
||||
if self.METRIC_DATA_ANCHOR not in self.content:
|
||||
if self._anchor_str("data") not in self.content:
|
||||
anchor = self.anchor("data")
|
||||
raise BadTemplateError(
|
||||
f"Template '{self.filename}' is not using "
|
||||
f"'{self.METRIC_DATA_ANCHOR}' anchor"
|
||||
f"Template '{self.filename}' is not using '{anchor}' anchor"
|
||||
)
|
||||
|
||||
if props.get("x"):
|
||||
@@ -59,19 +53,23 @@ class Template:
|
||||
|
||||
return content
|
||||
|
||||
@classmethod
|
||||
def anchor(cls, name):
|
||||
return cls.ANCHOR.format(name.upper())
|
||||
|
||||
def has_anchor(self, name):
|
||||
return self._anchor(name) in self.content
|
||||
return self._anchor_str(name) in self.content
|
||||
|
||||
@classmethod
|
||||
def _fill_anchor(cls, content, name, value):
|
||||
value_str = json.dumps(
|
||||
value, indent=cls.INDENT, separators=cls.SEPARATORS, sort_keys=True
|
||||
)
|
||||
return content.replace(cls._anchor(name), value_str)
|
||||
return content.replace(cls._anchor_str(name), value_str)
|
||||
|
||||
@classmethod
|
||||
def _anchor(cls, name):
|
||||
return cls.ANCHOR.format(name.upper())
|
||||
def _anchor_str(cls, name):
|
||||
return '"{}"'.format(cls.anchor(name))
|
||||
|
||||
@classmethod
|
||||
def _fill_metadata(cls, content, props):
|
||||
@@ -98,19 +96,19 @@ class DefaultLinearTemplate(Template):
|
||||
|
||||
DEFAULT_CONTENT = {
|
||||
"$schema": "https://vega.github.io/schema/vega-lite/v4.json",
|
||||
"data": {"values": Template.METRIC_DATA_ANCHOR},
|
||||
"title": Template.TITLE_ANCHOR,
|
||||
"data": {"values": Template.anchor("data")},
|
||||
"title": Template.anchor("title"),
|
||||
"mark": {"type": "line"},
|
||||
"encoding": {
|
||||
"x": {
|
||||
"field": Template.X_ANCHOR,
|
||||
"field": Template.anchor("x"),
|
||||
"type": "quantitative",
|
||||
"title": Template.X_LABEL_ANCHOR,
|
||||
"title": Template.anchor("x_label"),
|
||||
},
|
||||
"y": {
|
||||
"field": Template.Y_ANCHOR,
|
||||
"field": Template.anchor("y"),
|
||||
"type": "quantitative",
|
||||
"title": Template.Y_LABEL_ANCHOR,
|
||||
"title": Template.anchor("y_label"),
|
||||
"scale": {"zero": False},
|
||||
},
|
||||
"color": {"field": "rev", "type": "nominal"},
|
||||
@@ -122,21 +120,21 @@ class DefaultConfusionTemplate(Template):
|
||||
DEFAULT_NAME = "confusion"
|
||||
DEFAULT_CONTENT = {
|
||||
"$schema": "https://vega.github.io/schema/vega-lite/v4.json",
|
||||
"data": {"values": Template.METRIC_DATA_ANCHOR},
|
||||
"title": Template.TITLE_ANCHOR,
|
||||
"data": {"values": Template.anchor("data")},
|
||||
"title": Template.anchor("title"),
|
||||
"mark": "rect",
|
||||
"encoding": {
|
||||
"x": {
|
||||
"field": Template.X_ANCHOR,
|
||||
"field": Template.anchor("x"),
|
||||
"type": "nominal",
|
||||
"sort": "ascending",
|
||||
"title": Template.X_LABEL_ANCHOR,
|
||||
"title": Template.anchor("x_label"),
|
||||
},
|
||||
"y": {
|
||||
"field": Template.Y_ANCHOR,
|
||||
"field": Template.anchor("y"),
|
||||
"type": "nominal",
|
||||
"sort": "ascending",
|
||||
"title": Template.Y_LABEL_ANCHOR,
|
||||
"title": Template.anchor("y_label"),
|
||||
},
|
||||
"color": {"aggregate": "count", "type": "quantitative"},
|
||||
"facet": {"field": "rev", "type": "nominal"},
|
||||
@@ -148,19 +146,19 @@ class DefaultScatterTemplate(Template):
|
||||
DEFAULT_NAME = "scatter"
|
||||
DEFAULT_CONTENT = {
|
||||
"$schema": "https://vega.github.io/schema/vega-lite/v4.json",
|
||||
"data": {"values": Template.METRIC_DATA_ANCHOR},
|
||||
"title": Template.TITLE_ANCHOR,
|
||||
"data": {"values": Template.anchor("data")},
|
||||
"title": Template.anchor("title"),
|
||||
"mark": "point",
|
||||
"encoding": {
|
||||
"x": {
|
||||
"field": Template.X_ANCHOR,
|
||||
"field": Template.anchor("x"),
|
||||
"type": "quantitative",
|
||||
"title": Template.X_LABEL_ANCHOR,
|
||||
"title": Template.anchor("x_label"),
|
||||
},
|
||||
"y": {
|
||||
"field": Template.Y_ANCHOR,
|
||||
"field": Template.anchor("y"),
|
||||
"type": "quantitative",
|
||||
"title": Template.Y_LABEL_ANCHOR,
|
||||
"title": Template.anchor("y_label"),
|
||||
"scale": {"zero": False},
|
||||
},
|
||||
"color": {"field": "rev", "type": "nominal"},
|
||||
@@ -173,27 +171,27 @@ class SmoothLinearTemplate(Template):
|
||||
|
||||
DEFAULT_CONTENT = {
|
||||
"$schema": "https://vega.github.io/schema/vega-lite/v4.json",
|
||||
"data": {"values": Template.METRIC_DATA_ANCHOR},
|
||||
"title": Template.TITLE_ANCHOR,
|
||||
"data": {"values": Template.anchor("data")},
|
||||
"title": Template.anchor("title"),
|
||||
"mark": {"type": "line"},
|
||||
"encoding": {
|
||||
"x": {
|
||||
"field": Template.X_ANCHOR,
|
||||
"field": Template.anchor("x"),
|
||||
"type": "quantitative",
|
||||
"title": Template.X_LABEL_ANCHOR,
|
||||
"title": Template.anchor("x_label"),
|
||||
},
|
||||
"y": {
|
||||
"field": Template.Y_ANCHOR,
|
||||
"field": Template.anchor("y"),
|
||||
"type": "quantitative",
|
||||
"title": Template.Y_LABEL_ANCHOR,
|
||||
"title": Template.anchor("y_label"),
|
||||
"scale": {"zero": False},
|
||||
},
|
||||
"color": {"field": "rev", "type": "nominal"},
|
||||
},
|
||||
"transform": [
|
||||
{
|
||||
"loess": Template.Y_ANCHOR,
|
||||
"on": Template.X_ANCHOR,
|
||||
"loess": Template.anchor("y"),
|
||||
"on": Template.anchor("x"),
|
||||
"groupby": ["rev"],
|
||||
"bandwidth": 0.3,
|
||||
}
|
||||
|
||||
+30
-18
@@ -142,7 +142,7 @@ class DvcTree(BaseTree):
|
||||
files = []
|
||||
|
||||
root_len = len(root.parts)
|
||||
for key, out in trie.iteritems(prefix=root.parts):
|
||||
for key, out in trie.iteritems(prefix=root.parts): # noqa: B301
|
||||
if key == root.parts:
|
||||
continue
|
||||
|
||||
@@ -153,25 +153,29 @@ class DvcTree(BaseTree):
|
||||
|
||||
files.append(name)
|
||||
|
||||
if topdown:
|
||||
dirs = list(dirs)
|
||||
yield root.fspath, dirs, files
|
||||
assert topdown
|
||||
dirs = list(dirs)
|
||||
yield root.fspath, dirs, files
|
||||
|
||||
for dname in dirs:
|
||||
yield from self._walk(root / dname, trie)
|
||||
else:
|
||||
assert False
|
||||
for dname in dirs:
|
||||
yield from self._walk(root / dname, trie)
|
||||
|
||||
def walk(self, top, topdown=True, download_callback=None, **kwargs):
|
||||
def walk(
|
||||
self, top, topdown=True, onerror=None, download_callback=None, **kwargs
|
||||
):
|
||||
from pygtrie import Trie
|
||||
|
||||
assert topdown
|
||||
|
||||
if not self.exists(top):
|
||||
raise FileNotFoundError
|
||||
if onerror is not None:
|
||||
onerror(FileNotFoundError(top))
|
||||
return
|
||||
|
||||
if not self.isdir(top):
|
||||
raise NotADirectoryError
|
||||
if onerror is not None:
|
||||
onerror(NotADirectoryError(top))
|
||||
return
|
||||
|
||||
root = PathInfo(os.path.abspath(top))
|
||||
outs = self._find_outs(top, recursive=True, strict=False)
|
||||
@@ -333,12 +337,14 @@ class RepoTree(BaseTree):
|
||||
elif dirname in repo_set:
|
||||
yield from self._walk_one(repo_walk)
|
||||
|
||||
def walk(self, top, topdown=True, dvcfiles=False, **kwargs):
|
||||
def walk(self, top, topdown=True, onerror=None, dvcfiles=False, **kwargs):
|
||||
"""Walk and merge both DVC and repo trees.
|
||||
|
||||
Args:
|
||||
top: path to walk from
|
||||
topdown: if True, tree will be walked from top down.
|
||||
onerror: if set, onerror function will be called if an error
|
||||
occurs (by default errors are ignored).
|
||||
dvcfiles: if True, dvcfiles will be included in the files list
|
||||
for walked directories.
|
||||
|
||||
@@ -348,21 +354,27 @@ class RepoTree(BaseTree):
|
||||
assert topdown
|
||||
|
||||
if not self.exists(top):
|
||||
raise FileNotFoundError
|
||||
if onerror is not None:
|
||||
onerror(FileNotFoundError(top))
|
||||
return
|
||||
|
||||
if not self.isdir(top):
|
||||
raise NotADirectoryError
|
||||
if onerror is not None:
|
||||
onerror(NotADirectoryError(top))
|
||||
return
|
||||
|
||||
dvc_exists = self.dvctree and self.dvctree.exists(top)
|
||||
repo_exists = self.repo.tree.exists(top)
|
||||
if dvc_exists and not repo_exists:
|
||||
yield from self.dvctree.walk(top, topdown=topdown, **kwargs)
|
||||
yield from self.dvctree.walk(
|
||||
top, topdown=topdown, onerror=onerror, **kwargs
|
||||
)
|
||||
return
|
||||
if repo_exists and not dvc_exists:
|
||||
yield from self.repo.tree.walk(top, topdown=topdown)
|
||||
yield from self.repo.tree.walk(
|
||||
top, topdown=topdown, onerror=onerror
|
||||
)
|
||||
return
|
||||
if not dvc_exists and not repo_exists:
|
||||
raise FileNotFoundError
|
||||
|
||||
dvc_walk = self.dvctree.walk(top, topdown=topdown, **kwargs)
|
||||
repo_walk = self.repo.tree.walk(top, topdown=topdown)
|
||||
|
||||
@@ -51,9 +51,6 @@ def _edit_rwlock(lock_dir):
|
||||
yield lock
|
||||
with open(path, "w+") as fobj:
|
||||
json.dump(lock, fobj)
|
||||
# NOTE: flush and fsync to ensure that rwlock contents are saved
|
||||
fobj.flush()
|
||||
os.fsync(fobj.fileno())
|
||||
|
||||
|
||||
def _infos_to_str(infos):
|
||||
|
||||
+16
-18
@@ -37,24 +37,22 @@ class TqdmGit(Tqdm):
|
||||
def code2desc(op_code):
|
||||
from git import RootUpdateProgress as OP
|
||||
|
||||
ops = dict(
|
||||
(
|
||||
(OP.COUNTING, "Counting"),
|
||||
(OP.COMPRESSING, "Compressing"),
|
||||
(OP.WRITING, "Writing"),
|
||||
(OP.RECEIVING, "Receiving"),
|
||||
(OP.RESOLVING, "Resolving"),
|
||||
(OP.FINDING_SOURCES, "Finding sources"),
|
||||
(OP.CHECKING_OUT, "Checking out"),
|
||||
(OP.CLONE, "Cloning"),
|
||||
(OP.FETCH, "Fetching"),
|
||||
(OP.UPDWKTREE, "Updating working tree"),
|
||||
(OP.REMOVE, "Removing"),
|
||||
(OP.PATHCHANGE, "Changing path"),
|
||||
(OP.URLCHANGE, "Changing URL"),
|
||||
(OP.BRANCHCHANGE, "Changing branch"),
|
||||
)
|
||||
)
|
||||
ops = {
|
||||
OP.COUNTING: "Counting",
|
||||
OP.COMPRESSING: "Compressing",
|
||||
OP.WRITING: "Writing",
|
||||
OP.RECEIVING: "Receiving",
|
||||
OP.RESOLVING: "Resolving",
|
||||
OP.FINDING_SOURCES: "Finding sources",
|
||||
OP.CHECKING_OUT: "Checking out",
|
||||
OP.CLONE: "Cloning",
|
||||
OP.FETCH: "Fetching",
|
||||
OP.UPDWKTREE: "Updating working tree",
|
||||
OP.REMOVE: "Removing",
|
||||
OP.PATHCHANGE: "Changing path",
|
||||
OP.URLCHANGE: "Changing URL",
|
||||
OP.BRANCHCHANGE: "Changing branch",
|
||||
}
|
||||
return ops.get(op_code & OP.OP_MASK, "")
|
||||
|
||||
|
||||
|
||||
+8
-3
@@ -128,17 +128,22 @@ class GitTree(BaseTree):
|
||||
if not topdown:
|
||||
yield os.path.normpath(tree.abspath), dirs, nondirs
|
||||
|
||||
def walk(self, top, topdown=True):
|
||||
def walk(self, top, topdown=True, onerror=None):
|
||||
"""Directory tree generator.
|
||||
|
||||
See `os.walk` for the docs. Differences:
|
||||
- no support for symlinks
|
||||
- it could raise exceptions, there is no onerror argument
|
||||
"""
|
||||
|
||||
tree = self.git_object_by_path(top)
|
||||
if tree is None:
|
||||
raise OSError(errno.ENOENT, "No such file")
|
||||
if onerror is not None:
|
||||
onerror(OSError(errno.ENOENT, "No such file", top))
|
||||
return
|
||||
if tree.mode != GIT_MODE_DIR:
|
||||
if onerror is not None:
|
||||
onerror(NotADirectoryError(top))
|
||||
return
|
||||
|
||||
yield from self._walk(tree, topdown=topdown)
|
||||
|
||||
|
||||
+4
-9
@@ -21,12 +21,11 @@ class BaseTree:
|
||||
def isfile(self, path):
|
||||
"""Test whether a path is a regular file"""
|
||||
|
||||
def walk(self, top, topdown=True):
|
||||
def walk(self, top, topdown=True, onerror=None):
|
||||
"""Directory tree generator.
|
||||
|
||||
See `os.walk` for the docs. Differences:
|
||||
- no support for symlinks
|
||||
- it could raise exceptions, there is no onerror argument
|
||||
"""
|
||||
|
||||
def walk_files(self, top):
|
||||
@@ -39,7 +38,8 @@ class BaseTree:
|
||||
class WorkingTree(BaseTree):
|
||||
"""Proxies the repo file access methods to working tree files"""
|
||||
|
||||
def __init__(self, repo_root=os.getcwd()):
|
||||
def __init__(self, repo_root=None):
|
||||
repo_root = repo_root or os.getcwd()
|
||||
self.repo_root = repo_root
|
||||
|
||||
@property
|
||||
@@ -64,17 +64,12 @@ class WorkingTree(BaseTree):
|
||||
"""Test whether a path is a regular file"""
|
||||
return os.path.isfile(path)
|
||||
|
||||
def walk(self, top, topdown=True):
|
||||
def walk(self, top, topdown=True, onerror=None):
|
||||
"""Directory tree generator.
|
||||
|
||||
See `os.walk` for the docs. Differences:
|
||||
- no support for symlinks
|
||||
- it could raise exceptions, there is no onerror argument
|
||||
"""
|
||||
|
||||
def onerror(e):
|
||||
raise e
|
||||
|
||||
for root, dirs, files in os.walk(
|
||||
top, topdown=topdown, onerror=onerror
|
||||
):
|
||||
|
||||
+10
-3
@@ -3,7 +3,7 @@ import os
|
||||
import string
|
||||
from collections import defaultdict
|
||||
|
||||
from funcy import project
|
||||
from funcy import cached_property, project
|
||||
|
||||
import dvc.dependency as dependency
|
||||
import dvc.prompt as prompt
|
||||
@@ -127,6 +127,8 @@ class Stage(params.StageParams):
|
||||
@path.setter
|
||||
def path(self, path):
|
||||
self._path = path
|
||||
self.__dict__.pop("path_in_repo", None)
|
||||
self.__dict__.pop("relpath", None)
|
||||
|
||||
@property
|
||||
def dvcfile(self):
|
||||
@@ -172,11 +174,11 @@ class Stage(params.StageParams):
|
||||
and self.path_in_repo == other.path_in_repo
|
||||
)
|
||||
|
||||
@property
|
||||
@cached_property
|
||||
def path_in_repo(self):
|
||||
return relpath(self.path, self.repo.root_dir)
|
||||
|
||||
@property
|
||||
@cached_property
|
||||
def relpath(self):
|
||||
return relpath(self.path)
|
||||
|
||||
@@ -284,12 +286,17 @@ class Stage(params.StageParams):
|
||||
for out in self.outs:
|
||||
out.unprotect()
|
||||
|
||||
def ignore_remove_outs(self):
|
||||
for out in self.outs:
|
||||
out.ignore_remove()
|
||||
|
||||
@rwlocked(write=["outs"])
|
||||
def remove(self, force=False, remove_outs=True, purge=True):
|
||||
if remove_outs:
|
||||
self.remove_outs(ignore_remove=True, force=force)
|
||||
else:
|
||||
self.unprotect_outs()
|
||||
self.ignore_remove_outs()
|
||||
if purge:
|
||||
self.dvcfile.remove_stage(self)
|
||||
|
||||
|
||||
@@ -111,7 +111,7 @@ def _serialize_params_values(params: List[ParamsDependency]):
|
||||
def to_pipeline_file(stage: "PipelineStage"):
|
||||
wdir = resolve_wdir(stage.wdir, stage.path)
|
||||
params, deps = split_params_deps(stage)
|
||||
deps = sorted([d.def_path for d in deps])
|
||||
deps = sorted(d.def_path for d in deps)
|
||||
params = _serialize_params_keys(params)
|
||||
|
||||
outs, metrics, plots = _serialize_outs(stage.outs)
|
||||
|
||||
+1
-1
@@ -6,7 +6,7 @@
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
_BASE_VERSION = "1.0.0b4"
|
||||
_BASE_VERSION = "1.0.0b6"
|
||||
|
||||
|
||||
def _generate_version(base_version):
|
||||
|
||||
@@ -115,10 +115,10 @@ build_dvc() {
|
||||
# [1] https://github.com/iterative/dvc/issues/2585
|
||||
if [[ "$(uname)" == 'Linux' ]]; then
|
||||
mkdir -p $BUILD_DIR/$BASH_CMPLT_DIR
|
||||
cp scripts/completion/dvc.bash $BUILD_DIR/$BASH_CMPLT_DIR/dvc
|
||||
$LIB_DIR/dvc/dvc completion -s bash > $BUILD_DIR/$BASH_CMPLT_DIR/dvc
|
||||
|
||||
mkdir -p $BUILD_DIR/$ZSH_CMPLT_DIR
|
||||
cp scripts/completion/dvc.zsh $BUILD_DIR/$ZSH_CMPLT_DIR
|
||||
$LIB_DIR/dvc/dvc completion -s zsh > $BUILD_DIR/$ZSH_CMPLT_DIR/_dvc
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
@@ -1,146 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# References:
|
||||
# - https://www.gnu.org/software/bash/manual/html_node/Programmable-Completion.html
|
||||
# - https://opensource.com/article/18/3/creating-bash-completion-script
|
||||
# - https://stackoverflow.com/questions/12933362
|
||||
|
||||
_dvc_commands='add cache checkout commit config dag destroy diff fetch freeze \
|
||||
get-url get gc import-url import init install list metrics move params \
|
||||
plots pull push remote remove repro root run status unfreeze unprotect \
|
||||
update version'
|
||||
|
||||
_dvc_options='-h --help -V --version'
|
||||
_dvc_global_options='-h --help -q --quiet -v --verbose'
|
||||
|
||||
_dvc_add='-R --recursive -f --file --no-commit --external'
|
||||
_dvc_add_COMPGEN=_dvc_compgen_files
|
||||
_dvc_cache='dir'
|
||||
_dvc_cache_dir=' --global --system --local -u --unset'
|
||||
_dvc_checkout='-d --with-deps -R --recursive -f --force --relink --summary'
|
||||
_dvc_checkout_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_commit='-f --force -d --with-deps -R --recursive'
|
||||
_dvc_commit_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_config='-u --unset --local --system --global'
|
||||
_dvc_dag='--dot --full'
|
||||
_dvc_dag_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_destroy='-f --force'
|
||||
_dvc_diff='-t --show-json --show-hash --show-md'
|
||||
_dvc_fetch='-j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -R --recursive'
|
||||
_dvc_fetch_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_gc='-a --all-branches --all-commits -T --all-tags -w --workspace -c --cloud -r --remote -f --force -p --projects -j --jobs'
|
||||
_dvc_get='-o --out --rev --show-url'
|
||||
_dvc_get_url=''
|
||||
_dvc_import='-o --out --rev'
|
||||
_dvc_import_url='-f --file'
|
||||
_dvc_init='--no-scm -f --force'
|
||||
_dvc_install=''
|
||||
_dvc_list='-R --recursive --dvc-only --rev'
|
||||
_dvc_list_COMPGEN=_dvc_compgen_files
|
||||
_dvc_freeze=''
|
||||
_dvc_freeze_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_metrics='diff show'
|
||||
_dvc_metrics_diff='--targets -t --type -x --xpath -R --show-json --show-md --no-path --old'
|
||||
_dvc_metrics_show='-t --type -x --xpath -a --all-branches -T --all-tags -R --recursive'
|
||||
_dvc_metrics_show_COMPGEN=_dvc_compgen_files
|
||||
_dvc_move=''
|
||||
_dvc_move_COMPGEN=_dvc_compgen_files
|
||||
_dvc_params='diff'
|
||||
_dvc_params_diff='--all --show-json --show-md --no-path'
|
||||
_dvc_plots='show diff'
|
||||
_dvc_plots_show='-t --template -o --out -x -y --show-vega --no-csv-header --title --x-label --y-label'
|
||||
_dvc_plots_diff='-t --template --targets -o --out -x -y --show-vega --no-csv-header --title --x-label --y-label'
|
||||
_dvc_plots_modify='-t --template -x -y --no-csv-header --title --x-label --y-label'
|
||||
_dvc_pull='-j --jobs -r --remote -a --all-branches -T --all-tags -f --force -d --with-deps -R --recursive'
|
||||
_dvc_pull_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_push='-j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -R --recursive'
|
||||
_dvc_push_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_remote='add default list modify remove rename'
|
||||
_dvc_remote_add='--global --system --local -d --default -f --force'
|
||||
_dvc_remote_default='--global --system --local -u --unset'
|
||||
_dvc_remote_list='--global --system --local'
|
||||
_dvc_remote_modify='--global --system --local -u --unset'
|
||||
_dvc_remote_remove='--global --system --local'
|
||||
_dvc_remote_rename='--global --system --local'
|
||||
_dvc_remove='-o --outs -p --purge -f --force'
|
||||
_dvc_remove_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_repro='-f --force -s --single-item -c --cwd -m --metrics --dry -i --interactive -p --pipeline -P --all-pipelines --no-run-cache --force-downstream --no-commit -R --recursive --downstream'
|
||||
_dvc_repro_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_root=''
|
||||
_dvc_run='--no-exec -f --file -d --deps -o --outs -O --outs-no-cache --outs-persist --outs-persist-no-cache -m --metrics -M --metrics-no-cache --overwrite-dvcfile --no-run-cache --no-commit -w --wdir --external'
|
||||
_dvc_run_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_status='-j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -c --cloud'
|
||||
_dvc_status_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_unfreeze_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_unprotect_COMPGEN=_dvc_compgen_files
|
||||
_dvc_update='--rev'
|
||||
_dvc_update_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_version=''
|
||||
|
||||
# $1=COMP_WORDS[1]
|
||||
_dvc_compgen_DVCFiles() {
|
||||
compgen -f -X '!*?.dvc' -- $1
|
||||
compgen -d -S '/' -- $1 # recurse into subdirs
|
||||
# Note that the recurse into dirs is only for looking for DVC-files.
|
||||
# Since dirs themselves are not required, we need `-o nospace` at the bottom
|
||||
# unfortunately :(
|
||||
}
|
||||
|
||||
# $1=COMP_WORDS[1]
|
||||
_dvc_compgen_files() {
|
||||
compgen -f -- $1
|
||||
compgen -d -S '/' -- $1 # recurse into subdirs
|
||||
}
|
||||
|
||||
# $1=COMP_WORDS[1]
|
||||
_dvc_replace_hyphen() {
|
||||
echo $1 | sed 's/-/_/g'
|
||||
}
|
||||
|
||||
# $1=COMP_WORDS[1]
|
||||
_dvc_compgen_command() {
|
||||
local flags_list="_dvc_$(_dvc_replace_hyphen $1)"
|
||||
local args_gen="${flags_list}_COMPGEN"
|
||||
COMPREPLY=( $(compgen -W "$_dvc_global_options ${!flags_list}" -- "$word"; [ -n "${!args_gen}" ] && ${!args_gen} "$word") )
|
||||
}
|
||||
|
||||
# $1=COMP_WORDS[1]
|
||||
# $2=COMP_WORDS[2]
|
||||
_dvc_compgen_subcommand() {
|
||||
local flags_list="_dvc_$(_dvc_replace_hyphen $1)_$(_dvc_replace_hyphen $2)"
|
||||
local args_gen="${flags_list}_COMPGEN"
|
||||
[ -n "${!args_gen}" ] && local opts_more="$(${!args_gen} "$word")"
|
||||
local opts="${!flags_list}"
|
||||
if [ -z "$opts$opts_more" ]; then
|
||||
_dvc_compgen_command $1
|
||||
else
|
||||
COMPREPLY=( $(compgen -W "$_dvc_global_options $opts" -- "$word"; [ -n "$opts_more" ] && echo "$opts_more") )
|
||||
fi
|
||||
}
|
||||
|
||||
# Notes:
|
||||
# `COMPREPLY` contains what will be rendered after completion is triggered
|
||||
# `word` refers to the current typed word
|
||||
# `${!var}` is to evaluate the content of `var` and expand its content as a variable
|
||||
# hello="world"
|
||||
# x="hello"
|
||||
# ${!x} -> ${hello} -> "world"
|
||||
_dvc() {
|
||||
local word="${COMP_WORDS[COMP_CWORD]}"
|
||||
|
||||
COMPREPLY=()
|
||||
|
||||
if [ "${COMP_CWORD}" -eq 1 ]; then
|
||||
case "$word" in
|
||||
-*) COMPREPLY=($(compgen -W "$_dvc_options" -- "$word")) ;;
|
||||
*) COMPREPLY=($(compgen -W "$_dvc_commands" -- "$word")) ;;
|
||||
esac
|
||||
elif [ "${COMP_CWORD}" -eq 2 ]; then
|
||||
_dvc_compgen_command ${COMP_WORDS[1]}
|
||||
elif [ "${COMP_CWORD}" -ge 3 ]; then
|
||||
_dvc_compgen_subcommand ${COMP_WORDS[1]} ${COMP_WORDS[2]}
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
complete -o nospace -F _dvc dvc
|
||||
@@ -1,340 +0,0 @@
|
||||
#compdef dvc
|
||||
|
||||
#----------------------------------------------------------
|
||||
# Repository: https://github.com/iterative/dvc
|
||||
#
|
||||
# References:
|
||||
# - https://github.com/zsh-users/zsh-completions
|
||||
# - http://zsh.sourceforge.net/Doc/Release/Completion-System.html
|
||||
# - https://mads-hartmann.com/2017/08/06/writing-zsh-completion-scripts.html
|
||||
# - http://www.linux-mag.com/id/1106/
|
||||
#----------------------------------------------------------
|
||||
|
||||
_dvc_commands() {
|
||||
local _commands=(
|
||||
"add:Track data files or directories with DVC."
|
||||
"cache:Manage cache settings."
|
||||
"checkout:Checkout data files from cache."
|
||||
"commit:Save changed data to cache and update DVC-files."
|
||||
"config:Get or set config settings."
|
||||
"dag:Visualize DVC project DAG."
|
||||
"destroy:Remove DVC-files, local DVC config and data cache."
|
||||
"diff:Show added, modified, or deleted data between commits in the DVC repository, or between a commit and the workspace."
|
||||
"fetch:Get files or directories tracked by DVC from remote storage into the cache."
|
||||
"get-url:Download or copy files from URL."
|
||||
"get:Download data from DVC repository."
|
||||
"gc:Garbage collect unused objects from cache or remote storage."
|
||||
"import-url:Download or copy file from URL and take it under DVC control."
|
||||
"import:Download data from DVC repository and take it under DVC control."
|
||||
"init:Initialize DVC in the current directory."
|
||||
"install:Install DVC git hooks into the repository."
|
||||
"list:List repository contents, including files and directories tracked by DVC and by Git."
|
||||
"lock:Lock DVC-file."
|
||||
"metrics:Commands to display and compare metrics."
|
||||
"move:Rename or move a DVC controlled data file or a directory."
|
||||
"params:Commands to display params."
|
||||
"pull:Pull data files from a DVC remote storage."
|
||||
"push:Push data files to a DVC remote storage."
|
||||
"plots:Generate plot for metrics structured as JSON, CSV or TSV."
|
||||
"remote:Manage remote storage configuration."
|
||||
"remove:Remove outputs of DVC-file."
|
||||
"repro:Check for changes and reproduce DVC-file and dependencies."
|
||||
"root:Relative path to project's directory."
|
||||
"run:Generate a stage file from a command and execute the command."
|
||||
"status:Show changed stages, compare local cache and a remote storage."
|
||||
"unlock:Unlock DVC-file."
|
||||
"unprotect:Unprotect data file/directory."
|
||||
"update:Update data artifacts imported from other DVC repositories."
|
||||
"version:Show DVC version and system/environment information."
|
||||
)
|
||||
|
||||
_describe 'dvc commands' _commands
|
||||
}
|
||||
|
||||
_dvc_options=(
|
||||
"(-)"{-h,--help}"[Show help message.]"
|
||||
"(-)"{-V,--version}"[Show program's version]"
|
||||
)
|
||||
|
||||
_dvc_global_options=(
|
||||
"(-)"{-h,--help}"[Show help message related to the command.]"
|
||||
"(-)"{-q,--quiet}"[Be quiet.]"
|
||||
"(-)"{-v,--verbose}"[Be verbose.]"
|
||||
)
|
||||
|
||||
_dvc_add=(
|
||||
{-R,--recursive}"[Recursively add each file under the directory.]"
|
||||
"--no-commit[Don't put files/directories into cache.]"
|
||||
{-f,--file}"[Specify name of the DVC-file it generates.]:File:_files"
|
||||
"--external[Allow targets that are outside of the DVC project.]"
|
||||
"1:File:_files"
|
||||
)
|
||||
|
||||
_dvc_cache=(
|
||||
"1:Sub command:(dir)"
|
||||
)
|
||||
|
||||
_dvc_checkout=(
|
||||
{-d,--with-deps}"[Checkout all dependencies of the specified target.]"
|
||||
{-R,--recursive}"[Checkout all subdirectories of the specified directory.]"
|
||||
{-f,--force}"[Do not prompt when removing working directory files.]"
|
||||
"--relink[Recreate links or copies from cache to workspace.]"
|
||||
"--summary[Show summary of the changes.]"
|
||||
"1:Stages:_files -g '(*.dvc|Dvcfile)'"
|
||||
)
|
||||
|
||||
_dvc_commit=(
|
||||
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
|
||||
{-f,--force}"[Commit even if checksums for dependencies/outputs changed.]"
|
||||
{-d,--with-deps}"[Commit all dependencies of the specified target.]"
|
||||
{-R,--recursive}"[Commit cache for subdirectories of the specified directory.]"
|
||||
)
|
||||
|
||||
_dvc_config=(
|
||||
"--global[Use global config.]"
|
||||
"--system[Use system config.]"
|
||||
"--local[Use local config.]"
|
||||
{-u,--unset}"[Unset option.]"
|
||||
)
|
||||
|
||||
_dvc_dag=(
|
||||
"--dot[Print DAG in DOT format.]"
|
||||
"--full[Show full DAG that the target belongs too, instead of showing DAG consisting only of ancestors.]"
|
||||
"1:Stage:"
|
||||
)
|
||||
|
||||
_dvc_destroy=(
|
||||
{-f,--force}"[Force destruction.]"
|
||||
)
|
||||
|
||||
_dvc_diff=(
|
||||
"--show-json[Format the output into a JSON]"
|
||||
"--show-hash[Display hash value for each entry]"
|
||||
"--show-md[Format the output into a Markdown table]"
|
||||
"1:Old Git commit to compare (defaults to HEAD):"
|
||||
"2:New Git commit to compare (defaults to the current workspace):"
|
||||
)
|
||||
|
||||
_dvc_fetch=(
|
||||
{-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:"
|
||||
{-r,--remote}"[Remote repository to fetch from.]:Remote repository:"
|
||||
{-a,--all-branches}"[Fetch cache for all branches.]"
|
||||
{-T,--all-tags}"[Fetch cache for all tags.]"
|
||||
{-d,--with-deps}"[Fetch cache for all dependencies of the specified target.]"
|
||||
{-R,--recursive}"[Fetch cache for subdirectories of specified directory.]"
|
||||
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
|
||||
)
|
||||
|
||||
_dvc_geturl=(
|
||||
"1:URL:"
|
||||
"2:Output:"
|
||||
)
|
||||
|
||||
_dvc_get=(
|
||||
{-o,--out}"[Destination path to put data to.]:OUT:_files -/"
|
||||
"--rev[Git revision (e.g. SHA, branch, tag)]:Revision:"
|
||||
"--show-url[Returns path/url to the location in remote for given path]"
|
||||
"1:URL:"
|
||||
"2:Path:"
|
||||
)
|
||||
|
||||
_dvc_gc=(
|
||||
{-w,--workspace}"[Keep data files used in the current workspace.]"
|
||||
{-a,--all-branches}"[Keep data files for the tips of all Git branches.]"
|
||||
"--all-commits[Keep data files for all Git commits.]"
|
||||
{-T,--all-tags}"[Keep data files for all Git tags.]"
|
||||
{-c,--cloud}"[Collect garbage in remote repository.]"
|
||||
{-r,--remote}"[Remote storage to collect garbage in.]:Remote repository:"
|
||||
{-f,--force}"[Force garbage collection - automatically agree to all prompts.]:Repos:_files"
|
||||
{-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:"
|
||||
{-p,--projects}"[Keep data files required by these projects in addition to the current one.]:Repos:_files"
|
||||
)
|
||||
|
||||
_dvc_importurl=(
|
||||
{-f,--file}"[Specify name of the DVC-file it generates.]:File:_files"
|
||||
"1:URL:"
|
||||
"2:Output:"
|
||||
)
|
||||
|
||||
_dvc_import=(
|
||||
{-o,--out}"[Destination path to put data to.]:OUT:_files -/"
|
||||
"--rev[Git revision (e.g. SHA, branch, tag)]:Commit hash:"
|
||||
"1:URL:"
|
||||
"2:Path:"
|
||||
)
|
||||
|
||||
_dvc_init=(
|
||||
"--no-scm[Initiate dvc in directory that is not tracked by any scm tool.]"
|
||||
{-f,--force}"[Overwrite existing '.dvc' directory. This operation removes local cache.]"
|
||||
)
|
||||
|
||||
_dvc_install=()
|
||||
|
||||
_dvc_freeze=(
|
||||
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
|
||||
)
|
||||
|
||||
_dvc_list=(
|
||||
"--rev[Git revision (e.g. branch, tag, SHA)]:Revision:"
|
||||
{-R,--recursive}"[Recursively add each file under the directory.]"
|
||||
"--dvc-only[Only outputs DVC-outs.]"
|
||||
"1:URL:"
|
||||
"2:Path:"
|
||||
)
|
||||
|
||||
_dvc_metrics=(
|
||||
"1:Sub command:(show diff)"
|
||||
)
|
||||
|
||||
_dvc_move=(
|
||||
"1:Source:_files"
|
||||
"2:Destination:"
|
||||
)
|
||||
|
||||
_dvc_params=(
|
||||
"1:Sub command:(diff)"
|
||||
)
|
||||
|
||||
_dvc_pull=(
|
||||
{-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:"
|
||||
{-r,--remote}"[Remote repository to pull from.]:Remote repository:"
|
||||
{-a,--all-branches}"[Fetch cache for all branches.]"
|
||||
{-T,--all-tags}"[Fetch cache for all tags.]"
|
||||
{-d,--with-deps}"[Fetch cache for all dependencies of the specified target.]"
|
||||
{-f,--force}"[Do not prompt when removing working directory files.]"
|
||||
{-R,--recursive}"[Pull cache for subdirectories of the specified directory.]"
|
||||
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
|
||||
)
|
||||
|
||||
_dvc_push=(
|
||||
{-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:"
|
||||
{-r,--remote}"[Remote repository to push to.]:Remote repository:"
|
||||
{-a,--all-branches}"[Push cache for all branches.]"
|
||||
{-T,--all-tags}"[Push cache for all tags.]"
|
||||
{-d,--with-deps}"[Push cache for all dependencies of the specified target.]"
|
||||
{-R,--recursive}"[Push cache for subdirectories of specified directory.]"
|
||||
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
|
||||
)
|
||||
|
||||
_dvc_plots=(
|
||||
"1:Sub command:(show diff modify)"
|
||||
)
|
||||
|
||||
_dvc_remote=(
|
||||
"1:Sub command:(add default remove modify list rename)"
|
||||
)
|
||||
|
||||
_dvc_remove=(
|
||||
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
|
||||
"--dry[Only print the commands that would be executed without actually executing]"
|
||||
{-o,--outs}"[Only remove DVC-file outputs. (Default)]"
|
||||
{-p,--purge}"[Remove DVC-file and all its outputs.]"
|
||||
{-f,--force}"[Force purge.]"
|
||||
)
|
||||
|
||||
_dvc_repro=(
|
||||
{-f,--force}"[Reproduce even if dependencies were not changed.]"
|
||||
{-s,--single-item}"[Reproduce only single data item without recursive dependencies check.]"
|
||||
{-c,--cwd}"[Directory within your repo to reproduce from.]:CWD:_files -/"
|
||||
{-m,--metrics}"[Show metrics after reproduction.]"
|
||||
"--dry[Only print the commands that would be executed without actually executing]"
|
||||
{-i,--interactive}"[Ask for confirmation before reproducing each stage.]"
|
||||
{-p,--pipeline}"[Reproduce the whole pipeline that the specified stage file belongs to.]"
|
||||
{-P,--all-pipelines}"[Reproduce all pipelines in the repo.]"
|
||||
{-R,--recursive}"[Reproduce all stages in the specified directory.]"
|
||||
"--force-downstream[Reproduce all descendants of a changed stage even if their direct dependencies didn't change.]"
|
||||
"--no-run-cache[Run changed stage even if it has been already ran with the same command/dependencies/outputs/etc before.]"
|
||||
"--no-commit[Don't put files/directories into cache.]"
|
||||
"--downstream[Start from the specified stages when reproducing pipelines.]"
|
||||
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
|
||||
)
|
||||
|
||||
_dvc_root=()
|
||||
|
||||
_dvc_run=(
|
||||
"*"{-d,--deps}"[Declare dependencies for reproducible cmd.]:Dependency:_files"
|
||||
"*"{-o,--outs}"[Declare output file or directory.]:Output data:_files"
|
||||
"*"{-O,--outs-no-cache}"[Declare output file or directory (do not put into DVC cache).]:Output regular:_files"
|
||||
"*"{-m,--metrics}"[Declare output metric file or directory.]:Metrics:_files"
|
||||
"*"{-M,--metrics-no-cache}"[Declare output metric file or directory (do not put into DVC cache).]:Metrics (no cache):_files"
|
||||
{-f,--file}"[Specify name of the DVC-file it generates.]:File:_files"
|
||||
{-c,--cwd}"[Deprecated, use -w and -f instead.]:CWD:_files -/"
|
||||
{-w,--wdir}"[Directory within your repo to run your command in.]:WDIR:_files -/"
|
||||
"--no-exec[Only create stage file without actually running it.]"
|
||||
{-y,--yes}"[Deprecated, use --overwrite-dvcfile instead]"
|
||||
"--overwrite-dvcfile[Overwrite existing DVC-file without asking for confirmation.]"
|
||||
"--no-run-cache[Run this stage even if it has been already ran with the same command/dependencies/outputs/etc before.]"
|
||||
"--remove-outs[Deprecated, this is now the default behavior]"
|
||||
"--no-commit[Don't put files/directories into cache.]"
|
||||
"--outs-persist[Declare output file or directory that will not be removed upon repro.]:Output persistent:_files"
|
||||
"--outs-persist-no-cache[Declare output file or directory that will not be removed upon repro (do not put into DVC cache).]:Output persistent regular:_files"
|
||||
"--external[Allow outputs that are outside of the DVC project.]"
|
||||
)
|
||||
|
||||
_dvc_status=(
|
||||
{-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:"
|
||||
{-q,--quiet}"[Suppresses all output. Exit with 0 if pipelines are up to date, otherwise 1.]"
|
||||
{-c,--cloud}"[Show status of a local cache compared to a remote repository.]"
|
||||
{-r,--remote}"[Remote repository to compare local cache to.]:Remote repository:"
|
||||
{-a,--all-branches}"[Show status of a local cache compared to a remote repository for all branches.]"
|
||||
{-T,--all-tags}"[Show status of a local cache compared to a remote repository for all tags.]"
|
||||
{-d,--with-deps}"[Show status for all dependencies of the specified target.]"
|
||||
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
|
||||
)
|
||||
|
||||
_dvc_unfreeze=(
|
||||
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
|
||||
)
|
||||
|
||||
_dvc_unprotect=(
|
||||
"*:Data files:_files"
|
||||
)
|
||||
|
||||
_dvc_update=(
|
||||
"--rev[Git revision (e.g. SHA, branch, tag)]:Revision:"
|
||||
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
|
||||
)
|
||||
|
||||
typeset -A opt_args
|
||||
local context state line curcontext="$curcontext"
|
||||
|
||||
_arguments \
|
||||
$_dvc_options \
|
||||
'1: :_dvc_commands' \
|
||||
'*::args:->args'
|
||||
|
||||
case $words[1] in
|
||||
add) _arguments $_dvc_global_options $_dvc_add ;;
|
||||
cache) _arguments $_dvc_global_options $_dvc_cache ;;
|
||||
checkout) _arguments $_dvc_global_options $_dvc_checkout ;;
|
||||
commit) _arguments $_dvc_global_options $_dvc_commit ;;
|
||||
config) _arguments $_dvc_global_options $_dvc_config ;;
|
||||
dag) _arguments $_dvc_global_options $_dvc_dag ;;
|
||||
destroy) _arguments $_dvc_global_options $_dvc_destroy ;;
|
||||
diff) _arguments $_dvc_global_options $_dvc_diff ;;
|
||||
fetch) _arguments $_dvc_global_options $_dvc_fetch ;;
|
||||
freeze) _arguments $_dvc_global_options $_dvc_freeze ;;
|
||||
get-url) _arguments $_dvc_global_options $_dvc_geturl ;;
|
||||
get) _arguments $_dvc_global_options $_dvc_get ;;
|
||||
gc) _arguments $_dvc_global_options $_dvc_gc ;;
|
||||
import-url) _arguments $_dvc_global_options $_dvc_importurl ;;
|
||||
import) _arguments $_dvc_global_options $_dvc_import ;;
|
||||
init) _arguments $_dvc_global_options $_dvc_init ;;
|
||||
install) _arguments $_dvc_global_options $_dvc_install ;;
|
||||
list) _arguments $_dvc_global_options $_dvc_list ;;
|
||||
metrics) _arguments $_dvc_global_options $_dvc_metrics ;;
|
||||
move) _arguments $_dvc_global_options $_dvc_move ;;
|
||||
params) _arguments $_dvc_global_options $_dvc_params ;;
|
||||
pull) _arguments $_dvc_global_options $_dvc_pull ;;
|
||||
push) _arguments $_dvc_global_options $_dvc_push ;;
|
||||
plots) _arguments $_dvc_global_options $_dvc_plots ;;
|
||||
remote) _arguments $_dvc_global_options $_dvc_remote ;;
|
||||
remove) _arguments $_dvc_global_options $_dvc_remove ;;
|
||||
repro) _arguments $_dvc_global_options $_dvc_repro ;;
|
||||
root) _arguments $_dvc_global_options $_dvc_root ;;
|
||||
run) _arguments $_dvc_global_options $_dvc_run ;;
|
||||
status) _arguments $_dvc_global_options $_dvc_status ;;
|
||||
unfreeze) _arguments $_dvc_global_options $_dvc_unfreeze ;;
|
||||
unprotect) _arguments $_dvc_global_options $_dvc_unprotect ;;
|
||||
update) _arguments $_dvc_global_options $_dvc_update ;;
|
||||
esac
|
||||
@@ -1,6 +1,3 @@
|
||||
md5: 3cc286c534a71504476da009ed174423
|
||||
outs:
|
||||
- cache: true
|
||||
md5: 90104d9e83cfb825cf45507e90aadd27
|
||||
metric: false
|
||||
- md5: 90104d9e83cfb825cf45507e90aadd27
|
||||
path: dvc.ico
|
||||
|
||||
@@ -1,6 +1,3 @@
|
||||
md5: c7c2944695a3eb7f04a0a7ae20c9dc0c
|
||||
outs:
|
||||
- cache: true
|
||||
md5: 9106cda08aa427e73492389a0f17c72d
|
||||
metric: false
|
||||
- md5: 9106cda08aa427e73492389a0f17c72d
|
||||
path: dvc_left.bmp
|
||||
|
||||
@@ -1,6 +1,3 @@
|
||||
md5: cde0fe37927a5771b1ba8d1c909e120c
|
||||
outs:
|
||||
- cache: true
|
||||
md5: 94614d6650e062655f9f77507dc9c1f2
|
||||
metric: false
|
||||
- md5: 94614d6650e062655f9f77507dc9c1f2
|
||||
path: dvc_up.bmp
|
||||
|
||||
@@ -6,13 +6,18 @@ ignore=
|
||||
E203, # Whitespace before ':'
|
||||
E266, # Too many leading '#' for block comment
|
||||
W503, # Line break occurred before a binary operator
|
||||
P1, # unindexed parameters in the str.format, see:
|
||||
# https://pypi.org/project/flake8-string-format/
|
||||
max_line_length=79
|
||||
select=B,C,E,F,W,T4,B9
|
||||
max-complexity = 15
|
||||
select=B,C,E,F,W,T4,B902,T,P
|
||||
show_source=true
|
||||
count=true
|
||||
|
||||
[isort]
|
||||
include_trailing_comma=true
|
||||
known_first_party=dvc,tests
|
||||
known_third_party=PyInstaller,RangeHTTPServer,boto3,colorama,configobj,distro,dpath,flaky,flufl,funcy,git,google,grandalf,mock,mockssh,moto,nanotime,networkx,packaging,paramiko,pathspec,pytest,requests,ruamel,setuptools,shortuuid,tqdm,voluptuous,yaml,zc
|
||||
known_third_party=PyInstaller,RangeHTTPServer,boto3,colorama,configobj,distro,dpath,flaky,flufl,funcy,git,google,grandalf,mock,moto,nanotime,networkx,packaging,paramiko,pathspec,pytest,requests,ruamel,setuptools,shortuuid,shtab,tqdm,voluptuous,yaml,zc
|
||||
line_length=79
|
||||
force_grid_wrap=0
|
||||
use_parentheses=True
|
||||
|
||||
@@ -78,6 +78,7 @@ install_requires = [
|
||||
"tabulate>=0.8.7",
|
||||
"pygtrie==2.3.2",
|
||||
"dpath>=2.0.1,<3",
|
||||
"shtab>=0.0.2",
|
||||
]
|
||||
|
||||
|
||||
@@ -106,6 +107,7 @@ tests_requirements = [
|
||||
"pytest-cov>=2.6.1",
|
||||
"pytest-xdist>=1.26.1",
|
||||
"pytest-mock==1.11.2",
|
||||
"pytest-lazy-fixture",
|
||||
"flaky>=3.5.3",
|
||||
"mock>=3.0.0",
|
||||
"xmltodict>=0.11.0",
|
||||
@@ -122,6 +124,9 @@ tests_requirements = [
|
||||
"moto==1.3.14.dev464",
|
||||
"rangehttpserver==1.2.0",
|
||||
"beautifulsoup4==4.4.0",
|
||||
"flake8-bugbear",
|
||||
"flake8-comprehensions",
|
||||
"flake8-string-format",
|
||||
]
|
||||
|
||||
if (sys.version_info) >= (3, 6):
|
||||
|
||||
+8
-4
@@ -31,17 +31,21 @@ parts:
|
||||
snapcraftctl pull
|
||||
snapcraftctl set-version $(cd $SNAPCRAFT_PART_SRC && git describe --tags)
|
||||
git diff --quiet || error_dirty_build
|
||||
echo 'PKG = "snap"' > $SNAPCRAFT_PART_SRC/dvc/utils/build.py
|
||||
echo 'PKG = "snap"' > dvc/utils/build.py
|
||||
# install all optional extras
|
||||
sed -ri 's/(=install_requires)/\1+all_remotes+hdfs/' $SNAPCRAFT_PART_SRC/setup.py
|
||||
sed -ri 's/(=install_requires)/\1+all_remotes+hdfs/' setup.py
|
||||
# ensure dvc knows the state isn't really dirty
|
||||
sed -rin 's/.*git.*diff.*--quiet.*//' $SNAPCRAFT_PART_SRC/dvc/version.py
|
||||
sed -rin 's/.*git.*diff.*--quiet.*//' dvc/version.py
|
||||
override-build: |
|
||||
snapcraftctl build
|
||||
# prevent user site packages interfering with this snap - reference:
|
||||
# https://github.com/snapcore/snapcraft/blob/19393ef36cd773a28131cec10cc0bfb3bf9c7e77/tools/snapcraft-override-build.sh#L18
|
||||
sed -ri 's/^(ENABLE_USER_SITE = )None$/\1False/' $SNAPCRAFT_PART_INSTALL/usr/lib/python*/site.py
|
||||
cp $SNAPCRAFT_PART_BUILD/scripts/completion/dvc.bash $SNAPCRAFT_PART_INSTALL/completion.sh
|
||||
# make plugin aware that a file (generated later) needs to be staged
|
||||
touch $SNAPCRAFT_PART_INSTALL/completion.sh
|
||||
override-stage: |
|
||||
snapcraftctl stage
|
||||
usr/bin/python3 -m dvc completion -s bash > completion.sh
|
||||
apps:
|
||||
dvc:
|
||||
command: bin/dvc
|
||||
|
||||
+1
-34
@@ -1,12 +1,9 @@
|
||||
import os
|
||||
|
||||
import mockssh
|
||||
import pytest
|
||||
|
||||
from dvc.remote.ssh.connection import SSHConnection
|
||||
from tests.utils.httpd import PushRequestHandler, StaticFileServer
|
||||
|
||||
from .dir_helpers import * # noqa
|
||||
from .remotes import * # noqa
|
||||
|
||||
# Prevent updater and analytics from running their processes
|
||||
os.environ["DVC_TEST"] = "true"
|
||||
@@ -28,39 +25,9 @@ def reset_loglevel(request, caplog):
|
||||
yield
|
||||
|
||||
|
||||
here = os.path.abspath(os.path.dirname(__file__))
|
||||
|
||||
user = "user"
|
||||
key_path = os.path.join(here, f"{user}.key")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def ssh_server():
|
||||
users = {user: key_path}
|
||||
with mockssh.Server(users) as s:
|
||||
s.test_creds = {
|
||||
"host": s.host,
|
||||
"port": s.port,
|
||||
"username": user,
|
||||
"key_filename": key_path,
|
||||
}
|
||||
yield s
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def ssh(ssh_server):
|
||||
yield SSHConnection(**ssh_server.test_creds)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
def _close_pools():
|
||||
from dvc.remote.pool import close_pools
|
||||
|
||||
yield
|
||||
close_pools()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def http_server(tmp_dir):
|
||||
with StaticFileServer(handler_class=PushRequestHandler) as httpd:
|
||||
yield httpd
|
||||
|
||||
+21
-18
@@ -63,7 +63,6 @@ __all__ = [
|
||||
"run_head",
|
||||
"erepo_dir",
|
||||
"git_dir",
|
||||
"setup_remote",
|
||||
"git_init",
|
||||
]
|
||||
|
||||
@@ -177,6 +176,27 @@ class TmpDir(pathlib.Path):
|
||||
if commit:
|
||||
self.scm.commit(commit)
|
||||
|
||||
def add_remote(
|
||||
self, *, url=None, config=None, name="upstream", default=True
|
||||
):
|
||||
self._require("dvc")
|
||||
|
||||
assert bool(url) ^ bool(config)
|
||||
|
||||
if url:
|
||||
config = {"url": url}
|
||||
|
||||
with self.dvc.config.edit() as conf:
|
||||
conf["remote"][name] = config
|
||||
if default:
|
||||
conf["core"]["remote"] = name
|
||||
|
||||
if hasattr(self, "scm"):
|
||||
self.scm.add(self.dvc.config.files["repo"])
|
||||
self.scm.commit(f"add '{name}' remote")
|
||||
|
||||
return url or config["url"]
|
||||
|
||||
# contexts
|
||||
@contextmanager
|
||||
def chdir(self):
|
||||
@@ -315,20 +335,3 @@ def git_dir(make_tmp_dir):
|
||||
path = make_tmp_dir("git-erepo", scm=True)
|
||||
path.scm.commit("init repo")
|
||||
return path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def setup_remote(make_tmp_dir):
|
||||
def create(repo, url=None, name="upstream", default=True):
|
||||
if not url:
|
||||
url = os.fspath(make_tmp_dir("local_remote"))
|
||||
with repo.config.edit() as conf:
|
||||
conf["remote"][name] = {"url": url}
|
||||
if default:
|
||||
conf["core"]["remote"] = name
|
||||
|
||||
repo.scm.add(repo.config.files["repo"])
|
||||
repo.scm.commit(f"add '{name}' remote")
|
||||
return url
|
||||
|
||||
return create
|
||||
|
||||
+32
-90
@@ -5,93 +5,42 @@ import pytest
|
||||
from dvc import api
|
||||
from dvc.api import UrlNotDvcRepoError
|
||||
from dvc.exceptions import FileMissingError
|
||||
from dvc.main import main
|
||||
from dvc.path_info import URLInfo
|
||||
from dvc.utils.fs import remove
|
||||
from tests.remotes import (
|
||||
GCP,
|
||||
HDFS,
|
||||
OSS,
|
||||
S3,
|
||||
SSH,
|
||||
TEST_REMOTE,
|
||||
Azure,
|
||||
GDrive,
|
||||
Local,
|
||||
)
|
||||
|
||||
remote_params = [S3, GCP, Azure, GDrive, OSS, SSH, HDFS]
|
||||
all_remote_params = [Local] + remote_params
|
||||
cloud_names = [
|
||||
"s3",
|
||||
"gs",
|
||||
"azure",
|
||||
"gdrive",
|
||||
"oss",
|
||||
"ssh",
|
||||
"hdfs",
|
||||
"http",
|
||||
]
|
||||
clouds = [pytest.lazy_fixture(cloud) for cloud in cloud_names]
|
||||
all_clouds = [pytest.lazy_fixture("local")] + clouds
|
||||
remotes = [pytest.lazy_fixture(f"{cloud}_remote") for cloud in cloud_names]
|
||||
all_remotes = [pytest.lazy_fixture("local_remote")] + remotes
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def remote_url(request):
|
||||
if not request.param.should_test():
|
||||
raise pytest.skip()
|
||||
return request.param.get_url()
|
||||
|
||||
|
||||
def run_dvc(*argv):
|
||||
assert main(argv) == 0
|
||||
|
||||
|
||||
def ensure_dir(dvc, url):
|
||||
if url.startswith("gdrive://"):
|
||||
GDrive.create_dir(dvc, url)
|
||||
run_dvc(
|
||||
"remote",
|
||||
"modify",
|
||||
TEST_REMOTE,
|
||||
"gdrive_service_account_email",
|
||||
"test",
|
||||
)
|
||||
run_dvc(
|
||||
"remote",
|
||||
"modify",
|
||||
TEST_REMOTE,
|
||||
"gdrive_service_account_p12_file_path",
|
||||
"test.p12",
|
||||
)
|
||||
run_dvc(
|
||||
"remote",
|
||||
"modify",
|
||||
TEST_REMOTE,
|
||||
"gdrive_use_service_account",
|
||||
"True",
|
||||
)
|
||||
|
||||
|
||||
def ensure_dir_scm(dvc, url):
|
||||
if url.startswith("gdrive://"):
|
||||
GDrive.create_dir(dvc, url)
|
||||
with dvc.config.edit() as conf:
|
||||
conf["remote"][TEST_REMOTE].update(
|
||||
gdrive_service_account_email="test",
|
||||
gdrive_service_account_p12_file_path="test.p12",
|
||||
gdrive_use_service_account=True,
|
||||
)
|
||||
dvc.scm.add(dvc.config.files["repo"])
|
||||
dvc.scm.commit(f"modify '{TEST_REMOTE}' remote")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("remote_url", remote_params, indirect=True)
|
||||
def test_get_url(tmp_dir, dvc, remote_url):
|
||||
run_dvc("remote", "add", "-d", TEST_REMOTE, remote_url)
|
||||
@pytest.mark.parametrize("remote", remotes)
|
||||
def test_get_url(tmp_dir, dvc, request, remote):
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
|
||||
expected_url = URLInfo(remote_url) / "ac/bd18db4cc2f85cedef654fccc4a4d8"
|
||||
expected_url = URLInfo(remote.url) / "ac/bd18db4cc2f85cedef654fccc4a4d8"
|
||||
assert api.get_url("foo") == expected_url
|
||||
|
||||
|
||||
@pytest.mark.parametrize("remote_url", remote_params, indirect=True)
|
||||
def test_get_url_external(erepo_dir, remote_url, setup_remote):
|
||||
setup_remote(erepo_dir.dvc, url=remote_url)
|
||||
@pytest.mark.parametrize("cloud", clouds)
|
||||
def test_get_url_external(erepo_dir, cloud):
|
||||
erepo_dir.add_remote(config=cloud.config)
|
||||
with erepo_dir.chdir():
|
||||
erepo_dir.dvc_gen("foo", "foo", commit="add foo")
|
||||
|
||||
# Using file url to force clone to tmp repo
|
||||
repo_url = f"file://{erepo_dir}"
|
||||
expected_url = URLInfo(remote_url) / "ac/bd18db4cc2f85cedef654fccc4a4d8"
|
||||
expected_url = URLInfo(cloud.url) / "ac/bd18db4cc2f85cedef654fccc4a4d8"
|
||||
assert api.get_url("foo", repo=repo_url) == expected_url
|
||||
|
||||
|
||||
@@ -105,12 +54,10 @@ def test_get_url_requires_dvc(tmp_dir, scm):
|
||||
api.get_url("foo", repo=f"file://{tmp_dir}")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("remote_url", all_remote_params, indirect=True)
|
||||
def test_open(remote_url, tmp_dir, dvc):
|
||||
run_dvc("remote", "add", "-d", TEST_REMOTE, remote_url)
|
||||
ensure_dir(dvc, remote_url)
|
||||
@pytest.mark.parametrize("remote", all_remotes)
|
||||
def test_open(tmp_dir, dvc, remote):
|
||||
tmp_dir.dvc_gen("foo", "foo-text")
|
||||
run_dvc("push")
|
||||
dvc.push()
|
||||
|
||||
# Remove cache to force download
|
||||
remove(dvc.cache.local.cache_dir)
|
||||
@@ -119,10 +66,9 @@ def test_open(remote_url, tmp_dir, dvc):
|
||||
assert fd.read() == "foo-text"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("remote_url", all_remote_params, indirect=True)
|
||||
def test_open_external(remote_url, erepo_dir, setup_remote):
|
||||
setup_remote(erepo_dir.dvc, url=remote_url)
|
||||
ensure_dir_scm(erepo_dir.dvc, remote_url)
|
||||
@pytest.mark.parametrize("cloud", clouds)
|
||||
def test_open_external(erepo_dir, cloud):
|
||||
erepo_dir.add_remote(config=cloud.config)
|
||||
|
||||
with erepo_dir.chdir():
|
||||
erepo_dir.dvc_gen("version", "master", commit="add version")
|
||||
@@ -144,12 +90,10 @@ def test_open_external(remote_url, erepo_dir, setup_remote):
|
||||
assert api.read("version", repo=repo_url, rev="branch") == "branchver"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("remote_url", all_remote_params, indirect=True)
|
||||
def test_open_granular(remote_url, tmp_dir, dvc):
|
||||
run_dvc("remote", "add", "-d", TEST_REMOTE, remote_url)
|
||||
ensure_dir(dvc, remote_url)
|
||||
@pytest.mark.parametrize("remote", all_remotes)
|
||||
def test_open_granular(tmp_dir, dvc, remote):
|
||||
tmp_dir.dvc_gen({"dir": {"foo": "foo-text"}})
|
||||
run_dvc("push")
|
||||
dvc.push()
|
||||
|
||||
# Remove cache to force download
|
||||
remove(dvc.cache.local.cache_dir)
|
||||
@@ -158,11 +102,9 @@ def test_open_granular(remote_url, tmp_dir, dvc):
|
||||
assert fd.read() == "foo-text"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("remote_url", all_remote_params, indirect=True)
|
||||
def test_missing(remote_url, tmp_dir, dvc):
|
||||
@pytest.mark.parametrize("remote", all_remotes)
|
||||
def test_missing(tmp_dir, dvc, remote):
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
run_dvc("remote", "add", "-d", TEST_REMOTE, remote_url)
|
||||
ensure_dir(dvc, remote_url)
|
||||
|
||||
# Remove cache to make foo missing
|
||||
remove(dvc.cache.local.cache_dir)
|
||||
|
||||
+142
-578
@@ -1,497 +1,163 @@
|
||||
import copy
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import uuid
|
||||
from unittest import SkipTest
|
||||
|
||||
import pytest
|
||||
from flaky.flaky_decorator import flaky
|
||||
|
||||
from dvc.cache import NamedCache
|
||||
from dvc.data_cloud import DataCloud
|
||||
from dvc.external_repo import clean_repos
|
||||
from dvc.main import main
|
||||
from dvc.remote.azure import AzureRemoteTree
|
||||
from dvc.remote.base import STATUS_DELETED, STATUS_NEW, STATUS_OK
|
||||
from dvc.remote.gdrive import GDriveRemoteTree
|
||||
from dvc.remote.gs import GSRemoteTree
|
||||
from dvc.remote.hdfs import HDFSRemoteTree
|
||||
from dvc.remote.http import HTTPRemoteTree
|
||||
from dvc.remote.local import LocalRemoteTree
|
||||
from dvc.remote.oss import OSSRemoteTree
|
||||
from dvc.remote.s3 import S3RemoteTree
|
||||
from dvc.remote.ssh import SSHRemoteTree
|
||||
from dvc.stage.exceptions import StageNotFound
|
||||
from dvc.utils import file_md5
|
||||
from dvc.utils.fs import remove
|
||||
from dvc.utils.yaml import dump_yaml, load_yaml
|
||||
from tests.basic_env import TestDvc
|
||||
from tests.remotes import (
|
||||
GCP,
|
||||
HDFS,
|
||||
HTTP,
|
||||
OSS,
|
||||
S3,
|
||||
TEST_CONFIG,
|
||||
TEST_GCP_CREDS_FILE,
|
||||
TEST_REMOTE,
|
||||
Azure,
|
||||
GDrive,
|
||||
Local,
|
||||
SSHMocked,
|
||||
)
|
||||
|
||||
from .test_api import all_remotes
|
||||
|
||||
|
||||
class TestDataCloud(TestDvc):
|
||||
def _test_cloud(self, config, cl):
|
||||
self.dvc.config = config
|
||||
cloud = DataCloud(self.dvc)
|
||||
self.assertIsInstance(cloud.get_remote().tree, cl)
|
||||
@pytest.mark.parametrize("remote", all_remotes)
|
||||
def test_cloud(tmp_dir, dvc, remote):
|
||||
(stage,) = tmp_dir.dvc_gen("foo", "foo")
|
||||
out = stage.outs[0]
|
||||
cache = out.cache_path
|
||||
md5 = out.checksum
|
||||
info = out.get_used_cache()
|
||||
|
||||
def test(self):
|
||||
config = copy.deepcopy(TEST_CONFIG)
|
||||
|
||||
clist = [
|
||||
("s3://mybucket/", S3RemoteTree),
|
||||
("gs://mybucket/", GSRemoteTree),
|
||||
("ssh://user@localhost:/", SSHRemoteTree),
|
||||
("http://localhost:8000/", HTTPRemoteTree),
|
||||
("azure://ContainerName=mybucket;conn_string;", AzureRemoteTree),
|
||||
("oss://mybucket/", OSSRemoteTree),
|
||||
(TestDvc.mkdtemp(), LocalRemoteTree),
|
||||
]
|
||||
|
||||
for scheme, cl in clist:
|
||||
remote_url = scheme + str(uuid.uuid4())
|
||||
config["remote"][TEST_REMOTE]["url"] = remote_url
|
||||
self._test_cloud(config, cl)
|
||||
|
||||
|
||||
class TestDataCloudBase(TestDvc):
|
||||
def _get_cloud_class(self):
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def should_test():
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def get_url():
|
||||
return NotImplementedError
|
||||
|
||||
def _get_keyfile(self):
|
||||
return None
|
||||
|
||||
def _ensure_should_run(self):
|
||||
if not self.should_test():
|
||||
raise SkipTest(f"Test {self.__class__.__name__} is disabled")
|
||||
|
||||
def _setup_cloud(self):
|
||||
self._ensure_should_run()
|
||||
|
||||
repo = self.get_url()
|
||||
keyfile = self._get_keyfile()
|
||||
|
||||
config = copy.deepcopy(TEST_CONFIG)
|
||||
config["remote"][TEST_REMOTE] = {"url": repo, "keyfile": keyfile}
|
||||
self.dvc.config = config
|
||||
self.cloud = DataCloud(self.dvc)
|
||||
|
||||
self.assertIsInstance(
|
||||
self.cloud.get_remote().tree, self._get_cloud_class()
|
||||
)
|
||||
|
||||
def _test_cloud(self):
|
||||
self._setup_cloud()
|
||||
|
||||
stages = self.dvc.add(self.FOO)
|
||||
self.assertEqual(len(stages), 1)
|
||||
stage = stages[0]
|
||||
self.assertTrue(stage is not None)
|
||||
out = stage.outs[0]
|
||||
cache = out.cache_path
|
||||
md5 = out.checksum
|
||||
info = out.get_used_cache()
|
||||
|
||||
stages = self.dvc.add(self.DATA_DIR)
|
||||
self.assertEqual(len(stages), 1)
|
||||
stage_dir = stages[0]
|
||||
self.assertTrue(stage_dir is not None)
|
||||
out_dir = stage_dir.outs[0]
|
||||
cache_dir = out_dir.cache_path
|
||||
name_dir = str(out_dir)
|
||||
md5_dir = out_dir.checksum
|
||||
info_dir = NamedCache.make(out_dir.scheme, md5_dir, name_dir)
|
||||
|
||||
with self.cloud.repo.state:
|
||||
# Check status
|
||||
status = self.cloud.status(info, show_checksums=True)
|
||||
expected = {md5: {"name": md5, "status": STATUS_NEW}}
|
||||
self.assertEqual(status, expected)
|
||||
|
||||
status_dir = self.cloud.status(info_dir, show_checksums=True)
|
||||
expected = {md5_dir: {"name": md5_dir, "status": STATUS_NEW}}
|
||||
self.assertEqual(status_dir, expected)
|
||||
|
||||
# Push and check status
|
||||
self.cloud.push(info)
|
||||
self.assertTrue(os.path.exists(cache))
|
||||
self.assertTrue(os.path.isfile(cache))
|
||||
|
||||
self.cloud.push(info_dir)
|
||||
self.assertTrue(os.path.isfile(cache_dir))
|
||||
|
||||
status = self.cloud.status(info, show_checksums=True)
|
||||
expected = {md5: {"name": md5, "status": STATUS_OK}}
|
||||
self.assertEqual(status, expected)
|
||||
|
||||
status_dir = self.cloud.status(info_dir, show_checksums=True)
|
||||
expected = {md5_dir: {"name": md5_dir, "status": STATUS_OK}}
|
||||
self.assertEqual(status_dir, expected)
|
||||
|
||||
# Remove and check status
|
||||
remove(self.dvc.cache.local.cache_dir)
|
||||
|
||||
status = self.cloud.status(info, show_checksums=True)
|
||||
expected = {md5: {"name": md5, "status": STATUS_DELETED}}
|
||||
self.assertEqual(status, expected)
|
||||
|
||||
status_dir = self.cloud.status(info_dir, show_checksums=True)
|
||||
expected = {md5_dir: {"name": md5_dir, "status": STATUS_DELETED}}
|
||||
self.assertEqual(status_dir, expected)
|
||||
|
||||
# Pull and check status
|
||||
self.cloud.pull(info)
|
||||
self.assertTrue(os.path.exists(cache))
|
||||
self.assertTrue(os.path.isfile(cache))
|
||||
with open(cache) as fd:
|
||||
self.assertEqual(fd.read(), self.FOO_CONTENTS)
|
||||
|
||||
self.cloud.pull(info_dir)
|
||||
self.assertTrue(os.path.isfile(cache_dir))
|
||||
|
||||
status = self.cloud.status(info, show_checksums=True)
|
||||
expected = {md5: {"name": md5, "status": STATUS_OK}}
|
||||
self.assertEqual(status, expected)
|
||||
|
||||
status_dir = self.cloud.status(info_dir, show_checksums=True)
|
||||
expected = {md5_dir: {"name": md5_dir, "status": STATUS_OK}}
|
||||
self.assertTrue(status_dir, expected)
|
||||
|
||||
def test(self):
|
||||
self._ensure_should_run()
|
||||
self._test_cloud()
|
||||
|
||||
|
||||
class TestS3Remote(S3, TestDataCloudBase):
|
||||
def _get_cloud_class(self):
|
||||
return S3RemoteTree
|
||||
|
||||
|
||||
class TestGDriveRemote(GDrive, TestDataCloudBase):
|
||||
def _setup_cloud(self):
|
||||
self._ensure_should_run()
|
||||
|
||||
url = self.get_url()
|
||||
self.create_dir(self.dvc, url)
|
||||
|
||||
config = copy.deepcopy(TEST_CONFIG)
|
||||
config["remote"][TEST_REMOTE] = {
|
||||
"url": url,
|
||||
"gdrive_service_account_email": "test",
|
||||
"gdrive_service_account_p12_file_path": "test.p12",
|
||||
"gdrive_use_service_account": True,
|
||||
(stage_dir,) = tmp_dir.dvc_gen(
|
||||
{
|
||||
"data_dir": {
|
||||
"data_sub_dir": {"data_sub": "data_sub"},
|
||||
"data": "data",
|
||||
}
|
||||
}
|
||||
)
|
||||
out_dir = stage_dir.outs[0]
|
||||
cache_dir = out_dir.cache_path
|
||||
name_dir = str(out_dir)
|
||||
md5_dir = out_dir.checksum
|
||||
info_dir = NamedCache.make(out_dir.scheme, md5_dir, name_dir)
|
||||
|
||||
self.dvc.config = config
|
||||
self.cloud = DataCloud(self.dvc)
|
||||
remote = self.cloud.get_remote()
|
||||
self.assertIsInstance(remote.tree, self._get_cloud_class())
|
||||
with dvc.state:
|
||||
# Check status
|
||||
status = dvc.cloud.status(info, show_checksums=True)
|
||||
expected = {md5: {"name": md5, "status": STATUS_NEW}}
|
||||
assert status == expected
|
||||
|
||||
def _get_cloud_class(self):
|
||||
return GDriveRemoteTree
|
||||
status_dir = dvc.cloud.status(info_dir, show_checksums=True)
|
||||
expected = {md5_dir: {"name": md5_dir, "status": STATUS_NEW}}
|
||||
assert status_dir == expected
|
||||
|
||||
# Push and check status
|
||||
dvc.cloud.push(info)
|
||||
assert os.path.exists(cache)
|
||||
assert os.path.isfile(cache)
|
||||
|
||||
class TestGSRemote(GCP, TestDataCloudBase):
|
||||
def _setup_cloud(self):
|
||||
self._ensure_should_run()
|
||||
dvc.cloud.push(info_dir)
|
||||
assert os.path.isfile(cache_dir)
|
||||
|
||||
repo = self.get_url()
|
||||
status = dvc.cloud.status(info, show_checksums=True)
|
||||
expected = {md5: {"name": md5, "status": STATUS_OK}}
|
||||
assert status == expected
|
||||
|
||||
config = copy.deepcopy(TEST_CONFIG)
|
||||
config["remote"][TEST_REMOTE] = {
|
||||
"url": repo,
|
||||
"credentialpath": TEST_GCP_CREDS_FILE,
|
||||
}
|
||||
self.dvc.config = config
|
||||
self.cloud = DataCloud(self.dvc)
|
||||
status_dir = dvc.cloud.status(info_dir, show_checksums=True)
|
||||
expected = {md5_dir: {"name": md5_dir, "status": STATUS_OK}}
|
||||
assert status_dir == expected
|
||||
|
||||
self.assertIsInstance(
|
||||
self.cloud.get_remote().tree, self._get_cloud_class()
|
||||
)
|
||||
# Remove and check status
|
||||
remove(dvc.cache.local.cache_dir)
|
||||
|
||||
def _get_cloud_class(self):
|
||||
return GSRemoteTree
|
||||
status = dvc.cloud.status(info, show_checksums=True)
|
||||
expected = {md5: {"name": md5, "status": STATUS_DELETED}}
|
||||
assert status == expected
|
||||
|
||||
status_dir = dvc.cloud.status(info_dir, show_checksums=True)
|
||||
expected = {md5_dir: {"name": md5_dir, "status": STATUS_DELETED}}
|
||||
assert status_dir == expected
|
||||
|
||||
class TestAzureRemote(Azure, TestDataCloudBase):
|
||||
def _get_cloud_class(self):
|
||||
return AzureRemoteTree
|
||||
|
||||
|
||||
class TestOSSRemote(OSS, TestDataCloudBase):
|
||||
def _get_cloud_class(self):
|
||||
return OSSRemoteTree
|
||||
|
||||
|
||||
class TestLocalRemote(Local, TestDataCloudBase):
|
||||
def _get_cloud_class(self):
|
||||
return LocalRemoteTree
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("ssh_server")
|
||||
class TestSSHRemoteMocked(SSHMocked, TestDataCloudBase):
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup_method_fixture(self, request, ssh_server):
|
||||
self.ssh_server = ssh_server
|
||||
self.method_name = request.function.__name__
|
||||
|
||||
def _setup_cloud(self):
|
||||
self._ensure_should_run()
|
||||
|
||||
repo = self.get_url()
|
||||
keyfile = self._get_keyfile()
|
||||
|
||||
self._get_cloud_class().CAN_TRAVERSE = False
|
||||
config = copy.deepcopy(TEST_CONFIG)
|
||||
config["remote"][TEST_REMOTE] = {
|
||||
"url": repo,
|
||||
"keyfile": keyfile,
|
||||
}
|
||||
self.dvc.config = config
|
||||
self.cloud = DataCloud(self.dvc)
|
||||
|
||||
self.assertIsInstance(
|
||||
self.cloud.get_remote().tree, self._get_cloud_class()
|
||||
)
|
||||
|
||||
def get_url(self):
|
||||
user = self.ssh_server.test_creds["username"]
|
||||
return super().get_url(user, self.ssh_server.port)
|
||||
|
||||
def _get_keyfile(self):
|
||||
return self.ssh_server.test_creds["key_filename"]
|
||||
|
||||
def _get_cloud_class(self):
|
||||
return SSHRemoteTree
|
||||
|
||||
|
||||
class TestHDFSRemote(HDFS, TestDataCloudBase):
|
||||
def _get_cloud_class(self):
|
||||
return HDFSRemoteTree
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("http_server")
|
||||
class TestHTTPRemote(HTTP, TestDataCloudBase):
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup_method_fixture(self, request, http_server):
|
||||
self.http_server = http_server
|
||||
self.method_name = request.function.__name__
|
||||
|
||||
def get_url(self):
|
||||
return super().get_url(self.http_server.server_port)
|
||||
|
||||
def _get_cloud_class(self):
|
||||
return HTTPRemoteTree
|
||||
|
||||
|
||||
class TestDataCloudCLIBase(TestDvc):
|
||||
def main(self, args):
|
||||
ret = main(args)
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
@staticmethod
|
||||
def should_test():
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def get_url():
|
||||
raise NotImplementedError
|
||||
|
||||
def _setup_cloud(self):
|
||||
pass
|
||||
|
||||
def _test_cloud(self, remote=None):
|
||||
self._setup_cloud()
|
||||
|
||||
args = ["-v", "-j", "2"]
|
||||
if remote:
|
||||
args += ["-r", remote]
|
||||
else:
|
||||
args += []
|
||||
|
||||
stages = self.dvc.add(self.FOO)
|
||||
self.assertEqual(len(stages), 1)
|
||||
stage = stages[0]
|
||||
self.assertTrue(stage is not None)
|
||||
cache = stage.outs[0].cache_path
|
||||
|
||||
stages = self.dvc.add(self.DATA_DIR)
|
||||
self.assertEqual(len(stages), 1)
|
||||
stage_dir = stages[0]
|
||||
self.assertTrue(stage_dir is not None)
|
||||
cache_dir = stage_dir.outs[0].cache_path
|
||||
|
||||
# FIXME check status output
|
||||
|
||||
self.main(["push"] + args)
|
||||
self.assertTrue(os.path.exists(cache))
|
||||
self.assertTrue(os.path.isfile(cache))
|
||||
self.assertTrue(os.path.isfile(cache_dir))
|
||||
|
||||
remove(self.dvc.cache.local.cache_dir)
|
||||
|
||||
self.main(["fetch"] + args)
|
||||
self.assertTrue(os.path.exists(cache))
|
||||
self.assertTrue(os.path.isfile(cache))
|
||||
self.assertTrue(os.path.isfile(cache_dir))
|
||||
|
||||
self.main(["pull"] + args)
|
||||
self.assertTrue(os.path.exists(cache))
|
||||
self.assertTrue(os.path.isfile(cache))
|
||||
self.assertTrue(os.path.isfile(cache_dir))
|
||||
self.assertTrue(os.path.isfile(self.FOO))
|
||||
self.assertTrue(os.path.isdir(self.DATA_DIR))
|
||||
|
||||
# Pull and check status
|
||||
dvc.cloud.pull(info)
|
||||
assert os.path.exists(cache)
|
||||
assert os.path.isfile(cache)
|
||||
with open(cache) as fd:
|
||||
self.assertEqual(fd.read(), self.FOO_CONTENTS)
|
||||
self.assertTrue(os.path.isfile(cache_dir))
|
||||
assert fd.read() == "foo"
|
||||
|
||||
# NOTE: check if remote gc works correctly on directories
|
||||
self.main(["gc", "-cw", "-f"] + args)
|
||||
shutil.move(
|
||||
self.dvc.cache.local.cache_dir,
|
||||
self.dvc.cache.local.cache_dir + ".back",
|
||||
)
|
||||
dvc.cloud.pull(info_dir)
|
||||
assert os.path.isfile(cache_dir)
|
||||
|
||||
self.main(["fetch"] + args)
|
||||
status = dvc.cloud.status(info, show_checksums=True)
|
||||
expected = {md5: {"name": md5, "status": STATUS_OK}}
|
||||
assert status == expected
|
||||
|
||||
self.main(["pull", "-f"] + args)
|
||||
self.assertTrue(os.path.exists(cache))
|
||||
self.assertTrue(os.path.isfile(cache))
|
||||
self.assertTrue(os.path.isfile(cache_dir))
|
||||
self.assertTrue(os.path.isfile(self.FOO))
|
||||
self.assertTrue(os.path.isdir(self.DATA_DIR))
|
||||
|
||||
def _test(self):
|
||||
pass
|
||||
|
||||
def test(self):
|
||||
if not self.should_test():
|
||||
raise SkipTest(f"Test {self.__class__.__name__} is disabled")
|
||||
self._test()
|
||||
status_dir = dvc.cloud.status(info_dir, show_checksums=True)
|
||||
expected = {md5_dir: {"name": md5_dir, "status": STATUS_OK}}
|
||||
assert status_dir == expected
|
||||
|
||||
|
||||
class TestLocalRemoteCLI(Local, TestDataCloudCLIBase):
|
||||
def _test(self):
|
||||
url = self.get_url()
|
||||
@pytest.mark.parametrize("remote", all_remotes)
|
||||
def test_cloud_cli(tmp_dir, dvc, remote):
|
||||
args = ["-v", "-j", "2"]
|
||||
|
||||
self.main(["remote", "add", TEST_REMOTE, url])
|
||||
(stage,) = tmp_dir.dvc_gen("foo", "foo")
|
||||
cache = stage.outs[0].cache_path
|
||||
|
||||
self._test_cloud(TEST_REMOTE)
|
||||
(stage_dir,) = tmp_dir.dvc_gen(
|
||||
{
|
||||
"data_dir": {
|
||||
"data_sub_dir": {"data_sub": "data_sub"},
|
||||
"data": "data",
|
||||
}
|
||||
}
|
||||
)
|
||||
assert stage_dir is not None
|
||||
cache_dir = stage_dir.outs[0].cache_path
|
||||
|
||||
# FIXME check status output
|
||||
|
||||
class TestRemoteHDFSCLI(HDFS, TestDataCloudCLIBase):
|
||||
def _test(self):
|
||||
url = self.get_url()
|
||||
assert main(["push"] + args) == 0
|
||||
assert os.path.exists(cache)
|
||||
assert os.path.isfile(cache)
|
||||
assert os.path.isfile(cache_dir)
|
||||
|
||||
self.main(["remote", "add", TEST_REMOTE, url])
|
||||
remove(dvc.cache.local.cache_dir)
|
||||
|
||||
self._test_cloud(TEST_REMOTE)
|
||||
assert main(["fetch"] + args) == 0
|
||||
assert os.path.exists(cache)
|
||||
assert os.path.isfile(cache)
|
||||
assert os.path.isfile(cache_dir)
|
||||
|
||||
assert main(["pull"] + args) == 0
|
||||
assert os.path.exists(cache)
|
||||
assert os.path.isfile(cache)
|
||||
assert os.path.isfile(cache_dir)
|
||||
assert os.path.isfile("foo")
|
||||
assert os.path.isdir("data_dir")
|
||||
|
||||
class TestS3RemoteCLI(S3, TestDataCloudCLIBase):
|
||||
def _test(self):
|
||||
url = self.get_url()
|
||||
with open(cache) as fd:
|
||||
assert fd.read() == "foo"
|
||||
assert os.path.isfile(cache_dir)
|
||||
|
||||
self.main(["remote", "add", TEST_REMOTE, url])
|
||||
# NOTE: http doesn't support gc yet
|
||||
if remote.url.startswith("http"):
|
||||
return
|
||||
|
||||
self._test_cloud(TEST_REMOTE)
|
||||
# NOTE: check if remote gc works correctly on directories
|
||||
assert main(["gc", "-cw", "-f"] + args) == 0
|
||||
shutil.move(
|
||||
dvc.cache.local.cache_dir, dvc.cache.local.cache_dir + ".back",
|
||||
)
|
||||
|
||||
assert main(["fetch"] + args) == 0
|
||||
|
||||
class TestGDriveRemoteCLI(GDrive, TestDataCloudCLIBase):
|
||||
def _test(self):
|
||||
url = self.get_url()
|
||||
|
||||
self.create_dir(self.dvc, url)
|
||||
|
||||
self.main(["remote", "add", TEST_REMOTE, url])
|
||||
self.main(
|
||||
[
|
||||
"remote",
|
||||
"modify",
|
||||
TEST_REMOTE,
|
||||
"gdrive_service_account_email",
|
||||
"test",
|
||||
]
|
||||
)
|
||||
self.main(
|
||||
[
|
||||
"remote",
|
||||
"modify",
|
||||
TEST_REMOTE,
|
||||
"gdrive_service_account_p12_file_path",
|
||||
"test.p12",
|
||||
]
|
||||
)
|
||||
self.main(
|
||||
[
|
||||
"remote",
|
||||
"modify",
|
||||
TEST_REMOTE,
|
||||
"gdrive_use_service_account",
|
||||
"True",
|
||||
]
|
||||
)
|
||||
|
||||
self._test_cloud(TEST_REMOTE)
|
||||
|
||||
|
||||
class TestGSRemoteCLI(GCP, TestDataCloudCLIBase):
|
||||
def _test(self):
|
||||
url = self.get_url()
|
||||
|
||||
self.main(["remote", "add", TEST_REMOTE, url])
|
||||
self.main(
|
||||
[
|
||||
"remote",
|
||||
"modify",
|
||||
TEST_REMOTE,
|
||||
"credentialpath",
|
||||
TEST_GCP_CREDS_FILE,
|
||||
]
|
||||
)
|
||||
|
||||
self._test_cloud(TEST_REMOTE)
|
||||
|
||||
|
||||
class TestAzureRemoteCLI(Azure, TestDataCloudCLIBase):
|
||||
def _test(self):
|
||||
url = self.get_url()
|
||||
|
||||
self.main(["remote", "add", TEST_REMOTE, url])
|
||||
|
||||
self._test_cloud(TEST_REMOTE)
|
||||
|
||||
|
||||
class TestOSSRemoteCLI(OSS, TestDataCloudCLIBase):
|
||||
def _test(self):
|
||||
url = self.get_url()
|
||||
|
||||
self.main(["remote", "add", TEST_REMOTE, url])
|
||||
|
||||
self._test_cloud(TEST_REMOTE)
|
||||
assert main(["pull", "-f"] + args) == 0
|
||||
assert os.path.exists(cache)
|
||||
assert os.path.isfile(cache)
|
||||
assert os.path.isfile(cache_dir)
|
||||
assert os.path.isfile("foo")
|
||||
assert os.path.isdir("data_dir")
|
||||
|
||||
|
||||
class TestDataCloudErrorCLI(TestDvc):
|
||||
@@ -507,124 +173,30 @@ class TestDataCloudErrorCLI(TestDvc):
|
||||
self.main_fail(["fetch", f])
|
||||
|
||||
|
||||
class TestWarnOnOutdatedStage(TestDvc):
|
||||
def main(self, args):
|
||||
ret = main(args)
|
||||
self.assertEqual(ret, 0)
|
||||
def test_warn_on_outdated_stage(tmp_dir, dvc, local_remote, caplog):
|
||||
stage = dvc.run(outs=["bar"], cmd="echo bar > bar", single_stage=True)
|
||||
assert main(["push"]) == 0
|
||||
|
||||
def _test(self):
|
||||
url = Local.get_url()
|
||||
self.main(["remote", "add", "-d", TEST_REMOTE, url])
|
||||
stage_file_path = stage.relpath
|
||||
content = load_yaml(stage_file_path)
|
||||
del content["outs"][0]["md5"]
|
||||
dump_yaml(stage_file_path, content)
|
||||
|
||||
stage = self.dvc.run(
|
||||
outs=["bar"], cmd="echo bar > bar", single_stage=True
|
||||
)
|
||||
self.main(["push"])
|
||||
|
||||
stage_file_path = stage.relpath
|
||||
content = load_yaml(stage_file_path)
|
||||
del content["outs"][0]["md5"]
|
||||
dump_yaml(stage_file_path, content)
|
||||
|
||||
with self._caplog.at_level(logging.WARNING, logger="dvc"):
|
||||
self._caplog.clear()
|
||||
self.main(["status", "-c"])
|
||||
expected_warning = (
|
||||
"Output 'bar'(stage: 'bar.dvc') is missing version info. "
|
||||
"Cache for it will not be collected. "
|
||||
"Use `dvc repro` to get your pipeline up to date."
|
||||
)
|
||||
|
||||
assert expected_warning in self._caplog.text
|
||||
|
||||
def test(self):
|
||||
self._test()
|
||||
|
||||
|
||||
class TestRecursiveSyncOperations(Local, TestDataCloudBase):
|
||||
def main(self, args):
|
||||
ret = main(args)
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
def _get_cloud_class(self):
|
||||
return LocalRemoteTree
|
||||
|
||||
def _prepare_repo(self):
|
||||
remote = self.cloud.get_remote()
|
||||
self.main(
|
||||
["remote", "add", "-d", TEST_REMOTE, remote.path_info.fspath]
|
||||
with caplog.at_level(logging.WARNING, logger="dvc"):
|
||||
caplog.clear()
|
||||
assert main(["status", "-c"]) == 0
|
||||
expected_warning = (
|
||||
"Output 'bar'(stage: 'bar.dvc') is missing version info. "
|
||||
"Cache for it will not be collected. "
|
||||
"Use `dvc repro` to get your pipeline up to date."
|
||||
)
|
||||
|
||||
self.dvc.add(self.DATA)
|
||||
self.dvc.add(self.DATA_SUB)
|
||||
|
||||
def _remove_local_data_files(self):
|
||||
os.remove(self.DATA)
|
||||
os.remove(self.DATA_SUB)
|
||||
|
||||
def _test_recursive_pull(self):
|
||||
self._remove_local_data_files()
|
||||
self._clear_local_cache()
|
||||
|
||||
self.assertFalse(os.path.exists(self.DATA))
|
||||
self.assertFalse(os.path.exists(self.DATA_SUB))
|
||||
|
||||
self.main(["pull", "-R", self.DATA_DIR])
|
||||
|
||||
self.assertTrue(os.path.exists(self.DATA))
|
||||
self.assertTrue(os.path.exists(self.DATA_SUB))
|
||||
|
||||
def _clear_local_cache(self):
|
||||
remove(self.dvc.cache.local.cache_dir)
|
||||
|
||||
def _test_recursive_fetch(self, data_md5, data_sub_md5):
|
||||
self._clear_local_cache()
|
||||
|
||||
local_cache = self.dvc.cache.local
|
||||
local_cache_data_path = local_cache.hash_to_path_info(data_md5)
|
||||
local_cache_data_sub_path = local_cache.hash_to_path_info(data_sub_md5)
|
||||
|
||||
self.assertFalse(os.path.exists(local_cache_data_path))
|
||||
self.assertFalse(os.path.exists(local_cache_data_sub_path))
|
||||
|
||||
self.main(["fetch", "-R", self.DATA_DIR])
|
||||
|
||||
self.assertTrue(os.path.exists(local_cache_data_path))
|
||||
self.assertTrue(os.path.exists(local_cache_data_sub_path))
|
||||
|
||||
def _test_recursive_push(self, data_md5, data_sub_md5):
|
||||
remote = self.cloud.get_remote()
|
||||
cloud_data_path = remote.hash_to_path_info(data_md5)
|
||||
cloud_data_sub_path = remote.hash_to_path_info(data_sub_md5)
|
||||
|
||||
self.assertFalse(os.path.exists(cloud_data_path))
|
||||
self.assertFalse(os.path.exists(cloud_data_sub_path))
|
||||
|
||||
self.main(["push", "-R", self.DATA_DIR])
|
||||
|
||||
self.assertTrue(os.path.exists(cloud_data_path))
|
||||
self.assertTrue(os.path.exists(cloud_data_sub_path))
|
||||
|
||||
def test(self):
|
||||
self._setup_cloud()
|
||||
self._prepare_repo()
|
||||
|
||||
data_md5 = file_md5(self.DATA)[0]
|
||||
data_sub_md5 = file_md5(self.DATA_SUB)[0]
|
||||
|
||||
self._test_recursive_push(data_md5, data_sub_md5)
|
||||
|
||||
self._test_recursive_fetch(data_md5, data_sub_md5)
|
||||
|
||||
self._test_recursive_pull()
|
||||
assert expected_warning in caplog.text
|
||||
|
||||
|
||||
def test_hash_recalculation(mocker, dvc, tmp_dir):
|
||||
def test_hash_recalculation(mocker, dvc, tmp_dir, local_remote):
|
||||
tmp_dir.gen({"foo": "foo"})
|
||||
test_get_file_hash = mocker.spy(LocalRemoteTree, "get_file_hash")
|
||||
url = Local.get_url()
|
||||
ret = main(["remote", "add", "-d", TEST_REMOTE, url])
|
||||
assert ret == 0
|
||||
ret = main(["config", "cache.type", "hardlink"])
|
||||
assert ret == 0
|
||||
ret = main(["add", "foo"])
|
||||
@@ -687,10 +259,8 @@ class TestShouldWarnOnNoChecksumInLocalAndRemoteCache(TestDvc):
|
||||
|
||||
|
||||
def test_verify_hashes(
|
||||
tmp_dir, scm, dvc, mocker, tmp_path_factory, setup_remote
|
||||
tmp_dir, scm, dvc, mocker, tmp_path_factory, local_remote
|
||||
):
|
||||
|
||||
setup_remote(dvc, name="upstream")
|
||||
tmp_dir.dvc_gen({"file": "file1 content"}, commit="add file")
|
||||
tmp_dir.dvc_gen({"dir": {"subfile": "file2 content"}}, commit="add dir")
|
||||
dvc.push()
|
||||
@@ -782,8 +352,7 @@ def recurse_list_dir(d):
|
||||
]
|
||||
|
||||
|
||||
def test_dvc_pull_pipeline_stages(tmp_dir, dvc, run_copy, setup_remote):
|
||||
setup_remote(dvc)
|
||||
def test_dvc_pull_pipeline_stages(tmp_dir, dvc, run_copy, local_remote):
|
||||
(stage0,) = tmp_dir.dvc_gen("foo", "foo")
|
||||
stage1 = run_copy("foo", "bar", single_stage=True)
|
||||
stage2 = run_copy("bar", "foobar", name="copy-bar-foobar")
|
||||
@@ -814,8 +383,8 @@ def test_dvc_pull_pipeline_stages(tmp_dir, dvc, run_copy, setup_remote):
|
||||
assert set(stats["added"]) == set(outs)
|
||||
|
||||
|
||||
def test_pipeline_file_target_ops(tmp_dir, dvc, run_copy, setup_remote):
|
||||
remote_path = setup_remote(dvc)
|
||||
def test_pipeline_file_target_ops(tmp_dir, dvc, run_copy, local_remote):
|
||||
path = local_remote.url
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
run_copy("foo", "bar", single_stage=True)
|
||||
|
||||
@@ -834,7 +403,7 @@ def test_pipeline_file_target_ops(tmp_dir, dvc, run_copy, setup_remote):
|
||||
outs = ["foo", "bar", "lorem", "ipsum", "baz", "lorem2"]
|
||||
|
||||
# each one's a copy of other, hence 3
|
||||
assert len(recurse_list_dir(remote_path)) == 3
|
||||
assert len(recurse_list_dir(path)) == 3
|
||||
|
||||
clean(outs, dvc)
|
||||
assert set(dvc.pull(["dvc.yaml"])["added"]) == {"lorem2", "baz"}
|
||||
@@ -845,13 +414,13 @@ def test_pipeline_file_target_ops(tmp_dir, dvc, run_copy, setup_remote):
|
||||
# clean everything in remote and push
|
||||
from tests.dir_helpers import TmpDir
|
||||
|
||||
clean(TmpDir(remote_path).iterdir())
|
||||
clean(TmpDir(path).iterdir())
|
||||
dvc.push(["dvc.yaml:copy-ipsum-baz"])
|
||||
assert len(recurse_list_dir(remote_path)) == 1
|
||||
assert len(recurse_list_dir(path)) == 1
|
||||
|
||||
clean(TmpDir(remote_path).iterdir())
|
||||
clean(TmpDir(path).iterdir())
|
||||
dvc.push(["dvc.yaml"])
|
||||
assert len(recurse_list_dir(remote_path)) == 2
|
||||
assert len(recurse_list_dir(path)) == 2
|
||||
|
||||
with pytest.raises(StageNotFound):
|
||||
dvc.push(["dvc.yaml:StageThatDoesNotExist"])
|
||||
@@ -868,8 +437,7 @@ def test_pipeline_file_target_ops(tmp_dir, dvc, run_copy, setup_remote):
|
||||
({}, "Everything is up to date"),
|
||||
],
|
||||
)
|
||||
def test_push_stats(tmp_dir, dvc, fs, msg, caplog, setup_remote):
|
||||
setup_remote(dvc)
|
||||
def test_push_stats(tmp_dir, dvc, fs, msg, caplog, local_remote):
|
||||
tmp_dir.dvc_gen(fs)
|
||||
|
||||
caplog.clear()
|
||||
@@ -886,8 +454,7 @@ def test_push_stats(tmp_dir, dvc, fs, msg, caplog, setup_remote):
|
||||
({}, "Everything is up to date."),
|
||||
],
|
||||
)
|
||||
def test_fetch_stats(tmp_dir, dvc, fs, msg, caplog, setup_remote):
|
||||
setup_remote(dvc)
|
||||
def test_fetch_stats(tmp_dir, dvc, fs, msg, caplog, local_remote):
|
||||
tmp_dir.dvc_gen(fs)
|
||||
dvc.push()
|
||||
clean(list(fs.keys()), dvc)
|
||||
@@ -897,8 +464,7 @@ def test_fetch_stats(tmp_dir, dvc, fs, msg, caplog, setup_remote):
|
||||
assert msg in caplog.text
|
||||
|
||||
|
||||
def test_pull_stats(tmp_dir, dvc, caplog, setup_remote):
|
||||
setup_remote(dvc)
|
||||
def test_pull_stats(tmp_dir, dvc, caplog, local_remote):
|
||||
tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
|
||||
dvc.push()
|
||||
clean(["foo", "bar"], dvc)
|
||||
@@ -919,8 +485,7 @@ def test_pull_stats(tmp_dir, dvc, caplog, setup_remote):
|
||||
@pytest.mark.parametrize(
|
||||
"key,expected", [("all_tags", 2), ("all_branches", 3), ("all_commits", 3)]
|
||||
)
|
||||
def test_push_pull_all(tmp_dir, scm, dvc, setup_remote, key, expected):
|
||||
setup_remote(dvc)
|
||||
def test_push_pull_all(tmp_dir, scm, dvc, local_remote, key, expected):
|
||||
tmp_dir.dvc_gen({"foo": "foo"}, commit="first")
|
||||
scm.tag("v1")
|
||||
dvc.remove("foo.dvc")
|
||||
@@ -936,13 +501,12 @@ def test_push_pull_all(tmp_dir, scm, dvc, setup_remote, key, expected):
|
||||
assert dvc.pull(**{key: True})["fetched"] == expected
|
||||
|
||||
|
||||
def test_push_pull_fetch_pipeline_stages(tmp_dir, dvc, run_copy, setup_remote):
|
||||
remote_path = setup_remote(dvc)
|
||||
def test_push_pull_fetch_pipeline_stages(tmp_dir, dvc, run_copy, local_remote):
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
run_copy("foo", "bar", no_commit=True, name="copy-foo-bar")
|
||||
|
||||
dvc.push("copy-foo-bar")
|
||||
assert len(recurse_list_dir(remote_path)) == 1
|
||||
assert len(recurse_list_dir(local_remote.url)) == 1
|
||||
# pushing everything so as we can check pull/fetch only downloads
|
||||
# from specified targets
|
||||
dvc.push()
|
||||
|
||||
@@ -157,7 +157,7 @@ def test_stage_collection(tmp_dir, dvc):
|
||||
always_changed=True,
|
||||
single_stage=True,
|
||||
)
|
||||
assert {s for s in dvc.stages} == {stage1, stage3, stage2}
|
||||
assert set(dvc.stages) == {stage1, stage3, stage2}
|
||||
|
||||
|
||||
def test_remove_stage(tmp_dir, dvc, run_copy):
|
||||
|
||||
@@ -43,8 +43,8 @@ def test_source_change(erepo_dir):
|
||||
assert old_rev != new_rev
|
||||
|
||||
|
||||
def test_cache_reused(erepo_dir, mocker, setup_remote):
|
||||
setup_remote(erepo_dir.dvc)
|
||||
def test_cache_reused(erepo_dir, mocker, local_cloud):
|
||||
erepo_dir.add_remote(config=local_cloud.config)
|
||||
with erepo_dir.chdir():
|
||||
erepo_dir.dvc_gen("file", "text", commit="add file")
|
||||
erepo_dir.dvc.push()
|
||||
@@ -99,7 +99,7 @@ def test_pull_subdir_file(tmp_dir, erepo_dir):
|
||||
assert dest.read_text() == "contents"
|
||||
|
||||
|
||||
def test_relative_remote(erepo_dir, tmp_dir, setup_remote):
|
||||
def test_relative_remote(erepo_dir, tmp_dir):
|
||||
# these steps reproduce the script on this issue:
|
||||
# https://github.com/iterative/dvc/issues/2756
|
||||
with erepo_dir.chdir():
|
||||
@@ -107,7 +107,7 @@ def test_relative_remote(erepo_dir, tmp_dir, setup_remote):
|
||||
|
||||
upstream_dir = tmp_dir
|
||||
upstream_url = relpath(upstream_dir, erepo_dir)
|
||||
setup_remote(erepo_dir.dvc, url=upstream_url, name="upstream")
|
||||
erepo_dir.add_remote(url=upstream_url)
|
||||
|
||||
erepo_dir.dvc.push()
|
||||
|
||||
|
||||
@@ -52,7 +52,7 @@ class TestGC(TestDvcGit):
|
||||
class TestGCBranchesTags(TestDvcGit):
|
||||
def _check_cache(self, num):
|
||||
total = 0
|
||||
for root, dirs, files in os.walk(os.path.join(".dvc", "cache")):
|
||||
for _, _, files in os.walk(os.path.join(".dvc", "cache")):
|
||||
total += len(files)
|
||||
self.assertEqual(total, num)
|
||||
|
||||
@@ -112,7 +112,7 @@ class TestGCBranchesTags(TestDvcGit):
|
||||
class TestGCMultipleDvcRepos(TestDvcGit):
|
||||
def _check_cache(self, num):
|
||||
total = 0
|
||||
for root, dirs, files in os.walk(os.path.join(".dvc", "cache")):
|
||||
for _, _, files in os.walk(os.path.join(".dvc", "cache")):
|
||||
total += len(files)
|
||||
self.assertEqual(total, num)
|
||||
|
||||
@@ -240,9 +240,9 @@ def test_gc_without_workspace_raises_error(tmp_dir, dvc):
|
||||
dvc.gc(force=True, workspace=False)
|
||||
|
||||
|
||||
def test_gc_cloud_with_or_without_specifier(tmp_dir, erepo_dir, setup_remote):
|
||||
def test_gc_cloud_with_or_without_specifier(tmp_dir, erepo_dir, local_cloud):
|
||||
erepo_dir.add_remote(config=local_cloud.config)
|
||||
dvc = erepo_dir.dvc
|
||||
setup_remote(dvc)
|
||||
from dvc.exceptions import InvalidArgumentError
|
||||
|
||||
with pytest.raises(InvalidArgumentError):
|
||||
@@ -297,9 +297,7 @@ def test_gc_with_possible_args_positive(tmp_dir, dvc):
|
||||
assert main(["gc", "-vf", flag]) == 0
|
||||
|
||||
|
||||
def test_gc_cloud_positive(tmp_dir, dvc, tmp_path_factory, setup_remote):
|
||||
setup_remote(dvc)
|
||||
|
||||
def test_gc_cloud_positive(tmp_dir, dvc, tmp_path_factory, local_remote):
|
||||
for flag in ["-cw", "-ca", "-cT", "-caT", "-cwT"]:
|
||||
assert main(["gc", "-vf", flag]) == 0
|
||||
|
||||
|
||||
@@ -201,8 +201,8 @@ def test_get_file_from_dir(tmp_dir, erepo_dir):
|
||||
assert (tmp_dir / "X").read_text() == "foo"
|
||||
|
||||
|
||||
def test_get_url_positive(tmp_dir, erepo_dir, caplog, setup_remote):
|
||||
setup_remote(erepo_dir.dvc)
|
||||
def test_get_url_positive(tmp_dir, erepo_dir, caplog, local_cloud):
|
||||
erepo_dir.add_remote(config=local_cloud.config)
|
||||
with erepo_dir.chdir():
|
||||
erepo_dir.dvc_gen("foo", "foo")
|
||||
erepo_dir.dvc.push()
|
||||
@@ -238,11 +238,10 @@ def test_get_url_git_only_repo(tmp_dir, scm, caplog):
|
||||
|
||||
|
||||
def test_get_pipeline_tracked_outs(
|
||||
tmp_dir, dvc, scm, git_dir, run_copy, setup_remote
|
||||
tmp_dir, dvc, scm, git_dir, run_copy, local_remote
|
||||
):
|
||||
from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
|
||||
|
||||
setup_remote(dvc)
|
||||
tmp_dir.gen("foo", "foo")
|
||||
run_copy("foo", "bar", name="copy-foo-bar")
|
||||
dvc.push()
|
||||
|
||||
@@ -166,3 +166,10 @@ def test_ignore_subrepo(tmp_dir, scm, dvc):
|
||||
|
||||
for _ in subrepo.brancher(all_commits=True):
|
||||
assert subrepo.tree.exists(subrepo_dir / "foo")
|
||||
|
||||
|
||||
def test_ignore_blank_line(tmp_dir, dvc):
|
||||
tmp_dir.gen({"dir": {"ignored": "text", "other": "text2"}})
|
||||
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "foo\n\ndir/ignored")
|
||||
|
||||
assert _files_set("dir", dvc.tree) == {"dir/other"}
|
||||
|
||||
@@ -307,13 +307,13 @@ def test_pull_no_rev_lock(erepo_dir, tmp_dir, dvc):
|
||||
|
||||
|
||||
def test_import_from_bare_git_repo(
|
||||
tmp_dir, make_tmp_dir, erepo_dir, setup_remote
|
||||
tmp_dir, make_tmp_dir, erepo_dir, local_cloud
|
||||
):
|
||||
import git
|
||||
|
||||
git.Repo.init(os.fspath(tmp_dir), bare=True)
|
||||
|
||||
setup_remote(erepo_dir.dvc)
|
||||
erepo_dir.add_remote(config=local_cloud.config)
|
||||
with erepo_dir.chdir():
|
||||
erepo_dir.dvc_gen({"foo": "foo"}, commit="initial")
|
||||
erepo_dir.dvc.push()
|
||||
@@ -327,11 +327,10 @@ def test_import_from_bare_git_repo(
|
||||
|
||||
|
||||
def test_import_pipeline_tracked_outs(
|
||||
tmp_dir, dvc, scm, erepo_dir, run_copy, setup_remote
|
||||
tmp_dir, dvc, scm, erepo_dir, run_copy, local_remote
|
||||
):
|
||||
from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
|
||||
|
||||
setup_remote(dvc)
|
||||
tmp_dir.gen("foo", "foo")
|
||||
run_copy("foo", "bar", name="copy-foo-bar")
|
||||
dvc.push()
|
||||
|
||||
@@ -173,9 +173,7 @@ def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc):
|
||||
assert hash1 == hash2
|
||||
|
||||
|
||||
def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory, setup_remote):
|
||||
setup_remote(dvc, name="upstream")
|
||||
|
||||
def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory, local_remote):
|
||||
foo = tmp_dir.dvc_gen({"foo": "foo content"})[0].outs[0]
|
||||
bar = tmp_dir.dvc_gen({"bar": "bar content"})[0].outs[0]
|
||||
baz = tmp_dir.dvc_gen({"baz": {"foo": "baz content"}})[0].outs[0]
|
||||
@@ -211,9 +209,8 @@ def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory, setup_remote):
|
||||
|
||||
|
||||
def test_raise_on_too_many_open_files(
|
||||
tmp_dir, dvc, tmp_path_factory, mocker, setup_remote
|
||||
tmp_dir, dvc, tmp_path_factory, mocker, local_remote
|
||||
):
|
||||
setup_remote(dvc)
|
||||
tmp_dir.dvc_gen({"file": "file content"})
|
||||
|
||||
mocker.patch.object(
|
||||
@@ -246,8 +243,7 @@ def test_external_dir_resource_on_no_cache(tmp_dir, dvc, tmp_path_factory):
|
||||
)
|
||||
|
||||
|
||||
def test_push_order(tmp_dir, dvc, tmp_path_factory, mocker, setup_remote):
|
||||
setup_remote(dvc)
|
||||
def test_push_order(tmp_dir, dvc, tmp_path_factory, mocker, local_remote):
|
||||
tmp_dir.dvc_gen({"foo": {"bar": "bar content"}})
|
||||
tmp_dir.dvc_gen({"baz": "baz content"})
|
||||
|
||||
@@ -387,8 +383,7 @@ def test_remote_default(dvc):
|
||||
assert local_config["core"]["remote"] == new_name
|
||||
|
||||
|
||||
def test_protect_local_remote(tmp_dir, dvc, setup_remote):
|
||||
setup_remote(dvc, name="upstream")
|
||||
def test_protect_local_remote(tmp_dir, dvc, local_remote):
|
||||
(stage,) = tmp_dir.dvc_gen("file", "file content")
|
||||
|
||||
dvc.push()
|
||||
|
||||
@@ -7,14 +7,19 @@ from dvc.main import main
|
||||
from dvc.stage.exceptions import StageFileDoesNotExistError
|
||||
from dvc.system import System
|
||||
from dvc.utils.fs import remove
|
||||
from tests.utils import get_gitignore_content
|
||||
|
||||
|
||||
@pytest.mark.parametrize("remove_outs", [True, False])
|
||||
def test_remove(tmp_dir, dvc, run_copy, remove_outs):
|
||||
def test_remove(tmp_dir, scm, dvc, run_copy, remove_outs):
|
||||
(stage1,) = tmp_dir.dvc_gen("foo", "foo")
|
||||
stage2 = run_copy("foo", "bar", single_stage=True)
|
||||
stage3 = run_copy("bar", "foobar", name="copy-bar-foobar")
|
||||
|
||||
assert "/foo" in get_gitignore_content()
|
||||
assert "/bar" in get_gitignore_content()
|
||||
assert "/foobar" in get_gitignore_content()
|
||||
|
||||
for stage in [stage1, stage2, stage3]:
|
||||
dvc.remove(stage.addressing, outs=remove_outs)
|
||||
out_exists = (out.exists for out in stage.outs)
|
||||
@@ -24,6 +29,8 @@ def test_remove(tmp_dir, dvc, run_copy, remove_outs):
|
||||
else:
|
||||
assert all(out_exists)
|
||||
|
||||
assert not any(out in get_gitignore_content() for out in stage.outs)
|
||||
|
||||
|
||||
def test_remove_non_existent_file(tmp_dir, dvc):
|
||||
with pytest.raises(StageFileDoesNotExistError):
|
||||
|
||||
@@ -1737,12 +1737,14 @@ def test_downstream(dvc):
|
||||
reason="external output scenario is not supported on Windows",
|
||||
)
|
||||
def test_ssh_dir_out(tmp_dir, dvc, ssh_server):
|
||||
from tests.remotes.ssh import TEST_SSH_USER, TEST_SSH_KEY_PATH
|
||||
|
||||
tmp_dir.gen({"foo": "foo content"})
|
||||
|
||||
# Set up remote and cache
|
||||
user = ssh_server.test_creds["username"]
|
||||
user = TEST_SSH_USER
|
||||
port = ssh_server.port
|
||||
keyfile = ssh_server.test_creds["key_filename"]
|
||||
keyfile = TEST_SSH_KEY_PATH
|
||||
|
||||
remote_url = SSHMocked.get_url(user, port)
|
||||
assert main(["remote", "add", "upstream", remote_url]) == 0
|
||||
|
||||
@@ -449,7 +449,7 @@ def test_repro_when_new_outs_added_does_not_exist(tmp_dir, dvc):
|
||||
{
|
||||
"stages": {
|
||||
"run-copy": {
|
||||
"cmd": "python copy {} {}".format("foo", "foobar"),
|
||||
"cmd": "python copy.py {} {}".format("foo", "foobar"),
|
||||
"deps": ["foo"],
|
||||
"outs": ["foobar", "bar"],
|
||||
}
|
||||
|
||||
@@ -8,12 +8,11 @@ def _recurse_count_files(path):
|
||||
return len([os.path.join(r, f) for r, _, fs in os.walk(path) for f in fs])
|
||||
|
||||
|
||||
def test_push_pull(tmp_dir, dvc, erepo_dir, run_copy, setup_remote):
|
||||
def test_push_pull(tmp_dir, dvc, erepo_dir, run_copy, local_remote):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
run_copy("foo", "bar", name="copy-foo-bar")
|
||||
url = setup_remote(dvc)
|
||||
assert dvc.push(run_cache=True) == 2
|
||||
setup_remote(erepo_dir.dvc, url)
|
||||
erepo_dir.add_remote(config=local_remote.config)
|
||||
with erepo_dir.chdir():
|
||||
assert not os.path.exists(erepo_dir.dvc.stage_cache.cache_dir)
|
||||
assert erepo_dir.dvc.pull(run_cache=True)["fetched"] == 2
|
||||
|
||||
@@ -220,6 +220,10 @@ def test_stage_strings_representation(tmp_dir, dvc, run_copy):
|
||||
folder = tmp_dir / "dir"
|
||||
folder.mkdir()
|
||||
with folder.chdir():
|
||||
# `Stage` caches `relpath` results, forcing it to reset
|
||||
stage1.path = stage1.path
|
||||
stage2.path = stage2.path
|
||||
|
||||
rel_path = os.path.relpath(stage1.path)
|
||||
assert stage1.addressing == rel_path
|
||||
assert repr(stage1) == f"Stage: '{rel_path}'"
|
||||
|
||||
@@ -179,8 +179,7 @@ class TestWalkInGit(AssertWalkEqualMixin, TestGit):
|
||||
)
|
||||
|
||||
|
||||
def test_repotree_walk_fetch(tmp_dir, dvc, scm, setup_remote):
|
||||
setup_remote(dvc)
|
||||
def test_repotree_walk_fetch(tmp_dir, dvc, scm, local_remote):
|
||||
out = tmp_dir.dvc_gen({"dir": {"foo": "foo"}}, commit="init")[0].outs[0]
|
||||
dvc.push()
|
||||
remove(dvc.cache.local.cache_dir)
|
||||
@@ -196,12 +195,12 @@ def test_repotree_walk_fetch(tmp_dir, dvc, scm, setup_remote):
|
||||
assert os.path.exists(dvc.cache.local.hash_to_path_info(hash_))
|
||||
|
||||
|
||||
def test_repotree_cache_save(tmp_dir, dvc, scm, erepo_dir, setup_remote):
|
||||
def test_repotree_cache_save(tmp_dir, dvc, scm, erepo_dir, local_cloud):
|
||||
with erepo_dir.chdir():
|
||||
erepo_dir.gen({"dir": {"subdir": {"foo": "foo"}, "bar": "bar"}})
|
||||
erepo_dir.dvc_add("dir/subdir", commit="subdir")
|
||||
erepo_dir.scm_add("dir", commit="dir")
|
||||
setup_remote(erepo_dir.dvc)
|
||||
erepo_dir.add_remote(config=local_cloud.config)
|
||||
erepo_dir.dvc.push()
|
||||
|
||||
# test only cares that either fetch or stream are set so that DVC dirs are
|
||||
|
||||
@@ -1,295 +0,0 @@
|
||||
import getpass
|
||||
import os
|
||||
import platform
|
||||
import uuid
|
||||
from contextlib import contextmanager
|
||||
from subprocess import CalledProcessError, Popen, check_output
|
||||
|
||||
from moto.s3 import mock_s3
|
||||
|
||||
from dvc.remote.base import Remote
|
||||
from dvc.remote.gdrive import GDriveRemoteTree
|
||||
from dvc.remote.gs import GSRemoteTree
|
||||
from dvc.remote.s3 import S3RemoteTree
|
||||
from dvc.utils import env2bool
|
||||
from tests.basic_env import TestDvc
|
||||
|
||||
TEST_REMOTE = "upstream"
|
||||
TEST_CONFIG = {
|
||||
"cache": {},
|
||||
"core": {"remote": TEST_REMOTE},
|
||||
"remote": {TEST_REMOTE: {"url": ""}},
|
||||
}
|
||||
|
||||
TEST_AWS_REPO_BUCKET = os.environ.get("DVC_TEST_AWS_REPO_BUCKET", "dvc-temp")
|
||||
TEST_GCP_REPO_BUCKET = os.environ.get("DVC_TEST_GCP_REPO_BUCKET", "dvc-test")
|
||||
TEST_GDRIVE_REPO_BUCKET = "root"
|
||||
TEST_OSS_REPO_BUCKET = "dvc-test"
|
||||
|
||||
TEST_GCP_CREDS_FILE = os.path.abspath(
|
||||
os.environ.get(
|
||||
"GOOGLE_APPLICATION_CREDENTIALS",
|
||||
os.path.join("scripts", "ci", "gcp-creds.json"),
|
||||
)
|
||||
)
|
||||
# Ensure that absolute path is used
|
||||
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = TEST_GCP_CREDS_FILE
|
||||
|
||||
always_test = staticmethod(lambda: True)
|
||||
|
||||
|
||||
class Local:
|
||||
should_test = always_test
|
||||
|
||||
@staticmethod
|
||||
def get_storagepath():
|
||||
return TestDvc.mkdtemp()
|
||||
|
||||
@staticmethod
|
||||
def get_url():
|
||||
return Local.get_storagepath()
|
||||
|
||||
|
||||
class S3:
|
||||
@staticmethod
|
||||
def should_test():
|
||||
do_test = env2bool("DVC_TEST_AWS", undefined=None)
|
||||
if do_test is not None:
|
||||
return do_test
|
||||
|
||||
if os.getenv("AWS_ACCESS_KEY_ID") and os.getenv(
|
||||
"AWS_SECRET_ACCESS_KEY"
|
||||
):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def get_storagepath():
|
||||
return TEST_AWS_REPO_BUCKET + "/" + str(uuid.uuid4())
|
||||
|
||||
@staticmethod
|
||||
def get_url():
|
||||
return "s3://" + S3.get_storagepath()
|
||||
|
||||
|
||||
class S3Mocked(S3):
|
||||
should_test = always_test
|
||||
|
||||
@classmethod
|
||||
@contextmanager
|
||||
def remote(cls, repo):
|
||||
with mock_s3():
|
||||
yield Remote(S3RemoteTree(repo, {"url": cls.get_url()}))
|
||||
|
||||
@staticmethod
|
||||
def put_objects(remote, objects):
|
||||
s3 = remote.tree.s3
|
||||
bucket = remote.path_info.bucket
|
||||
s3.create_bucket(Bucket=bucket)
|
||||
for key, body in objects.items():
|
||||
s3.put_object(
|
||||
Bucket=bucket, Key=(remote.path_info / key).path, Body=body
|
||||
)
|
||||
|
||||
|
||||
class GCP:
|
||||
@staticmethod
|
||||
def should_test():
|
||||
do_test = env2bool("DVC_TEST_GCP", undefined=None)
|
||||
if do_test is not None:
|
||||
return do_test
|
||||
|
||||
if not os.path.exists(TEST_GCP_CREDS_FILE):
|
||||
return False
|
||||
|
||||
try:
|
||||
check_output(
|
||||
[
|
||||
"gcloud",
|
||||
"auth",
|
||||
"activate-service-account",
|
||||
"--key-file",
|
||||
TEST_GCP_CREDS_FILE,
|
||||
]
|
||||
)
|
||||
except (CalledProcessError, OSError):
|
||||
return False
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def get_storagepath():
|
||||
return TEST_GCP_REPO_BUCKET + "/" + str(uuid.uuid4())
|
||||
|
||||
@staticmethod
|
||||
def get_url():
|
||||
return "gs://" + GCP.get_storagepath()
|
||||
|
||||
@classmethod
|
||||
@contextmanager
|
||||
def remote(cls, repo):
|
||||
yield Remote(GSRemoteTree(repo, {"url": cls.get_url()}))
|
||||
|
||||
@staticmethod
|
||||
def put_objects(remote, objects):
|
||||
client = remote.tree.gs
|
||||
bucket = client.get_bucket(remote.path_info.bucket)
|
||||
for key, body in objects.items():
|
||||
bucket.blob((remote.path_info / key).path).upload_from_string(body)
|
||||
|
||||
|
||||
class GDrive:
|
||||
@staticmethod
|
||||
def should_test():
|
||||
return os.getenv(GDriveRemoteTree.GDRIVE_CREDENTIALS_DATA) is not None
|
||||
|
||||
@staticmethod
|
||||
def create_dir(dvc, url):
|
||||
config = {
|
||||
"url": url,
|
||||
"gdrive_service_account_email": "test",
|
||||
"gdrive_service_account_p12_file_path": "test.p12",
|
||||
"gdrive_use_service_account": True,
|
||||
}
|
||||
tree = GDriveRemoteTree(dvc, config)
|
||||
tree._gdrive_create_dir("root", tree.path_info.path)
|
||||
|
||||
@staticmethod
|
||||
def get_storagepath():
|
||||
return TEST_GDRIVE_REPO_BUCKET + "/" + str(uuid.uuid4())
|
||||
|
||||
@staticmethod
|
||||
def get_url():
|
||||
# NOTE: `get_url` should always return new random url
|
||||
return "gdrive://" + GDrive.get_storagepath()
|
||||
|
||||
|
||||
class Azure:
|
||||
@staticmethod
|
||||
def should_test():
|
||||
do_test = env2bool("DVC_TEST_AZURE", undefined=None)
|
||||
if do_test is not None:
|
||||
return do_test
|
||||
|
||||
return os.getenv("AZURE_STORAGE_CONTAINER_NAME") and os.getenv(
|
||||
"AZURE_STORAGE_CONNECTION_STRING"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_url():
|
||||
container_name = os.getenv("AZURE_STORAGE_CONTAINER_NAME")
|
||||
assert container_name is not None
|
||||
return "azure://{}/{}".format(container_name, str(uuid.uuid4()))
|
||||
|
||||
|
||||
class OSS:
|
||||
@staticmethod
|
||||
def should_test():
|
||||
do_test = env2bool("DVC_TEST_OSS", undefined=None)
|
||||
if do_test is not None:
|
||||
return do_test
|
||||
|
||||
return (
|
||||
os.getenv("OSS_ENDPOINT")
|
||||
and os.getenv("OSS_ACCESS_KEY_ID")
|
||||
and os.getenv("OSS_ACCESS_KEY_SECRET")
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_storagepath():
|
||||
return f"{TEST_OSS_REPO_BUCKET}/{uuid.uuid4()}"
|
||||
|
||||
@staticmethod
|
||||
def get_url():
|
||||
return f"oss://{OSS.get_storagepath()}"
|
||||
|
||||
|
||||
class SSH:
|
||||
@staticmethod
|
||||
def should_test():
|
||||
do_test = env2bool("DVC_TEST_SSH", undefined=None)
|
||||
if do_test is not None:
|
||||
return do_test
|
||||
|
||||
# FIXME: enable on windows
|
||||
if os.name == "nt":
|
||||
return False
|
||||
|
||||
try:
|
||||
check_output(["ssh", "-o", "BatchMode=yes", "127.0.0.1", "ls"])
|
||||
except (CalledProcessError, OSError):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def get_url():
|
||||
return "ssh://{}@127.0.0.1:22{}".format(
|
||||
getpass.getuser(), Local.get_storagepath()
|
||||
)
|
||||
|
||||
|
||||
class SSHMocked:
|
||||
should_test = always_test
|
||||
|
||||
@staticmethod
|
||||
def get_url(user, port):
|
||||
path = Local.get_storagepath()
|
||||
if os.name == "nt":
|
||||
# NOTE: On Windows Local.get_storagepath() will return an
|
||||
# ntpath that looks something like `C:\some\path`, which is not
|
||||
# compatible with SFTP paths [1], so we need to convert it to
|
||||
# a proper posixpath.
|
||||
# To do that, we should construct a posixpath that would be
|
||||
# relative to the server's root.
|
||||
# In our case our ssh server is running with `c:/` as a root,
|
||||
# and our URL format requires absolute paths, so the
|
||||
# resulting path would look like `/some/path`.
|
||||
#
|
||||
# [1]https://tools.ietf.org/html/draft-ietf-secsh-filexfer-13#section-6
|
||||
drive, path = os.path.splitdrive(path)
|
||||
assert drive.lower() == "c:"
|
||||
path = path.replace("\\", "/")
|
||||
url = f"ssh://{user}@127.0.0.1:{port}{path}"
|
||||
return url
|
||||
|
||||
|
||||
class HDFS:
|
||||
@staticmethod
|
||||
def should_test():
|
||||
if platform.system() != "Linux":
|
||||
return False
|
||||
|
||||
try:
|
||||
check_output(
|
||||
["hadoop", "version"],
|
||||
shell=True,
|
||||
executable=os.getenv("SHELL"),
|
||||
)
|
||||
except (CalledProcessError, OSError):
|
||||
return False
|
||||
|
||||
p = Popen(
|
||||
"hadoop fs -ls hdfs://127.0.0.1/",
|
||||
shell=True,
|
||||
executable=os.getenv("SHELL"),
|
||||
)
|
||||
p.communicate()
|
||||
if p.returncode != 0:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def get_url():
|
||||
return "hdfs://{}@127.0.0.1{}".format(
|
||||
getpass.getuser(), Local.get_storagepath()
|
||||
)
|
||||
|
||||
|
||||
class HTTP:
|
||||
should_test = always_test
|
||||
|
||||
@staticmethod
|
||||
def get_url(port):
|
||||
return f"http://127.0.0.1:{port}"
|
||||
@@ -0,0 +1,35 @@
|
||||
TEST_REMOTE = "upstream"
|
||||
TEST_CONFIG = {
|
||||
"cache": {},
|
||||
"core": {"remote": TEST_REMOTE},
|
||||
"remote": {TEST_REMOTE: {"url": ""}},
|
||||
}
|
||||
|
||||
from .azure import Azure, azure, azure_remote # noqa: F401
|
||||
from .hdfs import HDFS, hdfs, hdfs_remote # noqa: F401
|
||||
from .http import HTTP, http, http_remote, http_server # noqa: F401
|
||||
from .local import Local, local_cloud, local_remote # noqa: F401
|
||||
from .oss import OSS, TEST_OSS_REPO_BUCKET, oss, oss_remote # noqa: F401
|
||||
from .s3 import S3, TEST_AWS_REPO_BUCKET, S3Mocked, s3, s3_remote # noqa: F401
|
||||
|
||||
from .gdrive import ( # noqa: F401; noqa: F401
|
||||
TEST_GDRIVE_REPO_BUCKET,
|
||||
GDrive,
|
||||
gdrive,
|
||||
gdrive_remote,
|
||||
)
|
||||
from .gs import ( # noqa: F401; noqa: F401
|
||||
GCP,
|
||||
TEST_GCP_CREDS_FILE,
|
||||
TEST_GCP_REPO_BUCKET,
|
||||
gs,
|
||||
gs_remote,
|
||||
)
|
||||
from .ssh import ( # noqa: F401; noqa: F401
|
||||
SSH,
|
||||
SSHMocked,
|
||||
ssh,
|
||||
ssh_connection,
|
||||
ssh_remote,
|
||||
ssh_server,
|
||||
)
|
||||
@@ -0,0 +1,39 @@
|
||||
import os
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
|
||||
from dvc.utils import env2bool
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class Azure(Base):
|
||||
@staticmethod
|
||||
def should_test():
|
||||
do_test = env2bool("DVC_TEST_AZURE", undefined=None)
|
||||
if do_test is not None:
|
||||
return do_test
|
||||
|
||||
return os.getenv("AZURE_STORAGE_CONTAINER_NAME") and os.getenv(
|
||||
"AZURE_STORAGE_CONNECTION_STRING"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_url():
|
||||
container_name = os.getenv("AZURE_STORAGE_CONTAINER_NAME")
|
||||
assert container_name is not None
|
||||
return "azure://{}/{}".format(container_name, str(uuid.uuid4()))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def azure():
|
||||
if not Azure.should_test():
|
||||
pytest.skip("no azure running")
|
||||
yield Azure()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def azure_remote(tmp_dir, dvc, azure):
|
||||
tmp_dir.add_remote(config=azure.config)
|
||||
yield azure
|
||||
@@ -0,0 +1,23 @@
|
||||
from funcy import cached_property
|
||||
|
||||
|
||||
class Base:
|
||||
@staticmethod
|
||||
def should_test():
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def get_storagepath():
|
||||
raise NotImplementedError
|
||||
|
||||
@staticmethod
|
||||
def get_url():
|
||||
raise NotImplementedError
|
||||
|
||||
@cached_property
|
||||
def url(self):
|
||||
return self.get_url()
|
||||
|
||||
@cached_property
|
||||
def config(self):
|
||||
return {"url": self.url}
|
||||
@@ -0,0 +1,55 @@
|
||||
import os
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
from funcy import cached_property
|
||||
|
||||
from dvc.remote.gdrive import GDriveRemoteTree
|
||||
|
||||
from .base import Base
|
||||
|
||||
TEST_GDRIVE_REPO_BUCKET = "root"
|
||||
|
||||
|
||||
class GDrive(Base):
|
||||
@staticmethod
|
||||
def should_test():
|
||||
return os.getenv(GDriveRemoteTree.GDRIVE_CREDENTIALS_DATA) is not None
|
||||
|
||||
@cached_property
|
||||
def config(self):
|
||||
return {
|
||||
"url": self.url,
|
||||
"gdrive_service_account_email": "test",
|
||||
"gdrive_service_account_p12_file_path": "test.p12",
|
||||
"gdrive_use_service_account": True,
|
||||
}
|
||||
|
||||
def __init__(self, dvc):
|
||||
tree = GDriveRemoteTree(dvc, self.config)
|
||||
tree._gdrive_create_dir("root", tree.path_info.path)
|
||||
|
||||
@staticmethod
|
||||
def get_storagepath():
|
||||
return TEST_GDRIVE_REPO_BUCKET + "/" + str(uuid.uuid4())
|
||||
|
||||
@staticmethod
|
||||
def get_url():
|
||||
# NOTE: `get_url` should always return new random url
|
||||
return "gdrive://" + GDrive.get_storagepath()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def gdrive(make_tmp_dir):
|
||||
if not GDrive.should_test():
|
||||
pytest.skip("no gdrive")
|
||||
|
||||
# NOTE: temporary workaround
|
||||
tmp_dir = make_tmp_dir("gdrive", dvc=True)
|
||||
return GDrive(tmp_dir.dvc)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def gdrive_remote(tmp_dir, dvc, gdrive):
|
||||
tmp_dir.add_remote(config=gdrive.config)
|
||||
return gdrive
|
||||
@@ -0,0 +1,90 @@
|
||||
import os
|
||||
import uuid
|
||||
from contextlib import contextmanager
|
||||
|
||||
import pytest
|
||||
from funcy import cached_property
|
||||
|
||||
from dvc.remote.base import Remote
|
||||
from dvc.remote.gs import GSRemoteTree
|
||||
from dvc.utils import env2bool
|
||||
|
||||
from .base import Base
|
||||
|
||||
TEST_GCP_REPO_BUCKET = os.environ.get("DVC_TEST_GCP_REPO_BUCKET", "dvc-test")
|
||||
|
||||
TEST_GCP_CREDS_FILE = os.path.abspath(
|
||||
os.environ.get(
|
||||
"GOOGLE_APPLICATION_CREDENTIALS",
|
||||
os.path.join("scripts", "ci", "gcp-creds.json"),
|
||||
)
|
||||
)
|
||||
# Ensure that absolute path is used
|
||||
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = TEST_GCP_CREDS_FILE
|
||||
|
||||
|
||||
class GCP(Base):
|
||||
@staticmethod
|
||||
def should_test():
|
||||
from subprocess import CalledProcessError, check_output
|
||||
|
||||
do_test = env2bool("DVC_TEST_GCP", undefined=None)
|
||||
if do_test is not None:
|
||||
return do_test
|
||||
|
||||
if not os.path.exists(TEST_GCP_CREDS_FILE):
|
||||
return False
|
||||
|
||||
try:
|
||||
check_output(
|
||||
[
|
||||
"gcloud",
|
||||
"auth",
|
||||
"activate-service-account",
|
||||
"--key-file",
|
||||
TEST_GCP_CREDS_FILE,
|
||||
]
|
||||
)
|
||||
except (CalledProcessError, OSError):
|
||||
return False
|
||||
return True
|
||||
|
||||
@cached_property
|
||||
def config(self):
|
||||
return {
|
||||
"url": self.url,
|
||||
"credentialpath": TEST_GCP_CREDS_FILE,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def get_storagepath():
|
||||
return TEST_GCP_REPO_BUCKET + "/" + str(uuid.uuid4())
|
||||
|
||||
@staticmethod
|
||||
def get_url():
|
||||
return "gs://" + GCP.get_storagepath()
|
||||
|
||||
@classmethod
|
||||
@contextmanager
|
||||
def remote(cls, repo):
|
||||
yield Remote(GSRemoteTree(repo, {"url": cls.get_url()}))
|
||||
|
||||
@staticmethod
|
||||
def put_objects(remote, objects):
|
||||
client = remote.tree.gs
|
||||
bucket = client.get_bucket(remote.path_info.bucket)
|
||||
for key, body in objects.items():
|
||||
bucket.blob((remote.path_info / key).path).upload_from_string(body)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def gs():
|
||||
if not GCP.should_test():
|
||||
pytest.skip("no gs")
|
||||
yield GCP()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def gs_remote(tmp_dir, dvc, gs):
|
||||
tmp_dir.add_remote(config=gs.config)
|
||||
yield gs
|
||||
@@ -0,0 +1,55 @@
|
||||
import getpass
|
||||
import os
|
||||
import platform
|
||||
from subprocess import CalledProcessError, Popen, check_output
|
||||
|
||||
import pytest
|
||||
|
||||
from .base import Base
|
||||
from .local import Local
|
||||
|
||||
|
||||
class HDFS(Base):
|
||||
@staticmethod
|
||||
def should_test():
|
||||
if platform.system() != "Linux":
|
||||
return False
|
||||
|
||||
try:
|
||||
check_output(
|
||||
["hadoop", "version"],
|
||||
shell=True,
|
||||
executable=os.getenv("SHELL"),
|
||||
)
|
||||
except (CalledProcessError, OSError):
|
||||
return False
|
||||
|
||||
p = Popen(
|
||||
"hadoop fs -ls hdfs://127.0.0.1/",
|
||||
shell=True,
|
||||
executable=os.getenv("SHELL"),
|
||||
)
|
||||
p.communicate()
|
||||
if p.returncode != 0:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def get_url():
|
||||
return "hdfs://{}@127.0.0.1{}".format(
|
||||
getpass.getuser(), Local.get_storagepath()
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def hdfs():
|
||||
if not HDFS.should_test():
|
||||
pytest.skip("no hadoop running")
|
||||
yield HDFS()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def hdfs_remote(tmp_dir, dvc, hdfs):
|
||||
tmp_dir.add_remote(config=hdfs.config)
|
||||
yield hdfs
|
||||
@@ -0,0 +1,31 @@
|
||||
import pytest
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class HTTP(Base):
|
||||
@staticmethod
|
||||
def get_url(port):
|
||||
return f"http://127.0.0.1:{port}"
|
||||
|
||||
def __init__(self, server):
|
||||
self.url = self.get_url(server.server_port)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def http_server(tmp_dir):
|
||||
from tests.utils.httpd import PushRequestHandler, StaticFileServer
|
||||
|
||||
with StaticFileServer(handler_class=PushRequestHandler) as httpd:
|
||||
yield httpd
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def http(http_server):
|
||||
yield HTTP(http_server)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def http_remote(tmp_dir, dvc, http):
|
||||
tmp_dir.add_remote(config=http.config)
|
||||
yield http
|
||||
@@ -0,0 +1,26 @@
|
||||
import pytest
|
||||
|
||||
from tests.basic_env import TestDvc
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class Local(Base):
|
||||
@staticmethod
|
||||
def get_storagepath():
|
||||
return TestDvc.mkdtemp()
|
||||
|
||||
@staticmethod
|
||||
def get_url():
|
||||
return Local.get_storagepath()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def local_cloud():
|
||||
yield Local()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def local_remote(tmp_dir, dvc, local_cloud):
|
||||
tmp_dir.add_remote(config=local_cloud.config)
|
||||
yield local_cloud
|
||||
@@ -0,0 +1,45 @@
|
||||
import os
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
|
||||
from dvc.utils import env2bool
|
||||
|
||||
from .base import Base
|
||||
|
||||
TEST_OSS_REPO_BUCKET = "dvc-test"
|
||||
|
||||
|
||||
class OSS(Base):
|
||||
@staticmethod
|
||||
def should_test():
|
||||
do_test = env2bool("DVC_TEST_OSS", undefined=None)
|
||||
if do_test is not None:
|
||||
return do_test
|
||||
|
||||
return (
|
||||
os.getenv("OSS_ENDPOINT")
|
||||
and os.getenv("OSS_ACCESS_KEY_ID")
|
||||
and os.getenv("OSS_ACCESS_KEY_SECRET")
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_storagepath():
|
||||
return f"{TEST_OSS_REPO_BUCKET}/{uuid.uuid4()}"
|
||||
|
||||
@staticmethod
|
||||
def get_url():
|
||||
return f"oss://{OSS.get_storagepath()}"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def oss():
|
||||
if not OSS.should_test():
|
||||
pytest.skip("no oss running")
|
||||
yield OSS()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def oss_remote(tmp_dir, dvc, oss):
|
||||
tmp_dir.add_remote(config=oss.config)
|
||||
yield oss
|
||||
@@ -0,0 +1,68 @@
|
||||
import os
|
||||
import uuid
|
||||
from contextlib import contextmanager
|
||||
|
||||
import pytest
|
||||
from moto.s3 import mock_s3
|
||||
|
||||
from dvc.remote.base import Remote
|
||||
from dvc.remote.s3 import S3RemoteTree
|
||||
from dvc.utils import env2bool
|
||||
|
||||
from .base import Base
|
||||
|
||||
TEST_AWS_REPO_BUCKET = os.environ.get("DVC_TEST_AWS_REPO_BUCKET", "dvc-temp")
|
||||
|
||||
|
||||
class S3(Base):
|
||||
@staticmethod
|
||||
def should_test():
|
||||
do_test = env2bool("DVC_TEST_AWS", undefined=None)
|
||||
if do_test is not None:
|
||||
return do_test
|
||||
|
||||
if os.getenv("AWS_ACCESS_KEY_ID") and os.getenv(
|
||||
"AWS_SECRET_ACCESS_KEY"
|
||||
):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def get_storagepath():
|
||||
return TEST_AWS_REPO_BUCKET + "/" + str(uuid.uuid4())
|
||||
|
||||
@staticmethod
|
||||
def get_url():
|
||||
return "s3://" + S3.get_storagepath()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def s3():
|
||||
if not S3.should_test():
|
||||
pytest.skip("no s3")
|
||||
yield S3()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def s3_remote(tmp_dir, dvc, s3):
|
||||
tmp_dir.add_remote(config=s3.config)
|
||||
yield s3
|
||||
|
||||
|
||||
class S3Mocked(S3):
|
||||
@classmethod
|
||||
@contextmanager
|
||||
def remote(cls, repo):
|
||||
with mock_s3():
|
||||
yield Remote(S3RemoteTree(repo, {"url": cls.get_url()}))
|
||||
|
||||
@staticmethod
|
||||
def put_objects(remote, objects):
|
||||
s3 = remote.tree.s3
|
||||
bucket = remote.path_info.bucket
|
||||
s3.create_bucket(Bucket=bucket)
|
||||
for key, body in objects.items():
|
||||
s3.put_object(
|
||||
Bucket=bucket, Key=(remote.path_info / key).path, Body=body
|
||||
)
|
||||
@@ -0,0 +1,115 @@
|
||||
import getpass
|
||||
import os
|
||||
from subprocess import CalledProcessError, check_output
|
||||
|
||||
import pytest
|
||||
from funcy import cached_property
|
||||
|
||||
from dvc.utils import env2bool
|
||||
|
||||
from .base import Base
|
||||
from .local import Local
|
||||
|
||||
TEST_SSH_USER = "user"
|
||||
TEST_SSH_KEY_PATH = os.path.join(
|
||||
os.path.abspath(os.path.dirname(__file__)), f"{TEST_SSH_USER}.key"
|
||||
)
|
||||
|
||||
|
||||
class SSH:
|
||||
@staticmethod
|
||||
def should_test():
|
||||
do_test = env2bool("DVC_TEST_SSH", undefined=None)
|
||||
if do_test is not None:
|
||||
return do_test
|
||||
|
||||
# FIXME: enable on windows
|
||||
if os.name == "nt":
|
||||
return False
|
||||
|
||||
try:
|
||||
check_output(["ssh", "-o", "BatchMode=yes", "127.0.0.1", "ls"])
|
||||
except (CalledProcessError, OSError):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def get_url():
|
||||
return "ssh://{}@127.0.0.1:22{}".format(
|
||||
getpass.getuser(), Local.get_storagepath()
|
||||
)
|
||||
|
||||
|
||||
class SSHMocked(Base):
|
||||
@staticmethod
|
||||
def get_url(user, port):
|
||||
path = Local.get_storagepath()
|
||||
if os.name == "nt":
|
||||
# NOTE: On Windows Local.get_storagepath() will return an
|
||||
# ntpath that looks something like `C:\some\path`, which is not
|
||||
# compatible with SFTP paths [1], so we need to convert it to
|
||||
# a proper posixpath.
|
||||
# To do that, we should construct a posixpath that would be
|
||||
# relative to the server's root.
|
||||
# In our case our ssh server is running with `c:/` as a root,
|
||||
# and our URL format requires absolute paths, so the
|
||||
# resulting path would look like `/some/path`.
|
||||
#
|
||||
# [1]https://tools.ietf.org/html/draft-ietf-secsh-filexfer-13#section-6
|
||||
drive, path = os.path.splitdrive(path)
|
||||
assert drive.lower() == "c:"
|
||||
path = path.replace("\\", "/")
|
||||
url = f"ssh://{user}@127.0.0.1:{port}{path}"
|
||||
return url
|
||||
|
||||
def __init__(self, server):
|
||||
self.server = server
|
||||
|
||||
@cached_property
|
||||
def url(self):
|
||||
return self.get_url(TEST_SSH_USER, self.server.port)
|
||||
|
||||
@cached_property
|
||||
def config(self):
|
||||
return {
|
||||
"url": self.url,
|
||||
"keyfile": TEST_SSH_KEY_PATH,
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
def ssh_server():
|
||||
import mockssh
|
||||
|
||||
users = {TEST_SSH_USER: TEST_SSH_KEY_PATH}
|
||||
with mockssh.Server(users) as s:
|
||||
yield s
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def ssh_connection(ssh_server):
|
||||
from dvc.remote.ssh.connection import SSHConnection
|
||||
|
||||
yield SSHConnection(
|
||||
host=ssh_server.host,
|
||||
port=ssh_server.port,
|
||||
username=TEST_SSH_USER,
|
||||
key_filename=TEST_SSH_KEY_PATH,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def ssh(ssh_server, monkeypatch):
|
||||
from dvc.remote.ssh import SSHRemoteTree
|
||||
|
||||
# NOTE: see http://github.com/iterative/dvc/pull/3501
|
||||
monkeypatch.setattr(SSHRemoteTree, "CAN_TRAVERSE", False)
|
||||
|
||||
return SSHMocked(ssh_server)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def ssh_remote(tmp_dir, dvc, ssh):
|
||||
tmp_dir.add_remote(config=ssh.config)
|
||||
yield ssh
|
||||
@@ -12,39 +12,39 @@ from dvc.system import System
|
||||
here = os.path.abspath(os.path.dirname(__file__))
|
||||
|
||||
|
||||
def test_isdir(ssh):
|
||||
assert ssh.isdir(here)
|
||||
assert not ssh.isdir(__file__)
|
||||
def test_isdir(ssh_connection):
|
||||
assert ssh_connection.isdir(here)
|
||||
assert not ssh_connection.isdir(__file__)
|
||||
|
||||
|
||||
def test_exists(ssh):
|
||||
assert not ssh.exists("/path/to/non/existent/file")
|
||||
assert ssh.exists(__file__)
|
||||
def test_exists(ssh_connection):
|
||||
assert not ssh_connection.exists("/path/to/non/existent/file")
|
||||
assert ssh_connection.exists(__file__)
|
||||
|
||||
|
||||
def test_isfile(ssh):
|
||||
assert ssh.isfile(__file__)
|
||||
assert not ssh.isfile(here)
|
||||
def test_isfile(ssh_connection):
|
||||
assert ssh_connection.isfile(__file__)
|
||||
assert not ssh_connection.isfile(here)
|
||||
|
||||
|
||||
def test_makedirs(tmp_path, ssh):
|
||||
def test_makedirs(tmp_path, ssh_connection):
|
||||
tmp = tmp_path.absolute().as_posix()
|
||||
path = posixpath.join(tmp, "dir", "subdir")
|
||||
ssh.makedirs(path)
|
||||
ssh_connection.makedirs(path)
|
||||
assert os.path.isdir(path)
|
||||
|
||||
|
||||
def test_remove_dir(tmp_path, ssh):
|
||||
def test_remove_dir(tmp_path, ssh_connection):
|
||||
dpath = tmp_path / "dir"
|
||||
dpath.mkdir()
|
||||
(dpath / "file").write_text("file")
|
||||
(dpath / "subdir").mkdir()
|
||||
(dpath / "subdir" / "subfile").write_text("subfile")
|
||||
ssh.remove(dpath.absolute().as_posix())
|
||||
ssh_connection.remove(dpath.absolute().as_posix())
|
||||
assert not dpath.exists()
|
||||
|
||||
|
||||
def test_walk(tmp_path, ssh):
|
||||
def test_walk(tmp_path, ssh_connection):
|
||||
root_path = tmp_path
|
||||
dir_path = root_path / "dir"
|
||||
subdir_path = dir_path / "subdir"
|
||||
@@ -75,7 +75,9 @@ def test_walk(tmp_path, ssh):
|
||||
expected = {entry.absolute().as_posix() for entry in entries}
|
||||
|
||||
paths = set()
|
||||
for root, dirs, files in ssh.walk(root_path.absolute().as_posix()):
|
||||
for root, dirs, files in ssh_connection.walk(
|
||||
root_path.absolute().as_posix()
|
||||
):
|
||||
for entry in dirs + files:
|
||||
paths.add(posixpath.join(root, entry))
|
||||
|
||||
@@ -87,9 +89,9 @@ def test_walk(tmp_path, ssh):
|
||||
not in ["xfs", "apfs", "btrfs"],
|
||||
reason="Reflinks only work in specified file systems",
|
||||
)
|
||||
def test_reflink(tmp_dir, ssh):
|
||||
def test_reflink(tmp_dir, ssh_connection):
|
||||
tmp_dir.gen("foo", "foo content")
|
||||
ssh.reflink("foo", "link")
|
||||
ssh_connection.reflink("foo", "link")
|
||||
assert filecmp.cmp("foo", "link")
|
||||
assert not System.is_symlink("link")
|
||||
assert not System.is_hardlink("link")
|
||||
@@ -99,9 +101,9 @@ def test_reflink(tmp_dir, ssh):
|
||||
platform.system() == "Windows",
|
||||
reason="sftp symlink is not supported on Windows",
|
||||
)
|
||||
def test_symlink(tmp_dir, ssh):
|
||||
def test_symlink(tmp_dir, ssh_connection):
|
||||
tmp_dir.gen("foo", "foo content")
|
||||
ssh.symlink("foo", "link")
|
||||
ssh_connection.symlink("foo", "link")
|
||||
assert System.is_symlink("link")
|
||||
|
||||
|
||||
@@ -109,9 +111,9 @@ def test_symlink(tmp_dir, ssh):
|
||||
platform.system() == "Windows",
|
||||
reason="hardlink is temporarily not supported on Windows",
|
||||
)
|
||||
def test_hardlink(tmp_dir, ssh):
|
||||
def test_hardlink(tmp_dir, ssh_connection):
|
||||
tmp_dir.gen("foo", "foo content")
|
||||
ssh.hardlink("foo", "link")
|
||||
ssh_connection.hardlink("foo", "link")
|
||||
assert System.is_hardlink("link")
|
||||
|
||||
|
||||
@@ -119,14 +121,14 @@ def test_hardlink(tmp_dir, ssh):
|
||||
platform.system() == "Windows",
|
||||
reason="copy is temporarily not supported on Windows",
|
||||
)
|
||||
def test_copy(tmp_dir, ssh):
|
||||
def test_copy(tmp_dir, ssh_connection):
|
||||
tmp_dir.gen("foo", "foo content")
|
||||
ssh.copy("foo", "link")
|
||||
ssh_connection.copy("foo", "link")
|
||||
assert filecmp.cmp("foo", "link")
|
||||
|
||||
|
||||
def test_move(tmp_dir, ssh):
|
||||
def test_move(tmp_dir, ssh_connection):
|
||||
tmp_dir.gen("foo", "foo content")
|
||||
ssh.move("foo", "copy")
|
||||
ssh_connection.move("foo", "copy")
|
||||
assert os.path.exists("copy")
|
||||
assert not os.path.exists("foo")
|
||||
|
||||
@@ -2,13 +2,19 @@ import pytest
|
||||
|
||||
from dvc.remote.pool import get_connection
|
||||
from dvc.remote.ssh.connection import SSHConnection
|
||||
from tests.remotes.ssh import TEST_SSH_KEY_PATH, TEST_SSH_USER
|
||||
|
||||
|
||||
def test_doesnt_swallow_errors(ssh_server):
|
||||
class MyError(Exception):
|
||||
pass
|
||||
|
||||
with pytest.raises(MyError), get_connection(
|
||||
SSHConnection, **ssh_server.test_creds
|
||||
):
|
||||
raise MyError
|
||||
with pytest.raises(MyError):
|
||||
with get_connection(
|
||||
SSHConnection,
|
||||
host=ssh_server.host,
|
||||
port=ssh_server.port,
|
||||
username=TEST_SSH_USER,
|
||||
key_filename=TEST_SSH_KEY_PATH,
|
||||
):
|
||||
raise MyError
|
||||
|
||||
@@ -7,7 +7,6 @@ from mock import mock_open, patch
|
||||
|
||||
from dvc.remote.ssh import SSHRemoteTree
|
||||
from dvc.system import System
|
||||
from tests.remotes import SSHMocked
|
||||
|
||||
|
||||
def test_url(dvc):
|
||||
@@ -184,17 +183,8 @@ def test_ssh_gss_auth(mock_file, mock_exists, dvc, config, expected_gss_auth):
|
||||
assert tree.gss_auth == expected_gss_auth
|
||||
|
||||
|
||||
def test_hardlink_optimization(dvc, tmp_dir, ssh_server):
|
||||
port = ssh_server.test_creds["port"]
|
||||
user = ssh_server.test_creds["username"]
|
||||
|
||||
config = {
|
||||
"url": SSHMocked.get_url(user, port),
|
||||
"port": port,
|
||||
"user": user,
|
||||
"keyfile": ssh_server.test_creds["key_filename"],
|
||||
}
|
||||
tree = SSHRemoteTree(dvc, config)
|
||||
def test_hardlink_optimization(dvc, tmp_dir, ssh):
|
||||
tree = SSHRemoteTree(dvc, ssh.config)
|
||||
|
||||
from_info = tree.path_info / "empty"
|
||||
to_info = tree.path_info / "link"
|
||||
|
||||
@@ -145,6 +145,28 @@ def test_walk(tmp_dir, dvc, dvcfiles, extra_expected):
|
||||
assert len(actual) == len(expected)
|
||||
|
||||
|
||||
def test_walk_onerror(tmp_dir, dvc):
|
||||
def onerror(exc):
|
||||
raise exc
|
||||
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
tree = RepoTree(dvc)
|
||||
|
||||
# path does not exist
|
||||
for _ in tree.walk("dir"):
|
||||
pass
|
||||
with pytest.raises(OSError):
|
||||
for _ in tree.walk("dir", onerror=onerror):
|
||||
pass
|
||||
|
||||
# path is not a directory
|
||||
for _ in tree.walk("foo"):
|
||||
pass
|
||||
with pytest.raises(OSError):
|
||||
for _ in tree.walk("foo", onerror=onerror):
|
||||
pass
|
||||
|
||||
|
||||
def test_isdvc(tmp_dir, dvc):
|
||||
tmp_dir.gen({"foo": "foo", "bar": "bar", "dir": {"baz": "baz"}})
|
||||
dvc.add("foo")
|
||||
|
||||
@@ -173,6 +173,28 @@ def test_walk_dir(tmp_dir, dvc, fetch, expected):
|
||||
assert len(actual) == len(expected)
|
||||
|
||||
|
||||
def test_walk_onerror(tmp_dir, dvc):
|
||||
def onerror(exc):
|
||||
raise exc
|
||||
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
tree = DvcTree(dvc)
|
||||
|
||||
# path does not exist
|
||||
for _ in tree.walk("dir"):
|
||||
pass
|
||||
with pytest.raises(OSError):
|
||||
for _ in tree.walk("dir", onerror=onerror):
|
||||
pass
|
||||
|
||||
# path is not a directory
|
||||
for _ in tree.walk("foo"):
|
||||
pass
|
||||
with pytest.raises(OSError):
|
||||
for _ in tree.walk("foo", onerror=onerror):
|
||||
pass
|
||||
|
||||
|
||||
def test_isdvc(tmp_dir, dvc):
|
||||
tmp_dir.gen({"foo": "foo", "bar": "bar"})
|
||||
dvc.add("foo")
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.basic_env import TestDvcGit
|
||||
|
||||
|
||||
@@ -37,6 +39,30 @@ def test_walk_with_submodules(tmp_dir, scm, git_dir):
|
||||
assert set(files) == {".gitmodules", "submodule"}
|
||||
|
||||
|
||||
def test_walk_onerror(tmp_dir, scm):
|
||||
def onerror(exc):
|
||||
raise exc
|
||||
|
||||
tmp_dir.scm_gen(
|
||||
{"foo": "foo"}, commit="init",
|
||||
)
|
||||
tree = scm.get_tree("HEAD")
|
||||
|
||||
# path does not exist
|
||||
for _ in tree.walk("dir"):
|
||||
pass
|
||||
with pytest.raises(OSError):
|
||||
for _ in tree.walk("dir", onerror=onerror):
|
||||
pass
|
||||
|
||||
# path is not a directory
|
||||
for _ in tree.walk("foo"):
|
||||
pass
|
||||
with pytest.raises(OSError):
|
||||
for _ in tree.walk("foo", onerror=onerror):
|
||||
pass
|
||||
|
||||
|
||||
def test_is_tracked(tmp_dir, scm):
|
||||
tmp_dir.scm_gen(
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user