mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-18 23:10:14 +01:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7c6351d039 | ||
|
|
613ca0bdb5 | ||
|
|
0fb9870de6 | ||
|
|
73e60a8c13 | ||
|
|
9d1dde4051 | ||
|
|
38dc0f85d4 | ||
|
|
d2dbe9649c | ||
|
|
ece6f2d3ee | ||
|
|
b672622324 | ||
|
|
c9391df313 | ||
|
|
a0c229cf9b | ||
|
|
a59f90fac9 | ||
|
|
bba502314d | ||
|
|
bc4bf66d17 | ||
|
|
f7348af1e0 | ||
|
|
4d140a9944 | ||
|
|
2ba73daedc | ||
|
|
7a31a35cbf | ||
|
|
24a14563cc | ||
|
|
a1d3440f85 | ||
|
|
45dcb7c27d | ||
|
|
9ff6fef53a | ||
|
|
9e7dc2640e | ||
|
|
584c97a58d | ||
|
|
b94a18cfb9 | ||
|
|
9ead641c95 | ||
|
|
652f5ab439 | ||
|
|
28a123e727 | ||
|
|
566472f088 | ||
|
|
2e12a31c5a | ||
|
|
d7788887f0 | ||
|
|
87276faef2 | ||
|
|
4042d5232d | ||
|
|
fda14c82d3 | ||
|
|
64be262d18 | ||
|
|
1d6c80001d | ||
|
|
91a7194e4f | ||
|
|
afe539aaba | ||
|
|
f259bc52cd | ||
|
|
db69def834 | ||
|
|
5efc3bbee2 | ||
|
|
7230d87137 | ||
|
|
56833387ce | ||
|
|
593e67318b | ||
|
|
afc183f4e7 | ||
|
|
d201fad1e3 | ||
|
|
5be12b5aac | ||
|
|
c1e5b10cbd | ||
|
|
b47b2c5673 | ||
|
|
939f78948a | ||
|
|
215ea4944c | ||
|
|
4e12e76e19 | ||
|
|
33092e8614 | ||
|
|
ae7b604531 | ||
|
|
b7c174641e | ||
|
|
cb43f9cdda | ||
|
|
791ce22759 | ||
|
|
c70adebc75 | ||
|
|
2c9cb87679 | ||
|
|
b57077af11 | ||
|
|
803bb3eb61 | ||
|
|
99b58d3d05 | ||
|
|
5bbaa28cae | ||
|
|
74bd8137ba | ||
|
|
72a576cefe | ||
|
|
036ce9398e | ||
|
|
13012f9c0c | ||
|
|
a65c7989c8 | ||
|
|
3590fe8414 | ||
|
|
05100122e1 | ||
|
|
9a55b909f8 | ||
|
|
ff54ac1bfb | ||
|
|
57e920aa39 | ||
|
|
ac4f351daf | ||
|
|
2c0116ccf9 | ||
|
|
c03ce4aa0f | ||
|
|
72dcbc0a9e | ||
|
|
fd65f44b7f | ||
|
|
7504b68c12 | ||
|
|
86f3457dec | ||
|
|
219662597b | ||
|
|
536966f9d1 | ||
|
|
076eadceba | ||
|
|
d1e08b6157 | ||
|
|
edfa1741db | ||
|
|
9ee62ef8f0 | ||
|
|
0bb49d2549 | ||
|
|
516d82e411 | ||
|
|
ec312d8601 | ||
|
|
f504245f35 | ||
|
|
bb50fb8664 | ||
|
|
777a9f80ae | ||
|
|
02a52d038a | ||
|
|
7d0a06248d | ||
|
|
2d8c6043ca | ||
|
|
3f996048e8 | ||
|
|
8d9435e8ee | ||
|
|
889a17e2fc | ||
|
|
63972c3d12 | ||
|
|
7027c631c5 | ||
|
|
e8f6cd7f5b | ||
|
|
10f2bb93fa | ||
|
|
2fecc66cb9 | ||
|
|
b47998463f | ||
|
|
d53b740f35 | ||
|
|
9198fa7459 | ||
|
|
6954efc6e6 | ||
|
|
d77af8196e | ||
|
|
c5920b00ab | ||
|
|
ce8036e535 | ||
|
|
b7ad2dc841 | ||
|
|
5ab948f8b6 | ||
|
|
f9fbe97509 | ||
|
|
2fba2e030a | ||
|
|
3e951162b8 | ||
|
|
d74358eecd | ||
|
|
58c9ff0de2 | ||
|
|
f0e991790e | ||
|
|
b7cd785eba | ||
|
|
21a3efe2c7 | ||
|
|
865d355554 | ||
|
|
84712c4acc | ||
|
|
0f16e15453 | ||
|
|
61f30ed5ba | ||
|
|
cde15e3d3e | ||
|
|
7672e324d2 | ||
|
|
130b76c7a5 | ||
|
|
040458a726 | ||
|
|
75e795c491 | ||
|
|
4dac2092c9 | ||
|
|
19eaced304 | ||
|
|
184028f176 | ||
|
|
5f63d538d2 | ||
|
|
324ff4be59 | ||
|
|
3f846a1794 | ||
|
|
b52c925d5f |
@@ -1,10 +1,18 @@
|
||||
---
|
||||
name: Bug report
|
||||
about: Create a report to help us improve
|
||||
|
||||
name: "\U0001F41B Bug Report"
|
||||
about: Create a bug report to help us improve DVC
|
||||
---
|
||||
|
||||
## Bug Report
|
||||
|
||||
**Please provide information about your setup**
|
||||
DVC version(i.e. `dvc --version`), Platform and method of installation (pip, homebrew, pkg Mac, exe (Windows), DEB(Linux), RPM(Linux))
|
||||
### Please provide information about your setup
|
||||
|
||||
**Output of `dvc version`:**
|
||||
|
||||
```console
|
||||
$ dvc version
|
||||
```
|
||||
|
||||
**Additional Information (if any):**
|
||||
|
||||
If applicable, please also provide a `--verbose` output of the command, eg: `dvc add --verbose`.
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
blank_issues_enabled: false
|
||||
contact_links:
|
||||
- name: "🤗 Need help?"
|
||||
url: https://dvc.org/chat
|
||||
about: If you have a question, ask us on Discord. Please join with this invite 👉
|
||||
@@ -1,6 +1,5 @@
|
||||
---
|
||||
name: Feature request
|
||||
name: "\U0001F680 Feature Request"
|
||||
about: Suggest an idea for this project
|
||||
|
||||
---
|
||||
|
||||
|
||||
+26
-7
@@ -27,18 +27,37 @@ repos:
|
||||
repo: https://github.com/lovesegfault/beautysh
|
||||
rev: master
|
||||
- hooks:
|
||||
- id: dvc-pre-commit
|
||||
language_version: python3
|
||||
- args:
|
||||
- git-hook
|
||||
- pre-commit
|
||||
entry: dvc
|
||||
id: dvc-pre-commit
|
||||
language: system
|
||||
name: DVC pre-commit
|
||||
stages:
|
||||
- commit
|
||||
- id: dvc-pre-push
|
||||
language_version: python3
|
||||
verbose: true
|
||||
require_serial: true
|
||||
- args:
|
||||
- git-hook
|
||||
- pre-push
|
||||
entry: dvc
|
||||
id: dvc-pre-push
|
||||
language: system
|
||||
name: DVC pre-push
|
||||
stages:
|
||||
- push
|
||||
require_serial: true
|
||||
- always_run: true
|
||||
args:
|
||||
- git-hook
|
||||
- post-checkout
|
||||
entry: dvc
|
||||
id: dvc-post-checkout
|
||||
language_version: python3
|
||||
language: system
|
||||
minimum_pre_commit_version: 2.2.0
|
||||
name: DVC post-checkout
|
||||
stages:
|
||||
- post-checkout
|
||||
repo: https://github.com/iterative/dvc
|
||||
rev: master
|
||||
require_serial: true
|
||||
repo: local
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
language: python
|
||||
language_version: python3
|
||||
name: DVC pre-commit
|
||||
require_serial: true
|
||||
stages:
|
||||
- commit
|
||||
verbose: true
|
||||
@@ -17,6 +18,7 @@
|
||||
language: python
|
||||
language_version: python3
|
||||
name: DVC pre-push
|
||||
require_serial: true
|
||||
stages:
|
||||
- push
|
||||
- always_run: true
|
||||
@@ -29,5 +31,6 @@
|
||||
language_version: python3
|
||||
minimum_pre_commit_version: 2.2.0
|
||||
name: DVC post-checkout
|
||||
require_serial: true
|
||||
stages:
|
||||
- post-checkout
|
||||
|
||||
+14
-3
@@ -98,12 +98,15 @@ jobs:
|
||||
- name: lxd
|
||||
channel: stable
|
||||
env:
|
||||
- SNAPCRAFT_IMAGE_INFO: '{"build_url": "$TRAVIS_BUILD_URL"}'
|
||||
- SNAPCRAFT_IMAGE_INFO: |
|
||||
'{"build_url": "$TRAVIS_JOB_WEB_URL"}'
|
||||
- SNAPCRAFT_BUILD_ENVIRONMENT: lxd
|
||||
- SNAPCRAFT_BUILD_INFO: 1 # https://snapcraft.io/blog/introducing-developer-notifications-for-snap-security-updates
|
||||
install:
|
||||
- sudo usermod --append --groups lxd $USER
|
||||
- sudo /snap/bin/lxd.migrate -yes
|
||||
- sudo /snap/bin/lxd waitready
|
||||
- sudo /snap/bin/lxd init --auto
|
||||
- git fetch --tags
|
||||
script:
|
||||
- ./scripts/build_snap.sh
|
||||
after_failure:
|
||||
@@ -154,6 +157,14 @@ deploy:
|
||||
script: "(echo $SNAP_TOKEN | snapcraft login --with -) && timeout 600 snapcraft push dvc_*.snap --release $SNAP_CHANNEL || echo timed out"
|
||||
on:
|
||||
all_branches: true
|
||||
condition: "$(./scripts/ci/deploy_condition.sh dvc_*.snap) && ($TRAVIS_BRANCH = master || -n $TRAVIS_TAG)"
|
||||
condition: "$(./scripts/ci/deploy_condition.sh dvc_*.snap) && ($TRAVIS_BRANCH = master || -n $TRAVIS_TAG) && -n $SNAP_CHANNEL"
|
||||
repo: iterative/dvc
|
||||
stage: build
|
||||
- provider: script
|
||||
skip_cleanup: true
|
||||
script: "(echo $SNAP_TOKEN | snapcraft login --with -) && timeout 600 snapcraft push dvc_*.snap --release $SNAP_CHANNEL_MAJOR || echo timed out"
|
||||
on:
|
||||
all_branches: true
|
||||
condition: "$(./scripts/ci/deploy_condition.sh dvc_*.snap) && ($TRAVIS_BRANCH = master || -n $TRAVIS_TAG) && -n $SNAP_CHANNEL_MAJOR"
|
||||
repo: iterative/dvc
|
||||
stage: build
|
||||
|
||||
+2
-1
@@ -101,7 +101,8 @@ Snap (Snapcraft/Linux)
|
||||
snap install dvc --classic
|
||||
|
||||
This corresponds to the latest tagged release.
|
||||
Add ``--edge`` for the latest ``master`` version.
|
||||
Add ``--beta`` for the latest tagged release candidate,
|
||||
or ``--edge`` for the latest ``master`` version.
|
||||
|
||||
Choco (Chocolatey/Windows)
|
||||
--------------------------
|
||||
|
||||
+1
-1
@@ -30,7 +30,7 @@ def get_url(path, repo=None, rev=None, remote=None):
|
||||
raise UrlNotDvcRepoError(_repo.url)
|
||||
out = _repo.find_out_by_relpath(path)
|
||||
remote_obj = _repo.cloud.get_remote(remote)
|
||||
return str(remote_obj.checksum_to_path_info(out.checksum))
|
||||
return str(remote_obj.hash_to_path_info(out.checksum))
|
||||
|
||||
|
||||
def open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
|
||||
|
||||
+8
-4
@@ -28,13 +28,15 @@ def _make_remote_property(name):
|
||||
"""
|
||||
|
||||
def getter(self):
|
||||
from dvc.remote import Remote
|
||||
from dvc.remote import get_cloud_tree
|
||||
from dvc.remote.base import CloudCache
|
||||
|
||||
remote = self.config.get(name)
|
||||
if not remote:
|
||||
return None
|
||||
|
||||
return Remote(self.repo, name=remote)
|
||||
tree = get_cloud_tree(self.repo, name=remote)
|
||||
return CloudCache(tree)
|
||||
|
||||
getter.__name__ = name
|
||||
return cached_property(getter)
|
||||
@@ -50,7 +52,8 @@ class Cache:
|
||||
CACHE_DIR = "cache"
|
||||
|
||||
def __init__(self, repo):
|
||||
from dvc.remote import Remote
|
||||
from dvc.remote import get_cloud_tree
|
||||
from dvc.remote.local import LocalCache
|
||||
|
||||
self.repo = repo
|
||||
self.config = config = repo.config["cache"]
|
||||
@@ -62,7 +65,8 @@ class Cache:
|
||||
else:
|
||||
settings = {**config, "url": config["dir"]}
|
||||
|
||||
self.local = Remote(repo, **settings)
|
||||
tree = get_cloud_tree(repo, **settings)
|
||||
self.local = LocalCache(tree)
|
||||
|
||||
s3 = _make_remote_property("s3")
|
||||
gs = _make_remote_property("gs")
|
||||
|
||||
+23
-12
@@ -10,9 +10,11 @@ from .command import (
|
||||
commit,
|
||||
config,
|
||||
daemon,
|
||||
dag,
|
||||
data_sync,
|
||||
destroy,
|
||||
diff,
|
||||
freeze,
|
||||
gc,
|
||||
get,
|
||||
get_url,
|
||||
@@ -21,12 +23,10 @@ from .command import (
|
||||
imp_url,
|
||||
init,
|
||||
install,
|
||||
lock,
|
||||
ls,
|
||||
metrics,
|
||||
move,
|
||||
params,
|
||||
pipeline,
|
||||
plots,
|
||||
remote,
|
||||
remove,
|
||||
@@ -66,8 +66,8 @@ COMMANDS = [
|
||||
install,
|
||||
root,
|
||||
ls,
|
||||
lock,
|
||||
pipeline,
|
||||
freeze,
|
||||
dag,
|
||||
daemon,
|
||||
commit,
|
||||
diff,
|
||||
@@ -81,18 +81,33 @@ COMMANDS = [
|
||||
class DvcParser(argparse.ArgumentParser):
|
||||
"""Custom parser class for dvc CLI."""
|
||||
|
||||
def error(self, message):
|
||||
def error(self, message, command=None):
|
||||
"""Custom error method.
|
||||
Args:
|
||||
message (str): error message.
|
||||
|
||||
command (str): subcommand name for help message
|
||||
Raises:
|
||||
dvc.exceptions.DvcParser: dvc parser exception.
|
||||
|
||||
"""
|
||||
logger.error(message)
|
||||
if command is not None:
|
||||
for action in self._actions:
|
||||
if action.dest == "cmd" and command in action.choices:
|
||||
subparser = action.choices[command]
|
||||
subparser.print_help()
|
||||
raise DvcParserError()
|
||||
self.print_help()
|
||||
raise DvcParserError()
|
||||
|
||||
# override this to send subcommand name to error method
|
||||
def parse_args(self, args=None, namespace=None):
|
||||
args, argv = self.parse_known_args(args, namespace)
|
||||
if argv:
|
||||
msg = "unrecognized arguments: %s"
|
||||
self.error(msg % " ".join(argv), args.cmd)
|
||||
return args
|
||||
|
||||
|
||||
class VersionAction(argparse.Action): # pragma: no cover
|
||||
# pylint: disable=too-few-public-methods
|
||||
@@ -116,14 +131,10 @@ def get_parent_parser():
|
||||
|
||||
log_level_group = parent_parser.add_mutually_exclusive_group()
|
||||
log_level_group.add_argument(
|
||||
"-q", "--quiet", action="store_true", default=False, help="Be quiet."
|
||||
"-q", "--quiet", action="count", default=0, help="Be quiet."
|
||||
)
|
||||
log_level_group.add_argument(
|
||||
"-v",
|
||||
"--verbose",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Be verbose.",
|
||||
"-v", "--verbose", action="count", default=0, help="Be verbose."
|
||||
)
|
||||
|
||||
return parent_parser
|
||||
|
||||
+7
-1
@@ -18,6 +18,7 @@ class CmdAdd(CmdBase):
|
||||
recursive=self.args.recursive,
|
||||
no_commit=self.args.no_commit,
|
||||
fname=self.args.file,
|
||||
external=self.args.external,
|
||||
)
|
||||
|
||||
except DvcException:
|
||||
@@ -50,7 +51,12 @@ def add_parser(subparsers, parent_parser):
|
||||
help="Don't put files/directories into cache.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-f",
|
||||
"--external",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Allow targets that are outside of the DVC repository.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--file",
|
||||
help="Specify name of the DVC-file this command will generate.",
|
||||
metavar="<filename>",
|
||||
|
||||
@@ -50,21 +50,22 @@ class CmdConfig(CmdBaseNoRepo):
|
||||
|
||||
|
||||
parent_config_parser = argparse.ArgumentParser(add_help=False)
|
||||
parent_config_parser.add_argument(
|
||||
level_group = parent_config_parser.add_mutually_exclusive_group()
|
||||
level_group.add_argument(
|
||||
"--global",
|
||||
dest="level",
|
||||
action="store_const",
|
||||
const="global",
|
||||
help="Use global config.",
|
||||
)
|
||||
parent_config_parser.add_argument(
|
||||
level_group.add_argument(
|
||||
"--system",
|
||||
dest="level",
|
||||
action="store_const",
|
||||
const="system",
|
||||
help="Use system config.",
|
||||
)
|
||||
parent_config_parser.add_argument(
|
||||
level_group.add_argument(
|
||||
"--local",
|
||||
dest="level",
|
||||
action="store_const",
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _show_ascii(G):
|
||||
from dvc.repo.graph import get_pipelines
|
||||
from dvc.dagascii import draw
|
||||
|
||||
pipelines = get_pipelines(G)
|
||||
|
||||
ret = []
|
||||
for pipeline in pipelines:
|
||||
ret.append(draw(pipeline.nodes, pipeline.edges))
|
||||
|
||||
return "\n".join(ret)
|
||||
|
||||
|
||||
def _show_dot(G):
|
||||
import io
|
||||
from networkx.drawing.nx_pydot import write_dot
|
||||
|
||||
dot_file = io.StringIO()
|
||||
write_dot(G, dot_file)
|
||||
return dot_file.getvalue()
|
||||
|
||||
|
||||
def _build(G, target=None, full=False):
|
||||
import networkx as nx
|
||||
from dvc.repo.graph import get_pipeline, get_pipelines
|
||||
|
||||
if target:
|
||||
H = get_pipeline(get_pipelines(G), target)
|
||||
if not full:
|
||||
descendants = nx.descendants(G, target)
|
||||
descendants.add(target)
|
||||
H.remove_nodes_from(set(G.nodes()) - descendants)
|
||||
else:
|
||||
H = G
|
||||
|
||||
def _relabel(stage):
|
||||
return stage.addressing
|
||||
|
||||
return nx.relabel_nodes(H, _relabel, copy=False)
|
||||
|
||||
|
||||
class CmdDAG(CmdBase):
|
||||
def run(self):
|
||||
try:
|
||||
target = None
|
||||
if self.args.target:
|
||||
stages = self.repo.collect(self.args.target)
|
||||
if len(stages) > 1:
|
||||
logger.error(
|
||||
f"'{self.args.target}' contains more than one stage "
|
||||
"{stages}, please specify one stage"
|
||||
)
|
||||
return 1
|
||||
target = stages[0]
|
||||
|
||||
G = _build(self.repo.graph, target=target, full=self.args.full,)
|
||||
|
||||
if self.args.dot:
|
||||
logger.info(_show_dot(G))
|
||||
else:
|
||||
from dvc.utils.pager import pager
|
||||
|
||||
pager(_show_ascii(G))
|
||||
|
||||
return 0
|
||||
except DvcException:
|
||||
msg = "failed to show "
|
||||
if self.args.target:
|
||||
msg += f"a pipeline for '{target}'"
|
||||
else:
|
||||
msg += "pipelines"
|
||||
logger.exception(msg)
|
||||
return 1
|
||||
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
DAG_HELP = "Visualize DVC project DAG."
|
||||
dag_parser = subparsers.add_parser(
|
||||
"dag",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(DAG_HELP, "dag"),
|
||||
help=DAG_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
dag_parser.add_argument(
|
||||
"--dot",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Print DAG with .dot format.",
|
||||
)
|
||||
dag_parser.add_argument(
|
||||
"--full",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help=(
|
||||
"Show full DAG that the target belongs too, instead of "
|
||||
"showing DAG consisting only of ancestors."
|
||||
),
|
||||
)
|
||||
dag_parser.add_argument(
|
||||
"target",
|
||||
nargs="?",
|
||||
help="Stage or output to show pipeline for (optional). "
|
||||
"Finds all stages in the workspace by default.",
|
||||
)
|
||||
dag_parser.set_defaults(func=CmdDAG)
|
||||
@@ -363,5 +363,11 @@ def add_parser(subparsers, _parent_parser):
|
||||
default=False,
|
||||
help="Show status of all stages in the specified directory.",
|
||||
)
|
||||
status_parser.add_argument(
|
||||
"--show-json",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Show status in JSON format.",
|
||||
)
|
||||
|
||||
status_parser.set_defaults(func=CmdDataStatus)
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CmdFreezeBase(CmdBase):
|
||||
def _run(self, func, name):
|
||||
ret = 0
|
||||
for target in self.args.targets:
|
||||
try:
|
||||
func(target)
|
||||
except DvcException:
|
||||
logger.exception(f"failed to {name} '{target}'")
|
||||
ret = 1
|
||||
return ret
|
||||
|
||||
|
||||
class CmdFreeze(CmdFreezeBase):
|
||||
def run(self):
|
||||
return self._run(self.repo.freeze, "freeze")
|
||||
|
||||
|
||||
class CmdUnfreeze(CmdFreezeBase):
|
||||
def run(self):
|
||||
return self._run(self.repo.unfreeze, "unfreeze")
|
||||
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
FREEZE_HELP = "Freeze DVC-files."
|
||||
freeze_parser = subparsers.add_parser(
|
||||
"freeze",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(FREEZE_HELP, "freeze"),
|
||||
help=FREEZE_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
freeze_parser.add_argument(
|
||||
"targets", nargs="+", help="DVC-files to freeze."
|
||||
)
|
||||
freeze_parser.set_defaults(func=CmdFreeze)
|
||||
|
||||
UNFREEZE_HELP = "Unfreeze DVC-files."
|
||||
unfreeze_parser = subparsers.add_parser(
|
||||
"unfreeze",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(UNFREEZE_HELP, "unfreeze"),
|
||||
help=UNFREEZE_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
unfreeze_parser.add_argument(
|
||||
"targets", nargs="+", help="DVC-files to unfreeze."
|
||||
)
|
||||
unfreeze_parser.set_defaults(func=CmdUnfreeze)
|
||||
+2
-2
@@ -21,7 +21,7 @@ class CmdGC(CmdBase):
|
||||
|
||||
msg = "This will remove all cache except items used in "
|
||||
|
||||
msg += "the working tree"
|
||||
msg += "the workspace"
|
||||
if self.args.all_commits:
|
||||
msg += " and all git commits"
|
||||
elif self.args.all_branches and self.args.all_tags:
|
||||
@@ -77,7 +77,7 @@ def add_parser(subparsers, parent_parser):
|
||||
"--workspace",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Keep data files used in the current workspace (working tree).",
|
||||
help="Keep data files used in the current workspace.",
|
||||
)
|
||||
gc_parser.add_argument(
|
||||
"-a",
|
||||
|
||||
@@ -57,7 +57,6 @@ def add_parser(subparsers, parent_parser):
|
||||
"out", nargs="?", help="Destination path to put files to."
|
||||
)
|
||||
import_parser.add_argument(
|
||||
"-f",
|
||||
"--file",
|
||||
help="Specify name of the DVC-file this command will generate.",
|
||||
metavar="<filename>",
|
||||
|
||||
@@ -1,59 +0,0 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CmdLockBase(CmdBase):
|
||||
def _run(self, unlock):
|
||||
for target in self.args.targets:
|
||||
try:
|
||||
self.repo.lock_stage(target, unlock=unlock)
|
||||
except DvcException:
|
||||
logger.exception(
|
||||
"failed to {}lock '{}'".format(
|
||||
"un" if unlock else "", target
|
||||
)
|
||||
)
|
||||
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
class CmdLock(CmdLockBase):
|
||||
def run(self):
|
||||
return self._run(False)
|
||||
|
||||
|
||||
class CmdUnlock(CmdLockBase):
|
||||
def run(self):
|
||||
return self._run(True)
|
||||
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
LOCK_HELP = "Lock DVC-files."
|
||||
lock_parser = subparsers.add_parser(
|
||||
"lock",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(LOCK_HELP, "lock"),
|
||||
help=LOCK_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
lock_parser.add_argument("targets", nargs="+", help="DVC-files to lock.")
|
||||
lock_parser.set_defaults(func=CmdLock)
|
||||
|
||||
UNLOCK_HELP = "Unlock DVC-files."
|
||||
unlock_parser = subparsers.add_parser(
|
||||
"unlock",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(UNLOCK_HELP, "unlock"),
|
||||
help=UNLOCK_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
unlock_parser.add_argument(
|
||||
"targets", nargs="+", help="DVC-files to unlock."
|
||||
)
|
||||
unlock_parser.set_defaults(func=CmdUnlock)
|
||||
@@ -31,7 +31,7 @@ class CmdList(CmdBaseNoRepo):
|
||||
self.args.path,
|
||||
rev=self.args.rev,
|
||||
recursive=self.args.recursive,
|
||||
outs_only=self.args.outs_only,
|
||||
dvc_only=self.args.dvc_only,
|
||||
)
|
||||
if entries:
|
||||
entries = _prettify(entries, sys.stdout.isatty())
|
||||
@@ -62,7 +62,7 @@ def add_parser(subparsers, parent_parser):
|
||||
help="Recursively list files.",
|
||||
)
|
||||
list_parser.add_argument(
|
||||
"--outs-only", action="store_true", help="Show only DVC outputs."
|
||||
"--dvc-only", action="store_true", help="Show only DVC outputs."
|
||||
)
|
||||
list_parser.add_argument(
|
||||
"--rev",
|
||||
|
||||
+75
-70
@@ -7,6 +7,9 @@ from dvc.exceptions import BadMetricError, DvcException
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
DEFAULT_PRECISION = 5
|
||||
|
||||
|
||||
def show_metrics(
|
||||
metrics, all_branches=False, all_tags=False, all_commits=False
|
||||
):
|
||||
@@ -63,49 +66,41 @@ class CmdMetricsShow(CmdBase):
|
||||
return 0
|
||||
|
||||
|
||||
class CmdMetricsAdd(CmdBase):
|
||||
def run(self):
|
||||
try:
|
||||
self.repo.metrics.add(self.args.path)
|
||||
except DvcException:
|
||||
msg = f"failed to add metric file '{self.args.path}'"
|
||||
logger.exception(msg)
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
class CmdMetricsRemove(CmdBase):
|
||||
def run(self):
|
||||
try:
|
||||
self.repo.metrics.remove(self.args.path)
|
||||
except DvcException:
|
||||
msg = f"failed to remove metric file '{self.args.path}'"
|
||||
logger.exception(msg)
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def _show_diff(diff, markdown=False):
|
||||
def _show_diff(diff, markdown=False, no_path=False, old=False, precision=None):
|
||||
from collections import OrderedDict
|
||||
|
||||
from dvc.utils.diff import table
|
||||
|
||||
if precision is None:
|
||||
precision = DEFAULT_PRECISION
|
||||
|
||||
def _round(val):
|
||||
if isinstance(val, float):
|
||||
return round(val, precision)
|
||||
|
||||
return val
|
||||
|
||||
rows = []
|
||||
for fname, mdiff in diff.items():
|
||||
sorted_mdiff = OrderedDict(sorted(mdiff.items()))
|
||||
for metric, change in sorted_mdiff.items():
|
||||
rows.append(
|
||||
[
|
||||
fname,
|
||||
metric,
|
||||
change["new"],
|
||||
change.get("diff", "diff not supported"),
|
||||
]
|
||||
)
|
||||
row = [] if no_path else [fname]
|
||||
row.append(metric)
|
||||
if old:
|
||||
row.append(_round(change.get("old")))
|
||||
row.append(_round(change["new"]))
|
||||
row.append(_round(change.get("diff", "diff not supported")))
|
||||
rows.append(row)
|
||||
|
||||
return table(["Path", "Metric", "Value", "Change"], rows, markdown)
|
||||
header = [] if no_path else ["Path"]
|
||||
header.append("Metric")
|
||||
if old:
|
||||
header.extend(["Old", "New"])
|
||||
else:
|
||||
header.append("Value")
|
||||
header.append("Change")
|
||||
|
||||
return table(header, rows, markdown)
|
||||
|
||||
|
||||
class CmdMetricsDiff(CmdBase):
|
||||
@@ -124,7 +119,13 @@ class CmdMetricsDiff(CmdBase):
|
||||
|
||||
logger.info(json.dumps(diff))
|
||||
else:
|
||||
table = _show_diff(diff, self.args.show_md)
|
||||
table = _show_diff(
|
||||
diff,
|
||||
self.args.show_md,
|
||||
self.args.no_path,
|
||||
self.args.old,
|
||||
precision=self.args.precision,
|
||||
)
|
||||
if table:
|
||||
logger.info(table)
|
||||
|
||||
@@ -136,7 +137,7 @@ class CmdMetricsDiff(CmdBase):
|
||||
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
METRICS_HELP = "Commands to add, manage, collect, and display metrics."
|
||||
METRICS_HELP = "Commands to display and compare metrics."
|
||||
|
||||
metrics_parser = subparsers.add_parser(
|
||||
"metrics",
|
||||
@@ -153,17 +154,6 @@ def add_parser(subparsers, parent_parser):
|
||||
|
||||
fix_subparsers(metrics_subparsers)
|
||||
|
||||
METRICS_ADD_HELP = "Mark a DVC-tracked file as a metric."
|
||||
metrics_add_parser = metrics_subparsers.add_parser(
|
||||
"add",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(METRICS_ADD_HELP, "metrics/add"),
|
||||
help=METRICS_ADD_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
metrics_add_parser.add_argument("path", help="Path to a metric file.")
|
||||
metrics_add_parser.set_defaults(func=CmdMetricsAdd)
|
||||
|
||||
METRICS_SHOW_HELP = "Print metrics, with optional formatting."
|
||||
metrics_show_parser = metrics_subparsers.add_parser(
|
||||
"show",
|
||||
@@ -175,7 +165,10 @@ def add_parser(subparsers, parent_parser):
|
||||
metrics_show_parser.add_argument(
|
||||
"targets",
|
||||
nargs="*",
|
||||
help="Metric files or directories (see -R) to show",
|
||||
help=(
|
||||
"Limit command scope to these metric files. Using -R, "
|
||||
"directories to search metric files in can also be given."
|
||||
),
|
||||
)
|
||||
metrics_show_parser.add_argument(
|
||||
"-a",
|
||||
@@ -197,6 +190,12 @@ def add_parser(subparsers, parent_parser):
|
||||
default=False,
|
||||
help="Show metrics for all commits.",
|
||||
)
|
||||
metrics_show_parser.add_argument(
|
||||
"--show-json",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Show output in JSON format.",
|
||||
)
|
||||
metrics_show_parser.add_argument(
|
||||
"-R",
|
||||
"--recursive",
|
||||
@@ -207,16 +206,12 @@ def add_parser(subparsers, parent_parser):
|
||||
"metric files."
|
||||
),
|
||||
)
|
||||
metrics_show_parser.add_argument(
|
||||
"--show-json",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Show output in JSON format.",
|
||||
)
|
||||
metrics_show_parser.set_defaults(func=CmdMetricsShow)
|
||||
|
||||
METRICS_DIFF_HELP = "Show changes in metrics between commits"
|
||||
" in the DVC repository, or between a commit and the workspace."
|
||||
METRICS_DIFF_HELP = (
|
||||
"Show changes in metrics between commits in the DVC repository, or "
|
||||
"between a commit and the workspace."
|
||||
)
|
||||
metrics_diff_parser = metrics_subparsers.add_parser(
|
||||
"diff",
|
||||
parents=[parent_parser],
|
||||
@@ -230,14 +225,14 @@ def add_parser(subparsers, parent_parser):
|
||||
metrics_diff_parser.add_argument(
|
||||
"b_rev",
|
||||
nargs="?",
|
||||
help=("New Git commit to compare (defaults to the current workspace)"),
|
||||
help="New Git commit to compare (defaults to the current workspace)",
|
||||
)
|
||||
metrics_diff_parser.add_argument(
|
||||
"--targets",
|
||||
nargs="*",
|
||||
help=(
|
||||
"Metric files or directories (see -R) to show diff for. "
|
||||
"Shows diff for all metric files by default."
|
||||
"Limit command scope to these metric files. Using -R, "
|
||||
"directories to search metric files in can also be given."
|
||||
),
|
||||
metavar="<paths>",
|
||||
)
|
||||
@@ -269,15 +264,25 @@ def add_parser(subparsers, parent_parser):
|
||||
default=False,
|
||||
help="Show tabulated output in the Markdown format (GFM).",
|
||||
)
|
||||
metrics_diff_parser.set_defaults(func=CmdMetricsDiff)
|
||||
|
||||
METRICS_REMOVE_HELP = "Remove metric mark on a DVC-tracked file."
|
||||
metrics_remove_parser = metrics_subparsers.add_parser(
|
||||
"remove",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(METRICS_REMOVE_HELP, "metrics/remove"),
|
||||
help=METRICS_REMOVE_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
metrics_diff_parser.add_argument(
|
||||
"--old",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Show old metric value.",
|
||||
)
|
||||
metrics_remove_parser.add_argument("path", help="Path to a metric file.")
|
||||
metrics_remove_parser.set_defaults(func=CmdMetricsRemove)
|
||||
metrics_diff_parser.add_argument(
|
||||
"--no-path",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Don't show metric path.",
|
||||
)
|
||||
metrics_diff_parser.add_argument(
|
||||
"--precision",
|
||||
type=int,
|
||||
help=(
|
||||
"Round metrics to `n` digits precision after the decimal point. "
|
||||
f"Rounds to {DEFAULT_PRECISION} digits by default."
|
||||
),
|
||||
metavar="<n>",
|
||||
)
|
||||
metrics_diff_parser.set_defaults(func=CmdMetricsDiff)
|
||||
|
||||
+19
-4
@@ -8,16 +8,25 @@ from dvc.exceptions import DvcException
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _show_diff(diff, markdown=False):
|
||||
def _show_diff(diff, markdown=False, no_path=False):
|
||||
from dvc.utils.diff import table
|
||||
|
||||
rows = []
|
||||
for fname, pdiff in diff.items():
|
||||
sorted_pdiff = OrderedDict(sorted(pdiff.items()))
|
||||
for param, change in sorted_pdiff.items():
|
||||
rows.append([fname, param, change["old"], change["new"]])
|
||||
row = [] if no_path else [fname]
|
||||
row.append(param)
|
||||
row.append(change["old"])
|
||||
row.append(change["new"])
|
||||
rows.append(row)
|
||||
|
||||
return table(["Path", "Param", "Old", "New"], rows, markdown)
|
||||
header = [] if no_path else ["Path"]
|
||||
header.append("Param")
|
||||
header.append("Old")
|
||||
header.append("New")
|
||||
|
||||
return table(header, rows, markdown)
|
||||
|
||||
|
||||
class CmdParamsDiff(CmdBase):
|
||||
@@ -34,7 +43,7 @@ class CmdParamsDiff(CmdBase):
|
||||
|
||||
logger.info(json.dumps(diff))
|
||||
else:
|
||||
table = _show_diff(diff, self.args.show_md)
|
||||
table = _show_diff(diff, self.args.show_md, self.args.no_path)
|
||||
if table:
|
||||
logger.info(table)
|
||||
|
||||
@@ -100,4 +109,10 @@ def add_parser(subparsers, parent_parser):
|
||||
default=False,
|
||||
help="Show tabulated output in the Markdown format (GFM).",
|
||||
)
|
||||
params_diff_parser.add_argument(
|
||||
"--no-path",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Don't show params path.",
|
||||
)
|
||||
params_diff_parser.set_defaults(func=CmdParamsDiff)
|
||||
|
||||
@@ -1,267 +0,0 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CmdPipelineShow(CmdBase):
|
||||
def _show(self, target, commands, outs, locked):
|
||||
import networkx
|
||||
from dvc import dvcfile
|
||||
from dvc.utils import parse_target
|
||||
|
||||
path, name = parse_target(target)
|
||||
stage = dvcfile.Dvcfile(self.repo, path).stages[name]
|
||||
G = self.repo.graph
|
||||
stages = networkx.dfs_postorder_nodes(G, stage)
|
||||
if locked:
|
||||
stages = [s for s in stages if s.locked]
|
||||
|
||||
for stage in stages:
|
||||
if commands:
|
||||
if stage.cmd is None:
|
||||
continue
|
||||
logger.info(stage.cmd)
|
||||
elif outs:
|
||||
for out in stage.outs:
|
||||
logger.info(str(out))
|
||||
else:
|
||||
logger.info(stage.addressing)
|
||||
|
||||
@staticmethod
|
||||
def _build_output_graph(G, target_stage):
|
||||
import networkx
|
||||
from itertools import product
|
||||
|
||||
nodes = {str(out) for out in target_stage.outs}
|
||||
edges = []
|
||||
|
||||
for from_stage, to_stage in networkx.edge_dfs(G, target_stage):
|
||||
from_stage_deps = {dep.path_info.parts for dep in from_stage.deps}
|
||||
to_outs = {
|
||||
to_out
|
||||
for to_out in to_stage.outs
|
||||
if to_out.path_info.parts in from_stage_deps
|
||||
}
|
||||
from_outs = {
|
||||
from_out
|
||||
for from_out in from_stage.outs
|
||||
if str(from_out) in nodes
|
||||
}
|
||||
nodes |= {str(to_out) for to_out in to_outs}
|
||||
for from_out, to_out in product(from_outs, to_outs):
|
||||
edges.append((str(from_out), str(to_out)))
|
||||
return nodes, edges
|
||||
|
||||
def _build_graph(self, target, commands=False, outs=False):
|
||||
import networkx
|
||||
from dvc import dvcfile
|
||||
from dvc.repo.graph import get_pipeline
|
||||
from dvc.utils import parse_target
|
||||
|
||||
path, name = parse_target(target)
|
||||
target_stage = dvcfile.Dvcfile(self.repo, path).stages[name]
|
||||
G = get_pipeline(self.repo.pipelines, target_stage)
|
||||
|
||||
nodes = set()
|
||||
for stage in networkx.dfs_preorder_nodes(G, target_stage):
|
||||
if commands:
|
||||
if stage.cmd is None:
|
||||
continue
|
||||
nodes.add(stage.cmd)
|
||||
elif not outs:
|
||||
nodes.add(stage.addressing)
|
||||
|
||||
edges = []
|
||||
for from_stage, to_stage in networkx.edge_dfs(G, target_stage):
|
||||
if commands:
|
||||
if to_stage.cmd is None:
|
||||
continue
|
||||
edges.append((from_stage.cmd, to_stage.cmd))
|
||||
elif not outs:
|
||||
edges.append((from_stage.addressing, to_stage.addressing))
|
||||
|
||||
if outs:
|
||||
nodes, edges = self._build_output_graph(G, target_stage)
|
||||
|
||||
return list(nodes), edges, networkx.is_tree(G)
|
||||
|
||||
def _show_ascii(self, target, commands, outs):
|
||||
from dvc.dagascii import draw
|
||||
|
||||
nodes, edges, _ = self._build_graph(target, commands, outs)
|
||||
|
||||
if not nodes:
|
||||
return
|
||||
|
||||
draw(nodes, edges)
|
||||
|
||||
def _show_dependencies_tree(self, target, commands, outs):
|
||||
from treelib import Tree
|
||||
|
||||
nodes, edges, is_tree = self._build_graph(target, commands, outs)
|
||||
if not nodes:
|
||||
return
|
||||
if not is_tree:
|
||||
raise DvcException(
|
||||
"DAG is not a tree, can not print it in tree-structure way, "
|
||||
"please use --ascii instead"
|
||||
)
|
||||
|
||||
tree = Tree()
|
||||
tree.create_node(target, target) # Root node
|
||||
observe_list = [target]
|
||||
while len(observe_list) > 0:
|
||||
current_root = observe_list[0]
|
||||
for edge in edges:
|
||||
if edge[0] == current_root:
|
||||
tree.create_node(edge[1], edge[1], parent=current_root)
|
||||
observe_list.append(edge[1])
|
||||
observe_list.pop(0)
|
||||
tree.show()
|
||||
|
||||
def _write_dot(self, target, commands, outs):
|
||||
import io
|
||||
import networkx
|
||||
from networkx.drawing.nx_pydot import write_dot
|
||||
|
||||
_, edges, _ = self._build_graph(target, commands, outs)
|
||||
edges = [edge[::-1] for edge in edges]
|
||||
|
||||
simple_g = networkx.DiGraph()
|
||||
simple_g.add_edges_from(edges)
|
||||
|
||||
dot_file = io.StringIO()
|
||||
write_dot(simple_g, dot_file)
|
||||
logger.info(dot_file.getvalue())
|
||||
|
||||
def run(self):
|
||||
from dvc.dvcfile import DVC_FILE
|
||||
|
||||
if not self.args.targets:
|
||||
self.args.targets = [DVC_FILE]
|
||||
|
||||
for target in self.args.targets:
|
||||
try:
|
||||
if self.args.ascii:
|
||||
self._show_ascii(
|
||||
target, self.args.commands, self.args.outs
|
||||
)
|
||||
elif self.args.dot:
|
||||
self._write_dot(target, self.args.commands, self.args.outs)
|
||||
elif self.args.tree:
|
||||
self._show_dependencies_tree(
|
||||
target, self.args.commands, self.args.outs
|
||||
)
|
||||
else:
|
||||
self._show(
|
||||
target,
|
||||
self.args.commands,
|
||||
self.args.outs,
|
||||
self.args.locked,
|
||||
)
|
||||
except DvcException:
|
||||
msg = f"failed to show pipeline for '{target}'"
|
||||
logger.exception(msg)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
class CmdPipelineList(CmdBase):
|
||||
def run(self):
|
||||
pipelines = self.repo.pipelines
|
||||
for pipeline in pipelines:
|
||||
for stage in pipeline:
|
||||
logger.info(stage.addressing)
|
||||
if len(pipeline) != 0:
|
||||
logger.info("=" * 80)
|
||||
logger.info("{} pipelines total".format(len(pipelines)))
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
PIPELINE_HELP = "Manage pipelines."
|
||||
pipeline_parser = subparsers.add_parser(
|
||||
"pipeline",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(PIPELINE_HELP, "pipeline"),
|
||||
help=PIPELINE_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
|
||||
pipeline_subparsers = pipeline_parser.add_subparsers(
|
||||
dest="cmd",
|
||||
help="Use `dvc pipeline CMD --help` for command-specific help.",
|
||||
)
|
||||
|
||||
fix_subparsers(pipeline_subparsers)
|
||||
|
||||
PIPELINE_LIST_HELP = "List connected groups of stages (pipelines)."
|
||||
pipeline_list_parser = pipeline_subparsers.add_parser(
|
||||
"list",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(PIPELINE_LIST_HELP, "pipeline/list"),
|
||||
help=PIPELINE_LIST_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
pipeline_list_parser.set_defaults(func=CmdPipelineList)
|
||||
|
||||
PIPELINE_SHOW_HELP = "Show stages in a pipeline."
|
||||
pipeline_show_parser = pipeline_subparsers.add_parser(
|
||||
"show",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(PIPELINE_SHOW_HELP, "pipeline/show"),
|
||||
help=PIPELINE_SHOW_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
pipeline_show_group = pipeline_show_parser.add_mutually_exclusive_group()
|
||||
pipeline_show_group.add_argument(
|
||||
"-c",
|
||||
"--commands",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Print commands instead of paths to DVC-files.",
|
||||
)
|
||||
pipeline_show_group.add_argument(
|
||||
"-o",
|
||||
"--outs",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Print output files instead of paths to DVC-files.",
|
||||
)
|
||||
pipeline_show_parser.add_argument(
|
||||
"-l",
|
||||
"--locked",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Print locked DVC stages",
|
||||
)
|
||||
pipeline_show_parser.add_argument(
|
||||
"--ascii",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Output DAG as ASCII.",
|
||||
)
|
||||
pipeline_show_parser.add_argument(
|
||||
"--dot",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Print DAG with .dot format.",
|
||||
)
|
||||
pipeline_show_parser.add_argument(
|
||||
"--tree",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Output DAG as Dependencies Tree.",
|
||||
)
|
||||
pipeline_show_parser.add_argument(
|
||||
"targets",
|
||||
nargs="*",
|
||||
help="DVC-files to show pipeline for. Optional. "
|
||||
"(Finds all DVC-files in the workspace by default.)",
|
||||
)
|
||||
pipeline_show_parser.set_defaults(func=CmdPipelineShow)
|
||||
+115
-112
@@ -4,6 +4,8 @@ import os
|
||||
|
||||
from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.schema import PLOT_PROPS
|
||||
from dvc.utils import format_link
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -31,32 +33,28 @@ class CmdPlots(CmdBase):
|
||||
def _func(self, *args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
def _props(self):
|
||||
# Pass only props specified by user, to not shadow ones from plot def
|
||||
props = {p: getattr(self.args, p) for p in PLOT_PROPS}
|
||||
return {k: v for k, v in props.items() if v is not None}
|
||||
|
||||
def run(self):
|
||||
fields = None
|
||||
jsonpath = None
|
||||
if self.args.select:
|
||||
if self.args.select.startswith("$"):
|
||||
jsonpath = self.args.select
|
||||
else:
|
||||
fields = set(self.args.select.split(","))
|
||||
if self.args.show_vega:
|
||||
if not self.args.targets:
|
||||
logger.error("please specify a target for `--show-vega`")
|
||||
return 1
|
||||
if len(self.args.targets) > 1:
|
||||
logger.error(
|
||||
"you can only specify one target for `--show-vega`"
|
||||
)
|
||||
return 1
|
||||
|
||||
try:
|
||||
plots = self._func(
|
||||
targets=self.args.targets,
|
||||
template=self.args.template,
|
||||
fields=fields,
|
||||
x_field=self.args.x,
|
||||
y_field=self.args.y,
|
||||
path=jsonpath,
|
||||
csv_header=not self.args.no_csv_header,
|
||||
title=self.args.title,
|
||||
x_title=self.args.xlab,
|
||||
y_title=self.args.ylab,
|
||||
)
|
||||
plots = self._func(targets=self.args.targets, props=self._props())
|
||||
|
||||
if self.args.show_json:
|
||||
import json
|
||||
|
||||
logger.info(json.dumps(plots))
|
||||
if self.args.show_vega:
|
||||
target = self.args.targets[0]
|
||||
logger.info(plots[target])
|
||||
return 0
|
||||
|
||||
divs = [
|
||||
@@ -90,10 +88,18 @@ class CmdPlotsDiff(CmdPlots):
|
||||
return self.repo.plots.diff(*args, revs=self.args.revisions, **kwargs)
|
||||
|
||||
|
||||
class CmdPlotsModify(CmdPlots):
|
||||
def run(self):
|
||||
self.repo.plots.modify(
|
||||
self.args.target, props=self._props(), unset=self.args.unset,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
PLOTS_HELP = (
|
||||
"Generating plots for metrics stored in structured files "
|
||||
"(JSON, CSV, TSV)."
|
||||
"Commands to visualize and compare plot metrics in structured files "
|
||||
"(JSON, YAML, CSV, TSV)"
|
||||
)
|
||||
|
||||
plots_parser = subparsers.add_parser(
|
||||
@@ -110,7 +116,7 @@ def add_parser(subparsers, parent_parser):
|
||||
|
||||
fix_subparsers(plots_subparsers)
|
||||
|
||||
SHOW_HELP = "Generate a plots image file from a metrics file."
|
||||
SHOW_HELP = "Generate plots from metric files."
|
||||
plots_show_parser = plots_subparsers.add_parser(
|
||||
"show",
|
||||
parents=[parent_parser],
|
||||
@@ -118,57 +124,18 @@ def add_parser(subparsers, parent_parser):
|
||||
help=SHOW_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
plots_show_parser.add_argument(
|
||||
"-t",
|
||||
"--template",
|
||||
nargs="?",
|
||||
default=None,
|
||||
help="File to be injected with data.",
|
||||
)
|
||||
plots_show_parser.add_argument(
|
||||
"-o", "--out", default=None, help="Destination path to save plots to.",
|
||||
)
|
||||
plots_show_parser.add_argument(
|
||||
"-s",
|
||||
"--select",
|
||||
default=None,
|
||||
help="Choose which field(s) or JSONPath to include in the plots.",
|
||||
)
|
||||
plots_show_parser.add_argument(
|
||||
"-x", default=None, help="Field name for x axis."
|
||||
)
|
||||
plots_show_parser.add_argument(
|
||||
"-y", default=None, help="Field name for y axis."
|
||||
)
|
||||
plots_show_parser.add_argument(
|
||||
"--no-csv-header",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Required when CSV or TSV datafile does not have a header.",
|
||||
)
|
||||
plots_show_parser.add_argument(
|
||||
"--show-json",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Show output in JSON format.",
|
||||
)
|
||||
plots_show_parser.add_argument("--title", default=None, help="Plot title.")
|
||||
plots_show_parser.add_argument(
|
||||
"--xlab", default=None, help="X axis title."
|
||||
)
|
||||
plots_show_parser.add_argument(
|
||||
"--ylab", default=None, help="Y axis title."
|
||||
)
|
||||
plots_show_parser.add_argument(
|
||||
"targets",
|
||||
nargs="*",
|
||||
help="Metrics files to visualize. Shows all plots by default.",
|
||||
help="Plots files to visualize. Shows all plots by default.",
|
||||
)
|
||||
_add_props_arguments(plots_show_parser)
|
||||
_add_output_arguments(plots_show_parser)
|
||||
plots_show_parser.set_defaults(func=CmdPlotsShow)
|
||||
|
||||
PLOTS_DIFF_HELP = (
|
||||
"Plot differences in metrics between commits in the DVC "
|
||||
"repository, or between the last commit and the workspace."
|
||||
"Show multiple versions of plot metrics "
|
||||
"by plotting them in a single image."
|
||||
)
|
||||
plots_diff_parser = plots_subparsers.add_parser(
|
||||
"diff",
|
||||
@@ -177,53 +144,89 @@ def add_parser(subparsers, parent_parser):
|
||||
help=PLOTS_DIFF_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"-t",
|
||||
"--template",
|
||||
nargs="?",
|
||||
default=None,
|
||||
help="File to be injected with data.",
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"--targets",
|
||||
nargs="*",
|
||||
help="Metrics file to visualize. Shows all plots by default.",
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"-o", "--out", default=None, help="Destination path to save plots to.",
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"-s",
|
||||
"--select",
|
||||
default=None,
|
||||
help="Choose which field(s) or JSONPath to include in the plot.",
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"-x", default=None, help="Field name for x axis."
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"-y", default=None, help="Field name for y axis."
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"--no-csv-header",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Provided CSV ot TSV datafile does not have a header.",
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"--show-json",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Show output in JSON format.",
|
||||
)
|
||||
plots_diff_parser.add_argument("--title", default=None, help="Plot title.")
|
||||
plots_diff_parser.add_argument(
|
||||
"--xlab", default=None, help="X axis title."
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"--ylab", default=None, help="Y axis title."
|
||||
help="Plots file to visualize. Shows all plots by default.",
|
||||
metavar="<path>",
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"revisions", nargs="*", default=None, help="Git commits to plot from",
|
||||
)
|
||||
_add_props_arguments(plots_diff_parser)
|
||||
_add_output_arguments(plots_diff_parser)
|
||||
plots_diff_parser.set_defaults(func=CmdPlotsDiff)
|
||||
|
||||
PLOTS_MODIFY_HELP = "Modify display properties of plot metric files."
|
||||
plots_modify_parser = plots_subparsers.add_parser(
|
||||
"modify",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(PLOTS_MODIFY_HELP, "plots/modify"),
|
||||
help=PLOTS_MODIFY_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
plots_modify_parser.add_argument(
|
||||
"target", help="Metric file to set properties to",
|
||||
)
|
||||
_add_props_arguments(plots_modify_parser)
|
||||
plots_modify_parser.add_argument(
|
||||
"--unset",
|
||||
nargs="*",
|
||||
metavar="<property>",
|
||||
help="Unset one or more display properties.",
|
||||
)
|
||||
plots_modify_parser.set_defaults(func=CmdPlotsModify)
|
||||
|
||||
|
||||
def _add_props_arguments(parser):
|
||||
parser.add_argument(
|
||||
"-t",
|
||||
"--template",
|
||||
nargs="?",
|
||||
default=None,
|
||||
help=(
|
||||
"Special JSON or HTML schema file to inject with the data. "
|
||||
"See {}".format(
|
||||
format_link("https://man.dvc.org/plots#plot-templates")
|
||||
)
|
||||
),
|
||||
metavar="<path>",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-x", default=None, help="Field name for X axis.", metavar="<field>"
|
||||
)
|
||||
parser.add_argument(
|
||||
"-y", default=None, help="Field name for Y axis.", metavar="<field>"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-header",
|
||||
action="store_false",
|
||||
dest="header",
|
||||
default=None, # Use default None to distinguish when it's not used
|
||||
help="Provided CSV or TSV datafile does not have a header.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--title", default=None, metavar="<text>", help="Plot title."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--x-label", default=None, help="X axis label", metavar="<text>"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--y-label", default=None, help="Y axis label", metavar="<text>"
|
||||
)
|
||||
|
||||
|
||||
def _add_output_arguments(parser):
|
||||
parser.add_argument(
|
||||
"-o",
|
||||
"--out",
|
||||
default=None,
|
||||
help="Destination path to save plots to",
|
||||
metavar="<path>",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--show-vega",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Show output in Vega format.",
|
||||
)
|
||||
|
||||
+45
-1
@@ -62,7 +62,11 @@ class CmdRemoteModify(CmdRemote):
|
||||
def run(self):
|
||||
with self.config.edit(self.args.level) as conf:
|
||||
self._check_exists(conf)
|
||||
conf["remote"][self.args.name][self.args.option] = self.args.value
|
||||
section = conf["remote"][self.args.name]
|
||||
if self.args.unset:
|
||||
section.pop(self.args.option, None)
|
||||
else:
|
||||
section[self.args.option] = self.args.value
|
||||
return 0
|
||||
|
||||
|
||||
@@ -104,6 +108,35 @@ class CmdRemoteList(CmdRemote):
|
||||
return 0
|
||||
|
||||
|
||||
class CmdRemoteRename(CmdRemote):
|
||||
def _rename_default(self, conf):
|
||||
if conf["core"].get("remote") == self.args.name:
|
||||
conf["core"]["remote"] = self.args.new
|
||||
|
||||
def run(self):
|
||||
all_config = self.config.load_config_to_level(None)
|
||||
if self.args.new in all_config.get("remote", {}):
|
||||
raise ConfigError(
|
||||
"Rename failed. Remote name '{}' already exists.".format(
|
||||
{self.args.new}
|
||||
)
|
||||
)
|
||||
|
||||
with self.config.edit(self.args.level) as conf:
|
||||
self._check_exists(conf)
|
||||
conf["remote"][self.args.new] = conf["remote"][self.args.name]
|
||||
del conf["remote"][self.args.name]
|
||||
self._rename_default(conf)
|
||||
|
||||
for level in reversed(self.config.LEVELS):
|
||||
if level == self.args.level:
|
||||
break
|
||||
with self.config.edit(level) as level_conf:
|
||||
self._rename_default(level_conf)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
from dvc.command.config import parent_config_parser
|
||||
|
||||
@@ -220,3 +253,14 @@ def add_parser(subparsers, parent_parser):
|
||||
"name", help="Name of the remote to remove."
|
||||
)
|
||||
remote_remove_parser.set_defaults(func=CmdRemoteRemove)
|
||||
REMOTE_RENAME_HELP = "Rename a DVC remote"
|
||||
remote_rename_parser = remote_subparsers.add_parser(
|
||||
"rename",
|
||||
parents=[parent_config_parser, parent_parser],
|
||||
description=append_doc_link(REMOTE_RENAME_HELP, "remote/rename"),
|
||||
help=REMOTE_RENAME_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
remote_rename_parser.add_argument("name", help="Remote to be renamed")
|
||||
remote_rename_parser.add_argument("new", help="New name of the remote")
|
||||
remote_rename_parser.set_defaults(func=CmdRemoteRename)
|
||||
|
||||
+4
-41
@@ -1,7 +1,6 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
import dvc.prompt as prompt
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
@@ -9,30 +8,10 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CmdRemove(CmdBase):
|
||||
def _is_outs_only(self, target):
|
||||
if not self.args.purge:
|
||||
return True
|
||||
|
||||
if self.args.force:
|
||||
return False
|
||||
|
||||
msg = "Are you sure you want to remove '{}' with its outputs?".format(
|
||||
target
|
||||
)
|
||||
|
||||
if prompt.confirm(msg):
|
||||
return False
|
||||
|
||||
raise DvcException(
|
||||
"Cannot purge without a confirmation from the user."
|
||||
" Use `-f` to force."
|
||||
)
|
||||
|
||||
def run(self):
|
||||
for target in self.args.targets:
|
||||
try:
|
||||
outs_only = self._is_outs_only(target)
|
||||
self.repo.remove(target, outs_only=outs_only)
|
||||
self.repo.remove(target, outs=self.args.outs)
|
||||
except DvcException:
|
||||
logger.exception(f"failed to remove '{target}'")
|
||||
return 1
|
||||
@@ -40,7 +19,7 @@ class CmdRemove(CmdBase):
|
||||
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
REMOVE_HELP = "Remove DVC-tracked files or directories."
|
||||
REMOVE_HELP = "Remove stage entry and unprotect outputs"
|
||||
remove_parser = subparsers.add_parser(
|
||||
"remove",
|
||||
parents=[parent_parser],
|
||||
@@ -48,27 +27,11 @@ def add_parser(subparsers, parent_parser):
|
||||
help=REMOVE_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
remove_parser_group = remove_parser.add_mutually_exclusive_group()
|
||||
remove_parser_group.add_argument(
|
||||
"-o",
|
||||
remove_parser.add_argument(
|
||||
"--outs",
|
||||
action="store_true",
|
||||
default=True,
|
||||
help="Only remove DVC-file outputs. (Default)",
|
||||
)
|
||||
remove_parser_group.add_argument(
|
||||
"-p",
|
||||
"--purge",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Remove DVC-file and all its outputs.",
|
||||
)
|
||||
remove_parser.add_argument(
|
||||
"-f",
|
||||
"--force",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Force purge.",
|
||||
help="Remove outputs as well.",
|
||||
)
|
||||
remove_parser.add_argument(
|
||||
"targets", nargs="+", help="DVC-files to remove."
|
||||
|
||||
@@ -5,6 +5,7 @@ import os
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.command.metrics import show_metrics
|
||||
from dvc.command.status import CmdDataStatus
|
||||
from dvc.dvcfile import PIPELINE_FILE
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -69,7 +70,7 @@ def add_parser(subparsers, parent_parser):
|
||||
repro_parser.add_argument(
|
||||
"targets",
|
||||
nargs="*",
|
||||
help="DVC-file to reproduce. 'Dvcfile' by default.",
|
||||
help=f"Stages to reproduce. '{PIPELINE_FILE}' by default.",
|
||||
)
|
||||
repro_parser.add_argument(
|
||||
"-f",
|
||||
|
||||
+13
-7
@@ -45,7 +45,7 @@ class CmdRun(CmdBase):
|
||||
fname=self.args.file,
|
||||
wdir=self.args.wdir,
|
||||
no_exec=self.args.no_exec,
|
||||
overwrite=self.args.overwrite_dvcfile,
|
||||
force=self.args.force,
|
||||
run_cache=not self.args.no_run_cache,
|
||||
no_commit=self.args.no_commit,
|
||||
outs_persist=self.args.outs_persist,
|
||||
@@ -53,6 +53,7 @@ class CmdRun(CmdBase):
|
||||
always_changed=self.args.always_changed,
|
||||
name=self.args.name,
|
||||
single_stage=self.args.single_stage,
|
||||
external=self.args.external,
|
||||
)
|
||||
except DvcException:
|
||||
logger.exception("")
|
||||
@@ -132,7 +133,7 @@ def add_parser(subparsers, parent_parser):
|
||||
"--metrics",
|
||||
action="append",
|
||||
default=[],
|
||||
help="Declare output metric file or directory.",
|
||||
help="Declare output metric file.",
|
||||
metavar="<path>",
|
||||
)
|
||||
run_parser.add_argument(
|
||||
@@ -140,8 +141,7 @@ def add_parser(subparsers, parent_parser):
|
||||
"--metrics-no-cache",
|
||||
action="append",
|
||||
default=[],
|
||||
help="Declare output metric file or directory "
|
||||
"(do not put into DVC cache).",
|
||||
help="Declare output metric file (do not put into DVC cache).",
|
||||
metavar="<path>",
|
||||
)
|
||||
run_parser.add_argument(
|
||||
@@ -159,7 +159,6 @@ def add_parser(subparsers, parent_parser):
|
||||
metavar="<path>",
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"-f",
|
||||
"--file",
|
||||
help="Specify name of the DVC-file this command will generate.",
|
||||
metavar="<filename>",
|
||||
@@ -177,10 +176,11 @@ def add_parser(subparsers, parent_parser):
|
||||
help="Only create stage file without actually running it.",
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"--overwrite-dvcfile",
|
||||
"-f",
|
||||
"--force",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Overwrite existing DVC-file without asking for confirmation.",
|
||||
help="Overwrite existing stage",
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"--no-run-cache",
|
||||
@@ -225,6 +225,12 @@ def add_parser(subparsers, parent_parser):
|
||||
default=False,
|
||||
help=argparse.SUPPRESS,
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"--external",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Allow outputs that are outside of the DVC repository.",
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"command", nargs=argparse.REMAINDER, help="Command to execute."
|
||||
)
|
||||
|
||||
+10
-5
@@ -50,11 +50,16 @@ class CmdDataStatus(CmdDataBase):
|
||||
with_deps=self.args.with_deps,
|
||||
recursive=self.args.recursive,
|
||||
)
|
||||
if st:
|
||||
if self.args.quiet:
|
||||
return 1
|
||||
else:
|
||||
self._show(st, indent)
|
||||
|
||||
if self.args.quiet:
|
||||
return bool(st)
|
||||
|
||||
if self.args.show_json:
|
||||
import json
|
||||
|
||||
logger.info(json.dumps(st))
|
||||
elif st:
|
||||
self._show(st, indent)
|
||||
else:
|
||||
logger.info(self.UP_TO_DATE_MSG)
|
||||
|
||||
|
||||
@@ -122,12 +122,12 @@ class CmdVersion(CmdBaseNoRepo):
|
||||
|
||||
@staticmethod
|
||||
def get_supported_remotes():
|
||||
from dvc.remote import REMOTES
|
||||
from dvc.remote import TREES
|
||||
|
||||
supported_remotes = []
|
||||
for remote in REMOTES:
|
||||
if not remote.get_missing_deps():
|
||||
supported_remotes.append(remote.scheme)
|
||||
for tree_cls in TREES:
|
||||
if not tree_cls.get_missing_deps():
|
||||
supported_remotes.append(tree_cls.scheme)
|
||||
|
||||
return ", ".join(supported_remotes)
|
||||
|
||||
|
||||
@@ -46,7 +46,6 @@ def _spawn_posix(cmd, env):
|
||||
os._exit(1) # pylint: disable=protected-access
|
||||
|
||||
os.setsid()
|
||||
os.umask(0)
|
||||
|
||||
try:
|
||||
pid = os.fork()
|
||||
|
||||
+2
-3
@@ -96,10 +96,9 @@ class AsciiCanvas:
|
||||
|
||||
def draw(self):
|
||||
"""Draws ASCII canvas on the screen."""
|
||||
pager = find_pager()
|
||||
lines = map("".join, self.canvas)
|
||||
joined_lines = os.linesep.join(lines)
|
||||
pager(joined_lines)
|
||||
return joined_lines
|
||||
|
||||
def point(self, x, y, char):
|
||||
"""Create a point on ASCII canvas.
|
||||
@@ -316,4 +315,4 @@ def draw(vertexes, edges):
|
||||
int(round(x - minx)) + 1, int(round(y - miny)) + 1, vertex.data
|
||||
)
|
||||
|
||||
canvas.draw()
|
||||
return canvas.draw()
|
||||
|
||||
+5
-5
@@ -3,7 +3,7 @@
|
||||
import logging
|
||||
|
||||
from dvc.config import NoRemoteError
|
||||
from dvc.remote import Remote
|
||||
from dvc.remote import get_remote
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -45,7 +45,7 @@ class DataCloud:
|
||||
raise NoRemoteError(error_msg)
|
||||
|
||||
def _init_remote(self, name):
|
||||
return Remote(self.repo, name=name)
|
||||
return get_remote(self.repo, name=name)
|
||||
|
||||
def push(
|
||||
self, cache, jobs=None, remote=None, show_checksums=False,
|
||||
@@ -85,15 +85,15 @@ class DataCloud:
|
||||
cache, jobs=jobs, remote=remote, show_checksums=show_checksums
|
||||
)
|
||||
|
||||
if not remote.verify:
|
||||
if not remote.tree.verify:
|
||||
self._save_pulled_checksums(cache)
|
||||
|
||||
return downloaded_items_num
|
||||
|
||||
def _save_pulled_checksums(self, cache):
|
||||
for checksum in cache.scheme_keys("local"):
|
||||
cache_file = self.repo.cache.local.checksum_to_path_info(checksum)
|
||||
if self.repo.cache.local.exists(cache_file):
|
||||
cache_file = self.repo.cache.local.hash_to_path_info(checksum)
|
||||
if self.repo.cache.local.tree.exists(cache_file):
|
||||
# We can safely save here, as existing corrupted files will
|
||||
# be removed upon status, while files corrupted during
|
||||
# download will not be moved from tmp_file
|
||||
|
||||
+29
-22
@@ -2,6 +2,7 @@ from collections import defaultdict
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import dvc.output as output
|
||||
from dvc.dependency.azure import AzureDependency
|
||||
from dvc.dependency.gs import GSDependency
|
||||
from dvc.dependency.hdfs import HDFSDependency
|
||||
from dvc.dependency.http import HTTPDependency
|
||||
@@ -11,12 +12,13 @@ from dvc.dependency.param import ParamsDependency
|
||||
from dvc.dependency.s3 import S3Dependency
|
||||
from dvc.dependency.ssh import SSHDependency
|
||||
from dvc.output.base import BaseOutput
|
||||
from dvc.remote import Remote
|
||||
from dvc.remote import get_remote
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
from .repo import RepoDependency
|
||||
|
||||
DEPS = [
|
||||
AzureDependency,
|
||||
GSDependency,
|
||||
HDFSDependency,
|
||||
HTTPDependency,
|
||||
@@ -30,6 +32,7 @@ DEP_MAP = {
|
||||
Schemes.LOCAL: LocalDependency,
|
||||
Schemes.SSH: SSHDependency,
|
||||
Schemes.S3: S3Dependency,
|
||||
Schemes.AZURE: AzureDependency,
|
||||
Schemes.GS: GSDependency,
|
||||
Schemes.HDFS: HDFSDependency,
|
||||
Schemes.HTTP: HTTPDependency,
|
||||
@@ -51,7 +54,7 @@ SCHEMA.update(ParamsDependency.PARAM_SCHEMA)
|
||||
def _get(stage, p, info):
|
||||
parsed = urlparse(p) if p else None
|
||||
if parsed and parsed.scheme == "remote":
|
||||
remote = Remote(stage.repo, name=parsed.netloc)
|
||||
remote = get_remote(stage.repo, name=parsed.netloc)
|
||||
return DEP_MAP[remote.scheme](stage, p, info, remote=remote)
|
||||
|
||||
if info and info.get(RepoDependency.PARAM_REPO):
|
||||
@@ -77,6 +80,7 @@ def loadd_from(stage, d_list):
|
||||
|
||||
|
||||
def loads_from(stage, s_list, erepo=None):
|
||||
assert isinstance(s_list, list)
|
||||
ret = []
|
||||
for s in s_list:
|
||||
info = {RepoDependency.PARAM_REPO: erepo} if erepo else {}
|
||||
@@ -84,26 +88,29 @@ def loads_from(stage, s_list, erepo=None):
|
||||
return ret
|
||||
|
||||
|
||||
def _parse_params(path_params):
|
||||
path, _, params_str = path_params.rpartition(":")
|
||||
params = params_str.split(",")
|
||||
return path, params
|
||||
def _merge_params(s_list):
|
||||
d = defaultdict(list)
|
||||
default_file = ParamsDependency.DEFAULT_PARAMS_FILE
|
||||
for key in s_list:
|
||||
if isinstance(key, str):
|
||||
d[default_file].append(key)
|
||||
continue
|
||||
if not isinstance(key, dict):
|
||||
msg = "Only list of str/dict is supported. Got: "
|
||||
msg += f"'{type(key).__name__}'."
|
||||
raise ValueError(msg)
|
||||
|
||||
for k, params in key.items():
|
||||
if not isinstance(params, list):
|
||||
msg = "Expected list of params for custom params file "
|
||||
msg += f"'{k}', got '{type(params).__name__}'."
|
||||
raise ValueError(msg)
|
||||
d[k].extend(params)
|
||||
return d
|
||||
|
||||
|
||||
def loads_params(stage, s_list):
|
||||
# Creates an object for each unique file that is referenced in the list
|
||||
params_by_path = defaultdict(list)
|
||||
for s in s_list:
|
||||
path, params = _parse_params(s)
|
||||
params_by_path[path].extend(params)
|
||||
|
||||
d_list = []
|
||||
for path, params in params_by_path.items():
|
||||
d_list.append(
|
||||
{
|
||||
BaseOutput.PARAM_PATH: path,
|
||||
ParamsDependency.PARAM_PARAMS: params,
|
||||
}
|
||||
)
|
||||
|
||||
return loadd_from(stage, d_list)
|
||||
d = _merge_params(s_list)
|
||||
return [
|
||||
ParamsDependency(stage, path, params) for path, params in d.items()
|
||||
]
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
from dvc.dependency.base import BaseDependency
|
||||
from dvc.output.base import BaseOutput
|
||||
from dvc.remote.azure import AzureRemoteTree
|
||||
|
||||
|
||||
class AzureDependency(BaseDependency, BaseOutput):
|
||||
TREE_CLS = AzureRemoteTree
|
||||
@@ -1,7 +1,7 @@
|
||||
from dvc.dependency.base import BaseDependency
|
||||
from dvc.output.base import BaseOutput
|
||||
from dvc.remote.http import HTTPRemote
|
||||
from dvc.remote.http import HTTPRemoteTree
|
||||
|
||||
|
||||
class HTTPDependency(BaseDependency, BaseOutput):
|
||||
REMOTE = HTTPRemote
|
||||
TREE_CLS = HTTPRemoteTree
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from dvc.remote.https import HTTPSRemote
|
||||
from dvc.remote.https import HTTPSRemoteTree
|
||||
|
||||
from .http import HTTPDependency
|
||||
|
||||
|
||||
class HTTPSDependency(HTTPDependency):
|
||||
REMOTE = HTTPSRemote
|
||||
TREE_CLS = HTTPSRemoteTree
|
||||
|
||||
@@ -40,7 +40,7 @@ class ParamsDependency(LocalDependency):
|
||||
info=info,
|
||||
)
|
||||
|
||||
def _dyn_load(self, values=None):
|
||||
def fill_values(self, values=None):
|
||||
"""Load params values dynamically."""
|
||||
if not values:
|
||||
return
|
||||
|
||||
+16
-7
@@ -50,13 +50,22 @@ class RepoDependency(LocalDependency):
|
||||
return external_repo(d["url"], rev=rev)
|
||||
|
||||
def _get_checksum(self, locked=True):
|
||||
from dvc.repo.tree import RepoTree
|
||||
|
||||
with self._make_repo(locked=locked) as repo:
|
||||
try:
|
||||
return repo.find_out_by_relpath(self.def_path).info["md5"]
|
||||
except OutputNotFoundError:
|
||||
path = PathInfo(os.path.join(repo.root_dir, self.def_path))
|
||||
|
||||
# we want stream but not fetch, so DVC out directories are
|
||||
# walked, but dir contents is not fetched
|
||||
tree = RepoTree(repo, stream=True)
|
||||
|
||||
# We are polluting our repo cache with some dir listing here
|
||||
return self.repo.cache.local.get_checksum(path)
|
||||
if tree.isdir(path):
|
||||
return self.repo.cache.local.get_hash(path, tree)
|
||||
return tree.get_file_hash(path)
|
||||
|
||||
def status(self):
|
||||
current_checksum = self._get_checksum(locked=True)
|
||||
@@ -76,16 +85,16 @@ class RepoDependency(LocalDependency):
|
||||
def download(self, to):
|
||||
with self._make_repo() as repo:
|
||||
if self.def_repo.get(self.PARAM_REV_LOCK) is None:
|
||||
self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()
|
||||
self.def_repo[self.PARAM_REV_LOCK] = repo.get_rev()
|
||||
|
||||
if hasattr(repo, "cache"):
|
||||
repo.cache.local.cache_dir = self.repo.cache.local.cache_dir
|
||||
|
||||
repo.pull_to(self.def_path, to.path_info)
|
||||
cache = self.repo.cache.local
|
||||
with repo.use_cache(cache):
|
||||
_, _, cache_infos = repo.fetch_external([self.def_path])
|
||||
cache.checkout(to.path_info, cache_infos[0])
|
||||
|
||||
def update(self, rev=None):
|
||||
if rev:
|
||||
self.def_repo[self.PARAM_REV] = rev
|
||||
|
||||
with self._make_repo(locked=False) as repo:
|
||||
self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()
|
||||
self.def_repo[self.PARAM_REV_LOCK] = repo.get_rev()
|
||||
|
||||
+66
-35
@@ -4,11 +4,9 @@ import os
|
||||
|
||||
from voluptuous import MultipleInvalid
|
||||
|
||||
import dvc.prompt as prompt
|
||||
from dvc import serialize
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.stage import serialize
|
||||
from dvc.stage.exceptions import (
|
||||
StageFileAlreadyExistsError,
|
||||
StageFileBadNameError,
|
||||
StageFileDoesNotExistError,
|
||||
StageFileFormatError,
|
||||
@@ -17,11 +15,7 @@ from dvc.stage.exceptions import (
|
||||
from dvc.stage.loader import SingleStageLoader, StageLoader
|
||||
from dvc.utils import relpath
|
||||
from dvc.utils.collections import apply_diff
|
||||
from dvc.utils.stage import (
|
||||
dump_stage_file,
|
||||
parse_stage,
|
||||
parse_stage_for_update,
|
||||
)
|
||||
from dvc.utils.yaml import dump_yaml, parse_yaml, parse_yaml_for_update
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -101,7 +95,7 @@ class FileMixin:
|
||||
|
||||
with self.repo.tree.open(self.path) as fd:
|
||||
stage_text = fd.read()
|
||||
d = parse_stage(stage_text, self.path)
|
||||
d = parse_yaml(stage_text, self.path)
|
||||
self.validate(d, self.relpath)
|
||||
return d, stage_text
|
||||
|
||||
@@ -113,9 +107,6 @@ class FileMixin:
|
||||
except MultipleInvalid as exc:
|
||||
raise StageFileFormatError(f"'{fname}' format error: {exc}")
|
||||
|
||||
def remove_with_prompt(self, force=False):
|
||||
raise NotImplementedError
|
||||
|
||||
def remove(self, force=False):
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
os.unlink(self.path)
|
||||
@@ -149,20 +140,10 @@ class SingleStageFile(FileMixin):
|
||||
logger.debug(
|
||||
"Saving information to '{file}'.".format(file=relpath(self.path))
|
||||
)
|
||||
dump_stage_file(self.path, serialize.to_single_stage_file(stage))
|
||||
self.repo.scm.track_file(relpath(self.path))
|
||||
|
||||
def remove_with_prompt(self, force=False):
|
||||
if not self.exists():
|
||||
return
|
||||
|
||||
msg = (
|
||||
"'{}' already exists. Do you wish to run the command and "
|
||||
"overwrite it?".format(relpath(self.path))
|
||||
)
|
||||
if not (force or prompt.confirm(msg)):
|
||||
raise StageFileAlreadyExistsError(self.path)
|
||||
dump_yaml(self.path, serialize.to_single_stage_file(stage))
|
||||
self.repo.scm.track_file(self.relpath)
|
||||
|
||||
def remove_stage(self, stage):
|
||||
self.remove()
|
||||
|
||||
|
||||
@@ -175,16 +156,19 @@ class PipelineFile(FileMixin):
|
||||
def _lockfile(self):
|
||||
return Lockfile(self.repo, os.path.splitext(self.path)[0] + ".lock")
|
||||
|
||||
def dump(self, stage, update_pipeline=False, **kwargs):
|
||||
def dump(self, stage, update_pipeline=False, no_lock=False):
|
||||
"""Dumps given stage appropriately in the dvcfile."""
|
||||
from dvc.stage import PipelineStage
|
||||
|
||||
assert isinstance(stage, PipelineStage)
|
||||
check_dvc_filename(self.path)
|
||||
self._dump_lockfile(stage)
|
||||
|
||||
if update_pipeline and not stage.is_data_source:
|
||||
self._dump_pipeline_file(stage)
|
||||
|
||||
if not no_lock:
|
||||
self._dump_lockfile(stage)
|
||||
|
||||
def _dump_lockfile(self, stage):
|
||||
self._lockfile.dump(stage)
|
||||
|
||||
@@ -192,20 +176,28 @@ class PipelineFile(FileMixin):
|
||||
data = {}
|
||||
if self.exists():
|
||||
with open(self.path) as fd:
|
||||
data = parse_stage_for_update(fd.read(), self.path)
|
||||
data = parse_yaml_for_update(fd.read(), self.path)
|
||||
else:
|
||||
logger.info("Creating '%s'", self.relpath)
|
||||
open(self.path, "w+").close()
|
||||
|
||||
data["stages"] = data.get("stages", {})
|
||||
stage_data = serialize.to_pipeline_file(stage)
|
||||
if data["stages"].get(stage.name):
|
||||
existing_entry = stage.name in data["stages"]
|
||||
|
||||
action = "Modifying" if existing_entry else "Adding"
|
||||
logger.info("%s stage '%s' in '%s'", action, stage.name, self.relpath)
|
||||
|
||||
if existing_entry:
|
||||
orig_stage_data = data["stages"][stage.name]
|
||||
if "meta" in orig_stage_data:
|
||||
stage_data[stage.name]["meta"] = orig_stage_data["meta"]
|
||||
apply_diff(stage_data[stage.name], orig_stage_data)
|
||||
else:
|
||||
data["stages"].update(stage_data)
|
||||
|
||||
dump_stage_file(self.path, data)
|
||||
self.repo.scm.track_file(relpath(self.path))
|
||||
dump_yaml(self.path, data)
|
||||
self.repo.scm.track_file(self.relpath)
|
||||
|
||||
@property
|
||||
def stage(self):
|
||||
@@ -227,6 +219,22 @@ class PipelineFile(FileMixin):
|
||||
super().remove()
|
||||
self._lockfile.remove()
|
||||
|
||||
def remove_stage(self, stage):
|
||||
self._lockfile.remove_stage(stage)
|
||||
if not self.exists():
|
||||
return
|
||||
|
||||
with open(self.path, "r") as f:
|
||||
d = parse_yaml_for_update(f.read(), self.path)
|
||||
|
||||
self.validate(d, self.path)
|
||||
if stage.name not in d.get("stages", {}):
|
||||
return
|
||||
|
||||
logger.debug("Removing '%s' from '%s'", stage.name, self.path)
|
||||
del d["stages"][stage.name]
|
||||
dump_yaml(self.path, d)
|
||||
|
||||
|
||||
class Lockfile(FileMixin):
|
||||
from dvc.schema import COMPILED_LOCKFILE_SCHEMA as SCHEMA
|
||||
@@ -235,7 +243,7 @@ class Lockfile(FileMixin):
|
||||
if not self.exists():
|
||||
return {}
|
||||
with self.repo.tree.open(self.path) as fd:
|
||||
data = parse_stage(fd.read(), self.path)
|
||||
data = parse_yaml(fd.read(), self.path)
|
||||
try:
|
||||
self.validate(data, fname=self.relpath)
|
||||
except StageFileFormatError:
|
||||
@@ -247,15 +255,38 @@ class Lockfile(FileMixin):
|
||||
def dump(self, stage, **kwargs):
|
||||
stage_data = serialize.to_lockfile(stage)
|
||||
if not self.exists():
|
||||
modified = True
|
||||
logger.info("Generating lock file '%s'", self.relpath)
|
||||
data = stage_data
|
||||
open(self.path, "w+").close()
|
||||
else:
|
||||
with self.repo.tree.open(self.path, "r") as fd:
|
||||
data = parse_stage_for_update(fd.read(), self.path)
|
||||
data = parse_yaml_for_update(fd.read(), self.path)
|
||||
modified = data.get(stage.name, {}) != stage_data.get(
|
||||
stage.name, {}
|
||||
)
|
||||
if modified:
|
||||
logger.info("Updating lock file '%s'", self.relpath)
|
||||
data.update(stage_data)
|
||||
dump_yaml(self.path, data)
|
||||
if modified:
|
||||
self.repo.scm.track_file(self.relpath)
|
||||
|
||||
dump_stage_file(self.path, data)
|
||||
self.repo.scm.track_file(relpath(self.path))
|
||||
def remove_stage(self, stage):
|
||||
if not self.exists():
|
||||
return
|
||||
|
||||
with open(self.path) as f:
|
||||
d = parse_yaml_for_update(f.read(), self.path)
|
||||
self.validate(d, self.path)
|
||||
|
||||
if stage.name not in d:
|
||||
return
|
||||
|
||||
logger.debug("Removing '%s' from '%s'", stage.name, self.path)
|
||||
del d[stage.name]
|
||||
|
||||
dump_yaml(self.path, d)
|
||||
|
||||
|
||||
class Dvcfile:
|
||||
|
||||
+33
-15
@@ -173,17 +173,24 @@ class BadMetricError(DvcException):
|
||||
class NoMetricsError(DvcException):
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
"no metric files in this repository. "
|
||||
"Use `dvc metrics add` to add a metric file to track."
|
||||
"no metric files in this repository. Use `-m/-M` options for "
|
||||
"`dvc run` to mark stage outputs as metrics."
|
||||
)
|
||||
|
||||
|
||||
class StageFileCorruptedError(DvcException):
|
||||
class NoPlotsError(DvcException):
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
"no plots in this repository. Use `--plots/--plots-no-cache` "
|
||||
"options for `dvc run` to mark stage outputs as plots."
|
||||
)
|
||||
|
||||
|
||||
class YAMLFileCorruptedError(DvcException):
|
||||
def __init__(self, path):
|
||||
path = relpath(path)
|
||||
super().__init__(
|
||||
"unable to read DVC-file: {} "
|
||||
"YAML file structure is corrupted".format(path)
|
||||
f"unable to read: '{path}', YAML file structure is corrupted"
|
||||
)
|
||||
|
||||
|
||||
@@ -215,9 +222,12 @@ class ETagMismatchError(DvcException):
|
||||
|
||||
|
||||
class FileMissingError(DvcException):
|
||||
def __init__(self, path):
|
||||
def __init__(self, path, hint=None):
|
||||
self.path = path
|
||||
super().__init__(f"Can't find '{path}' neither locally nor on remote")
|
||||
hint = "" if hint is None else f". {hint}"
|
||||
super().__init__(
|
||||
f"Can't find '{path}' neither locally nor on remote{hint}"
|
||||
)
|
||||
|
||||
|
||||
class DvcIgnoreInCollectedDirError(DvcException):
|
||||
@@ -294,19 +304,15 @@ class PathMissingError(DvcException):
|
||||
"The path '{}' does not exist in the target repository '{}'"
|
||||
" neither as a DVC output nor as a Git-tracked file."
|
||||
)
|
||||
default_msg_output_only = (
|
||||
default_msg_dvc_only = (
|
||||
"The path '{}' does not exist in the target repository '{}'"
|
||||
" as an DVC output."
|
||||
)
|
||||
|
||||
def __init__(self, path, repo, output_only=False):
|
||||
msg = (
|
||||
self.default_msg
|
||||
if not output_only
|
||||
else self.default_msg_output_only
|
||||
)
|
||||
def __init__(self, path, repo, dvc_only=False):
|
||||
msg = self.default_msg if not dvc_only else self.default_msg_dvc_only
|
||||
super().__init__(msg.format(path, repo))
|
||||
self.output_only = output_only
|
||||
self.dvc_only = dvc_only
|
||||
|
||||
|
||||
class RemoteCacheRequiredError(DvcException):
|
||||
@@ -326,3 +332,15 @@ class RemoteCacheRequiredError(DvcException):
|
||||
|
||||
class IsADirectoryError(DvcException):
|
||||
"""Raised when a file operation is requested on a directory."""
|
||||
|
||||
|
||||
class NoOutputOrStageError(DvcException):
|
||||
"""
|
||||
Raised when the target is neither an output nor a stage name in dvc.yaml
|
||||
"""
|
||||
|
||||
def __init__(self, target, file):
|
||||
super().__init__(
|
||||
f"'{target}' "
|
||||
f"does not exist as an output or a stage name in '{file}'"
|
||||
)
|
||||
|
||||
+116
-81
@@ -4,12 +4,12 @@ import tempfile
|
||||
import threading
|
||||
from contextlib import contextmanager
|
||||
from distutils.dir_util import copy_tree
|
||||
from typing import Iterable
|
||||
|
||||
from funcy import cached_property, retry, suppress, wrap_with
|
||||
from funcy import cached_property, retry, wrap_with
|
||||
|
||||
from dvc.config import NoRemoteError, NotDvcRepoError
|
||||
from dvc.exceptions import (
|
||||
CheckoutError,
|
||||
FileMissingError,
|
||||
NoOutputInExternalRepoError,
|
||||
NoRemoteInExternalRepoError,
|
||||
@@ -18,9 +18,10 @@ from dvc.exceptions import (
|
||||
)
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.repo import Repo
|
||||
from dvc.repo.tree import RepoTree
|
||||
from dvc.scm.git import Git
|
||||
from dvc.utils import tmp_fname
|
||||
from dvc.utils.fs import fs_copy, move, remove
|
||||
from dvc.scm.tree import is_working_tree
|
||||
from dvc.utils.fs import remove
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -29,10 +30,12 @@ logger = logging.getLogger(__name__)
|
||||
def external_repo(url, rev=None, for_write=False):
|
||||
logger.debug("Creating external repo %s@%s", url, rev)
|
||||
path = _cached_clone(url, rev, for_write=for_write)
|
||||
if not rev:
|
||||
rev = "HEAD"
|
||||
try:
|
||||
repo = ExternalRepo(path, url)
|
||||
repo = ExternalRepo(path, url, rev, for_write=for_write)
|
||||
except NotDvcRepoError:
|
||||
repo = ExternalGitRepo(path, url)
|
||||
repo = ExternalGitRepo(path, url, rev)
|
||||
|
||||
try:
|
||||
yield repo
|
||||
@@ -64,59 +67,100 @@ def clean_repos():
|
||||
_remove(path)
|
||||
|
||||
|
||||
class ExternalRepo(Repo):
|
||||
def __init__(self, root_dir, url):
|
||||
super().__init__(root_dir)
|
||||
class BaseExternalRepo:
|
||||
|
||||
_local_cache = None
|
||||
|
||||
@property
|
||||
def local_cache(self):
|
||||
if hasattr(self, "cache"):
|
||||
return self.cache.local
|
||||
return self._local_cache
|
||||
|
||||
@contextmanager
|
||||
def use_cache(self, cache):
|
||||
"""Use the specified cache in place of default tmpdir cache for
|
||||
download operations.
|
||||
"""
|
||||
if hasattr(self, "cache"):
|
||||
save_cache = self.cache.local
|
||||
self.cache.local = cache
|
||||
self._local_cache = cache
|
||||
|
||||
yield
|
||||
|
||||
if hasattr(self, "cache"):
|
||||
self.cache.local = save_cache
|
||||
self._local_cache = None
|
||||
|
||||
@cached_property
|
||||
def repo_tree(self):
|
||||
return RepoTree(self, fetch=True)
|
||||
|
||||
def get_rev(self):
|
||||
if is_working_tree(self.tree):
|
||||
return self.scm.get_rev()
|
||||
if hasattr(self.tree, "tree"):
|
||||
return self.tree.tree.rev
|
||||
return self.tree.rev
|
||||
|
||||
def fetch_external(self, paths: Iterable, **kwargs):
|
||||
"""Fetch specified external repo paths into cache.
|
||||
|
||||
Returns 3-tuple in the form
|
||||
(downloaded, failed, list(cache_infos))
|
||||
where cache_infos can be used as checkout targets for the
|
||||
fetched paths.
|
||||
"""
|
||||
download_results = []
|
||||
failed = 0
|
||||
|
||||
paths = [PathInfo(self.root_dir) / path for path in paths]
|
||||
|
||||
def download_update(result):
|
||||
download_results.append(result)
|
||||
|
||||
save_infos = []
|
||||
for path in paths:
|
||||
if not self.repo_tree.exists(path):
|
||||
raise PathMissingError(path, self.url)
|
||||
save_info = self.local_cache.save(
|
||||
path,
|
||||
self.repo_tree,
|
||||
None,
|
||||
save_link=False,
|
||||
download_callback=download_update,
|
||||
)
|
||||
save_infos.append(save_info)
|
||||
|
||||
return sum(download_results), failed, save_infos
|
||||
|
||||
def get_external(self, path, dest):
|
||||
"""Convenience wrapper for fetch_external and checkout."""
|
||||
if self.local_cache:
|
||||
# fetch DVC and git files to tmpdir cache, then checkout
|
||||
_, _, save_infos = self.fetch_external([path])
|
||||
self.local_cache.checkout(PathInfo(dest), save_infos[0])
|
||||
else:
|
||||
# git-only erepo with no cache, just copy files directly
|
||||
# to dest
|
||||
path = PathInfo(self.root_dir) / path
|
||||
if not self.repo_tree.exists(path):
|
||||
raise PathMissingError(path, self.url)
|
||||
self.repo_tree.copytree(path, dest)
|
||||
|
||||
|
||||
class ExternalRepo(Repo, BaseExternalRepo):
|
||||
def __init__(self, root_dir, url, rev, for_write=False):
|
||||
if for_write:
|
||||
super().__init__(root_dir)
|
||||
else:
|
||||
root_dir = os.path.realpath(root_dir)
|
||||
super().__init__(root_dir, scm=Git(root_dir), rev=rev)
|
||||
self.url = url
|
||||
self._set_cache_dir()
|
||||
self._fix_upstream()
|
||||
|
||||
def pull_to(self, path, to_info):
|
||||
"""
|
||||
Pull the corresponding file or directory specified by `path` and
|
||||
checkout it into `to_info`.
|
||||
|
||||
It works with files tracked by Git and DVC, and also local files
|
||||
outside the repository.
|
||||
"""
|
||||
out = None
|
||||
path_info = PathInfo(self.root_dir) / path
|
||||
|
||||
with suppress(OutputNotFoundError):
|
||||
(out,) = self.find_outs_by_path(path_info, strict=False)
|
||||
|
||||
try:
|
||||
if out and out.use_cache:
|
||||
self._pull_cached(out, path_info, to_info)
|
||||
return
|
||||
|
||||
# Check if it is handled by Git (it can't have an absolute path)
|
||||
if os.path.isabs(path):
|
||||
raise FileNotFoundError
|
||||
|
||||
fs_copy(path_info, to_info)
|
||||
except FileNotFoundError:
|
||||
raise PathMissingError(path, self.url)
|
||||
|
||||
def _pull_cached(self, out, path_info, dest):
|
||||
with self.state:
|
||||
tmp = PathInfo(tmp_fname(dest))
|
||||
src = tmp / path_info.relative_to(out.path_info)
|
||||
|
||||
out.path_info = tmp
|
||||
|
||||
# Only pull unless all needed cache is present
|
||||
if out.changed_cache(filter_info=src):
|
||||
self.cloud.pull(out.get_used_cache(filter_info=src))
|
||||
|
||||
try:
|
||||
out.checkout(filter_info=src)
|
||||
except CheckoutError:
|
||||
raise FileNotFoundError
|
||||
|
||||
move(src, dest)
|
||||
remove(tmp)
|
||||
|
||||
@wrap_with(threading.Lock())
|
||||
def _set_cache_dir(self):
|
||||
try:
|
||||
@@ -125,6 +169,7 @@ class ExternalRepo(Repo):
|
||||
cache_dir = CACHE_DIRS[self.url] = tempfile.mkdtemp("dvc-cache")
|
||||
|
||||
self.cache.local.cache_dir = cache_dir
|
||||
self._local_cache = self.cache.local
|
||||
|
||||
def _fix_upstream(self):
|
||||
if not os.path.isdir(self.url):
|
||||
@@ -165,10 +210,11 @@ class ExternalRepo(Repo):
|
||||
self.config["core"]["remote"] = "auto-generated-upstream"
|
||||
|
||||
|
||||
class ExternalGitRepo:
|
||||
def __init__(self, root_dir, url):
|
||||
self.root_dir = root_dir
|
||||
class ExternalGitRepo(BaseExternalRepo):
|
||||
def __init__(self, root_dir, url, rev):
|
||||
self.root_dir = os.path.realpath(root_dir)
|
||||
self.url = url
|
||||
self.tree = self.scm.get_tree(rev)
|
||||
|
||||
@cached_property
|
||||
def scm(self):
|
||||
@@ -181,23 +227,15 @@ class ExternalGitRepo:
|
||||
def find_out_by_relpath(self, path):
|
||||
raise OutputNotFoundError(path, self)
|
||||
|
||||
def pull_to(self, path, to_info):
|
||||
try:
|
||||
# Git handled files can't have absolute path
|
||||
if os.path.isabs(path):
|
||||
raise FileNotFoundError
|
||||
|
||||
fs_copy(os.path.join(self.root_dir, path), to_info)
|
||||
except FileNotFoundError:
|
||||
raise PathMissingError(path, self.url)
|
||||
|
||||
@contextmanager
|
||||
def open_by_relpath(self, path, mode="r", encoding=None, **kwargs):
|
||||
"""Opens a specified resource as a file object."""
|
||||
tree = RepoTree(self)
|
||||
try:
|
||||
abs_path = os.path.join(self.root_dir, path)
|
||||
with open(abs_path, mode, encoding=encoding) as fd:
|
||||
yield fd
|
||||
with tree.open(
|
||||
path, mode=mode, encoding=encoding, **kwargs
|
||||
) as fobj:
|
||||
yield fobj
|
||||
except FileNotFoundError:
|
||||
raise PathMissingError(path, self.url)
|
||||
|
||||
@@ -209,14 +247,12 @@ def _cached_clone(url, rev, for_write=False):
|
||||
revision checked out. If for_write is set prevents reusing this dir via
|
||||
cache.
|
||||
"""
|
||||
if not for_write and Git.is_sha(rev) and (url, rev) in CLONES:
|
||||
return CLONES[url, rev]
|
||||
|
||||
# even if we have already cloned this repo, we may need to
|
||||
# fetch/fast-forward to get specified rev
|
||||
clone_path = _clone_default_branch(url, rev)
|
||||
rev_sha = Git(clone_path).resolve_rev(rev or "HEAD")
|
||||
|
||||
if not for_write and (url, rev_sha) in CLONES:
|
||||
return CLONES[url, rev_sha]
|
||||
if not for_write and (url) in CLONES:
|
||||
return CLONES[url]
|
||||
|
||||
# Copy to a new dir to keep the clone clean
|
||||
repo_path = tempfile.mkdtemp("dvc-erepo")
|
||||
@@ -224,11 +260,10 @@ def _cached_clone(url, rev, for_write=False):
|
||||
copy_tree(clone_path, repo_path)
|
||||
|
||||
# Check out the specified revision
|
||||
if rev is not None:
|
||||
if for_write:
|
||||
_git_checkout(repo_path, rev)
|
||||
|
||||
if not for_write:
|
||||
CLONES[url, rev_sha] = repo_path
|
||||
else:
|
||||
CLONES[url] = repo_path
|
||||
return repo_path
|
||||
|
||||
|
||||
|
||||
+120
-20
@@ -1,11 +1,15 @@
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from itertools import groupby
|
||||
|
||||
from funcy import cached_property
|
||||
from pathspec import PathSpec
|
||||
from pathspec.patterns import GitWildMatchPattern
|
||||
from pathspec.util import normalize_file
|
||||
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.scm.tree import BaseTree
|
||||
from dvc.system import System
|
||||
from dvc.utils import relpath
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -26,7 +30,16 @@ class DvcIgnorePatterns(DvcIgnore):
|
||||
self.dirname = os.path.normpath(os.path.dirname(ignore_file_path))
|
||||
|
||||
with tree.open(ignore_file_path, encoding="utf-8") as fobj:
|
||||
self.ignore_spec = PathSpec.from_lines(GitWildMatchPattern, fobj)
|
||||
path_spec_lines = fobj.readlines()
|
||||
regex_pattern_list = map(
|
||||
GitWildMatchPattern.pattern_to_regex, path_spec_lines
|
||||
)
|
||||
self.ignore_spec = [
|
||||
(ignore, re.compile("|".join(item[0] for item in group)))
|
||||
for ignore, group in groupby(
|
||||
regex_pattern_list, lambda x: x[1]
|
||||
)
|
||||
]
|
||||
|
||||
def __call__(self, root, dirs, files):
|
||||
files = [f for f in files if not self.matches(root, f)]
|
||||
@@ -35,12 +48,28 @@ class DvcIgnorePatterns(DvcIgnore):
|
||||
return dirs, files
|
||||
|
||||
def matches(self, dirname, basename):
|
||||
abs_path = os.path.join(dirname, basename)
|
||||
rel_path = relpath(abs_path, self.dirname)
|
||||
|
||||
if os.pardir + os.sep in rel_path:
|
||||
# NOTE: `relpath` is too slow, so we have to assume that both
|
||||
# `dirname` and `self.dirname` are relative or absolute together.
|
||||
prefix = self.dirname + os.sep
|
||||
if dirname == self.dirname:
|
||||
path = basename
|
||||
elif dirname.startswith(prefix):
|
||||
rel = dirname[len(prefix) :]
|
||||
# NOTE: `os.path.join` is ~x5.5 slower
|
||||
path = f"{rel}{os.sep}{basename}"
|
||||
else:
|
||||
return False
|
||||
return self.ignore_spec.match_file(rel_path)
|
||||
|
||||
if not System.is_unix():
|
||||
path = normalize_file(path)
|
||||
return self.ignore(path)
|
||||
|
||||
def ignore(self, path):
|
||||
result = False
|
||||
for ignore, pattern in self.ignore_spec:
|
||||
if pattern.match(path):
|
||||
result = ignore
|
||||
return result
|
||||
|
||||
def __hash__(self):
|
||||
return hash(self.ignore_file_path)
|
||||
@@ -76,7 +105,7 @@ class DvcIgnoreRepo(DvcIgnore):
|
||||
def is_dvc_repo(directory):
|
||||
from dvc.repo import Repo
|
||||
|
||||
return os.path.isdir(os.path.join(directory, Repo.DVC_DIR))
|
||||
return os.path.isdir(os.path.join(root, directory, Repo.DVC_DIR))
|
||||
|
||||
dirs = [d for d in dirs if not is_dvc_repo(d)]
|
||||
|
||||
@@ -84,13 +113,14 @@ class DvcIgnoreRepo(DvcIgnore):
|
||||
|
||||
|
||||
class DvcIgnoreFilter:
|
||||
def __init__(self, tree):
|
||||
def __init__(self, tree, root_dir):
|
||||
self.tree = tree
|
||||
self.root_dir = root_dir
|
||||
self.ignores = {
|
||||
DvcIgnoreDirs([".git", ".hg", ".dvc"]),
|
||||
DvcIgnoreRepo(),
|
||||
}
|
||||
for root, dirs, files in self.tree.walk(self.tree.tree_root):
|
||||
for root, dirs, files in self.tree.walk(self.root_dir):
|
||||
self._update(root)
|
||||
dirs[:], files[:] = self(root, dirs, files)
|
||||
|
||||
@@ -107,37 +137,107 @@ class DvcIgnoreFilter:
|
||||
|
||||
|
||||
class CleanTree(BaseTree):
|
||||
def __init__(self, tree):
|
||||
def __init__(self, tree, tree_root=None):
|
||||
self.tree = tree
|
||||
if tree_root:
|
||||
self._tree_root = tree_root
|
||||
else:
|
||||
self._tree_root = self.tree.tree_root
|
||||
|
||||
@cached_property
|
||||
def dvcignore(self):
|
||||
return DvcIgnoreFilter(self.tree)
|
||||
return DvcIgnoreFilter(self.tree, self.tree_root)
|
||||
|
||||
@property
|
||||
def tree_root(self):
|
||||
return self.tree.tree_root
|
||||
return self._tree_root
|
||||
|
||||
def open(self, path, mode="r", encoding="utf-8"):
|
||||
return self.tree.open(path, mode, encoding)
|
||||
if self.isfile(path):
|
||||
return self.tree.open(path, mode, encoding)
|
||||
raise FileNotFoundError
|
||||
|
||||
def exists(self, path):
|
||||
return self.tree.exists(path)
|
||||
if self.tree.exists(path) and self._parents_exist(path):
|
||||
if self.tree.isdir(path):
|
||||
return self._valid_dirname(path)
|
||||
return self._valid_filename(path)
|
||||
return False
|
||||
|
||||
def isdir(self, path):
|
||||
return self.tree.isdir(path)
|
||||
return (
|
||||
self.tree.isdir(path)
|
||||
and self._parents_exist(path)
|
||||
and self._valid_dirname(path)
|
||||
)
|
||||
|
||||
def _valid_dirname(self, path):
|
||||
path = os.path.abspath(path)
|
||||
if path == self.tree_root:
|
||||
return True
|
||||
dirname, basename = os.path.split(path)
|
||||
dirs, _ = self.dvcignore(dirname, [basename], [])
|
||||
if dirs:
|
||||
return True
|
||||
return False
|
||||
|
||||
def isfile(self, path):
|
||||
return self.tree.isfile(path)
|
||||
return (
|
||||
self.tree.isfile(path)
|
||||
and self._parents_exist(path)
|
||||
and self._valid_filename(path)
|
||||
)
|
||||
|
||||
def _valid_filename(self, path):
|
||||
dirname, basename = os.path.split(os.path.normpath(path))
|
||||
_, files = self.dvcignore(os.path.abspath(dirname), [], [basename])
|
||||
if files:
|
||||
return True
|
||||
return False
|
||||
|
||||
def isexec(self, path):
|
||||
return self.tree.isexec(path)
|
||||
return self.exists(path) and self.tree.isexec(path)
|
||||
|
||||
def _parents_exist(self, path):
|
||||
from dvc.repo import Repo
|
||||
|
||||
path = PathInfo(path)
|
||||
|
||||
# if parent is tree_root or inside a .dvc dir we can skip this check
|
||||
if path.parent == self.tree_root or Repo.DVC_DIR in path.parts:
|
||||
return True
|
||||
|
||||
# paths outside of the CleanTree root should be ignored
|
||||
path = relpath(path, self.tree_root)
|
||||
if path.startswith("..") or (
|
||||
os.name == "nt"
|
||||
and not os.path.commonprefix(
|
||||
[os.path.abspath(path), self.tree_root]
|
||||
)
|
||||
):
|
||||
return False
|
||||
|
||||
# check if parent directories are in our ignores, starting from
|
||||
# tree_root
|
||||
for parent_dir in reversed(PathInfo(path).parents):
|
||||
dirname, basename = os.path.split(parent_dir)
|
||||
if basename == ".":
|
||||
# parent_dir == tree_root
|
||||
continue
|
||||
dirs, _ = self.dvcignore(os.path.abspath(dirname), [basename], [])
|
||||
if not dirs:
|
||||
return False
|
||||
return True
|
||||
|
||||
def walk(self, top, topdown=True):
|
||||
for root, dirs, files in self.tree.walk(top, topdown):
|
||||
dirs[:], files[:] = self.dvcignore(root, dirs, files)
|
||||
dirs[:], files[:] = self.dvcignore(
|
||||
os.path.abspath(root), dirs, files
|
||||
)
|
||||
|
||||
yield root, dirs, files
|
||||
|
||||
def stat(self, path):
|
||||
return self.tree.stat(path)
|
||||
if self.exists(path):
|
||||
return self.tree.stat(path)
|
||||
raise FileNotFoundError
|
||||
|
||||
+6
-2
@@ -7,13 +7,17 @@
|
||||
TEXT_CHARS = bytes(range(32, 127)) + b"\n\r\t\f\b"
|
||||
|
||||
|
||||
def istextfile(fname, blocksize=512):
|
||||
def istextfile(fname, blocksize=512, tree=None):
|
||||
""" Uses heuristics to guess whether the given file is text or binary,
|
||||
by reading a single block of bytes from the file.
|
||||
If more than 30% of the chars in the block are non-text, or there
|
||||
are NUL ('\x00') bytes in the block, assume this is a binary file.
|
||||
"""
|
||||
with open(fname, "rb") as fobj:
|
||||
if tree:
|
||||
open_func = tree.open
|
||||
else:
|
||||
open_func = open
|
||||
with open_func(fname, "rb") as fobj:
|
||||
block = fobj.read(blocksize)
|
||||
|
||||
if not block:
|
||||
|
||||
+1
-1
@@ -47,7 +47,7 @@ class Lock:
|
||||
disable=not self._friendly,
|
||||
desc=(
|
||||
"If DVC froze, see `hardlink_lock` in {}".format(
|
||||
format_link("man.dvc.org/config#core")
|
||||
format_link("https://man.dvc.org/config#core")
|
||||
)
|
||||
),
|
||||
):
|
||||
|
||||
+50
-10
@@ -18,20 +18,45 @@ FOOTER = (
|
||||
)
|
||||
|
||||
|
||||
def addLoggingLevel(levelName, levelNum, methodName=None):
|
||||
"""
|
||||
Adds a new logging level to the `logging` module and the
|
||||
currently configured logging class.
|
||||
|
||||
Based on https://stackoverflow.com/questions/2183233
|
||||
"""
|
||||
if methodName is None:
|
||||
methodName = levelName.lower()
|
||||
|
||||
assert not hasattr(logging, levelName)
|
||||
assert not hasattr(logging, methodName)
|
||||
assert not hasattr(logging.getLoggerClass(), methodName)
|
||||
|
||||
def logForLevel(self, message, *args, **kwargs):
|
||||
if self.isEnabledFor(levelNum):
|
||||
self._log(levelNum, message, args, **kwargs)
|
||||
|
||||
def logToRoot(message, *args, **kwargs):
|
||||
logging.log(levelNum, message, *args, **kwargs)
|
||||
|
||||
logging.addLevelName(levelNum, levelName)
|
||||
setattr(logging, levelName, levelNum)
|
||||
setattr(logging.getLoggerClass(), methodName, logForLevel)
|
||||
setattr(logging, methodName, logToRoot)
|
||||
|
||||
|
||||
class LoggingException(Exception):
|
||||
def __init__(self, record):
|
||||
msg = "failed to log {}".format(str(record))
|
||||
super().__init__(msg)
|
||||
|
||||
|
||||
class ExcludeErrorsFilter(logging.Filter):
|
||||
def filter(self, record):
|
||||
return record.levelno < logging.WARNING
|
||||
def excludeFilter(level):
|
||||
class ExcludeLevelFilter(logging.Filter):
|
||||
def filter(self, record):
|
||||
return record.levelno < level
|
||||
|
||||
|
||||
class ExcludeInfoFilter(logging.Filter):
|
||||
def filter(self, record):
|
||||
return record.levelno < logging.INFO
|
||||
return ExcludeLevelFilter
|
||||
|
||||
|
||||
class ColorFormatter(logging.Formatter):
|
||||
@@ -47,6 +72,7 @@ class ColorFormatter(logging.Formatter):
|
||||
"""
|
||||
|
||||
color_code = {
|
||||
"TRACE": colorama.Fore.GREEN,
|
||||
"DEBUG": colorama.Fore.BLUE,
|
||||
"WARNING": colorama.Fore.YELLOW,
|
||||
"ERROR": colorama.Fore.RED,
|
||||
@@ -116,7 +142,11 @@ class LoggerHandler(logging.StreamHandler):
|
||||
|
||||
|
||||
def _is_verbose():
|
||||
return logging.getLogger("dvc").getEffectiveLevel() == logging.DEBUG
|
||||
return (
|
||||
logging.NOTSET
|
||||
< logging.getLogger("dvc").getEffectiveLevel()
|
||||
<= logging.DEBUG
|
||||
)
|
||||
|
||||
|
||||
def _iter_causes(exc):
|
||||
@@ -152,12 +182,14 @@ def disable_other_loggers():
|
||||
def setup(level=logging.INFO):
|
||||
colorama.init()
|
||||
|
||||
addLoggingLevel("TRACE", logging.DEBUG - 5)
|
||||
logging.config.dictConfig(
|
||||
{
|
||||
"version": 1,
|
||||
"filters": {
|
||||
"exclude_errors": {"()": ExcludeErrorsFilter},
|
||||
"exclude_info": {"()": ExcludeInfoFilter},
|
||||
"exclude_errors": {"()": excludeFilter(logging.WARNING)},
|
||||
"exclude_info": {"()": excludeFilter(logging.INFO)},
|
||||
"exclude_debug": {"()": excludeFilter(logging.DEBUG)},
|
||||
},
|
||||
"formatters": {"color": {"()": ColorFormatter}},
|
||||
"handlers": {
|
||||
@@ -175,6 +207,13 @@ def setup(level=logging.INFO):
|
||||
"stream": "ext://sys.stdout",
|
||||
"filters": ["exclude_info"],
|
||||
},
|
||||
"console_trace": {
|
||||
"class": "dvc.logger.LoggerHandler",
|
||||
"level": "TRACE",
|
||||
"formatter": "color",
|
||||
"stream": "ext://sys.stdout",
|
||||
"filters": ["exclude_debug"],
|
||||
},
|
||||
"console_errors": {
|
||||
"class": "dvc.logger.LoggerHandler",
|
||||
"level": "WARNING",
|
||||
@@ -188,6 +227,7 @@ def setup(level=logging.INFO):
|
||||
"handlers": [
|
||||
"console_info",
|
||||
"console_debug",
|
||||
"console_trace",
|
||||
"console_errors",
|
||||
],
|
||||
},
|
||||
|
||||
+11
-6
@@ -18,7 +18,6 @@ from dvc.utils import format_link
|
||||
|
||||
"".encode("idna")
|
||||
|
||||
|
||||
logger = logging.getLogger("dvc")
|
||||
|
||||
|
||||
@@ -38,11 +37,17 @@ def main(argv=None):
|
||||
try:
|
||||
args = parse_args(argv)
|
||||
|
||||
if args.quiet:
|
||||
logger.setLevel(logging.CRITICAL)
|
||||
|
||||
elif args.verbose:
|
||||
logger.setLevel(logging.DEBUG)
|
||||
verbosity = args.verbose - args.quiet
|
||||
if verbosity:
|
||||
logger.setLevel(
|
||||
{
|
||||
-2: logging.CRITICAL,
|
||||
-1: logging.ERROR,
|
||||
1: logging.DEBUG,
|
||||
2: logging.TRACE,
|
||||
}[max(-2, min(verbosity, 2))]
|
||||
)
|
||||
logger.trace(args)
|
||||
|
||||
cmd = args.func(args)
|
||||
ret = cmd.run()
|
||||
|
||||
+55
-9
@@ -1,5 +1,7 @@
|
||||
from collections import defaultdict
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from funcy import collecting, project
|
||||
from voluptuous import And, Any, Coerce, Length, Lower, Required, SetTo
|
||||
|
||||
from dvc.output.base import BaseOutput
|
||||
@@ -8,10 +10,10 @@ from dvc.output.hdfs import HDFSOutput
|
||||
from dvc.output.local import LocalOutput
|
||||
from dvc.output.s3 import S3Output
|
||||
from dvc.output.ssh import SSHOutput
|
||||
from dvc.remote import Remote
|
||||
from dvc.remote.hdfs import HDFSRemote
|
||||
from dvc.remote.local import LocalRemote
|
||||
from dvc.remote.s3 import S3Remote
|
||||
from dvc.remote import get_remote
|
||||
from dvc.remote.hdfs import HDFSRemoteTree
|
||||
from dvc.remote.local import LocalRemoteTree
|
||||
from dvc.remote.s3 import S3RemoteTree
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
OUTS = [
|
||||
@@ -45,9 +47,9 @@ CHECKSUM_SCHEMA = Any(
|
||||
# so when a few types of outputs share the same name, we only need
|
||||
# specify it once.
|
||||
CHECKSUMS_SCHEMA = {
|
||||
LocalRemote.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
|
||||
S3Remote.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
|
||||
HDFSRemote.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
|
||||
LocalRemoteTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
|
||||
S3RemoteTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
|
||||
HDFSRemoteTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
|
||||
}
|
||||
|
||||
SCHEMA = CHECKSUMS_SCHEMA.copy()
|
||||
@@ -58,11 +60,13 @@ SCHEMA[BaseOutput.PARAM_PLOT] = bool
|
||||
SCHEMA[BaseOutput.PARAM_PERSIST] = bool
|
||||
|
||||
|
||||
def _get(stage, p, info, cache, metric, plot=False, persist=False):
|
||||
def _get(
|
||||
stage, p, info=None, cache=True, metric=False, plot=False, persist=False
|
||||
):
|
||||
parsed = urlparse(p)
|
||||
|
||||
if parsed.scheme == "remote":
|
||||
remote = Remote(stage.repo, name=parsed.netloc)
|
||||
remote = get_remote(stage.repo, name=parsed.netloc)
|
||||
return OUTS_MAP[remote.scheme](
|
||||
stage,
|
||||
p,
|
||||
@@ -135,3 +139,45 @@ def loads_from(
|
||||
)
|
||||
for s in s_list
|
||||
]
|
||||
|
||||
|
||||
def _split_dict(d, keys):
|
||||
return project(d, keys), project(d, d.keys() - keys)
|
||||
|
||||
|
||||
def _merge_data(s_list):
|
||||
d = defaultdict(dict)
|
||||
for key in s_list:
|
||||
if isinstance(key, str):
|
||||
d[key].update({})
|
||||
continue
|
||||
if not isinstance(key, dict):
|
||||
raise ValueError(f"'{type(key).__name__}' not supported.")
|
||||
|
||||
for k, flags in key.items():
|
||||
if not isinstance(flags, dict):
|
||||
raise ValueError(
|
||||
f"Expected dict for '{k}', got: '{type(flags).__name__}'"
|
||||
)
|
||||
d[k].update(flags)
|
||||
return d
|
||||
|
||||
|
||||
@collecting
|
||||
def load_from_pipeline(stage, s_list, typ="outs"):
|
||||
if typ not in (stage.PARAM_OUTS, stage.PARAM_METRICS, stage.PARAM_PLOTS):
|
||||
raise ValueError(f"'{typ}' key is not allowed for pipeline files.")
|
||||
|
||||
metric = typ == stage.PARAM_METRICS
|
||||
plot = typ == stage.PARAM_PLOTS
|
||||
|
||||
d = _merge_data(s_list)
|
||||
|
||||
for path, flags in d.items():
|
||||
plt_d = {}
|
||||
if plot:
|
||||
from dvc.schema import PLOT_PROPS
|
||||
|
||||
plt_d, flags = _split_dict(flags, keys=PLOT_PROPS.keys())
|
||||
extra = project(flags, ["cache", "persist"])
|
||||
yield _get(stage, path, {}, plot=plt_d or plot, metric=metric, **extra)
|
||||
|
||||
+66
-44
@@ -12,7 +12,7 @@ from dvc.exceptions import (
|
||||
DvcException,
|
||||
RemoteCacheRequiredError,
|
||||
)
|
||||
from dvc.remote.base import BaseRemote
|
||||
from dvc.remote.base import BaseRemoteTree, Remote
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -31,7 +31,11 @@ class OutputIsNotFileOrDirError(DvcException):
|
||||
|
||||
class OutputAlreadyTrackedError(DvcException):
|
||||
def __init__(self, path):
|
||||
msg = f"output '{path}' is already tracked by SCM (e.g. Git)"
|
||||
msg = f""" output '{path}' is already tracked by SCM (e.g. Git).
|
||||
You can remove it from Git, then add to DVC.
|
||||
To stop tracking from Git:
|
||||
git rm -r --cached '{path}'
|
||||
git commit -m "stop tracking {path}" """
|
||||
super().__init__(msg)
|
||||
|
||||
|
||||
@@ -43,7 +47,8 @@ class OutputIsStageFileError(DvcException):
|
||||
class BaseOutput:
|
||||
IS_DEPENDENCY = False
|
||||
|
||||
REMOTE = BaseRemote
|
||||
REMOTE_CLS = Remote
|
||||
TREE_CLS = BaseRemoteTree
|
||||
|
||||
PARAM_PATH = "path"
|
||||
PARAM_CACHE = "cache"
|
||||
@@ -52,6 +57,12 @@ class BaseOutput:
|
||||
PARAM_METRIC_XPATH = "xpath"
|
||||
PARAM_PLOT = "plot"
|
||||
PARAM_PLOT_TEMPLATE = "template"
|
||||
PARAM_PLOT_X = "x"
|
||||
PARAM_PLOT_Y = "y"
|
||||
PARAM_PLOT_X_LABEL = "x_label"
|
||||
PARAM_PLOT_Y_LABEL = "y_label"
|
||||
PARAM_PLOT_TITLE = "title"
|
||||
PARAM_PLOT_HEADER = "header"
|
||||
PARAM_PERSIST = "persist"
|
||||
|
||||
METRIC_SCHEMA = Any(
|
||||
@@ -95,7 +106,11 @@ class BaseOutput:
|
||||
self.repo = stage.repo if stage else None
|
||||
self.def_path = path
|
||||
self.info = info
|
||||
self.remote = remote or self.REMOTE(self.repo, {})
|
||||
if remote:
|
||||
self.remote = remote
|
||||
else:
|
||||
tree = self.TREE_CLS(self.repo, {})
|
||||
self.remote = self.REMOTE_CLS(tree)
|
||||
self.use_cache = False if self.IS_DEPENDENCY else cache
|
||||
self.metric = False if self.IS_DEPENDENCY else metric
|
||||
self.plot = False if self.IS_DEPENDENCY else plot
|
||||
@@ -109,7 +124,7 @@ class BaseOutput:
|
||||
if remote:
|
||||
parsed = urlparse(path)
|
||||
return remote.path_info / parsed.path.lstrip("/")
|
||||
return self.REMOTE.path_cls(path)
|
||||
return self.TREE_CLS.PATH_CLS(path)
|
||||
|
||||
def __repr__(self):
|
||||
return "{class_name}: '{def_path}'".format(
|
||||
@@ -121,7 +136,7 @@ class BaseOutput:
|
||||
|
||||
@property
|
||||
def scheme(self):
|
||||
return self.REMOTE.scheme
|
||||
return self.TREE_CLS.scheme
|
||||
|
||||
@property
|
||||
def is_in_repo(self):
|
||||
@@ -144,34 +159,34 @@ class BaseOutput:
|
||||
|
||||
@classmethod
|
||||
def supported(cls, url):
|
||||
return cls.REMOTE.supported(url)
|
||||
return cls.TREE_CLS.supported(url)
|
||||
|
||||
@property
|
||||
def cache_path(self):
|
||||
return self.cache.checksum_to_path_info(self.checksum).url
|
||||
return self.cache.hash_to_path_info(self.checksum).url
|
||||
|
||||
@property
|
||||
def checksum_type(self):
|
||||
return self.remote.PARAM_CHECKSUM
|
||||
return self.remote.tree.PARAM_CHECKSUM
|
||||
|
||||
@property
|
||||
def checksum(self):
|
||||
return self.info.get(self.remote.PARAM_CHECKSUM)
|
||||
return self.info.get(self.remote.tree.PARAM_CHECKSUM)
|
||||
|
||||
@checksum.setter
|
||||
def checksum(self, checksum):
|
||||
self.info[self.remote.PARAM_CHECKSUM] = checksum
|
||||
self.info[self.remote.tree.PARAM_CHECKSUM] = checksum
|
||||
|
||||
def get_checksum(self):
|
||||
return self.remote.get_checksum(self.path_info)
|
||||
return self.remote.get_hash(self.path_info)
|
||||
|
||||
@property
|
||||
def is_dir_checksum(self):
|
||||
return self.remote.is_dir_checksum(self.checksum)
|
||||
return self.remote.is_dir_hash(self.checksum)
|
||||
|
||||
@property
|
||||
def exists(self):
|
||||
return self.remote.exists(self.path_info)
|
||||
return self.remote.tree.exists(self.path_info)
|
||||
|
||||
def save_info(self):
|
||||
return self.remote.save_info(self.path_info)
|
||||
@@ -207,13 +222,13 @@ class BaseOutput:
|
||||
|
||||
@property
|
||||
def is_empty(self):
|
||||
return self.remote.is_empty(self.path_info)
|
||||
return self.remote.tree.is_empty(self.path_info)
|
||||
|
||||
def isdir(self):
|
||||
return self.remote.isdir(self.path_info)
|
||||
return self.remote.tree.isdir(self.path_info)
|
||||
|
||||
def isfile(self):
|
||||
return self.remote.isfile(self.path_info)
|
||||
return self.remote.tree.isfile(self.path_info)
|
||||
|
||||
def ignore(self):
|
||||
if not self.use_scm_ignore:
|
||||
@@ -236,29 +251,28 @@ class BaseOutput:
|
||||
|
||||
self.ignore()
|
||||
|
||||
if self.metric or self.plot:
|
||||
self.verify_metric()
|
||||
|
||||
if not self.use_cache:
|
||||
self.info = self.save_info()
|
||||
if self.metric or self.plot:
|
||||
self.verify_metric()
|
||||
if not self.IS_DEPENDENCY:
|
||||
logger.info(
|
||||
"Output '{}' doesn't use cache. Skipping saving.".format(
|
||||
self
|
||||
)
|
||||
logger.debug(
|
||||
"Output '%s' doesn't use cache. Skipping saving.", self
|
||||
)
|
||||
return
|
||||
|
||||
assert not self.IS_DEPENDENCY
|
||||
|
||||
if not self.changed():
|
||||
logger.info(f"Output '{self}' didn't change. Skipping saving.")
|
||||
logger.debug("Output '%s' didn't change. Skipping saving.", self)
|
||||
return
|
||||
|
||||
self.info = self.save_info()
|
||||
|
||||
def commit(self):
|
||||
if self.use_cache:
|
||||
self.cache.save(self.path_info, self.info)
|
||||
self.cache.save(self.path_info, self.cache.tree, self.info)
|
||||
|
||||
def dumpd(self):
|
||||
ret = copy(self.info)
|
||||
@@ -267,7 +281,8 @@ class BaseOutput:
|
||||
if self.IS_DEPENDENCY:
|
||||
return ret
|
||||
|
||||
ret[self.PARAM_CACHE] = self.use_cache
|
||||
if not self.use_cache:
|
||||
ret[self.PARAM_CACHE] = self.use_cache
|
||||
|
||||
if isinstance(self.metric, dict):
|
||||
if (
|
||||
@@ -276,12 +291,14 @@ class BaseOutput:
|
||||
):
|
||||
del self.metric[self.PARAM_METRIC_XPATH]
|
||||
|
||||
ret[self.PARAM_METRIC] = self.metric
|
||||
if self.metric:
|
||||
ret[self.PARAM_METRIC] = self.metric
|
||||
|
||||
if self.plot:
|
||||
ret[self.PARAM_PLOT] = self.plot
|
||||
|
||||
ret[self.PARAM_PERSIST] = self.persist
|
||||
if self.persist:
|
||||
ret[self.PARAM_PERSIST] = self.persist
|
||||
|
||||
return ret
|
||||
|
||||
@@ -289,7 +306,7 @@ class BaseOutput:
|
||||
raise DvcException(f"verify metric is not supported for {self.scheme}")
|
||||
|
||||
def download(self, to):
|
||||
self.remote.download(self.path_info, to.path_info)
|
||||
self.remote.tree.download(self.path_info, to.path_info)
|
||||
|
||||
def checkout(
|
||||
self,
|
||||
@@ -315,7 +332,7 @@ class BaseOutput:
|
||||
)
|
||||
|
||||
def remove(self, ignore_remove=False):
|
||||
self.remote.remove(self.path_info)
|
||||
self.remote.tree.remove(self.path_info)
|
||||
if self.scheme != "local":
|
||||
return
|
||||
|
||||
@@ -326,7 +343,7 @@ class BaseOutput:
|
||||
if self.scheme == "local" and self.use_scm_ignore:
|
||||
self.repo.scm.ignore_remove(self.fspath)
|
||||
|
||||
self.remote.move(self.path_info, out.path_info)
|
||||
self.remote.tree.move(self.path_info, out.path_info)
|
||||
self.def_path = out.def_path
|
||||
self.path_info = out.path_info
|
||||
self.save()
|
||||
@@ -345,7 +362,18 @@ class BaseOutput:
|
||||
|
||||
def unprotect(self):
|
||||
if self.exists:
|
||||
self.remote.unprotect(self.path_info)
|
||||
self.remote.tree.unprotect(self.path_info)
|
||||
|
||||
def get_dir_cache(self, **kwargs):
|
||||
if not self.is_dir_checksum:
|
||||
raise DvcException("cannot get dir cache for file checksum")
|
||||
if self.cache.changed_cache_file(self.checksum):
|
||||
self.repo.cloud.pull(
|
||||
NamedCache.make("local", self.checksum, str(self)),
|
||||
show_checksums=False,
|
||||
**kwargs,
|
||||
)
|
||||
return self.dir_cache
|
||||
|
||||
def collect_used_dir_cache(
|
||||
self, remote=None, force=False, jobs=None, filter_info=None
|
||||
@@ -371,16 +399,10 @@ class BaseOutput:
|
||||
|
||||
cache = NamedCache()
|
||||
|
||||
if self.cache.changed_cache_file(self.checksum):
|
||||
try:
|
||||
self.repo.cloud.pull(
|
||||
NamedCache.make("local", self.checksum, str(self)),
|
||||
jobs=jobs,
|
||||
remote=remote,
|
||||
show_checksums=False,
|
||||
)
|
||||
except DvcException:
|
||||
logger.debug(f"failed to pull cache for '{self}'")
|
||||
try:
|
||||
self.get_dir_cache(jobs=jobs, remote=remote)
|
||||
except DvcException:
|
||||
logger.debug(f"failed to pull cache for '{self}'")
|
||||
|
||||
if self.cache.changed_cache_file(self.checksum):
|
||||
msg = (
|
||||
@@ -400,8 +422,8 @@ class BaseOutput:
|
||||
filter_path = str(filter_info) if filter_info else None
|
||||
is_win = os.name == "nt"
|
||||
for entry in self.dir_cache:
|
||||
checksum = entry[self.remote.PARAM_CHECKSUM]
|
||||
entry_relpath = entry[self.remote.PARAM_RELPATH]
|
||||
checksum = entry[self.remote.tree.PARAM_CHECKSUM]
|
||||
entry_relpath = entry[self.remote.tree.PARAM_RELPATH]
|
||||
if is_win:
|
||||
entry_relpath = entry_relpath.replace("/", os.sep)
|
||||
entry_path = os.path.join(path, entry_relpath)
|
||||
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
from dvc.output.s3 import S3Output
|
||||
from dvc.remote.gs import GSRemote
|
||||
from dvc.remote.gs import GSRemoteTree
|
||||
|
||||
|
||||
class GSOutput(S3Output):
|
||||
REMOTE = GSRemote
|
||||
TREE_CLS = GSRemoteTree
|
||||
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
from dvc.output.base import BaseOutput
|
||||
from dvc.remote.hdfs import HDFSRemote
|
||||
from dvc.remote.hdfs import HDFSRemoteTree
|
||||
|
||||
|
||||
class HDFSOutput(BaseOutput):
|
||||
REMOTE = HDFSRemote
|
||||
TREE_CLS = HDFSRemoteTree
|
||||
|
||||
+8
-7
@@ -5,7 +5,7 @@ from urllib.parse import urlparse
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.istextfile import istextfile
|
||||
from dvc.output.base import BaseOutput
|
||||
from dvc.remote.local import LocalRemote
|
||||
from dvc.remote.local import LocalRemote, LocalRemoteTree
|
||||
from dvc.utils import relpath
|
||||
from dvc.utils.fs import path_isin
|
||||
|
||||
@@ -13,7 +13,8 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LocalOutput(BaseOutput):
|
||||
REMOTE = LocalRemote
|
||||
REMOTE_CLS = LocalRemote
|
||||
TREE_CLS = LocalRemoteTree
|
||||
sep = os.sep
|
||||
|
||||
def __init__(self, stage, path, *args, **kwargs):
|
||||
@@ -33,15 +34,15 @@ class LocalOutput(BaseOutput):
|
||||
#
|
||||
# FIXME: if we have Windows path containing / or posix one with \
|
||||
# then we have #2059 bug and can't really handle that.
|
||||
p = self.REMOTE.path_cls(path)
|
||||
if not p.is_absolute():
|
||||
p = self.TREE_CLS.PATH_CLS(path)
|
||||
if self.stage and not p.is_absolute():
|
||||
p = self.stage.wdir / p
|
||||
|
||||
abs_p = os.path.abspath(os.path.normpath(p))
|
||||
return self.REMOTE.path_cls(abs_p)
|
||||
return self.TREE_CLS.PATH_CLS(abs_p)
|
||||
|
||||
def __str__(self):
|
||||
if not self.is_in_repo:
|
||||
if not self.repo or not self.is_in_repo:
|
||||
return str(self.def_path)
|
||||
|
||||
cur_dir = os.getcwd()
|
||||
@@ -81,7 +82,7 @@ class LocalOutput(BaseOutput):
|
||||
name = "metrics" if self.metric else "plot"
|
||||
if os.path.isdir(path):
|
||||
msg = "directory '{}' cannot be used as {}."
|
||||
raise DvcException(msg.format(self.path_info, name))
|
||||
raise IsADirectoryError(msg.format(self.path_info, name))
|
||||
|
||||
if not istextfile(path):
|
||||
msg = "binary file '{}' cannot be used as {}."
|
||||
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
from dvc.output.base import BaseOutput
|
||||
from dvc.remote.s3 import S3Remote
|
||||
from dvc.remote.s3 import S3RemoteTree
|
||||
|
||||
|
||||
class S3Output(BaseOutput):
|
||||
REMOTE = S3Remote
|
||||
TREE_CLS = S3RemoteTree
|
||||
|
||||
+3
-2
@@ -1,6 +1,7 @@
|
||||
from dvc.output.base import BaseOutput
|
||||
from dvc.remote.ssh import SSHRemote
|
||||
from dvc.remote.ssh import SSHRemote, SSHRemoteTree
|
||||
|
||||
|
||||
class SSHOutput(BaseOutput):
|
||||
REMOTE = SSHRemote
|
||||
REMOTE_CLS = SSHRemote
|
||||
TREE_CLS = SSHRemoteTree
|
||||
|
||||
+43
-29
@@ -1,46 +1,46 @@
|
||||
import posixpath
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from dvc.remote.azure import AzureRemote
|
||||
from dvc.remote.gdrive import GDriveRemote
|
||||
from dvc.remote.gs import GSRemote
|
||||
from dvc.remote.hdfs import HDFSRemote
|
||||
from dvc.remote.http import HTTPRemote
|
||||
from dvc.remote.https import HTTPSRemote
|
||||
from dvc.remote.local import LocalRemote
|
||||
from dvc.remote.oss import OSSRemote
|
||||
from dvc.remote.s3 import S3Remote
|
||||
from dvc.remote.ssh import SSHRemote
|
||||
from dvc.remote.azure import AzureRemoteTree
|
||||
from dvc.remote.base import Remote
|
||||
from dvc.remote.gdrive import GDriveRemoteTree
|
||||
from dvc.remote.gs import GSRemoteTree
|
||||
from dvc.remote.hdfs import HDFSRemoteTree
|
||||
from dvc.remote.http import HTTPRemoteTree
|
||||
from dvc.remote.https import HTTPSRemoteTree
|
||||
from dvc.remote.local import LocalRemote, LocalRemoteTree
|
||||
from dvc.remote.oss import OSSRemoteTree
|
||||
from dvc.remote.s3 import S3RemoteTree
|
||||
from dvc.remote.ssh import SSHRemote, SSHRemoteTree
|
||||
|
||||
REMOTES = [
|
||||
AzureRemote,
|
||||
GDriveRemote,
|
||||
GSRemote,
|
||||
HDFSRemote,
|
||||
HTTPRemote,
|
||||
HTTPSRemote,
|
||||
S3Remote,
|
||||
SSHRemote,
|
||||
OSSRemote,
|
||||
# NOTE: LocalRemote is the default
|
||||
TREES = [
|
||||
AzureRemoteTree,
|
||||
GDriveRemoteTree,
|
||||
GSRemoteTree,
|
||||
HDFSRemoteTree,
|
||||
HTTPRemoteTree,
|
||||
HTTPSRemoteTree,
|
||||
S3RemoteTree,
|
||||
SSHRemoteTree,
|
||||
OSSRemoteTree,
|
||||
# NOTE: LocalRemoteTree is the default
|
||||
]
|
||||
|
||||
|
||||
def _get(remote_conf):
|
||||
for remote in REMOTES:
|
||||
if remote.supported(remote_conf):
|
||||
return remote
|
||||
return LocalRemote
|
||||
def _get_tree(remote_conf):
|
||||
for tree_cls in TREES:
|
||||
if tree_cls.supported(remote_conf):
|
||||
return tree_cls
|
||||
return LocalRemoteTree
|
||||
|
||||
|
||||
def Remote(repo, **kwargs):
|
||||
def _get_conf(repo, **kwargs):
|
||||
name = kwargs.get("name")
|
||||
if name:
|
||||
remote_conf = repo.config["remote"][name.lower()]
|
||||
else:
|
||||
remote_conf = kwargs
|
||||
remote_conf = _resolve_remote_refs(repo.config, remote_conf)
|
||||
return _get(remote_conf)(repo, remote_conf)
|
||||
return _resolve_remote_refs(repo.config, remote_conf)
|
||||
|
||||
|
||||
def _resolve_remote_refs(config, remote_conf):
|
||||
@@ -72,3 +72,17 @@ def _resolve_remote_refs(config, remote_conf):
|
||||
base = config["remote"][parsed.netloc]
|
||||
url = posixpath.join(base["url"], parsed.path.lstrip("/"))
|
||||
return {**base, **remote_conf, "url": url}
|
||||
|
||||
|
||||
def get_cloud_tree(repo, **kwargs):
|
||||
remote_conf = _get_conf(repo, **kwargs)
|
||||
return _get_tree(remote_conf)(repo, remote_conf)
|
||||
|
||||
|
||||
def get_remote(repo, **kwargs):
|
||||
tree = get_cloud_tree(repo, **kwargs)
|
||||
if tree.scheme == "local":
|
||||
return LocalRemote(tree)
|
||||
if tree.scheme == "ssh":
|
||||
return SSHRemote(tree)
|
||||
return Remote(tree)
|
||||
|
||||
+52
-73
@@ -1,30 +1,21 @@
|
||||
import logging
|
||||
import os
|
||||
import posixpath
|
||||
import re
|
||||
import threading
|
||||
from datetime import datetime, timedelta
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from funcy import cached_property, wrap_prop
|
||||
|
||||
from dvc.path_info import CloudURLInfo
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.remote.base import BaseRemote
|
||||
from dvc.remote.base import BaseRemoteTree
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AzureRemote(BaseRemote):
|
||||
class AzureRemoteTree(BaseRemoteTree):
|
||||
scheme = Schemes.AZURE
|
||||
path_cls = CloudURLInfo
|
||||
REGEX = (
|
||||
r"azure://((?P<path>[^=;]*)?|("
|
||||
# backward compatibility
|
||||
r"(ContainerName=(?P<container_name>[^;]+);?)?"
|
||||
r"(?P<connection_string>.+)?)?)$"
|
||||
)
|
||||
PATH_CLS = CloudURLInfo
|
||||
REQUIRES = {"azure-storage-blob": "azure.storage.blob"}
|
||||
PARAM_CHECKSUM = "etag"
|
||||
COPY_POLL_SECONDS = 5
|
||||
@@ -34,31 +25,14 @@ class AzureRemote(BaseRemote):
|
||||
super().__init__(repo, config)
|
||||
|
||||
url = config.get("url", "azure://")
|
||||
self.path_info = self.PATH_CLS(url)
|
||||
|
||||
match = re.match(self.REGEX, url) # backward compatibility
|
||||
path = match.group("path")
|
||||
bucket = (
|
||||
urlparse(url if path else "").netloc
|
||||
or match.group("container_name") # backward compatibility
|
||||
or os.getenv("AZURE_STORAGE_CONTAINER_NAME")
|
||||
)
|
||||
if not self.path_info.bucket:
|
||||
container = os.getenv("AZURE_STORAGE_CONTAINER_NAME")
|
||||
self.path_info = self.PATH_CLS(f"azure://{container}")
|
||||
|
||||
self.connection_string = (
|
||||
config.get("connection_string")
|
||||
or match.group("connection_string") # backward compatibility
|
||||
or os.getenv("AZURE_STORAGE_CONNECTION_STRING")
|
||||
)
|
||||
|
||||
if not bucket:
|
||||
raise ValueError("azure storage container name missing")
|
||||
|
||||
if not self.connection_string:
|
||||
raise ValueError("azure storage connection string missing")
|
||||
|
||||
self.path_info = (
|
||||
self.path_cls(url)
|
||||
if path
|
||||
else self.path_cls.from_parts(scheme=self.scheme, netloc=bucket)
|
||||
self.connection_string = config.get("connection_string") or os.getenv(
|
||||
"AZURE_STORAGE_CONNECTION_STRING"
|
||||
)
|
||||
|
||||
@wrap_prop(threading.Lock())
|
||||
@@ -81,14 +55,33 @@ class AzureRemote(BaseRemote):
|
||||
blob_service.create_container(self.path_info.bucket)
|
||||
return blob_service
|
||||
|
||||
def remove(self, path_info):
|
||||
if path_info.scheme != self.scheme:
|
||||
raise NotImplementedError
|
||||
def get_etag(self, path_info):
|
||||
etag = self.blob_service.get_blob_properties(
|
||||
path_info.bucket, path_info.path
|
||||
).properties.etag
|
||||
return etag.strip('"')
|
||||
|
||||
logger.debug(f"Removing {path_info}")
|
||||
self.blob_service.delete_blob(path_info.bucket, path_info.path)
|
||||
def _generate_download_url(self, path_info, expires=3600):
|
||||
from azure.storage.blob import BlobPermissions
|
||||
|
||||
def _list_paths(self, bucket, prefix, progress_callback=None):
|
||||
expires_at = datetime.utcnow() + timedelta(seconds=expires)
|
||||
|
||||
sas_token = self.blob_service.generate_blob_shared_access_signature(
|
||||
path_info.bucket,
|
||||
path_info.path,
|
||||
permission=BlobPermissions.READ,
|
||||
expiry=expires_at,
|
||||
)
|
||||
download_url = self.blob_service.make_blob_url(
|
||||
path_info.bucket, path_info.path, sas_token=sas_token
|
||||
)
|
||||
return download_url
|
||||
|
||||
def exists(self, path_info):
|
||||
paths = self._list_paths(path_info.bucket, path_info.path)
|
||||
return any(path_info.path == path for path in paths)
|
||||
|
||||
def _list_paths(self, bucket, prefix):
|
||||
blob_service = self.blob_service
|
||||
next_marker = None
|
||||
while True:
|
||||
@@ -97,8 +90,6 @@ class AzureRemote(BaseRemote):
|
||||
)
|
||||
|
||||
for blob in blobs:
|
||||
if progress_callback:
|
||||
progress_callback()
|
||||
yield blob.name
|
||||
|
||||
if not blobs.next_marker:
|
||||
@@ -106,16 +97,24 @@ class AzureRemote(BaseRemote):
|
||||
|
||||
next_marker = blobs.next_marker
|
||||
|
||||
def list_cache_paths(self, prefix=None, progress_callback=None):
|
||||
if prefix:
|
||||
prefix = posixpath.join(
|
||||
self.path_info.path, prefix[:2], prefix[2:]
|
||||
)
|
||||
else:
|
||||
prefix = self.path_info.path
|
||||
return self._list_paths(
|
||||
self.path_info.bucket, prefix, progress_callback
|
||||
)
|
||||
def walk_files(self, path_info, **kwargs):
|
||||
for fname in self._list_paths(
|
||||
path_info.bucket, path_info.path, **kwargs
|
||||
):
|
||||
if fname.endswith("/"):
|
||||
continue
|
||||
|
||||
yield path_info.replace(path=fname)
|
||||
|
||||
def remove(self, path_info):
|
||||
if path_info.scheme != self.scheme:
|
||||
raise NotImplementedError
|
||||
|
||||
logger.debug(f"Removing {path_info}")
|
||||
self.blob_service.delete_blob(path_info.bucket, path_info.path)
|
||||
|
||||
def get_file_hash(self, path_info):
|
||||
return self.get_etag(path_info)
|
||||
|
||||
def _upload(
|
||||
self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
|
||||
@@ -138,23 +137,3 @@ class AzureRemote(BaseRemote):
|
||||
to_file,
|
||||
progress_callback=pbar.update_to,
|
||||
)
|
||||
|
||||
def exists(self, path_info):
|
||||
paths = self._list_paths(path_info.bucket, path_info.path)
|
||||
return any(path_info.path == path for path in paths)
|
||||
|
||||
def _generate_download_url(self, path_info, expires=3600):
|
||||
from azure.storage.blob import BlobPermissions
|
||||
|
||||
expires_at = datetime.utcnow() + timedelta(seconds=expires)
|
||||
|
||||
sas_token = self.blob_service.generate_blob_shared_access_signature(
|
||||
path_info.bucket,
|
||||
path_info.path,
|
||||
permission=BlobPermissions.READ,
|
||||
expiry=expires_at,
|
||||
)
|
||||
download_url = self.blob_service.make_blob_url(
|
||||
path_info.bucket, path_info.path, sas_token=sas_token
|
||||
)
|
||||
return download_url
|
||||
|
||||
+856
-748
File diff suppressed because it is too large
Load Diff
+63
-45
@@ -1,38 +1,35 @@
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import posixpath
|
||||
import re
|
||||
import threading
|
||||
from collections import defaultdict
|
||||
from contextlib import contextmanager
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from funcy import cached_property, retry, wrap_prop, wrap_with
|
||||
from funcy.py3 import cat
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.exceptions import DvcException, FileMissingError
|
||||
from dvc.path_info import CloudURLInfo
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.remote.base import BaseRemote
|
||||
from dvc.remote.base import BaseRemoteTree
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.utils import format_link, tmp_fname
|
||||
from dvc.utils.stream import IterStream
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"
|
||||
|
||||
|
||||
class GDrivePathNotFound(DvcException):
|
||||
def __init__(self, path_info, hint):
|
||||
hint = "" if hint is None else f" {hint}"
|
||||
super().__init__(f"GDrive path '{path_info}' not found.{hint}")
|
||||
|
||||
|
||||
class GDriveAuthError(DvcException):
|
||||
def __init__(self, cred_location):
|
||||
|
||||
if cred_location:
|
||||
message = (
|
||||
"GDrive remote auth failed with credentials in '{}'.\n"
|
||||
"Backup first, remove of fix them, and run DVC again.\n"
|
||||
"Backup first, remove or fix them, and run DVC again.\n"
|
||||
"It should do auth again and refresh the credentials.\n\n"
|
||||
"Details:".format(cred_location)
|
||||
)
|
||||
@@ -87,9 +84,9 @@ class GDriveURLInfo(CloudURLInfo):
|
||||
self._spath = re.sub("/{2,}", "/", self._spath.rstrip("/"))
|
||||
|
||||
|
||||
class GDriveRemote(BaseRemote):
|
||||
class GDriveRemoteTree(BaseRemoteTree):
|
||||
scheme = Schemes.GDRIVE
|
||||
path_cls = GDriveURLInfo
|
||||
PATH_CLS = GDriveURLInfo
|
||||
REQUIRES = {"pydrive2": "pydrive2"}
|
||||
DEFAULT_VERIFY = True
|
||||
# Always prefer traverse for GDrive since API usage quotas are a concern.
|
||||
@@ -104,7 +101,8 @@ class GDriveRemote(BaseRemote):
|
||||
|
||||
def __init__(self, repo, config):
|
||||
super().__init__(repo, config)
|
||||
self.path_info = self.path_cls(config["url"])
|
||||
|
||||
self.path_info = self.PATH_CLS(config["url"])
|
||||
|
||||
if not self.path_info.bucket:
|
||||
raise DvcException(
|
||||
@@ -131,11 +129,11 @@ class GDriveRemote(BaseRemote):
|
||||
self._validate_config()
|
||||
self._gdrive_user_credentials_path = (
|
||||
tmp_fname(os.path.join(self.repo.tmp_dir, ""))
|
||||
if os.getenv(GDriveRemote.GDRIVE_CREDENTIALS_DATA)
|
||||
if os.getenv(GDriveRemoteTree.GDRIVE_CREDENTIALS_DATA)
|
||||
else config.get(
|
||||
"gdrive_user_credentials_file",
|
||||
os.path.join(
|
||||
self.repo.tmp_dir, self.DEFAULT_USER_CREDENTIALS_FILE
|
||||
self.repo.tmp_dir, self.DEFAULT_USER_CREDENTIALS_FILE,
|
||||
),
|
||||
)
|
||||
)
|
||||
@@ -173,8 +171,8 @@ class GDriveRemote(BaseRemote):
|
||||
Useful for tests, exception messages, etc. Returns either env variable
|
||||
name if it's set or actual path to the credentials file.
|
||||
"""
|
||||
if os.getenv(GDriveRemote.GDRIVE_CREDENTIALS_DATA):
|
||||
return GDriveRemote.GDRIVE_CREDENTIALS_DATA
|
||||
if os.getenv(GDriveRemoteTree.GDRIVE_CREDENTIALS_DATA):
|
||||
return GDriveRemoteTree.GDRIVE_CREDENTIALS_DATA
|
||||
if os.path.exists(self._gdrive_user_credentials_path):
|
||||
return self._gdrive_user_credentials_path
|
||||
return None
|
||||
@@ -188,7 +186,7 @@ class GDriveRemote(BaseRemote):
|
||||
DVC config client id or secret but forgets to remove the cached
|
||||
credentials file.
|
||||
"""
|
||||
if not os.getenv(GDriveRemote.GDRIVE_CREDENTIALS_DATA):
|
||||
if not os.getenv(GDriveRemoteTree.GDRIVE_CREDENTIALS_DATA):
|
||||
if (
|
||||
settings["client_config"]["client_id"]
|
||||
!= auth.credentials.client_id
|
||||
@@ -211,10 +209,10 @@ class GDriveRemote(BaseRemote):
|
||||
from pydrive2.auth import GoogleAuth
|
||||
from pydrive2.drive import GoogleDrive
|
||||
|
||||
if os.getenv(GDriveRemote.GDRIVE_CREDENTIALS_DATA):
|
||||
if os.getenv(GDriveRemoteTree.GDRIVE_CREDENTIALS_DATA):
|
||||
with open(self._gdrive_user_credentials_path, "w") as cred_file:
|
||||
cred_file.write(
|
||||
os.getenv(GDriveRemote.GDRIVE_CREDENTIALS_DATA)
|
||||
os.getenv(GDriveRemoteTree.GDRIVE_CREDENTIALS_DATA)
|
||||
)
|
||||
|
||||
auth_settings = {
|
||||
@@ -261,7 +259,7 @@ class GDriveRemote(BaseRemote):
|
||||
gauth.ServiceAuth()
|
||||
else:
|
||||
gauth.CommandLineAuth()
|
||||
GDriveRemote._validate_credentials(gauth, auth_settings)
|
||||
GDriveRemoteTree._validate_credentials(gauth, auth_settings)
|
||||
|
||||
# Handle AuthenticationError, RefreshError and other auth failures
|
||||
# It's hard to come up with a narrow exception, since PyDrive throws
|
||||
@@ -270,7 +268,7 @@ class GDriveRemote(BaseRemote):
|
||||
except Exception as exc:
|
||||
raise GDriveAuthError(self.credentials_location) from exc
|
||||
finally:
|
||||
if os.getenv(GDriveRemote.GDRIVE_CREDENTIALS_DATA):
|
||||
if os.getenv(GDriveRemoteTree.GDRIVE_CREDENTIALS_DATA):
|
||||
os.remove(self._gdrive_user_credentials_path)
|
||||
|
||||
return GoogleDrive(gauth)
|
||||
@@ -340,7 +338,7 @@ class GDriveRemote(BaseRemote):
|
||||
"use the `gdrive_user_credentials_file` config\n"
|
||||
" option if you use multiple GDrive remotes with "
|
||||
"different email accounts.\n\nDetails".format(
|
||||
item_id, self.path_info, self._credentials_location
|
||||
item_id, self.path_info, self.credentials_location
|
||||
)
|
||||
) from exc
|
||||
raise
|
||||
@@ -393,8 +391,25 @@ class GDriveRemote(BaseRemote):
|
||||
) as pbar:
|
||||
gdrive_file.GetContentFile(to_file, callback=pbar.update_to)
|
||||
|
||||
@contextmanager
|
||||
@_gdrive_retry
|
||||
def _gdrive_delete_file(self, item_id):
|
||||
def open(self, path_info, mode="r", encoding=None):
|
||||
assert mode in {"r", "rt", "rb"}
|
||||
|
||||
item_id = self._get_item_id(path_info)
|
||||
param = {"id": item_id}
|
||||
# it does not create a file on the remote
|
||||
gdrive_file = self._drive.CreateFile(param)
|
||||
fd = gdrive_file.GetContentIOBuffer()
|
||||
stream = IterStream(iter(fd))
|
||||
|
||||
if mode != "rb":
|
||||
stream = io.TextIOWrapper(stream, encoding=encoding)
|
||||
|
||||
yield stream
|
||||
|
||||
@_gdrive_retry
|
||||
def gdrive_delete_file(self, item_id):
|
||||
from pydrive2.files import ApiRequestError
|
||||
|
||||
param = {"id": item_id}
|
||||
@@ -506,30 +521,17 @@ class GDriveRemote(BaseRemote):
|
||||
return min(item_ids)
|
||||
|
||||
assert not create
|
||||
raise GDrivePathNotFound(path_info, hint)
|
||||
raise FileMissingError(path_info, hint)
|
||||
|
||||
def exists(self, path_info):
|
||||
try:
|
||||
self._get_item_id(path_info)
|
||||
except GDrivePathNotFound:
|
||||
except FileMissingError:
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
|
||||
def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
|
||||
dirname = to_info.parent
|
||||
assert dirname
|
||||
parent_id = self._get_item_id(dirname, True)
|
||||
|
||||
self._gdrive_upload_file(
|
||||
parent_id, to_info.name, no_progress_bar, from_file, name
|
||||
)
|
||||
|
||||
def _download(self, from_info, to_file, name=None, no_progress_bar=False):
|
||||
item_id = self._get_item_id(from_info)
|
||||
self._gdrive_download_file(item_id, to_file, name, no_progress_bar)
|
||||
|
||||
def list_cache_paths(self, prefix=None, progress_callback=None):
|
||||
def _list_paths(self, prefix=None):
|
||||
if not self._ids_cache["ids"]:
|
||||
return
|
||||
|
||||
@@ -545,19 +547,35 @@ class GDriveRemote(BaseRemote):
|
||||
query = f"({parents_query}) and trashed=false"
|
||||
|
||||
for item in self._gdrive_list(query):
|
||||
if progress_callback:
|
||||
progress_callback()
|
||||
parent_id = item["parents"][0]["id"]
|
||||
yield posixpath.join(
|
||||
self._ids_cache["ids"][parent_id], item["title"]
|
||||
)
|
||||
|
||||
def walk_files(self, path_info, **kwargs):
|
||||
if path_info == self.path_info:
|
||||
prefix = None
|
||||
else:
|
||||
prefix = path_info.path
|
||||
for fname in self._list_paths(prefix=prefix, **kwargs):
|
||||
yield path_info.replace(fname)
|
||||
|
||||
def remove(self, path_info):
|
||||
item_id = self._get_item_id(path_info)
|
||||
self._gdrive_delete_file(item_id)
|
||||
self.gdrive_delete_file(item_id)
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
def get_file_hash(self, path_info):
|
||||
raise NotImplementedError
|
||||
|
||||
def walk_files(self, path_info):
|
||||
raise NotImplementedError
|
||||
def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
|
||||
dirname = to_info.parent
|
||||
assert dirname
|
||||
parent_id = self._get_item_id(dirname, True)
|
||||
|
||||
self._gdrive_upload_file(
|
||||
parent_id, to_info.name, no_progress_bar, from_file, name
|
||||
)
|
||||
|
||||
def _download(self, from_info, to_file, name=None, no_progress_bar=False):
|
||||
item_id = self._get_item_id(from_info)
|
||||
self._gdrive_download_file(item_id, to_file, name, no_progress_bar)
|
||||
|
||||
+71
-85
@@ -1,6 +1,5 @@
|
||||
import logging
|
||||
import os.path
|
||||
import posixpath
|
||||
import threading
|
||||
from datetime import timedelta
|
||||
from functools import wraps
|
||||
@@ -10,7 +9,7 @@ from funcy import cached_property, wrap_prop
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.path_info import CloudURLInfo
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.remote.base import BaseRemote
|
||||
from dvc.remote.base import BaseRemoteTree
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -65,9 +64,9 @@ def _upload_to_bucket(
|
||||
blob.upload_from_file(wrapped)
|
||||
|
||||
|
||||
class GSRemote(BaseRemote):
|
||||
class GSRemoteTree(BaseRemoteTree):
|
||||
scheme = Schemes.GS
|
||||
path_cls = CloudURLInfo
|
||||
PATH_CLS = CloudURLInfo
|
||||
REQUIRES = {"google-cloud-storage": "google.cloud.storage"}
|
||||
PARAM_CHECKSUM = "md5"
|
||||
|
||||
@@ -75,7 +74,7 @@ class GSRemote(BaseRemote):
|
||||
super().__init__(repo, config)
|
||||
|
||||
url = config.get("url", "gs:///")
|
||||
self.path_info = self.path_cls(url)
|
||||
self.path_info = self.PATH_CLS(url)
|
||||
|
||||
self.projectname = config.get("projectname", None)
|
||||
self.credentialpath = config.get("credentialpath")
|
||||
@@ -91,74 +90,22 @@ class GSRemote(BaseRemote):
|
||||
else Client(self.projectname)
|
||||
)
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
import base64
|
||||
import codecs
|
||||
def _generate_download_url(self, path_info, expires=3600):
|
||||
expiration = timedelta(seconds=int(expires))
|
||||
|
||||
bucket = path_info.bucket
|
||||
path = path_info.path
|
||||
blob = self.gs.bucket(bucket).get_blob(path)
|
||||
if not blob:
|
||||
return None
|
||||
bucket = self.gs.bucket(path_info.bucket)
|
||||
blob = bucket.get_blob(path_info.path)
|
||||
if blob is None:
|
||||
raise FileNotFoundError
|
||||
return blob.generate_signed_url(expiration=expiration)
|
||||
|
||||
b64_md5 = blob.md5_hash
|
||||
md5 = base64.b64decode(b64_md5)
|
||||
return codecs.getencoder("hex")(md5)[0].decode("utf-8")
|
||||
def exists(self, path_info):
|
||||
"""Check if the blob exists. If it does not exist,
|
||||
it could be a part of a directory path.
|
||||
|
||||
def copy(self, from_info, to_info):
|
||||
from_bucket = self.gs.bucket(from_info.bucket)
|
||||
blob = from_bucket.get_blob(from_info.path)
|
||||
if not blob:
|
||||
msg = f"'{from_info.path}' doesn't exist in the cloud"
|
||||
raise DvcException(msg)
|
||||
|
||||
to_bucket = self.gs.bucket(to_info.bucket)
|
||||
from_bucket.copy_blob(blob, to_bucket, new_name=to_info.path)
|
||||
|
||||
def remove(self, path_info):
|
||||
if path_info.scheme != "gs":
|
||||
raise NotImplementedError
|
||||
|
||||
logger.debug(f"Removing gs://{path_info}")
|
||||
blob = self.gs.bucket(path_info.bucket).get_blob(path_info.path)
|
||||
if not blob:
|
||||
return
|
||||
|
||||
blob.delete()
|
||||
|
||||
def _list_paths(
|
||||
self, path_info, max_items=None, prefix=None, progress_callback=None
|
||||
):
|
||||
if prefix:
|
||||
prefix = posixpath.join(path_info.path, prefix[:2], prefix[2:])
|
||||
else:
|
||||
prefix = path_info.path
|
||||
for blob in self.gs.bucket(path_info.bucket).list_blobs(
|
||||
prefix=path_info.path, max_results=max_items
|
||||
):
|
||||
if progress_callback:
|
||||
progress_callback()
|
||||
yield blob.name
|
||||
|
||||
def list_cache_paths(self, prefix=None, progress_callback=None):
|
||||
return self._list_paths(
|
||||
self.path_info, prefix=prefix, progress_callback=progress_callback
|
||||
)
|
||||
|
||||
def walk_files(self, path_info):
|
||||
for fname in self._list_paths(path_info / ""):
|
||||
# skip nested empty directories
|
||||
if fname.endswith("/"):
|
||||
continue
|
||||
yield path_info.replace(fname)
|
||||
|
||||
def makedirs(self, path_info):
|
||||
if not path_info.path:
|
||||
return
|
||||
|
||||
self.gs.bucket(path_info.bucket).blob(
|
||||
(path_info / "").path
|
||||
).upload_from_string("")
|
||||
eg: if `data/file.txt` exists, check for `data` should return True
|
||||
"""
|
||||
return self.isfile(path_info) or self.isdir(path_info)
|
||||
|
||||
def isdir(self, path_info):
|
||||
dir_path = path_info / ""
|
||||
@@ -171,13 +118,61 @@ class GSRemote(BaseRemote):
|
||||
blob = self.gs.bucket(path_info.bucket).blob(path_info.path)
|
||||
return blob.exists()
|
||||
|
||||
def exists(self, path_info):
|
||||
"""Check if the blob exists. If it does not exist,
|
||||
it could be a part of a directory path.
|
||||
def _list_paths(self, path_info, max_items=None):
|
||||
for blob in self.gs.bucket(path_info.bucket).list_blobs(
|
||||
prefix=path_info.path, max_results=max_items
|
||||
):
|
||||
yield blob.name
|
||||
|
||||
eg: if `data/file.txt` exists, check for `data` should return True
|
||||
"""
|
||||
return self.isfile(path_info) or self.isdir(path_info)
|
||||
def walk_files(self, path_info, **kwargs):
|
||||
for fname in self._list_paths(path_info / "", **kwargs):
|
||||
# skip nested empty directories
|
||||
if fname.endswith("/"):
|
||||
continue
|
||||
yield path_info.replace(fname)
|
||||
|
||||
def remove(self, path_info):
|
||||
if path_info.scheme != "gs":
|
||||
raise NotImplementedError
|
||||
|
||||
logger.debug(f"Removing gs://{path_info}")
|
||||
blob = self.gs.bucket(path_info.bucket).get_blob(path_info.path)
|
||||
if not blob:
|
||||
return
|
||||
|
||||
blob.delete()
|
||||
|
||||
def makedirs(self, path_info):
|
||||
if not path_info.path:
|
||||
return
|
||||
|
||||
self.gs.bucket(path_info.bucket).blob(
|
||||
(path_info / "").path
|
||||
).upload_from_string("")
|
||||
|
||||
def copy(self, from_info, to_info):
|
||||
from_bucket = self.gs.bucket(from_info.bucket)
|
||||
blob = from_bucket.get_blob(from_info.path)
|
||||
if not blob:
|
||||
msg = f"'{from_info.path}' doesn't exist in the cloud"
|
||||
raise DvcException(msg)
|
||||
|
||||
to_bucket = self.gs.bucket(to_info.bucket)
|
||||
from_bucket.copy_blob(blob, to_bucket, new_name=to_info.path)
|
||||
|
||||
def get_file_hash(self, path_info):
|
||||
import base64
|
||||
import codecs
|
||||
|
||||
bucket = path_info.bucket
|
||||
path = path_info.path
|
||||
blob = self.gs.bucket(bucket).get_blob(path)
|
||||
if not blob:
|
||||
return None
|
||||
|
||||
b64_md5 = blob.md5_hash
|
||||
md5 = base64.b64decode(b64_md5)
|
||||
return codecs.getencoder("hex")(md5)[0].decode("utf-8")
|
||||
|
||||
def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
|
||||
bucket = self.gs.bucket(to_info.bucket)
|
||||
@@ -201,12 +196,3 @@ class GSRemote(BaseRemote):
|
||||
disable=no_progress_bar,
|
||||
) as wrapped:
|
||||
blob.download_to_file(wrapped)
|
||||
|
||||
def _generate_download_url(self, path_info, expires=3600):
|
||||
expiration = timedelta(seconds=int(expires))
|
||||
|
||||
bucket = self.gs.bucket(path_info.bucket)
|
||||
blob = bucket.get_blob(path_info.path)
|
||||
if blob is None:
|
||||
raise FileNotFoundError
|
||||
return blob.generate_signed_url(expiration=expiration)
|
||||
|
||||
+82
-83
@@ -11,21 +11,22 @@ from urllib.parse import urlparse
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.utils import fix_env, tmp_fname
|
||||
|
||||
from .base import BaseRemote, RemoteCmdError
|
||||
from .base import BaseRemoteTree, RemoteCmdError
|
||||
from .pool import get_connection
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HDFSRemote(BaseRemote):
|
||||
class HDFSRemoteTree(BaseRemoteTree):
|
||||
scheme = Schemes.HDFS
|
||||
REQUIRES = {"pyarrow": "pyarrow"}
|
||||
REGEX = r"^hdfs://((?P<user>.*)@)?.*$"
|
||||
PARAM_CHECKSUM = "checksum"
|
||||
REQUIRES = {"pyarrow": "pyarrow"}
|
||||
TRAVERSE_PREFIX_LEN = 2
|
||||
|
||||
def __init__(self, repo, config):
|
||||
super().__init__(repo, config)
|
||||
|
||||
self.path_info = None
|
||||
url = config.get("url")
|
||||
if not url:
|
||||
@@ -34,7 +35,7 @@ class HDFSRemote(BaseRemote):
|
||||
parsed = urlparse(url)
|
||||
user = parsed.username or config.get("user")
|
||||
|
||||
self.path_info = self.path_cls.from_parts(
|
||||
self.path_info = self.PATH_CLS.from_parts(
|
||||
scheme=self.scheme,
|
||||
host=parsed.hostname,
|
||||
user=user,
|
||||
@@ -42,7 +43,8 @@ class HDFSRemote(BaseRemote):
|
||||
path=parsed.path,
|
||||
)
|
||||
|
||||
def hdfs(self, path_info):
|
||||
@staticmethod
|
||||
def hdfs(path_info):
|
||||
import pyarrow
|
||||
|
||||
return get_connection(
|
||||
@@ -52,6 +54,80 @@ class HDFSRemote(BaseRemote):
|
||||
user=path_info.user,
|
||||
)
|
||||
|
||||
@contextmanager
|
||||
def open(self, path_info, mode="r", encoding=None):
|
||||
assert mode in {"r", "rt", "rb"}
|
||||
|
||||
try:
|
||||
with self.hdfs(path_info) as hdfs, closing(
|
||||
hdfs.open(path_info.path, mode="rb")
|
||||
) as fd:
|
||||
if mode == "rb":
|
||||
yield fd
|
||||
else:
|
||||
yield io.TextIOWrapper(fd, encoding=encoding)
|
||||
except OSError as e:
|
||||
# Empty .errno and not specific enough error class in pyarrow,
|
||||
# see https://issues.apache.org/jira/browse/ARROW-6248
|
||||
if "file does not exist" in str(e):
|
||||
raise FileNotFoundError(*e.args)
|
||||
raise
|
||||
|
||||
def exists(self, path_info):
|
||||
assert not isinstance(path_info, list)
|
||||
assert path_info.scheme == "hdfs"
|
||||
with self.hdfs(path_info) as hdfs:
|
||||
return hdfs.exists(path_info.path)
|
||||
|
||||
def walk_files(self, path_info, **kwargs):
|
||||
if not self.exists(path_info):
|
||||
return
|
||||
|
||||
root = path_info.path
|
||||
dirs = deque([root])
|
||||
|
||||
with self.hdfs(self.path_info) as hdfs:
|
||||
if not hdfs.exists(root):
|
||||
return
|
||||
while dirs:
|
||||
try:
|
||||
entries = hdfs.ls(dirs.pop(), detail=True)
|
||||
for entry in entries:
|
||||
if entry["kind"] == "directory":
|
||||
dirs.append(urlparse(entry["name"]).path)
|
||||
elif entry["kind"] == "file":
|
||||
path = urlparse(entry["name"]).path
|
||||
yield path_info.replace(path=path)
|
||||
except OSError:
|
||||
# When searching for a specific prefix pyarrow raises an
|
||||
# exception if the specified cache dir does not exist
|
||||
pass
|
||||
|
||||
def remove(self, path_info):
|
||||
if path_info.scheme != "hdfs":
|
||||
raise NotImplementedError
|
||||
|
||||
if self.exists(path_info):
|
||||
logger.debug(f"Removing {path_info.path}")
|
||||
with self.hdfs(path_info) as hdfs:
|
||||
hdfs.rm(path_info.path)
|
||||
|
||||
def copy(self, from_info, to_info, **_kwargs):
|
||||
dname = posixpath.dirname(to_info.path)
|
||||
with self.hdfs(to_info) as hdfs:
|
||||
hdfs.mkdir(dname)
|
||||
# NOTE: this is how `hadoop fs -cp` works too: it copies through
|
||||
# your local machine.
|
||||
with hdfs.open(from_info.path, "rb") as from_fobj:
|
||||
tmp_info = to_info.parent / tmp_fname(to_info.name)
|
||||
try:
|
||||
with hdfs.open(tmp_info.path, "wb") as tmp_fobj:
|
||||
tmp_fobj.upload(from_fobj)
|
||||
hdfs.rename(tmp_info.path, to_info.path)
|
||||
except Exception:
|
||||
self.remove(tmp_info)
|
||||
raise
|
||||
|
||||
def hadoop_fs(self, cmd, user=None):
|
||||
cmd = "hadoop fs -" + cmd
|
||||
if user:
|
||||
@@ -83,7 +159,7 @@ class HDFSRemote(BaseRemote):
|
||||
assert match is not None
|
||||
return match.group(gname)
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
def get_file_hash(self, path_info):
|
||||
# NOTE: pyarrow doesn't support checksum, so we need to use hadoop
|
||||
regex = r".*\t.*\t(?P<checksum>.*)"
|
||||
stdout = self.hadoop_fs(
|
||||
@@ -91,37 +167,6 @@ class HDFSRemote(BaseRemote):
|
||||
)
|
||||
return self._group(regex, stdout, "checksum")
|
||||
|
||||
def copy(self, from_info, to_info, **_kwargs):
|
||||
dname = posixpath.dirname(to_info.path)
|
||||
with self.hdfs(to_info) as hdfs:
|
||||
hdfs.mkdir(dname)
|
||||
# NOTE: this is how `hadoop fs -cp` works too: it copies through
|
||||
# your local machine.
|
||||
with hdfs.open(from_info.path, "rb") as from_fobj:
|
||||
tmp_info = to_info.parent / tmp_fname(to_info.name)
|
||||
try:
|
||||
with hdfs.open(tmp_info.path, "wb") as tmp_fobj:
|
||||
tmp_fobj.upload(from_fobj)
|
||||
hdfs.rename(tmp_info.path, to_info.path)
|
||||
except Exception:
|
||||
self.remove(tmp_info)
|
||||
raise
|
||||
|
||||
def remove(self, path_info):
|
||||
if path_info.scheme != "hdfs":
|
||||
raise NotImplementedError
|
||||
|
||||
if self.exists(path_info):
|
||||
logger.debug(f"Removing {path_info.path}")
|
||||
with self.hdfs(path_info) as hdfs:
|
||||
hdfs.rm(path_info.path)
|
||||
|
||||
def exists(self, path_info):
|
||||
assert not isinstance(path_info, list)
|
||||
assert path_info.scheme == "hdfs"
|
||||
with self.hdfs(path_info) as hdfs:
|
||||
return hdfs.exists(path_info.path)
|
||||
|
||||
def _upload(self, from_file, to_info, **_kwargs):
|
||||
with self.hdfs(to_info) as hdfs:
|
||||
hdfs.mkdir(posixpath.dirname(to_info.path))
|
||||
@@ -134,49 +179,3 @@ class HDFSRemote(BaseRemote):
|
||||
with self.hdfs(from_info) as hdfs:
|
||||
with open(to_file, "wb+") as fobj:
|
||||
hdfs.download(from_info.path, fobj)
|
||||
|
||||
@contextmanager
|
||||
def open(self, path_info, mode="r", encoding=None):
|
||||
assert mode in {"r", "rt", "rb"}
|
||||
|
||||
try:
|
||||
with self.hdfs(path_info) as hdfs, closing(
|
||||
hdfs.open(path_info.path, mode="rb")
|
||||
) as fd:
|
||||
if mode == "rb":
|
||||
yield fd
|
||||
else:
|
||||
yield io.TextIOWrapper(fd, encoding=encoding)
|
||||
except OSError as e:
|
||||
# Empty .errno and not specific enough error class in pyarrow,
|
||||
# see https://issues.apache.org/jira/browse/ARROW-6248
|
||||
if "file does not exist" in str(e):
|
||||
raise FileNotFoundError(*e.args)
|
||||
raise
|
||||
|
||||
def list_cache_paths(self, prefix=None, progress_callback=None):
|
||||
if not self.exists(self.path_info):
|
||||
return
|
||||
|
||||
if prefix:
|
||||
root = posixpath.join(self.path_info.path, prefix[:2])
|
||||
else:
|
||||
root = self.path_info.path
|
||||
dirs = deque([root])
|
||||
|
||||
with self.hdfs(self.path_info) as hdfs:
|
||||
while dirs:
|
||||
try:
|
||||
entries = hdfs.ls(dirs.pop(), detail=True)
|
||||
for entry in entries:
|
||||
if entry["kind"] == "directory":
|
||||
dirs.append(urlparse(entry["name"]).path)
|
||||
elif entry["kind"] == "file":
|
||||
if progress_callback:
|
||||
progress_callback()
|
||||
yield urlparse(entry["name"]).path
|
||||
except OSError as e:
|
||||
# When searching for a specific prefix pyarrow raises an
|
||||
# exception if the specified cache dir does not exist
|
||||
if not prefix:
|
||||
raise e
|
||||
|
||||
+77
-78
@@ -8,7 +8,7 @@ import dvc.prompt as prompt
|
||||
from dvc.exceptions import DvcException, HTTPError
|
||||
from dvc.path_info import HTTPURLInfo
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.remote.base import BaseRemote
|
||||
from dvc.remote.base import BaseRemoteTree
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -23,22 +23,23 @@ def ask_password(host, user):
|
||||
)
|
||||
|
||||
|
||||
class HTTPRemote(BaseRemote):
|
||||
class HTTPRemoteTree(BaseRemoteTree):
|
||||
scheme = Schemes.HTTP
|
||||
path_cls = HTTPURLInfo
|
||||
PATH_CLS = HTTPURLInfo
|
||||
PARAM_CHECKSUM = "etag"
|
||||
CAN_TRAVERSE = False
|
||||
|
||||
SESSION_RETRIES = 5
|
||||
SESSION_BACKOFF_FACTOR = 0.1
|
||||
REQUEST_TIMEOUT = 10
|
||||
CHUNK_SIZE = 2 ** 16
|
||||
PARAM_CHECKSUM = "etag"
|
||||
CAN_TRAVERSE = False
|
||||
|
||||
def __init__(self, repo, config):
|
||||
super().__init__(repo, config)
|
||||
|
||||
url = config.get("url")
|
||||
if url:
|
||||
self.path_info = self.path_cls(url)
|
||||
self.path_info = self.PATH_CLS(url)
|
||||
user = config.get("user", None)
|
||||
if user:
|
||||
self.path_info.user = user
|
||||
@@ -51,74 +52,7 @@ class HTTPRemote(BaseRemote):
|
||||
self.ask_password = config.get("ask_password", False)
|
||||
self.headers = {}
|
||||
|
||||
def _download(self, from_info, to_file, name=None, no_progress_bar=False):
|
||||
response = self._request("GET", from_info.url, stream=True)
|
||||
if response.status_code != 200:
|
||||
raise HTTPError(response.status_code, response.reason)
|
||||
with open(to_file, "wb") as fd:
|
||||
with Tqdm.wrapattr(
|
||||
fd,
|
||||
"write",
|
||||
total=None
|
||||
if no_progress_bar
|
||||
else self._content_length(response),
|
||||
leave=False,
|
||||
desc=from_info.url if name is None else name,
|
||||
disable=no_progress_bar,
|
||||
) as fd_wrapped:
|
||||
for chunk in response.iter_content(chunk_size=self.CHUNK_SIZE):
|
||||
fd_wrapped.write(chunk)
|
||||
|
||||
def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
|
||||
def chunks():
|
||||
with open(from_file, "rb") as fd:
|
||||
with Tqdm.wrapattr(
|
||||
fd,
|
||||
"read",
|
||||
total=None
|
||||
if no_progress_bar
|
||||
else os.path.getsize(from_file),
|
||||
leave=False,
|
||||
desc=to_info.url if name is None else name,
|
||||
disable=no_progress_bar,
|
||||
) as fd_wrapped:
|
||||
while True:
|
||||
chunk = fd_wrapped.read(self.CHUNK_SIZE)
|
||||
if not chunk:
|
||||
break
|
||||
yield chunk
|
||||
|
||||
response = self._request("POST", to_info.url, data=chunks())
|
||||
if response.status_code not in (200, 201):
|
||||
raise HTTPError(response.status_code, response.reason)
|
||||
|
||||
def exists(self, path_info):
|
||||
return bool(self._request("HEAD", path_info.url))
|
||||
|
||||
def _content_length(self, response):
|
||||
res = response.headers.get("Content-Length")
|
||||
return int(res) if res else None
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
url = path_info.url
|
||||
headers = self._request("HEAD", url).headers
|
||||
etag = headers.get("ETag") or headers.get("Content-MD5")
|
||||
|
||||
if not etag:
|
||||
raise DvcException(
|
||||
"could not find an ETag or "
|
||||
"Content-MD5 header for '{url}'".format(url=url)
|
||||
)
|
||||
|
||||
if etag.startswith("W/"):
|
||||
raise DvcException(
|
||||
"Weak ETags are not supported."
|
||||
" (Etag: '{etag}', URL: '{url}')".format(etag=etag, url=url)
|
||||
)
|
||||
|
||||
return etag
|
||||
|
||||
def auth_method(self, path_info=None):
|
||||
def _auth_method(self, path_info=None):
|
||||
from requests.auth import HTTPBasicAuth, HTTPDigestAuth
|
||||
|
||||
if path_info is None:
|
||||
@@ -155,7 +89,7 @@ class HTTPRemote(BaseRemote):
|
||||
|
||||
return session
|
||||
|
||||
def _request(self, method, url, **kwargs):
|
||||
def request(self, method, url, **kwargs):
|
||||
import requests
|
||||
|
||||
kwargs.setdefault("allow_redirects", True)
|
||||
@@ -165,7 +99,7 @@ class HTTPRemote(BaseRemote):
|
||||
res = self._session.request(
|
||||
method,
|
||||
url,
|
||||
auth=self.auth_method(),
|
||||
auth=self._auth_method(),
|
||||
headers=self.headers,
|
||||
**kwargs,
|
||||
)
|
||||
@@ -187,5 +121,70 @@ class HTTPRemote(BaseRemote):
|
||||
except requests.exceptions.RequestException:
|
||||
raise DvcException(f"could not perform a {method} request")
|
||||
|
||||
def gc(self):
|
||||
raise NotImplementedError
|
||||
def exists(self, path_info):
|
||||
return bool(self.request("HEAD", path_info.url))
|
||||
|
||||
def get_file_hash(self, path_info):
|
||||
url = path_info.url
|
||||
headers = self.request("HEAD", url).headers
|
||||
etag = headers.get("ETag") or headers.get("Content-MD5")
|
||||
|
||||
if not etag:
|
||||
raise DvcException(
|
||||
"could not find an ETag or "
|
||||
"Content-MD5 header for '{url}'".format(url=url)
|
||||
)
|
||||
|
||||
if etag.startswith("W/"):
|
||||
raise DvcException(
|
||||
"Weak ETags are not supported."
|
||||
" (Etag: '{etag}', URL: '{url}')".format(etag=etag, url=url)
|
||||
)
|
||||
|
||||
return etag
|
||||
|
||||
def _download(self, from_info, to_file, name=None, no_progress_bar=False):
|
||||
response = self.request("GET", from_info.url, stream=True)
|
||||
if response.status_code != 200:
|
||||
raise HTTPError(response.status_code, response.reason)
|
||||
with open(to_file, "wb") as fd:
|
||||
with Tqdm.wrapattr(
|
||||
fd,
|
||||
"write",
|
||||
total=None
|
||||
if no_progress_bar
|
||||
else self._content_length(response),
|
||||
leave=False,
|
||||
desc=from_info.url if name is None else name,
|
||||
disable=no_progress_bar,
|
||||
) as fd_wrapped:
|
||||
for chunk in response.iter_content(chunk_size=self.CHUNK_SIZE):
|
||||
fd_wrapped.write(chunk)
|
||||
|
||||
def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
|
||||
def chunks():
|
||||
with open(from_file, "rb") as fd:
|
||||
with Tqdm.wrapattr(
|
||||
fd,
|
||||
"read",
|
||||
total=None
|
||||
if no_progress_bar
|
||||
else os.path.getsize(from_file),
|
||||
leave=False,
|
||||
desc=to_info.url if name is None else name,
|
||||
disable=no_progress_bar,
|
||||
) as fd_wrapped:
|
||||
while True:
|
||||
chunk = fd_wrapped.read(self.CHUNK_SIZE)
|
||||
if not chunk:
|
||||
break
|
||||
yield chunk
|
||||
|
||||
response = self.request("POST", to_info.url, data=chunks())
|
||||
if response.status_code not in (200, 201):
|
||||
raise HTTPError(response.status_code, response.reason)
|
||||
|
||||
@staticmethod
|
||||
def _content_length(response):
|
||||
res = response.headers.get("Content-Length")
|
||||
return int(res) if res else None
|
||||
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
from .http import HTTPRemote
|
||||
from .http import HTTPRemoteTree
|
||||
|
||||
|
||||
class HTTPSRemote(HTTPRemote):
|
||||
class HTTPSRemoteTree(HTTPRemoteTree):
|
||||
scheme = Schemes.HTTPS
|
||||
|
||||
+27
-31
@@ -25,15 +25,15 @@ class RemoteIndexNoop:
|
||||
def __iter__(self):
|
||||
return iter([])
|
||||
|
||||
def __contains__(self, checksum):
|
||||
def __contains__(self, hash_):
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def checksums():
|
||||
def hashes():
|
||||
return []
|
||||
|
||||
@staticmethod
|
||||
def dir_checksums():
|
||||
def dir_hashes():
|
||||
return []
|
||||
|
||||
def load(self):
|
||||
@@ -54,13 +54,13 @@ class RemoteIndexNoop:
|
||||
|
||||
|
||||
class RemoteIndex:
|
||||
"""Class for indexing remote checksums in a sqlite3 database.
|
||||
"""Class for indexing remote hashes in a sqlite3 database.
|
||||
|
||||
Args:
|
||||
repo: repo for this remote index.
|
||||
name: name for this index. Index db will be loaded from and saved to
|
||||
``.dvc/tmp/index/{name}.idx``.
|
||||
dir_suffix: suffix used for naming directory checksums
|
||||
dir_suffix: suffix used for naming directory hashes
|
||||
"""
|
||||
|
||||
INDEX_SUFFIX = ".idx"
|
||||
@@ -79,8 +79,8 @@ class RemoteIndex:
|
||||
|
||||
def __iter__(self):
|
||||
cmd = f"SELECT checksum FROM {self.INDEX_TABLE}"
|
||||
for (checksum,) in self._execute(cmd):
|
||||
yield checksum
|
||||
for (hash_,) in self._execute(cmd):
|
||||
yield hash_
|
||||
|
||||
def __enter__(self):
|
||||
self.lock.acquire()
|
||||
@@ -90,25 +90,25 @@ class RemoteIndex:
|
||||
self.dump()
|
||||
self.lock.release()
|
||||
|
||||
def __contains__(self, checksum):
|
||||
def __contains__(self, hash_):
|
||||
cmd = "SELECT checksum FROM {} WHERE checksum = (?)".format(
|
||||
self.INDEX_TABLE
|
||||
)
|
||||
self._execute(cmd, (checksum,))
|
||||
self._execute(cmd, (hash_,))
|
||||
return self.cursor.fetchone() is not None
|
||||
|
||||
def checksums(self):
|
||||
"""Iterate over checksums stored in the index."""
|
||||
def hashes(self):
|
||||
"""Iterate over hashes stored in the index."""
|
||||
return iter(self)
|
||||
|
||||
def dir_checksums(self):
|
||||
"""Iterate over .dir checksums stored in the index."""
|
||||
def dir_hashes(self):
|
||||
"""Iterate over .dir hashes stored in the index."""
|
||||
cmd = f"SELECT checksum FROM {self.INDEX_TABLE} WHERE dir = 1"
|
||||
for (checksum,) in self._execute(cmd):
|
||||
yield checksum
|
||||
for (hash_,) in self._execute(cmd):
|
||||
yield hash_
|
||||
|
||||
def is_dir_checksum(self, checksum):
|
||||
return checksum.endswith(self.dir_suffix)
|
||||
def is_dir_hash(self, hash_):
|
||||
return hash_.endswith(self.dir_suffix)
|
||||
|
||||
def _execute(self, cmd, parameters=()):
|
||||
return self.cursor.execute(cmd, parameters)
|
||||
@@ -185,28 +185,24 @@ class RemoteIndex:
|
||||
cmd = f"DELETE FROM {self.INDEX_TABLE}"
|
||||
self._execute(cmd)
|
||||
|
||||
def update(self, dir_checksums, file_checksums):
|
||||
"""Update this index, adding the specified checksums.
|
||||
def update(self, dir_hashes, file_hashes):
|
||||
"""Update this index, adding the specified hashes.
|
||||
|
||||
Changes to the index will not committed until dump() is called.
|
||||
"""
|
||||
cmd = "INSERT OR IGNORE INTO {} (checksum, dir) VALUES (?, ?)".format(
|
||||
self.INDEX_TABLE
|
||||
)
|
||||
self._executemany(
|
||||
cmd, ((checksum, True) for checksum in dir_checksums)
|
||||
)
|
||||
self._executemany(
|
||||
cmd, ((checksum, False) for checksum in file_checksums)
|
||||
)
|
||||
self._executemany(cmd, ((hash_, True) for hash_ in dir_hashes))
|
||||
self._executemany(cmd, ((hash_, False) for hash_ in file_hashes))
|
||||
|
||||
def intersection(self, checksums):
|
||||
"""Iterate over values from `checksums` which exist in the index."""
|
||||
def intersection(self, hashes):
|
||||
"""Iterate over values from `hashes` which exist in the index."""
|
||||
# sqlite has a compile time limit of 999, see:
|
||||
# https://www.sqlite.org/c3ref/c_limit_attached.html#sqlitelimitvariablenumber
|
||||
for chunk in lchunks(999, checksums):
|
||||
for chunk in lchunks(999, hashes):
|
||||
cmd = "SELECT checksum FROM {} WHERE checksum IN ({})".format(
|
||||
self.INDEX_TABLE, ",".join("?" for checksum in chunk)
|
||||
self.INDEX_TABLE, ",".join("?" for hash_ in chunk)
|
||||
)
|
||||
for (checksum,) in self._execute(cmd, chunk):
|
||||
yield checksum
|
||||
for (hash_,) in self._execute(cmd, chunk):
|
||||
yield hash_
|
||||
|
||||
+306
-296
@@ -3,7 +3,7 @@ import logging
|
||||
import os
|
||||
import stat
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from functools import partial
|
||||
from functools import partial, wraps
|
||||
|
||||
from funcy import cached_property, concat
|
||||
from shortuuid import uuid
|
||||
@@ -16,104 +16,87 @@ from dvc.remote.base import (
|
||||
STATUS_MAP,
|
||||
STATUS_MISSING,
|
||||
STATUS_NEW,
|
||||
BaseRemote,
|
||||
BaseRemoteTree,
|
||||
CloudCache,
|
||||
Remote,
|
||||
index_locked,
|
||||
)
|
||||
from dvc.remote.index import RemoteIndexNoop
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.scm.tree import is_working_tree
|
||||
from dvc.scm.tree import WorkingTree, is_working_tree
|
||||
from dvc.system import System
|
||||
from dvc.utils import file_md5, relpath, tmp_fname
|
||||
from dvc.utils.fs import copyfile, makedirs, move, remove, walk_files
|
||||
from dvc.utils.fs import (
|
||||
copy_fobj_to_file,
|
||||
copyfile,
|
||||
makedirs,
|
||||
move,
|
||||
remove,
|
||||
walk_files,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LocalRemote(BaseRemote):
|
||||
class LocalRemoteTree(BaseRemoteTree):
|
||||
scheme = Schemes.LOCAL
|
||||
path_cls = PathInfo
|
||||
PATH_CLS = PathInfo
|
||||
PARAM_CHECKSUM = "md5"
|
||||
PARAM_PATH = "path"
|
||||
TRAVERSE_PREFIX_LEN = 2
|
||||
INDEX_CLS = RemoteIndexNoop
|
||||
|
||||
UNPACKED_DIR_SUFFIX = ".unpacked"
|
||||
|
||||
DEFAULT_CACHE_TYPES = ["reflink", "copy"]
|
||||
|
||||
CACHE_MODE = 0o444
|
||||
SHARED_MODE_MAP = {None: (0o644, 0o755), "group": (0o664, 0o775)}
|
||||
|
||||
def __init__(self, repo, config):
|
||||
super().__init__(repo, config)
|
||||
self.cache_dir = config.get("url")
|
||||
self._dir_info = {}
|
||||
url = config.get("url")
|
||||
self.path_info = self.PATH_CLS(url) if url else None
|
||||
|
||||
@property
|
||||
def state(self):
|
||||
return self.repo.state
|
||||
from dvc.state import StateNoop
|
||||
|
||||
@property
|
||||
def cache_dir(self):
|
||||
return self.path_info.fspath if self.path_info else None
|
||||
|
||||
@cache_dir.setter
|
||||
def cache_dir(self, value):
|
||||
self.path_info = PathInfo(value) if value else None
|
||||
|
||||
@classmethod
|
||||
def supported(cls, config):
|
||||
return True
|
||||
return self.repo.state if self.repo else StateNoop()
|
||||
|
||||
@cached_property
|
||||
def cache_path(self):
|
||||
return os.path.abspath(self.cache_dir)
|
||||
def work_tree(self):
|
||||
# When using repo.brancher, repo.tree may change to/from WorkingTree to
|
||||
# GitTree arbitarily. When repo.tree is GitTree, local cache needs to
|
||||
# use its own WorkingTree instance.
|
||||
if self.repo:
|
||||
return WorkingTree(self.repo.root_dir)
|
||||
return None
|
||||
|
||||
def checksum_to_path(self, checksum):
|
||||
return os.path.join(self.cache_path, checksum[0:2], checksum[2:])
|
||||
|
||||
def list_cache_paths(self, prefix=None, progress_callback=None):
|
||||
assert self.path_info is not None
|
||||
if prefix:
|
||||
path_info = self.path_info / prefix[:2]
|
||||
else:
|
||||
path_info = self.path_info
|
||||
if progress_callback:
|
||||
for path in walk_files(path_info):
|
||||
progress_callback()
|
||||
yield path
|
||||
else:
|
||||
yield from walk_files(path_info)
|
||||
|
||||
def get(self, md5):
|
||||
if not md5:
|
||||
return None
|
||||
|
||||
return self.checksum_to_path_info(md5).url
|
||||
@staticmethod
|
||||
def open(path_info, mode="r", encoding=None):
|
||||
return open(path_info, mode=mode, encoding=encoding)
|
||||
|
||||
def exists(self, path_info):
|
||||
assert is_working_tree(self.repo.tree)
|
||||
assert isinstance(path_info, str) or path_info.scheme == "local"
|
||||
return self.repo.tree.exists(path_info)
|
||||
if not self.repo:
|
||||
return os.path.exists(path_info)
|
||||
return self.work_tree.exists(path_info)
|
||||
|
||||
def makedirs(self, path_info):
|
||||
makedirs(path_info, exist_ok=True, mode=self._dir_mode)
|
||||
def isfile(self, path_info):
|
||||
if not self.repo:
|
||||
return os.path.isfile(path_info)
|
||||
return self.work_tree.isfile(path_info)
|
||||
|
||||
def already_cached(self, path_info):
|
||||
assert path_info.scheme in ["", "local"]
|
||||
def isdir(self, path_info):
|
||||
if not self.repo:
|
||||
return os.path.isdir(path_info)
|
||||
return self.work_tree.isdir(path_info)
|
||||
|
||||
current_md5 = self.get_checksum(path_info)
|
||||
def iscopy(self, path_info):
|
||||
return not (
|
||||
System.is_symlink(path_info) or System.is_hardlink(path_info)
|
||||
)
|
||||
|
||||
if not current_md5:
|
||||
return False
|
||||
|
||||
return not self.changed_cache(current_md5)
|
||||
|
||||
def _verify_link(self, path_info, link_type):
|
||||
if link_type == "hardlink" and self.getsize(path_info) == 0:
|
||||
return
|
||||
|
||||
super()._verify_link(path_info, link_type)
|
||||
def walk_files(self, path_info, **kwargs):
|
||||
for fname in self.work_tree.walk_files(path_info):
|
||||
yield PathInfo(fname)
|
||||
|
||||
def is_empty(self, path_info):
|
||||
path = path_info.fspath
|
||||
@@ -126,32 +109,6 @@ class LocalRemote(BaseRemote):
|
||||
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def isfile(path_info):
|
||||
return os.path.isfile(path_info)
|
||||
|
||||
@staticmethod
|
||||
def isdir(path_info):
|
||||
return os.path.isdir(path_info)
|
||||
|
||||
def iscopy(self, path_info):
|
||||
return not (
|
||||
System.is_symlink(path_info) or System.is_hardlink(path_info)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def getsize(path_info):
|
||||
return os.path.getsize(path_info)
|
||||
|
||||
def walk_files(self, path_info):
|
||||
assert is_working_tree(self.repo.tree)
|
||||
|
||||
for fname in self.repo.tree.walk_files(path_info):
|
||||
yield PathInfo(fname)
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
return file_md5(path_info)[0]
|
||||
|
||||
def remove(self, path_info):
|
||||
if isinstance(path_info, PathInfo):
|
||||
if path_info.scheme != "local":
|
||||
@@ -163,6 +120,9 @@ class LocalRemote(BaseRemote):
|
||||
if self.exists(path):
|
||||
remove(path)
|
||||
|
||||
def makedirs(self, path_info):
|
||||
makedirs(path_info, exist_ok=True, mode=self.dir_mode)
|
||||
|
||||
def move(self, from_info, to_info, mode=None):
|
||||
if from_info.scheme != "local" or to_info.scheme != "local":
|
||||
raise NotImplementedError
|
||||
@@ -171,9 +131,9 @@ class LocalRemote(BaseRemote):
|
||||
|
||||
if mode is None:
|
||||
if self.isfile(from_info):
|
||||
mode = self._file_mode
|
||||
mode = self.file_mode
|
||||
else:
|
||||
mode = self._dir_mode
|
||||
mode = self.dir_mode
|
||||
|
||||
move(from_info, to_info, mode=mode)
|
||||
|
||||
@@ -181,7 +141,18 @@ class LocalRemote(BaseRemote):
|
||||
tmp_info = to_info.parent / tmp_fname(to_info.name)
|
||||
try:
|
||||
System.copy(from_info, tmp_info)
|
||||
os.chmod(tmp_info, self._file_mode)
|
||||
os.chmod(tmp_info, self.file_mode)
|
||||
os.rename(tmp_info, to_info)
|
||||
except Exception:
|
||||
self.remove(tmp_info)
|
||||
raise
|
||||
|
||||
def copy_fobj(self, fobj, to_info):
|
||||
self.makedirs(to_info.parent)
|
||||
tmp_info = to_info.parent / tmp_fname(to_info.name)
|
||||
try:
|
||||
copy_fobj_to_file(fobj, tmp_info)
|
||||
os.chmod(tmp_info, self.file_mode)
|
||||
os.rename(tmp_info, to_info)
|
||||
except Exception:
|
||||
self.remove(tmp_info)
|
||||
@@ -200,7 +171,7 @@ class LocalRemote(BaseRemote):
|
||||
# and the cache type is `hardlink`, we might reach link limits and
|
||||
# will get something like: `too many links error`
|
||||
#
|
||||
# This is because all those empty files will have the same checksum
|
||||
# This is because all those empty files will have the same hash
|
||||
# (i.e. 68b329da9893e34099c7d8ad5cb9c940), therefore, they will be
|
||||
# linked to the same file in the cache.
|
||||
#
|
||||
@@ -230,20 +201,81 @@ class LocalRemote(BaseRemote):
|
||||
System.reflink(from_info, tmp_info)
|
||||
# NOTE: reflink has its own separate inode, so you can set permissions
|
||||
# that are different from the source.
|
||||
os.chmod(tmp_info, self._file_mode)
|
||||
os.chmod(tmp_info, self.file_mode)
|
||||
os.rename(tmp_info, to_info)
|
||||
|
||||
def cache_exists(self, checksums, jobs=None, name=None):
|
||||
return [
|
||||
checksum
|
||||
for checksum in Tqdm(
|
||||
checksums,
|
||||
unit="file",
|
||||
desc="Querying "
|
||||
+ ("cache in " + name if name else "local cache"),
|
||||
def _unprotect_file(self, path):
|
||||
if System.is_symlink(path) or System.is_hardlink(path):
|
||||
logger.debug(f"Unprotecting '{path}'")
|
||||
tmp = os.path.join(os.path.dirname(path), "." + uuid())
|
||||
|
||||
# The operations order is important here - if some application
|
||||
# would access the file during the process of copyfile then it
|
||||
# would get only the part of file. So, at first, the file should be
|
||||
# copied with the temporary name, and then original file should be
|
||||
# replaced by new.
|
||||
copyfile(path, tmp, name="Unprotecting '{}'".format(relpath(path)))
|
||||
remove(path)
|
||||
os.rename(tmp, path)
|
||||
|
||||
else:
|
||||
logger.debug(
|
||||
"Skipping copying for '{}', since it is not "
|
||||
"a symlink or a hardlink.".format(path)
|
||||
)
|
||||
if not self.changed_cache_file(checksum)
|
||||
]
|
||||
|
||||
os.chmod(path, self.file_mode)
|
||||
|
||||
def _unprotect_dir(self, path):
|
||||
assert is_working_tree(self.repo.tree)
|
||||
|
||||
for fname in self.repo.tree.walk_files(path):
|
||||
self._unprotect_file(fname)
|
||||
|
||||
def unprotect(self, path_info):
|
||||
path = path_info.fspath
|
||||
if not os.path.exists(path):
|
||||
raise DvcException(f"can't unprotect non-existing data '{path}'")
|
||||
|
||||
if os.path.isdir(path):
|
||||
self._unprotect_dir(path)
|
||||
else:
|
||||
self._unprotect_file(path)
|
||||
|
||||
def protect(self, path_info):
|
||||
path = os.fspath(path_info)
|
||||
mode = self.CACHE_MODE
|
||||
|
||||
try:
|
||||
os.chmod(path, mode)
|
||||
except OSError as exc:
|
||||
# There is nothing we need to do in case of a read-only file system
|
||||
if exc.errno == errno.EROFS:
|
||||
return
|
||||
|
||||
# In shared cache scenario, we might not own the cache file, so we
|
||||
# need to check if cache file is already protected.
|
||||
if exc.errno not in [errno.EPERM, errno.EACCES]:
|
||||
raise
|
||||
|
||||
actual = stat.S_IMODE(os.stat(path).st_mode)
|
||||
if actual != mode:
|
||||
raise
|
||||
|
||||
def is_protected(self, path_info):
|
||||
try:
|
||||
mode = os.stat(path_info).st_mode
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
|
||||
return stat.S_IMODE(mode) == self.CACHE_MODE
|
||||
|
||||
def get_file_hash(self, path_info):
|
||||
return file_md5(path_info)[0]
|
||||
|
||||
@staticmethod
|
||||
def getsize(path_info):
|
||||
return os.path.getsize(path_info)
|
||||
|
||||
def _upload(
|
||||
self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
|
||||
@@ -254,18 +286,122 @@ class LocalRemote(BaseRemote):
|
||||
copyfile(
|
||||
from_file, tmp_file, name=name, no_progress_bar=no_progress_bar
|
||||
)
|
||||
|
||||
self.protect(tmp_file)
|
||||
os.rename(tmp_file, to_info)
|
||||
|
||||
@staticmethod
|
||||
def _download(
|
||||
self, from_info, to_file, name=None, no_progress_bar=False, **_kwargs
|
||||
from_info, to_file, name=None, no_progress_bar=False, **_kwargs
|
||||
):
|
||||
copyfile(
|
||||
from_info, to_file, no_progress_bar=no_progress_bar, name=name
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def open(path_info, mode="r", encoding=None):
|
||||
return open(path_info, mode=mode, encoding=encoding)
|
||||
def list_paths(self, prefix=None, progress_callback=None):
|
||||
assert self.path_info is not None
|
||||
if prefix:
|
||||
path_info = self.path_info / prefix[:2]
|
||||
if not self.exists(path_info):
|
||||
return
|
||||
else:
|
||||
path_info = self.path_info
|
||||
# NOTE: use utils.fs walk_files since tree.walk_files will not follow
|
||||
# symlinks
|
||||
if progress_callback:
|
||||
for path in walk_files(path_info):
|
||||
progress_callback()
|
||||
yield path
|
||||
else:
|
||||
yield from walk_files(path_info)
|
||||
|
||||
def _remove_unpacked_dir(self, hash_):
|
||||
info = self.hash_to_path_info(hash_)
|
||||
path_info = info.with_name(info.name + self.UNPACKED_DIR_SUFFIX)
|
||||
self.remove(path_info)
|
||||
|
||||
|
||||
def _log_exceptions(func, operation):
|
||||
@wraps(func)
|
||||
def wrapper(from_info, to_info, *args, **kwargs):
|
||||
try:
|
||||
func(from_info, to_info, *args, **kwargs)
|
||||
return 0
|
||||
except Exception as exc:
|
||||
# NOTE: this means we ran out of file descriptors and there is no
|
||||
# reason to try to proceed, as we will hit this error anyways.
|
||||
if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
|
||||
raise
|
||||
|
||||
logger.exception(
|
||||
"failed to %s '%s' to '%s'", operation, from_info, to_info
|
||||
)
|
||||
return 1
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
class LocalRemote(Remote):
|
||||
INDEX_CLS = RemoteIndexNoop
|
||||
|
||||
|
||||
class LocalCache(CloudCache):
|
||||
DEFAULT_CACHE_TYPES = ["reflink", "copy"]
|
||||
CACHE_MODE = LocalRemoteTree.CACHE_MODE
|
||||
|
||||
def __init__(self, tree):
|
||||
super().__init__(tree)
|
||||
self.cache_dir = tree.config.get("url")
|
||||
|
||||
@property
|
||||
def cache_dir(self):
|
||||
return self.tree.path_info.fspath if self.tree.path_info else None
|
||||
|
||||
@cache_dir.setter
|
||||
def cache_dir(self, value):
|
||||
self.tree.path_info = PathInfo(value) if value else None
|
||||
|
||||
@classmethod
|
||||
def supported(cls, config):
|
||||
return True
|
||||
|
||||
@cached_property
|
||||
def cache_path(self):
|
||||
return os.path.abspath(self.cache_dir)
|
||||
|
||||
def hash_to_path(self, hash_):
|
||||
# NOTE: `self.cache_path` is already normalized so we can simply use
|
||||
# `os.sep` instead of `os.path.join`. This results in this helper
|
||||
# being ~5.5 times faster.
|
||||
return f"{self.cache_path}{os.sep}{hash_[0:2]}{os.sep}{hash_[2:]}"
|
||||
|
||||
def hashes_exist(self, hashes, jobs=None, name=None):
|
||||
return [
|
||||
hash_
|
||||
for hash_ in Tqdm(
|
||||
hashes,
|
||||
unit="file",
|
||||
desc="Querying "
|
||||
+ ("cache in " + name if name else "local cache"),
|
||||
)
|
||||
if not self.changed_cache_file(hash_)
|
||||
]
|
||||
|
||||
def already_cached(self, path_info):
|
||||
assert path_info.scheme in ["", "local"]
|
||||
|
||||
current_md5 = self.get_hash(path_info)
|
||||
|
||||
if not current_md5:
|
||||
return False
|
||||
|
||||
return not self.changed_cache(current_md5)
|
||||
|
||||
def _verify_link(self, path_info, link_type):
|
||||
if link_type == "hardlink" and self.tree.getsize(path_info) == 0:
|
||||
return
|
||||
|
||||
super()._verify_link(path_info, link_type)
|
||||
|
||||
@index_locked
|
||||
def status(
|
||||
@@ -298,7 +434,7 @@ class LocalRemote(BaseRemote):
|
||||
|
||||
dir_status_info contains status for .dir files, file_status_info
|
||||
contains status for all other files, and dir_contents is a dict of
|
||||
{dir_checksum: set(file_checksum, ...)} which can be used to map
|
||||
{dir_hash: set(file_hash, ...)} which can be used to map
|
||||
a .dir file to its file contents.
|
||||
"""
|
||||
logger.debug(f"Preparing to collect status from {remote.path_info}")
|
||||
@@ -306,7 +442,7 @@ class LocalRemote(BaseRemote):
|
||||
|
||||
logger.debug("Collecting information from local cache...")
|
||||
local_exists = frozenset(
|
||||
self.cache_exists(md5s, jobs=jobs, name=self.cache_dir)
|
||||
self.hashes_exist(md5s, jobs=jobs, name=self.cache_dir)
|
||||
)
|
||||
|
||||
# This is a performance optimization. We can safely assume that,
|
||||
@@ -321,12 +457,12 @@ class LocalRemote(BaseRemote):
|
||||
dir_md5s = set(named_cache.dir_keys(self.scheme))
|
||||
if dir_md5s:
|
||||
remote_exists.update(
|
||||
self._indexed_dir_checksums(named_cache, remote, dir_md5s)
|
||||
self._indexed_dir_hashes(named_cache, remote, dir_md5s)
|
||||
)
|
||||
md5s.difference_update(remote_exists)
|
||||
if md5s:
|
||||
remote_exists.update(
|
||||
remote.cache_exists(
|
||||
remote.hashes_exist(
|
||||
md5s, jobs=jobs, name=str(remote.path_info)
|
||||
)
|
||||
)
|
||||
@@ -337,23 +473,23 @@ class LocalRemote(BaseRemote):
|
||||
def _make_status(
|
||||
self, named_cache, show_checksums, local_exists, remote_exists
|
||||
):
|
||||
def make_names(checksum, names):
|
||||
return {"name": checksum if show_checksums else " ".join(names)}
|
||||
def make_names(hash_, names):
|
||||
return {"name": hash_ if show_checksums else " ".join(names)}
|
||||
|
||||
dir_status = {}
|
||||
file_status = {}
|
||||
dir_contents = {}
|
||||
for checksum, item in named_cache[self.scheme].items():
|
||||
for hash_, item in named_cache[self.scheme].items():
|
||||
if item.children:
|
||||
dir_status[checksum] = make_names(checksum, item.names)
|
||||
dir_contents[checksum] = set()
|
||||
for child_checksum, child in item.children.items():
|
||||
file_status[child_checksum] = make_names(
|
||||
child_checksum, child.names
|
||||
dir_status[hash_] = make_names(hash_, item.names)
|
||||
dir_contents[hash_] = set()
|
||||
for child_hash, child in item.children.items():
|
||||
file_status[child_hash] = make_names(
|
||||
child_hash, child.names
|
||||
)
|
||||
dir_contents[checksum].add(child_checksum)
|
||||
dir_contents[hash_].add(child_hash)
|
||||
else:
|
||||
file_status[checksum] = make_names(checksum, item.names)
|
||||
file_status[hash_] = make_names(hash_, item.names)
|
||||
|
||||
self._fill_statuses(dir_status, local_exists, remote_exists)
|
||||
self._fill_statuses(file_status, local_exists, remote_exists)
|
||||
@@ -362,50 +498,50 @@ class LocalRemote(BaseRemote):
|
||||
|
||||
return dir_status, file_status, dir_contents
|
||||
|
||||
def _indexed_dir_checksums(self, named_cache, remote, dir_md5s):
|
||||
# Validate our index by verifying all indexed .dir checksums
|
||||
def _indexed_dir_hashes(self, named_cache, remote, dir_md5s):
|
||||
# Validate our index by verifying all indexed .dir hashes
|
||||
# still exist on the remote
|
||||
indexed_dirs = set(remote.index.dir_checksums())
|
||||
indexed_dirs = set(remote.index.dir_hashes())
|
||||
indexed_dir_exists = set()
|
||||
if indexed_dirs:
|
||||
indexed_dir_exists.update(
|
||||
remote._cache_object_exists(indexed_dirs)
|
||||
remote.tree.list_hashes_exists(indexed_dirs)
|
||||
)
|
||||
missing_dirs = indexed_dirs.difference(indexed_dir_exists)
|
||||
if missing_dirs:
|
||||
logger.debug(
|
||||
"Remote cache missing indexed .dir checksums '{}', "
|
||||
"Remote cache missing indexed .dir hashes '{}', "
|
||||
"clearing remote index".format(", ".join(missing_dirs))
|
||||
)
|
||||
remote.index.clear()
|
||||
|
||||
# Check if non-indexed (new) dir checksums exist on remote
|
||||
# Check if non-indexed (new) dir hashes exist on remote
|
||||
dir_exists = dir_md5s.intersection(indexed_dir_exists)
|
||||
dir_exists.update(remote._cache_object_exists(dir_md5s - dir_exists))
|
||||
dir_exists.update(
|
||||
remote.tree.list_hashes_exists(dir_md5s - dir_exists)
|
||||
)
|
||||
|
||||
# If .dir checksum exists on the remote, assume directory contents
|
||||
# If .dir hash exists on the remote, assume directory contents
|
||||
# still exists on the remote
|
||||
for dir_checksum in dir_exists:
|
||||
file_checksums = list(
|
||||
named_cache.child_keys(self.scheme, dir_checksum)
|
||||
)
|
||||
if dir_checksum not in remote.index:
|
||||
for dir_hash in dir_exists:
|
||||
file_hashes = list(named_cache.child_keys(self.scheme, dir_hash))
|
||||
if dir_hash not in remote.index:
|
||||
logger.debug(
|
||||
"Indexing new .dir '{}' with '{}' nested files".format(
|
||||
dir_checksum, len(file_checksums)
|
||||
dir_hash, len(file_hashes)
|
||||
)
|
||||
)
|
||||
remote.index.update([dir_checksum], file_checksums)
|
||||
yield dir_checksum
|
||||
yield from file_checksums
|
||||
remote.index.update([dir_hash], file_hashes)
|
||||
yield dir_hash
|
||||
yield from file_hashes
|
||||
|
||||
@staticmethod
|
||||
def _fill_statuses(checksum_info_dir, local_exists, remote_exists):
|
||||
def _fill_statuses(hash_info_dir, local_exists, remote_exists):
|
||||
# Using sets because they are way faster for lookups
|
||||
local = set(local_exists)
|
||||
remote = set(remote_exists)
|
||||
|
||||
for md5, info in checksum_info_dir.items():
|
||||
for md5, info in hash_info_dir.items():
|
||||
status = STATUS_MAP[(md5 in local, md5 in remote)]
|
||||
info["status"] = status
|
||||
|
||||
@@ -413,15 +549,15 @@ class LocalRemote(BaseRemote):
|
||||
cache = []
|
||||
path_infos = []
|
||||
names = []
|
||||
checksums = []
|
||||
hashes = []
|
||||
for md5, info in Tqdm(
|
||||
status_info.items(), desc="Analysing status", unit="file"
|
||||
):
|
||||
if info["status"] == status:
|
||||
cache.append(self.checksum_to_path_info(md5))
|
||||
path_infos.append(remote.checksum_to_path_info(md5))
|
||||
cache.append(self.hash_to_path_info(md5))
|
||||
path_infos.append(remote.hash_to_path_info(md5))
|
||||
names.append(info["name"])
|
||||
checksums.append(md5)
|
||||
hashes.append(md5)
|
||||
|
||||
if download:
|
||||
to_infos = cache
|
||||
@@ -430,7 +566,7 @@ class LocalRemote(BaseRemote):
|
||||
to_infos = path_infos
|
||||
from_infos = cache
|
||||
|
||||
return from_infos, to_infos, names, checksums
|
||||
return from_infos, to_infos, names, hashes
|
||||
|
||||
def _process(
|
||||
self,
|
||||
@@ -449,19 +585,19 @@ class LocalRemote(BaseRemote):
|
||||
|
||||
if download:
|
||||
func = partial(
|
||||
remote.download,
|
||||
dir_mode=self._dir_mode,
|
||||
file_mode=self._file_mode,
|
||||
_log_exceptions(remote.tree.download, "download"),
|
||||
dir_mode=self.tree.dir_mode,
|
||||
file_mode=self.tree.file_mode,
|
||||
)
|
||||
status = STATUS_DELETED
|
||||
desc = "Downloading"
|
||||
else:
|
||||
func = remote.upload
|
||||
func = _log_exceptions(remote.tree.upload, "upload")
|
||||
status = STATUS_NEW
|
||||
desc = "Uploading"
|
||||
|
||||
if jobs is None:
|
||||
jobs = remote.JOBS
|
||||
jobs = remote.tree.JOBS
|
||||
|
||||
dir_status, file_status, dir_contents = self._status(
|
||||
named_cache,
|
||||
@@ -482,26 +618,28 @@ class LocalRemote(BaseRemote):
|
||||
func = pbar.wrap_fn(func)
|
||||
with ThreadPoolExecutor(max_workers=jobs) as executor:
|
||||
if download:
|
||||
fails = sum(executor.map(func, *dir_plans))
|
||||
fails += sum(executor.map(func, *file_plans))
|
||||
from_infos, to_infos, names, _ = (
|
||||
d + f for d, f in zip(dir_plans, file_plans)
|
||||
)
|
||||
fails = sum(
|
||||
executor.map(func, from_infos, to_infos, names)
|
||||
)
|
||||
else:
|
||||
# for uploads, push files first, and any .dir files last
|
||||
|
||||
file_futures = {}
|
||||
for from_info, to_info, name, checksum in zip(*file_plans):
|
||||
file_futures[checksum] = executor.submit(
|
||||
for from_info, to_info, name, hash_ in zip(*file_plans):
|
||||
file_futures[hash_] = executor.submit(
|
||||
func, from_info, to_info, name
|
||||
)
|
||||
dir_futures = {}
|
||||
for from_info, to_info, name, dir_checksum in zip(
|
||||
*dir_plans
|
||||
):
|
||||
for from_info, to_info, name, dir_hash in zip(*dir_plans):
|
||||
wait_futures = {
|
||||
future
|
||||
for file_checksum, future in file_futures.items()
|
||||
if file_checksum in dir_contents[dir_checksum]
|
||||
for file_hash, future in file_futures.items()
|
||||
if file_hash in dir_contents[dir_hash]
|
||||
}
|
||||
dir_futures[dir_checksum] = executor.submit(
|
||||
dir_futures[dir_hash] = executor.submit(
|
||||
self._dir_upload,
|
||||
func,
|
||||
wait_futures,
|
||||
@@ -524,16 +662,14 @@ class LocalRemote(BaseRemote):
|
||||
|
||||
if not download:
|
||||
# index successfully pushed dirs
|
||||
for dir_checksum, future in dir_futures.items():
|
||||
for dir_hash, future in dir_futures.items():
|
||||
if future.result() == 0:
|
||||
file_checksums = dir_contents[dir_checksum]
|
||||
file_hashes = dir_contents[dir_hash]
|
||||
logger.debug(
|
||||
"Indexing pushed dir '{}' with "
|
||||
"'{}' nested files".format(
|
||||
dir_checksum, len(file_checksums)
|
||||
)
|
||||
"'{}' nested files".format(dir_hash, len(file_hashes))
|
||||
)
|
||||
remote.index.update([dir_checksum], file_checksums)
|
||||
remote.index.update([dir_hash], file_hashes)
|
||||
|
||||
return len(dir_plans[0]) + len(file_plans[0])
|
||||
|
||||
@@ -572,10 +708,10 @@ class LocalRemote(BaseRemote):
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _log_missing_caches(checksum_info_dict):
|
||||
def _log_missing_caches(hash_info_dict):
|
||||
missing_caches = [
|
||||
(md5, info)
|
||||
for md5, info in checksum_info_dict.items()
|
||||
for md5, info in hash_info_dict.items()
|
||||
if info["status"] == STATUS_MISSING
|
||||
]
|
||||
if missing_caches:
|
||||
@@ -588,129 +724,3 @@ class LocalRemote(BaseRemote):
|
||||
"nor on remote. Missing cache files: {}".format(missing_desc)
|
||||
)
|
||||
logger.warning(msg)
|
||||
|
||||
def _unprotect_file(self, path):
|
||||
if System.is_symlink(path) or System.is_hardlink(path):
|
||||
logger.debug(f"Unprotecting '{path}'")
|
||||
tmp = os.path.join(os.path.dirname(path), "." + uuid())
|
||||
|
||||
# The operations order is important here - if some application
|
||||
# would access the file during the process of copyfile then it
|
||||
# would get only the part of file. So, at first, the file should be
|
||||
# copied with the temporary name, and then original file should be
|
||||
# replaced by new.
|
||||
copyfile(path, tmp, name="Unprotecting '{}'".format(relpath(path)))
|
||||
remove(path)
|
||||
os.rename(tmp, path)
|
||||
|
||||
else:
|
||||
logger.debug(
|
||||
"Skipping copying for '{}', since it is not "
|
||||
"a symlink or a hardlink.".format(path)
|
||||
)
|
||||
|
||||
os.chmod(path, self._file_mode)
|
||||
|
||||
def _unprotect_dir(self, path):
|
||||
assert is_working_tree(self.repo.tree)
|
||||
|
||||
for fname in self.repo.tree.walk_files(path):
|
||||
self._unprotect_file(fname)
|
||||
|
||||
def unprotect(self, path_info):
|
||||
path = path_info.fspath
|
||||
if not os.path.exists(path):
|
||||
raise DvcException(f"can't unprotect non-existing data '{path}'")
|
||||
|
||||
if os.path.isdir(path):
|
||||
self._unprotect_dir(path)
|
||||
else:
|
||||
self._unprotect_file(path)
|
||||
|
||||
def protect(self, path_info):
|
||||
path = os.fspath(path_info)
|
||||
mode = self.CACHE_MODE
|
||||
|
||||
try:
|
||||
os.chmod(path, mode)
|
||||
except OSError as exc:
|
||||
# There is nothing we need to do in case of a read-only file system
|
||||
if exc.errno == errno.EROFS:
|
||||
return
|
||||
|
||||
# In shared cache scenario, we might not own the cache file, so we
|
||||
# need to check if cache file is already protected.
|
||||
if exc.errno not in [errno.EPERM, errno.EACCES]:
|
||||
raise
|
||||
|
||||
actual = stat.S_IMODE(os.stat(path).st_mode)
|
||||
if actual != mode:
|
||||
raise
|
||||
|
||||
def _get_unpacked_dir_path_info(self, checksum):
|
||||
info = self.checksum_to_path_info(checksum)
|
||||
return info.with_name(info.name + self.UNPACKED_DIR_SUFFIX)
|
||||
|
||||
def _remove_unpacked_dir(self, checksum):
|
||||
path_info = self._get_unpacked_dir_path_info(checksum)
|
||||
self.remove(path_info)
|
||||
|
||||
def _path_info_changed(self, path_info):
|
||||
if self.exists(path_info) and self.state.get(path_info):
|
||||
return False
|
||||
return True
|
||||
|
||||
def _update_unpacked_dir(self, checksum):
|
||||
unpacked_dir_info = self._get_unpacked_dir_path_info(checksum)
|
||||
|
||||
if not self._path_info_changed(unpacked_dir_info):
|
||||
return
|
||||
|
||||
self.remove(unpacked_dir_info)
|
||||
|
||||
try:
|
||||
dir_info = self.get_dir_cache(checksum)
|
||||
self._create_unpacked_dir(checksum, dir_info, unpacked_dir_info)
|
||||
except DvcException:
|
||||
logger.warning(f"Could not create '{unpacked_dir_info}'")
|
||||
|
||||
self.remove(unpacked_dir_info)
|
||||
|
||||
def _create_unpacked_dir(self, checksum, dir_info, unpacked_dir_info):
|
||||
self.makedirs(unpacked_dir_info)
|
||||
|
||||
for entry in Tqdm(dir_info, desc="Creating unpacked dir", unit="file"):
|
||||
entry_cache_info = self.checksum_to_path_info(
|
||||
entry[self.PARAM_CHECKSUM]
|
||||
)
|
||||
relative_path = entry[self.PARAM_RELPATH]
|
||||
# In shared cache mode some cache files might not be owned by the
|
||||
# user, so we need to use symlinks because, unless
|
||||
# /proc/sys/fs/protected_hardlinks is disabled, the user is not
|
||||
# allowed to create hardlinks to files that he doesn't own.
|
||||
link_types = ["hardlink", "symlink"]
|
||||
self._link(
|
||||
entry_cache_info, unpacked_dir_info / relative_path, link_types
|
||||
)
|
||||
|
||||
self.state.save(unpacked_dir_info, checksum)
|
||||
|
||||
def _changed_unpacked_dir(self, checksum):
|
||||
status_unpacked_dir_info = self._get_unpacked_dir_path_info(checksum)
|
||||
|
||||
return not self.state.get(status_unpacked_dir_info)
|
||||
|
||||
def _get_unpacked_dir_names(self, checksums):
|
||||
unpacked = set()
|
||||
for c in checksums:
|
||||
if self.is_dir_checksum(c):
|
||||
unpacked.add(c + self.UNPACKED_DIR_SUFFIX)
|
||||
return unpacked
|
||||
|
||||
def is_protected(self, path_info):
|
||||
if not self.exists(path_info):
|
||||
return False
|
||||
|
||||
mode = os.stat(path_info).st_mode
|
||||
|
||||
return stat.S_IMODE(mode) == self.CACHE_MODE
|
||||
|
||||
+28
-31
@@ -1,19 +1,18 @@
|
||||
import logging
|
||||
import os
|
||||
import posixpath
|
||||
import threading
|
||||
|
||||
from funcy import cached_property, wrap_prop
|
||||
|
||||
from dvc.path_info import CloudURLInfo
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.remote.base import BaseRemote
|
||||
from dvc.remote.base import BaseRemoteTree
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class OSSRemote(BaseRemote):
|
||||
class OSSRemoteTree(BaseRemoteTree):
|
||||
"""
|
||||
oss2 document:
|
||||
https://www.alibabacloud.com/help/doc-detail/32026.htm
|
||||
@@ -33,7 +32,7 @@ class OSSRemote(BaseRemote):
|
||||
"""
|
||||
|
||||
scheme = Schemes.OSS
|
||||
path_cls = CloudURLInfo
|
||||
PATH_CLS = CloudURLInfo
|
||||
REQUIRES = {"oss2": "oss2"}
|
||||
PARAM_CHECKSUM = "etag"
|
||||
COPY_POLL_SECONDS = 5
|
||||
@@ -43,7 +42,7 @@ class OSSRemote(BaseRemote):
|
||||
super().__init__(repo, config)
|
||||
|
||||
url = config.get("url")
|
||||
self.path_info = self.path_cls(url) if url else None
|
||||
self.path_info = self.PATH_CLS(url) if url else None
|
||||
|
||||
self.endpoint = config.get("oss_endpoint") or os.getenv("OSS_ENDPOINT")
|
||||
|
||||
@@ -83,6 +82,30 @@ class OSSRemote(BaseRemote):
|
||||
)
|
||||
return bucket
|
||||
|
||||
def _generate_download_url(self, path_info, expires=3600):
|
||||
assert path_info.bucket == self.path_info.bucket
|
||||
|
||||
return self.oss_service.sign_url("GET", path_info.path, expires)
|
||||
|
||||
def exists(self, path_info):
|
||||
paths = self._list_paths(path_info)
|
||||
return any(path_info.path == path for path in paths)
|
||||
|
||||
def _list_paths(self, path_info):
|
||||
import oss2
|
||||
|
||||
for blob in oss2.ObjectIterator(
|
||||
self.oss_service, prefix=path_info.path
|
||||
):
|
||||
yield blob.key
|
||||
|
||||
def walk_files(self, path_info, **kwargs):
|
||||
for fname in self._list_paths(path_info):
|
||||
if fname.endswith("/"):
|
||||
continue
|
||||
|
||||
yield path_info.replace(path=fname)
|
||||
|
||||
def remove(self, path_info):
|
||||
if path_info.scheme != self.scheme:
|
||||
raise NotImplementedError
|
||||
@@ -90,23 +113,6 @@ class OSSRemote(BaseRemote):
|
||||
logger.debug(f"Removing oss://{path_info}")
|
||||
self.oss_service.delete_object(path_info.path)
|
||||
|
||||
def _list_paths(self, prefix, progress_callback=None):
|
||||
import oss2
|
||||
|
||||
for blob in oss2.ObjectIterator(self.oss_service, prefix=prefix):
|
||||
if progress_callback:
|
||||
progress_callback()
|
||||
yield blob.key
|
||||
|
||||
def list_cache_paths(self, prefix=None, progress_callback=None):
|
||||
if prefix:
|
||||
prefix = posixpath.join(
|
||||
self.path_info.path, prefix[:2], prefix[2:]
|
||||
)
|
||||
else:
|
||||
prefix = self.path_info.path
|
||||
return self._list_paths(prefix, progress_callback)
|
||||
|
||||
def _upload(
|
||||
self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
|
||||
):
|
||||
@@ -122,12 +128,3 @@ class OSSRemote(BaseRemote):
|
||||
self.oss_service.get_object_to_file(
|
||||
from_info.path, to_file, progress_callback=pbar.update_to
|
||||
)
|
||||
|
||||
def _generate_download_url(self, path_info, expires=3600):
|
||||
assert path_info.bucket == self.path_info.bucket
|
||||
|
||||
return self.oss_service.sign_url("GET", path_info.path, expires)
|
||||
|
||||
def exists(self, path_info):
|
||||
paths = self._list_paths(path_info.path)
|
||||
return any(path_info.path == path for path in paths)
|
||||
|
||||
+141
-159
@@ -1,6 +1,5 @@
|
||||
import logging
|
||||
import os
|
||||
import posixpath
|
||||
import threading
|
||||
|
||||
from funcy import cached_property, wrap_prop
|
||||
@@ -9,15 +8,15 @@ from dvc.config import ConfigError
|
||||
from dvc.exceptions import DvcException, ETagMismatchError
|
||||
from dvc.path_info import CloudURLInfo
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.remote.base import BaseRemote
|
||||
from dvc.remote.base import BaseRemoteTree
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class S3Remote(BaseRemote):
|
||||
class S3RemoteTree(BaseRemoteTree):
|
||||
scheme = Schemes.S3
|
||||
path_cls = CloudURLInfo
|
||||
PATH_CLS = CloudURLInfo
|
||||
REQUIRES = {"boto3": "boto3"}
|
||||
PARAM_CHECKSUM = "etag"
|
||||
|
||||
@@ -25,7 +24,7 @@ class S3Remote(BaseRemote):
|
||||
super().__init__(repo, config)
|
||||
|
||||
url = config.get("url", "s3://")
|
||||
self.path_info = self.path_cls(url)
|
||||
self.path_info = self.PATH_CLS(url)
|
||||
|
||||
self.region = config.get("region")
|
||||
self.profile = config.get("profile")
|
||||
@@ -77,9 +76,6 @@ class S3Remote(BaseRemote):
|
||||
|
||||
return obj["ETag"].strip('"')
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
return self.get_etag(self.s3, path_info.bucket, path_info.path)
|
||||
|
||||
@staticmethod
|
||||
def get_head_object(s3, bucket, path, *args, **kwargs):
|
||||
|
||||
@@ -89,6 +85,136 @@ class S3Remote(BaseRemote):
|
||||
raise DvcException(f"s3://{bucket}/{path} does not exist") from exc
|
||||
return obj
|
||||
|
||||
def _append_aws_grants_to_extra_args(self, config):
|
||||
# Keys for extra_args can be one of the following list:
|
||||
# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/customizations/s3.html#boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS
|
||||
"""
|
||||
ALLOWED_UPLOAD_ARGS = [
|
||||
'ACL', 'CacheControl', 'ContentDisposition', 'ContentEncoding',
|
||||
'ContentLanguage', 'ContentType', 'Expires', 'GrantFullControl',
|
||||
'GrantRead', 'GrantReadACP', 'GrantWriteACP', 'Metadata',
|
||||
'RequestPayer', 'ServerSideEncryption', 'StorageClass',
|
||||
'SSECustomerAlgorithm', 'SSECustomerKey', 'SSECustomerKeyMD5',
|
||||
'SSEKMSKeyId', 'WebsiteRedirectLocation'
|
||||
]
|
||||
"""
|
||||
|
||||
grants = {
|
||||
"grant_full_control": "GrantFullControl",
|
||||
"grant_read": "GrantRead",
|
||||
"grant_read_acp": "GrantReadACP",
|
||||
"grant_write_acp": "GrantWriteACP",
|
||||
}
|
||||
|
||||
for grant_option, extra_args_key in grants.items():
|
||||
if config.get(grant_option):
|
||||
if self.acl:
|
||||
raise ConfigError(
|
||||
"`acl` and `grant_*` AWS S3 config options "
|
||||
"are mutually exclusive"
|
||||
)
|
||||
|
||||
self.extra_args[extra_args_key] = config.get(grant_option)
|
||||
|
||||
def _generate_download_url(self, path_info, expires=3600):
|
||||
params = {"Bucket": path_info.bucket, "Key": path_info.path}
|
||||
return self.s3.generate_presigned_url(
|
||||
ClientMethod="get_object", Params=params, ExpiresIn=int(expires)
|
||||
)
|
||||
|
||||
def exists(self, path_info):
|
||||
"""Check if the blob exists. If it does not exist,
|
||||
it could be a part of a directory path.
|
||||
|
||||
eg: if `data/file.txt` exists, check for `data` should return True
|
||||
"""
|
||||
return self.isfile(path_info) or self.isdir(path_info)
|
||||
|
||||
def isdir(self, path_info):
|
||||
# S3 doesn't have a concept for directories.
|
||||
#
|
||||
# Using `head_object` with a path pointing to a directory
|
||||
# will throw a 404 error.
|
||||
#
|
||||
# A reliable way to know if a given path is a directory is by
|
||||
# checking if there are more files sharing the same prefix
|
||||
# with a `list_objects` call.
|
||||
#
|
||||
# We need to make sure that the path ends with a forward slash,
|
||||
# since we can end with false-positives like the following example:
|
||||
#
|
||||
# bucket
|
||||
# └── data
|
||||
# ├── alice
|
||||
# └── alpha
|
||||
#
|
||||
# Using `data/al` as prefix will return `[data/alice, data/alpha]`,
|
||||
# While `data/al/` will return nothing.
|
||||
#
|
||||
dir_path = path_info / ""
|
||||
return bool(list(self._list_paths(dir_path, max_items=1)))
|
||||
|
||||
def isfile(self, path_info):
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
if path_info.path.endswith("/"):
|
||||
return False
|
||||
|
||||
try:
|
||||
self.s3.head_object(Bucket=path_info.bucket, Key=path_info.path)
|
||||
except ClientError as exc:
|
||||
if exc.response["Error"]["Code"] != "404":
|
||||
raise
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def _list_objects(self, path_info, max_items=None):
|
||||
""" Read config for list object api, paginate through list objects."""
|
||||
kwargs = {
|
||||
"Bucket": path_info.bucket,
|
||||
"Prefix": path_info.path,
|
||||
"PaginationConfig": {"MaxItems": max_items},
|
||||
}
|
||||
paginator = self.s3.get_paginator(self.list_objects_api)
|
||||
for page in paginator.paginate(**kwargs):
|
||||
contents = page.get("Contents", ())
|
||||
yield from contents
|
||||
|
||||
def _list_paths(self, path_info, max_items=None):
|
||||
return (
|
||||
item["Key"] for item in self._list_objects(path_info, max_items)
|
||||
)
|
||||
|
||||
def walk_files(self, path_info, **kwargs):
|
||||
for fname in self._list_paths(path_info / "", **kwargs):
|
||||
if fname.endswith("/"):
|
||||
continue
|
||||
|
||||
yield path_info.replace(path=fname)
|
||||
|
||||
def remove(self, path_info):
|
||||
if path_info.scheme != "s3":
|
||||
raise NotImplementedError
|
||||
|
||||
logger.debug(f"Removing {path_info}")
|
||||
self.s3.delete_object(Bucket=path_info.bucket, Key=path_info.path)
|
||||
|
||||
def makedirs(self, path_info):
|
||||
# We need to support creating empty directories, which means
|
||||
# creating an object with an empty body and a trailing slash `/`.
|
||||
#
|
||||
# We are not creating directory objects for every parent prefix,
|
||||
# as it is not required.
|
||||
if not path_info.path:
|
||||
return
|
||||
|
||||
dir_path = path_info / ""
|
||||
self.s3.put_object(Bucket=path_info.bucket, Key=dir_path.path, Body="")
|
||||
|
||||
def copy(self, from_info, to_info):
|
||||
self._copy(self.s3, from_info, to_info, self.extra_args)
|
||||
|
||||
@classmethod
|
||||
def _copy_multipart(
|
||||
cls, s3, from_info, to_info, size, n_parts, extra_args
|
||||
@@ -101,7 +227,7 @@ class S3Remote(BaseRemote):
|
||||
parts = []
|
||||
byte_position = 0
|
||||
for i in range(1, n_parts + 1):
|
||||
obj = cls.get_head_object(
|
||||
obj = S3RemoteTree.get_head_object(
|
||||
s3, from_info.bucket, from_info.path, PartNumber=i
|
||||
)
|
||||
part_size = obj["ContentLength"]
|
||||
@@ -156,7 +282,9 @@ class S3Remote(BaseRemote):
|
||||
# object is transfered in the same chunks as it was originally.
|
||||
from boto3.s3.transfer import TransferConfig
|
||||
|
||||
obj = cls.get_head_object(s3, from_info.bucket, from_info.path)
|
||||
obj = S3RemoteTree.get_head_object(
|
||||
s3, from_info.bucket, from_info.path
|
||||
)
|
||||
etag = obj["ETag"].strip('"')
|
||||
size = obj["ContentLength"]
|
||||
|
||||
@@ -176,114 +304,12 @@ class S3Remote(BaseRemote):
|
||||
Config=TransferConfig(multipart_threshold=size + 1),
|
||||
)
|
||||
|
||||
cached_etag = cls.get_etag(s3, to_info.bucket, to_info.path)
|
||||
cached_etag = S3RemoteTree.get_etag(s3, to_info.bucket, to_info.path)
|
||||
if etag != cached_etag:
|
||||
raise ETagMismatchError(etag, cached_etag)
|
||||
|
||||
def copy(self, from_info, to_info):
|
||||
self._copy(self.s3, from_info, to_info, self.extra_args)
|
||||
|
||||
def remove(self, path_info):
|
||||
if path_info.scheme != "s3":
|
||||
raise NotImplementedError
|
||||
|
||||
logger.debug(f"Removing {path_info}")
|
||||
self.s3.delete_object(Bucket=path_info.bucket, Key=path_info.path)
|
||||
|
||||
def _list_objects(
|
||||
self, path_info, max_items=None, prefix=None, progress_callback=None
|
||||
):
|
||||
""" Read config for list object api, paginate through list objects."""
|
||||
kwargs = {
|
||||
"Bucket": path_info.bucket,
|
||||
"Prefix": path_info.path,
|
||||
"PaginationConfig": {"MaxItems": max_items},
|
||||
}
|
||||
if prefix:
|
||||
kwargs["Prefix"] = posixpath.join(path_info.path, prefix[:2])
|
||||
paginator = self.s3.get_paginator(self.list_objects_api)
|
||||
for page in paginator.paginate(**kwargs):
|
||||
contents = page.get("Contents", ())
|
||||
if progress_callback:
|
||||
for item in contents:
|
||||
progress_callback()
|
||||
yield item
|
||||
else:
|
||||
yield from contents
|
||||
|
||||
def _list_paths(
|
||||
self, path_info, max_items=None, prefix=None, progress_callback=None
|
||||
):
|
||||
return (
|
||||
item["Key"]
|
||||
for item in self._list_objects(
|
||||
path_info, max_items, prefix, progress_callback
|
||||
)
|
||||
)
|
||||
|
||||
def list_cache_paths(self, prefix=None, progress_callback=None):
|
||||
return self._list_paths(
|
||||
self.path_info, prefix=prefix, progress_callback=progress_callback
|
||||
)
|
||||
|
||||
def isfile(self, path_info):
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
if path_info.path.endswith("/"):
|
||||
return False
|
||||
|
||||
try:
|
||||
self.s3.head_object(Bucket=path_info.bucket, Key=path_info.path)
|
||||
except ClientError as exc:
|
||||
if exc.response["Error"]["Code"] != "404":
|
||||
raise
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def exists(self, path_info):
|
||||
"""Check if the blob exists. If it does not exist,
|
||||
it could be a part of a directory path.
|
||||
|
||||
eg: if `data/file.txt` exists, check for `data` should return True
|
||||
"""
|
||||
return self.isfile(path_info) or self.isdir(path_info)
|
||||
|
||||
def makedirs(self, path_info):
|
||||
# We need to support creating empty directories, which means
|
||||
# creating an object with an empty body and a trailing slash `/`.
|
||||
#
|
||||
# We are not creating directory objects for every parent prefix,
|
||||
# as it is not required.
|
||||
if not path_info.path:
|
||||
return
|
||||
|
||||
dir_path = path_info / ""
|
||||
self.s3.put_object(Bucket=path_info.bucket, Key=dir_path.path, Body="")
|
||||
|
||||
def isdir(self, path_info):
|
||||
# S3 doesn't have a concept for directories.
|
||||
#
|
||||
# Using `head_object` with a path pointing to a directory
|
||||
# will throw a 404 error.
|
||||
#
|
||||
# A reliable way to know if a given path is a directory is by
|
||||
# checking if there are more files sharing the same prefix
|
||||
# with a `list_objects` call.
|
||||
#
|
||||
# We need to make sure that the path ends with a forward slash,
|
||||
# since we can end with false-positives like the following example:
|
||||
#
|
||||
# bucket
|
||||
# └── data
|
||||
# ├── alice
|
||||
# └── alpha
|
||||
#
|
||||
# Using `data/al` as prefix will return `[data/alice, data/alpha]`,
|
||||
# While `data/al/` will return nothing.
|
||||
#
|
||||
dir_path = path_info / ""
|
||||
return bool(list(self._list_paths(dir_path, max_items=1)))
|
||||
def get_file_hash(self, path_info):
|
||||
return self.get_etag(self.s3, path_info.bucket, path_info.path)
|
||||
|
||||
def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
|
||||
total = os.path.getsize(from_file)
|
||||
@@ -311,47 +337,3 @@ class S3Remote(BaseRemote):
|
||||
self.s3.download_file(
|
||||
from_info.bucket, from_info.path, to_file, Callback=pbar.update
|
||||
)
|
||||
|
||||
def _generate_download_url(self, path_info, expires=3600):
|
||||
params = {"Bucket": path_info.bucket, "Key": path_info.path}
|
||||
return self.s3.generate_presigned_url(
|
||||
ClientMethod="get_object", Params=params, ExpiresIn=int(expires)
|
||||
)
|
||||
|
||||
def walk_files(self, path_info, max_items=None):
|
||||
for fname in self._list_paths(path_info / "", max_items):
|
||||
if fname.endswith("/"):
|
||||
continue
|
||||
|
||||
yield path_info.replace(path=fname)
|
||||
|
||||
def _append_aws_grants_to_extra_args(self, config):
|
||||
# Keys for extra_args can be one of the following list:
|
||||
# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/customizations/s3.html#boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS
|
||||
"""
|
||||
ALLOWED_UPLOAD_ARGS = [
|
||||
'ACL', 'CacheControl', 'ContentDisposition', 'ContentEncoding',
|
||||
'ContentLanguage', 'ContentType', 'Expires', 'GrantFullControl',
|
||||
'GrantRead', 'GrantReadACP', 'GrantWriteACP', 'Metadata',
|
||||
'RequestPayer', 'ServerSideEncryption', 'StorageClass',
|
||||
'SSECustomerAlgorithm', 'SSECustomerKey', 'SSECustomerKeyMD5',
|
||||
'SSEKMSKeyId', 'WebsiteRedirectLocation'
|
||||
]
|
||||
"""
|
||||
|
||||
grants = {
|
||||
"grant_full_control": "GrantFullControl",
|
||||
"grant_read": "GrantRead",
|
||||
"grant_read_acp": "GrantReadACP",
|
||||
"grant_write_acp": "GrantWriteACP",
|
||||
}
|
||||
|
||||
for grant_option, extra_args_key in grants.items():
|
||||
if config.get(grant_option):
|
||||
if self.acl:
|
||||
raise ConfigError(
|
||||
"`acl` and `grant_*` AWS S3 config options "
|
||||
"are mutually exclusive"
|
||||
)
|
||||
|
||||
self.extra_args[extra_args_key] = config.get(grant_option)
|
||||
|
||||
+63
-57
@@ -14,7 +14,7 @@ from funcy import first, memoize, silent, wrap_with
|
||||
|
||||
import dvc.prompt as prompt
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.remote.base import BaseRemote
|
||||
from dvc.remote.base import BaseRemoteTree, Remote
|
||||
from dvc.remote.pool import get_connection
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.utils import to_chunks
|
||||
@@ -33,21 +33,21 @@ def ask_password(host, user, port):
|
||||
)
|
||||
|
||||
|
||||
class SSHRemote(BaseRemote):
|
||||
class SSHRemoteTree(BaseRemoteTree):
|
||||
scheme = Schemes.SSH
|
||||
REQUIRES = {"paramiko": "paramiko"}
|
||||
|
||||
JOBS = 4
|
||||
|
||||
PARAM_CHECKSUM = "md5"
|
||||
DEFAULT_PORT = 22
|
||||
TIMEOUT = 1800
|
||||
# At any given time some of the connections will go over network and
|
||||
# paramiko stuff, so we would ideally have it double of server processors.
|
||||
# We use conservative setting of 4 instead to not exhaust max sessions.
|
||||
CHECKSUM_JOBS = 4
|
||||
DEFAULT_CACHE_TYPES = ["copy"]
|
||||
TRAVERSE_PREFIX_LEN = 2
|
||||
|
||||
DEFAULT_CACHE_TYPES = ["copy"]
|
||||
DEFAULT_PORT = 22
|
||||
TIMEOUT = 1800
|
||||
|
||||
def __init__(self, repo, config):
|
||||
super().__init__(repo, config)
|
||||
@@ -69,7 +69,7 @@ class SSHRemote(BaseRemote):
|
||||
or self._try_get_ssh_config_port(user_ssh_config)
|
||||
or self.DEFAULT_PORT
|
||||
)
|
||||
self.path_info = self.path_cls.from_parts(
|
||||
self.path_info = self.PATH_CLS.from_parts(
|
||||
scheme=self.scheme,
|
||||
host=host,
|
||||
user=user,
|
||||
@@ -103,7 +103,7 @@ class SSHRemote(BaseRemote):
|
||||
def _load_user_ssh_config(hostname):
|
||||
import paramiko
|
||||
|
||||
user_config_file = SSHRemote.ssh_config_filename()
|
||||
user_config_file = SSHRemoteTree.ssh_config_filename()
|
||||
user_ssh_config = {}
|
||||
if hostname and os.path.exists(user_config_file):
|
||||
ssh_config = paramiko.SSHConfig()
|
||||
@@ -148,17 +148,22 @@ class SSHRemote(BaseRemote):
|
||||
sock=self.sock,
|
||||
)
|
||||
|
||||
@contextmanager
|
||||
def open(self, path_info, mode="r", encoding=None):
|
||||
assert mode in {"r", "rt", "rb", "wb"}
|
||||
|
||||
with self.ssh(path_info) as ssh, closing(
|
||||
ssh.sftp.open(path_info.path, mode)
|
||||
) as fd:
|
||||
if "b" in mode:
|
||||
yield fd
|
||||
else:
|
||||
yield io.TextIOWrapper(fd, encoding=encoding)
|
||||
|
||||
def exists(self, path_info):
|
||||
with self.ssh(path_info) as ssh:
|
||||
return ssh.exists(path_info.path)
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
if path_info.scheme != self.scheme:
|
||||
raise NotImplementedError
|
||||
|
||||
with self.ssh(path_info) as ssh:
|
||||
return ssh.md5(path_info.path)
|
||||
|
||||
def isdir(self, path_info):
|
||||
with self.ssh(path_info) as ssh:
|
||||
return ssh.isdir(path_info.path)
|
||||
@@ -167,9 +172,29 @@ class SSHRemote(BaseRemote):
|
||||
with self.ssh(path_info) as ssh:
|
||||
return ssh.isfile(path_info.path)
|
||||
|
||||
def getsize(self, path_info):
|
||||
def walk_files(self, path_info, **kwargs):
|
||||
with self.ssh(path_info) as ssh:
|
||||
return ssh.getsize(path_info.path)
|
||||
for fname in ssh.walk_files(path_info.path):
|
||||
yield path_info.replace(path=fname)
|
||||
|
||||
def remove(self, path_info):
|
||||
if path_info.scheme != self.scheme:
|
||||
raise NotImplementedError
|
||||
|
||||
with self.ssh(path_info) as ssh:
|
||||
ssh.remove(path_info.path)
|
||||
|
||||
def makedirs(self, path_info):
|
||||
with self.ssh(path_info) as ssh:
|
||||
ssh.makedirs(path_info.path)
|
||||
|
||||
def move(self, from_info, to_info, mode=None):
|
||||
assert mode is None
|
||||
if from_info.scheme != self.scheme or to_info.scheme != self.scheme:
|
||||
raise NotImplementedError
|
||||
|
||||
with self.ssh(from_info) as ssh:
|
||||
ssh.move(from_info.path, to_info.path)
|
||||
|
||||
def copy(self, from_info, to_info):
|
||||
if not from_info.scheme == to_info.scheme == self.scheme:
|
||||
@@ -212,20 +237,16 @@ class SSHRemote(BaseRemote):
|
||||
with self.ssh(from_info) as ssh:
|
||||
ssh.reflink(from_info.path, to_info.path)
|
||||
|
||||
def remove(self, path_info):
|
||||
def get_file_hash(self, path_info):
|
||||
if path_info.scheme != self.scheme:
|
||||
raise NotImplementedError
|
||||
|
||||
with self.ssh(path_info) as ssh:
|
||||
ssh.remove(path_info.path)
|
||||
return ssh.md5(path_info.path)
|
||||
|
||||
def move(self, from_info, to_info, mode=None):
|
||||
assert mode is None
|
||||
if from_info.scheme != self.scheme or to_info.scheme != self.scheme:
|
||||
raise NotImplementedError
|
||||
|
||||
with self.ssh(from_info) as ssh:
|
||||
ssh.move(from_info.path, to_info.path)
|
||||
def getsize(self, path_info):
|
||||
with self.ssh(path_info) as ssh:
|
||||
return ssh.getsize(path_info.path)
|
||||
|
||||
def _download(self, from_info, to_file, name=None, no_progress_bar=False):
|
||||
assert from_info.isin(self.path_info)
|
||||
@@ -247,24 +268,14 @@ class SSHRemote(BaseRemote):
|
||||
no_progress_bar=no_progress_bar,
|
||||
)
|
||||
|
||||
@contextmanager
|
||||
def open(self, path_info, mode="r", encoding=None):
|
||||
assert mode in {"r", "rt", "rb", "wb"}
|
||||
|
||||
with self.ssh(path_info) as ssh, closing(
|
||||
ssh.sftp.open(path_info.path, mode)
|
||||
) as fd:
|
||||
if "b" in mode:
|
||||
yield fd
|
||||
else:
|
||||
yield io.TextIOWrapper(fd, encoding=encoding)
|
||||
|
||||
def list_cache_paths(self, prefix=None, progress_callback=None):
|
||||
def list_paths(self, prefix=None, progress_callback=None):
|
||||
if prefix:
|
||||
root = posixpath.join(self.path_info.path, prefix[:2])
|
||||
else:
|
||||
root = self.path_info.path
|
||||
with self.ssh(self.path_info) as ssh:
|
||||
if prefix and not ssh.exists(root):
|
||||
return
|
||||
# If we simply return an iterator then with above closes instantly
|
||||
if progress_callback:
|
||||
for path in ssh.walk_files(root):
|
||||
@@ -273,15 +284,8 @@ class SSHRemote(BaseRemote):
|
||||
else:
|
||||
yield from ssh.walk_files(root)
|
||||
|
||||
def walk_files(self, path_info):
|
||||
with self.ssh(path_info) as ssh:
|
||||
for fname in ssh.walk_files(path_info.path):
|
||||
yield path_info.replace(path=fname)
|
||||
|
||||
def makedirs(self, path_info):
|
||||
with self.ssh(path_info) as ssh:
|
||||
ssh.makedirs(path_info.path)
|
||||
|
||||
class SSHRemote(Remote):
|
||||
def batch_exists(self, path_infos, callback):
|
||||
def _exists(chunk_and_channel):
|
||||
chunk, channel = chunk_and_channel
|
||||
@@ -297,7 +301,7 @@ class SSHRemote(BaseRemote):
|
||||
callback(path)
|
||||
return ret
|
||||
|
||||
with self.ssh(path_infos[0]) as ssh:
|
||||
with self.tree.ssh(path_infos[0]) as ssh:
|
||||
channels = ssh.open_max_sftp_channels()
|
||||
max_workers = len(channels)
|
||||
|
||||
@@ -310,32 +314,34 @@ class SSHRemote(BaseRemote):
|
||||
|
||||
return results
|
||||
|
||||
def cache_exists(self, checksums, jobs=None, name=None):
|
||||
def hashes_exist(self, hashes, jobs=None, name=None):
|
||||
"""This is older implementation used in remote/base.py
|
||||
We are reusing it in RemoteSSH, because SSH's batch_exists proved to be
|
||||
faster than current approach (relying on exists(path_info)) applied in
|
||||
remote/base.
|
||||
"""
|
||||
if not self.CAN_TRAVERSE:
|
||||
return list(set(checksums) & set(self.all()))
|
||||
if not self.tree.CAN_TRAVERSE:
|
||||
return list(set(hashes) & set(self.tree.all()))
|
||||
|
||||
# possibly prompt for credentials before "Querying" progress output
|
||||
self.ensure_credentials()
|
||||
self.tree.ensure_credentials()
|
||||
|
||||
with Tqdm(
|
||||
desc="Querying "
|
||||
+ ("cache in " + name if name else "remote cache"),
|
||||
total=len(checksums),
|
||||
total=len(hashes),
|
||||
unit="file",
|
||||
) as pbar:
|
||||
|
||||
def exists_with_progress(chunks):
|
||||
return self.batch_exists(chunks, callback=pbar.update_msg)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=jobs or self.JOBS) as executor:
|
||||
path_infos = [self.checksum_to_path_info(x) for x in checksums]
|
||||
chunks = to_chunks(path_infos, num_chunks=self.JOBS)
|
||||
with ThreadPoolExecutor(
|
||||
max_workers=jobs or self.tree.JOBS
|
||||
) as executor:
|
||||
path_infos = [self.hash_to_path_info(x) for x in hashes]
|
||||
chunks = to_chunks(path_infos, num_chunks=self.tree.JOBS)
|
||||
results = executor.map(exists_with_progress, chunks)
|
||||
in_remote = itertools.chain.from_iterable(results)
|
||||
ret = list(itertools.compress(checksums, in_remote))
|
||||
ret = list(itertools.compress(hashes, in_remote))
|
||||
return ret
|
||||
|
||||
@@ -237,11 +237,15 @@ class SSHConnection:
|
||||
or channel.recv_ready()
|
||||
or channel.recv_stderr_ready()
|
||||
):
|
||||
import select
|
||||
import selectors
|
||||
|
||||
selector = selectors.DefaultSelector()
|
||||
selector.register(stdout.channel, selectors.EVENT_READ)
|
||||
|
||||
got_chunk = False
|
||||
readq, _, _ = select.select([stdout.channel], [], [], self.timeout)
|
||||
for c in readq:
|
||||
events = selector.select(self.timeout)
|
||||
for key, _ in events:
|
||||
c = key.fileobj
|
||||
if c.recv_ready():
|
||||
stdout_chunks.append(stdout.channel.recv(len(c.in_buffer)))
|
||||
got_chunk = True
|
||||
|
||||
+147
-57
@@ -1,3 +1,4 @@
|
||||
import logging
|
||||
import os
|
||||
from contextlib import contextmanager
|
||||
from functools import wraps
|
||||
@@ -5,17 +6,26 @@ from functools import wraps
|
||||
from funcy import cached_property, cat, first
|
||||
|
||||
from dvc.config import Config
|
||||
from dvc.dvcfile import PIPELINE_FILE, Dvcfile, is_valid_filename
|
||||
from dvc.exceptions import FileMissingError
|
||||
from dvc.exceptions import IsADirectoryError as DvcIsADirectoryError
|
||||
from dvc.exceptions import NotDvcRepoError, OutputNotFoundError
|
||||
from dvc.exceptions import (
|
||||
NoOutputOrStageError,
|
||||
NotDvcRepoError,
|
||||
OutputNotFoundError,
|
||||
)
|
||||
from dvc.ignore import CleanTree
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.repo.tree import RepoTree
|
||||
from dvc.scm.tree import is_working_tree
|
||||
from dvc.utils.fs import path_isin
|
||||
|
||||
from ..stage.exceptions import StageFileDoesNotExistError, StageNotFound
|
||||
from ..utils import parse_target
|
||||
from .graph import check_acyclic, get_pipeline, get_pipelines
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def locked(f):
|
||||
@wraps(f)
|
||||
@@ -38,7 +48,7 @@ class Repo:
|
||||
from dvc.repo.add import add
|
||||
from dvc.repo.remove import remove
|
||||
from dvc.repo.ls import ls
|
||||
from dvc.repo.lock import lock as lock_stage
|
||||
from dvc.repo.freeze import freeze, unfreeze
|
||||
from dvc.repo.move import move
|
||||
from dvc.repo.run import run
|
||||
from dvc.repo.imp import imp
|
||||
@@ -57,8 +67,8 @@ class Repo:
|
||||
from dvc.repo.get_url import get_url
|
||||
from dvc.repo.update import update
|
||||
|
||||
def __init__(self, root_dir=None):
|
||||
from dvc.state import State
|
||||
def __init__(self, root_dir=None, scm=None, rev=None):
|
||||
from dvc.state import State, StateNoop
|
||||
from dvc.lock import make_lock
|
||||
from dvc.scm import SCM
|
||||
from dvc.cache import Cache
|
||||
@@ -70,17 +80,24 @@ class Repo:
|
||||
from dvc.utils.fs import makedirs
|
||||
from dvc.stage.cache import StageCache
|
||||
|
||||
root_dir = self.find_root(root_dir)
|
||||
if scm:
|
||||
# use GitTree instead of WorkingTree as default repo tree instance
|
||||
tree = scm.get_tree(rev)
|
||||
self.root_dir = self.find_root(root_dir, tree)
|
||||
self.scm = scm
|
||||
self.tree = tree
|
||||
self.state = StateNoop()
|
||||
else:
|
||||
root_dir = self.find_root(root_dir)
|
||||
self.root_dir = os.path.abspath(os.path.realpath(root_dir))
|
||||
|
||||
self.root_dir = os.path.abspath(os.path.realpath(root_dir))
|
||||
self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR)
|
||||
|
||||
self.config = Config(self.dvc_dir)
|
||||
|
||||
no_scm = self.config["core"].get("no_scm", False)
|
||||
self.scm = SCM(self.root_dir, no_scm=no_scm)
|
||||
|
||||
self.tree = WorkingTree(self.root_dir)
|
||||
if not scm:
|
||||
no_scm = self.config["core"].get("no_scm", False)
|
||||
self.scm = SCM(self.root_dir, no_scm=no_scm)
|
||||
self.tree = WorkingTree(self.root_dir)
|
||||
|
||||
self.tmp_dir = os.path.join(self.dvc_dir, "tmp")
|
||||
self.index_dir = os.path.join(self.tmp_dir, "index")
|
||||
@@ -94,13 +111,15 @@ class Repo:
|
||||
friendly=True,
|
||||
)
|
||||
|
||||
# NOTE: storing state and link_state in the repository itself to avoid
|
||||
# any possible state corruption in 'shared cache dir' scenario.
|
||||
self.state = State(self)
|
||||
|
||||
self.cache = Cache(self)
|
||||
self.cloud = DataCloud(self)
|
||||
|
||||
if not scm:
|
||||
# NOTE: storing state and link_state in the repository itself to
|
||||
# avoid any possible state corruption in 'shared cache dir'
|
||||
# scenario.
|
||||
self.state = State(self.cache.local)
|
||||
|
||||
self.stage_cache = StageCache(self)
|
||||
|
||||
self.metrics = Metrics(self)
|
||||
@@ -115,7 +134,13 @@ class Repo:
|
||||
|
||||
@tree.setter
|
||||
def tree(self, tree):
|
||||
self._tree = tree if isinstance(tree, CleanTree) else CleanTree(tree)
|
||||
if is_working_tree(tree) or tree.tree_root == self.root_dir:
|
||||
root = None
|
||||
else:
|
||||
root = self.root_dir
|
||||
self._tree = (
|
||||
tree if isinstance(tree, CleanTree) else CleanTree(tree, root)
|
||||
)
|
||||
# Our graph cache is no longer valid, as it was based on the previous
|
||||
# tree.
|
||||
self._reset()
|
||||
@@ -124,9 +149,14 @@ class Repo:
|
||||
return f"{self.__class__.__name__}: '{self.root_dir}'"
|
||||
|
||||
@classmethod
|
||||
def find_root(cls, root=None):
|
||||
def find_root(cls, root=None, tree=None):
|
||||
root_dir = os.path.realpath(root or os.curdir)
|
||||
|
||||
if tree:
|
||||
if tree.isdir(os.path.join(root_dir, cls.DVC_DIR)):
|
||||
return root_dir
|
||||
raise NotDvcRepoError(f"'{root}' does not contain DVC directory")
|
||||
|
||||
if not os.path.isdir(root_dir):
|
||||
raise NotDvcRepoError(f"directory '{root}' does not exist")
|
||||
|
||||
@@ -157,7 +187,7 @@ class Repo:
|
||||
return Repo(root_dir)
|
||||
|
||||
def unprotect(self, target):
|
||||
return self.cache.local.unprotect(PathInfo(target))
|
||||
return self.cache.local.tree.unprotect(PathInfo(target))
|
||||
|
||||
def _ignore(self):
|
||||
flist = [self.config.files["local"], self.tmp_dir]
|
||||
@@ -167,6 +197,25 @@ class Repo:
|
||||
|
||||
self.scm.ignore_list(flist)
|
||||
|
||||
def get_stage(self, path=None, name=None):
|
||||
if not path:
|
||||
path = PIPELINE_FILE
|
||||
logger.debug("Assuming '%s' to be a stage inside '%s'", name, path)
|
||||
|
||||
dvcfile = Dvcfile(self, path)
|
||||
return dvcfile.stages[name]
|
||||
|
||||
def get_stages(self, path=None, name=None):
|
||||
if not path:
|
||||
path = PIPELINE_FILE
|
||||
logger.debug("Assuming '%s' to be a stage inside '%s'", name, path)
|
||||
|
||||
if name:
|
||||
return [self.get_stage(path, name)]
|
||||
|
||||
dvcfile = Dvcfile(self, path)
|
||||
return list(dvcfile.stages.values())
|
||||
|
||||
def check_modified_graph(self, new_stages):
|
||||
"""Generate graph including the new stage to check for errors"""
|
||||
# Building graph might be costly for the ones with many DVC-files,
|
||||
@@ -190,10 +239,9 @@ class Repo:
|
||||
stages = nx.dfs_postorder_nodes(graph)
|
||||
return [stage for stage in stages if path_isin(stage.path, path)]
|
||||
|
||||
def collect(self, target, with_deps=False, recursive=False, graph=None):
|
||||
import networkx as nx
|
||||
from ..dvcfile import Dvcfile
|
||||
|
||||
def collect(
|
||||
self, target=None, with_deps=False, recursive=False, graph=None
|
||||
):
|
||||
if not target:
|
||||
return list(graph) if graph else self.stages
|
||||
|
||||
@@ -203,36 +251,81 @@ class Repo:
|
||||
)
|
||||
|
||||
path, name = parse_target(target)
|
||||
dvcfile = Dvcfile(self, path)
|
||||
stages = list(dvcfile.stages.filter(name).values())
|
||||
stages = self.get_stages(path, name)
|
||||
if not with_deps:
|
||||
return stages
|
||||
|
||||
res = set()
|
||||
for stage in stages:
|
||||
pipeline = get_pipeline(get_pipelines(graph or self.graph), stage)
|
||||
res.update(nx.dfs_postorder_nodes(pipeline, stage))
|
||||
res.update(self._collect_pipeline(stage, graph=graph))
|
||||
return res
|
||||
|
||||
def collect_granular(self, target, *args, **kwargs):
|
||||
from ..dvcfile import Dvcfile, is_valid_filename
|
||||
def _collect_pipeline(self, stage, graph=None):
|
||||
import networkx as nx
|
||||
|
||||
pipeline = get_pipeline(get_pipelines(graph or self.graph), stage)
|
||||
return nx.dfs_postorder_nodes(pipeline, stage)
|
||||
|
||||
def _collect_from_default_dvcfile(self, target):
|
||||
dvcfile = Dvcfile(self, PIPELINE_FILE)
|
||||
if dvcfile.exists():
|
||||
return dvcfile.stages.get(target)
|
||||
|
||||
def collect_granular(
|
||||
self, target=None, with_deps=False, recursive=False, graph=None
|
||||
):
|
||||
"""
|
||||
Priority is in the order of following in case of ambiguity:
|
||||
- .dvc file or .yaml file
|
||||
- dir if recursive and directory exists
|
||||
- stage_name
|
||||
- output file
|
||||
"""
|
||||
if not target:
|
||||
return [(stage, None) for stage in self.stages]
|
||||
|
||||
file, name = parse_target(target)
|
||||
if is_valid_filename(file) and not kwargs.get("with_deps"):
|
||||
# Optimization: do not collect the graph for a specific .dvc target
|
||||
stages = Dvcfile(self, file).stages.filter(name)
|
||||
return [(stage, None) for stage in stages.values()]
|
||||
stages = []
|
||||
|
||||
try:
|
||||
(out,) = self.find_outs_by_path(file, strict=False)
|
||||
filter_info = PathInfo(os.path.abspath(file))
|
||||
return [(out.stage, filter_info)]
|
||||
except OutputNotFoundError:
|
||||
stages = self.collect(target, *args, **kwargs)
|
||||
return [(stage, None) for stage in stages]
|
||||
# Optimization: do not collect the graph for a specific target
|
||||
if not file:
|
||||
# parsing is ambiguous when it does not have a colon
|
||||
# or if it's not a dvcfile, as it can be a stage name
|
||||
# in `dvc.yaml` or, an output in a stage.
|
||||
logger.debug(
|
||||
"Checking if stage '%s' is in '%s'", target, PIPELINE_FILE
|
||||
)
|
||||
if not (recursive and os.path.isdir(target)):
|
||||
stage = self._collect_from_default_dvcfile(target)
|
||||
if stage:
|
||||
stages = (
|
||||
self._collect_pipeline(stage) if with_deps else [stage]
|
||||
)
|
||||
elif not with_deps and is_valid_filename(file):
|
||||
stages = self.get_stages(file, name)
|
||||
|
||||
if not stages:
|
||||
if not (recursive and os.path.isdir(target)):
|
||||
try:
|
||||
(out,) = self.find_outs_by_path(target, strict=False)
|
||||
filter_info = PathInfo(os.path.abspath(target))
|
||||
return [(out.stage, filter_info)]
|
||||
except OutputNotFoundError:
|
||||
pass
|
||||
|
||||
try:
|
||||
stages = self.collect(target, with_deps, recursive, graph)
|
||||
except StageFileDoesNotExistError as exc:
|
||||
# collect() might try to use `target` as a stage name
|
||||
# and throw error that dvc.yaml does not exist, whereas it
|
||||
# should say that both stage name and file does not exist.
|
||||
if file and is_valid_filename(file):
|
||||
raise
|
||||
raise NoOutputOrStageError(target, exc.file) from exc
|
||||
except StageNotFound as exc:
|
||||
raise NoOutputOrStageError(target, exc.file) from exc
|
||||
|
||||
return [(stage, None) for stage in stages]
|
||||
|
||||
def used_cache(
|
||||
self,
|
||||
@@ -348,7 +441,7 @@ class Repo:
|
||||
stages = stages or self.stages
|
||||
outs = Trie() # Use trie to efficiently find overlapping outs and deps
|
||||
|
||||
for stage in filter(bool, stages):
|
||||
for stage in filter(bool, stages): # bug? not using it later
|
||||
for out in stage.outs:
|
||||
out_key = out.path_info.parts
|
||||
|
||||
@@ -429,22 +522,18 @@ class Repo:
|
||||
return PlotTemplates(self.dvc_dir)
|
||||
|
||||
def _collect_stages(self):
|
||||
from dvc.dvcfile import Dvcfile, is_valid_filename
|
||||
|
||||
stages = []
|
||||
outs = set()
|
||||
|
||||
for root, dirs, files in self.tree.walk(self.root_dir):
|
||||
for file_name in filter(is_valid_filename, files):
|
||||
path = os.path.join(root, file_name)
|
||||
stage_loader = Dvcfile(self, path).stages
|
||||
stages.extend(stage_loader.values())
|
||||
new_stages = self.get_stages(os.path.join(root, file_name))
|
||||
stages.extend(new_stages)
|
||||
outs.update(
|
||||
out.fspath
|
||||
for stage in stages
|
||||
for out in (
|
||||
out for out in stage.outs if out.scheme == "local"
|
||||
)
|
||||
for stage in new_stages
|
||||
for out in stage.outs
|
||||
if out.scheme == "local"
|
||||
)
|
||||
dirs[:] = [d for d in dirs if os.path.join(root, d) not in outs]
|
||||
return stages
|
||||
@@ -488,17 +577,18 @@ class Repo:
|
||||
tree = RepoTree(self, stream=True)
|
||||
path = os.path.join(self.root_dir, path)
|
||||
try:
|
||||
with tree.open(
|
||||
os.path.join(self.root_dir, path),
|
||||
mode=mode,
|
||||
encoding=encoding,
|
||||
remote=remote,
|
||||
) as fobj:
|
||||
yield fobj
|
||||
with self.state:
|
||||
with tree.open(
|
||||
os.path.join(self.root_dir, path),
|
||||
mode=mode,
|
||||
encoding=encoding,
|
||||
remote=remote,
|
||||
) as fobj:
|
||||
yield fobj
|
||||
except FileNotFoundError as exc:
|
||||
raise FileMissingError(path) from exc
|
||||
except IsADirectoryError as exc:
|
||||
raise DvcIsADirectoryError from exc
|
||||
raise DvcIsADirectoryError(f"'{path}' is a directory") from exc
|
||||
|
||||
def close(self):
|
||||
self.scm.close()
|
||||
|
||||
+16
-5
@@ -21,7 +21,9 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
@locked
|
||||
@scm_context
|
||||
def add(repo, targets, recursive=False, no_commit=False, fname=None):
|
||||
def add(
|
||||
repo, targets, recursive=False, no_commit=False, fname=None, external=False
|
||||
):
|
||||
if recursive and fname:
|
||||
raise RecursiveAddingWhileUsingFilename()
|
||||
|
||||
@@ -52,7 +54,9 @@ def add(repo, targets, recursive=False, no_commit=False, fname=None):
|
||||
)
|
||||
)
|
||||
|
||||
stages = _create_stages(repo, sub_targets, fname, pbar=pbar)
|
||||
stages = _create_stages(
|
||||
repo, sub_targets, fname, pbar=pbar, external=external
|
||||
)
|
||||
|
||||
try:
|
||||
repo.check_modified_graph(stages)
|
||||
@@ -120,7 +124,7 @@ def _find_all_targets(repo, target, recursive):
|
||||
return [target]
|
||||
|
||||
|
||||
def _create_stages(repo, targets, fname, pbar=None):
|
||||
def _create_stages(repo, targets, fname, pbar=None, external=False):
|
||||
from dvc.stage import Stage, create_stage
|
||||
|
||||
stages = []
|
||||
@@ -132,9 +136,16 @@ def _create_stages(repo, targets, fname, pbar=None):
|
||||
unit="file",
|
||||
):
|
||||
path, wdir, out = resolve_paths(repo, out)
|
||||
stage = create_stage(Stage, repo, fname or path, wdir=wdir, outs=[out])
|
||||
stage = create_stage(
|
||||
Stage,
|
||||
repo,
|
||||
fname or path,
|
||||
wdir=wdir,
|
||||
outs=[out],
|
||||
external=external,
|
||||
)
|
||||
if stage:
|
||||
Dvcfile(repo, stage.path).remove_with_prompt(force=True)
|
||||
Dvcfile(repo, stage.path).remove()
|
||||
|
||||
repo._reset()
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@ def brancher( # noqa: E302
|
||||
str: the display name for the currently selected tree, it could be:
|
||||
- a git revision identifier
|
||||
- empty string it there is no branches to iterate over
|
||||
- "Working Tree" if there are uncommitted changes in the SCM repo
|
||||
- "workspace" if there are uncommitted changes in the SCM repo
|
||||
"""
|
||||
if not any([revs, all_branches, all_tags, all_commits]):
|
||||
yield ""
|
||||
@@ -30,10 +30,10 @@ def brancher( # noqa: E302
|
||||
scm = self.scm
|
||||
|
||||
self.tree = WorkingTree(self.root_dir)
|
||||
yield "working tree"
|
||||
yield "workspace"
|
||||
|
||||
if revs and "working tree" in revs:
|
||||
revs.remove("working tree")
|
||||
if revs and "workspace" in revs:
|
||||
revs.remove("workspace")
|
||||
|
||||
if all_commits:
|
||||
revs = scm.list_all_commits()
|
||||
@@ -48,6 +48,9 @@ def brancher( # noqa: E302
|
||||
if revs:
|
||||
for sha, names in group_by(scm.resolve_rev, revs).items():
|
||||
self.tree = scm.get_tree(sha)
|
||||
yield ", ".join(names)
|
||||
# ignore revs that don't contain repo root
|
||||
# (i.e. revs from before a subdir=True repo was init'ed)
|
||||
if self.tree.exists(self.root_dir):
|
||||
yield ", ".join(names)
|
||||
finally:
|
||||
self.tree = saved_tree
|
||||
|
||||
+14
-8
@@ -1,7 +1,11 @@
|
||||
import logging
|
||||
import os
|
||||
|
||||
from dvc.exceptions import CheckoutError, CheckoutErrorSuggestGit
|
||||
from dvc.exceptions import (
|
||||
CheckoutError,
|
||||
CheckoutErrorSuggestGit,
|
||||
NoOutputOrStageError,
|
||||
)
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.utils import relpath
|
||||
|
||||
@@ -18,11 +22,11 @@ def _get_unused_links(repo):
|
||||
return repo.state.get_unused_links(used)
|
||||
|
||||
|
||||
def _fspath_dir(path, root):
|
||||
def _fspath_dir(path):
|
||||
if not os.path.exists(str(path)):
|
||||
return str(path)
|
||||
|
||||
path = relpath(path, root)
|
||||
path = relpath(path)
|
||||
return os.path.join(path, "") if os.path.isdir(path) else path
|
||||
|
||||
|
||||
@@ -56,7 +60,7 @@ def _checkout(
|
||||
targets = [None]
|
||||
unused = _get_unused_links(self)
|
||||
|
||||
stats["deleted"] = [_fspath_dir(u, self.root_dir) for u in unused]
|
||||
stats["deleted"] = [_fspath_dir(u) for u in unused]
|
||||
self.state.remove_links(unused)
|
||||
|
||||
if isinstance(targets, str):
|
||||
@@ -70,7 +74,11 @@ def _checkout(
|
||||
target, with_deps=with_deps, recursive=recursive
|
||||
)
|
||||
)
|
||||
except (StageFileDoesNotExistError, StageFileBadNameError) as exc:
|
||||
except (
|
||||
StageFileDoesNotExistError,
|
||||
StageFileBadNameError,
|
||||
NoOutputOrStageError,
|
||||
) as exc:
|
||||
if not target:
|
||||
raise
|
||||
raise CheckoutErrorSuggestGit(target) from exc
|
||||
@@ -87,9 +95,7 @@ def _checkout(
|
||||
filter_info=filter_info,
|
||||
)
|
||||
for key, items in result.items():
|
||||
stats[key].extend(
|
||||
_fspath_dir(path, self.root_dir) for path in items
|
||||
)
|
||||
stats[key].extend(_fspath_dir(path) for path in items)
|
||||
|
||||
if stats.get("failed"):
|
||||
raise CheckoutError(stats["failed"], stats)
|
||||
|
||||
@@ -46,3 +46,4 @@ def commit(self, target, with_deps=False, recursive=False, force=False):
|
||||
stage.commit()
|
||||
|
||||
Dvcfile(self, stage.path).dump(stage)
|
||||
return stages
|
||||
|
||||
+2
-2
@@ -7,7 +7,7 @@ from dvc.scm.tree import is_working_tree
|
||||
@locked
|
||||
def diff(self, a_rev="HEAD", b_rev=None):
|
||||
"""
|
||||
By default, it compares the working tree with the last commit's tree.
|
||||
By default, it compares the workspace with the last commit's tree.
|
||||
|
||||
This implementation differs from `git diff` since DVC doesn't have
|
||||
the concept of `index`, but it keeps the same interface, thus,
|
||||
@@ -37,7 +37,7 @@ def diff(self, a_rev="HEAD", b_rev=None):
|
||||
|
||||
def _to_checksum(output):
|
||||
if on_working_tree:
|
||||
return self.cache.local.get_checksum(output.path_info)
|
||||
return self.cache.local.get_hash(output.path_info)
|
||||
return output.checksum
|
||||
|
||||
def _exists(output):
|
||||
|
||||
+7
-53
@@ -1,9 +1,7 @@
|
||||
import logging
|
||||
|
||||
from dvc.cache import NamedCache
|
||||
from dvc.config import NoRemoteError
|
||||
from dvc.exceptions import DownloadError, OutputNotFoundError
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.exceptions import DownloadError
|
||||
from dvc.scm.base import CloneError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -74,39 +72,19 @@ def _fetch(
|
||||
if failed:
|
||||
raise DownloadError(failed)
|
||||
|
||||
return downloaded
|
||||
return downloaded + len(used_run_cache)
|
||||
|
||||
|
||||
def _fetch_external(self, repo_url, repo_rev, files, jobs):
|
||||
from dvc.external_repo import external_repo, ExternalRepo
|
||||
from dvc.external_repo import external_repo
|
||||
|
||||
failed, downloaded = 0, 0
|
||||
try:
|
||||
with external_repo(repo_url, repo_rev) as repo:
|
||||
is_dvc_repo = isinstance(repo, ExternalRepo)
|
||||
# gather git-only tracked files if dvc repo
|
||||
git_files = [] if is_dvc_repo else files
|
||||
if is_dvc_repo:
|
||||
repo.cache.local.cache_dir = self.cache.local.cache_dir
|
||||
with repo.state:
|
||||
cache = NamedCache()
|
||||
for name in files:
|
||||
try:
|
||||
out = repo.find_out_by_relpath(name)
|
||||
except OutputNotFoundError:
|
||||
# try to add to cache if they are git-tracked files
|
||||
git_files.append(name)
|
||||
else:
|
||||
cache.update(out.get_used_cache())
|
||||
|
||||
try:
|
||||
downloaded += repo.cloud.pull(cache, jobs=jobs)
|
||||
except DownloadError as exc:
|
||||
failed += exc.amount
|
||||
|
||||
d, f = _git_to_cache(self.cache.local, repo.root_dir, git_files)
|
||||
downloaded += d
|
||||
failed += f
|
||||
with repo.use_cache(self.cache.local):
|
||||
d, f, _ = repo.fetch_external(files, jobs=jobs)
|
||||
downloaded += d
|
||||
failed += f
|
||||
except CloneError:
|
||||
failed += 1
|
||||
logger.exception(
|
||||
@@ -114,27 +92,3 @@ def _fetch_external(self, repo_url, repo_rev, files, jobs):
|
||||
)
|
||||
|
||||
return downloaded, failed
|
||||
|
||||
|
||||
def _git_to_cache(cache, repo_root, files):
|
||||
"""Save files from a git repo directly to the cache."""
|
||||
failed = set()
|
||||
num_downloads = 0
|
||||
repo_root = PathInfo(repo_root)
|
||||
for file in files:
|
||||
info = cache.save_info(repo_root / file)
|
||||
if info.get(cache.PARAM_CHECKSUM) is None:
|
||||
failed.add(file)
|
||||
continue
|
||||
|
||||
if cache.changed_cache(info[cache.PARAM_CHECKSUM]):
|
||||
logger.debug("fetched '%s' from '%s' repo", file, repo_root)
|
||||
num_downloads += 1
|
||||
cache.save(repo_root / file, info, save_link=False)
|
||||
|
||||
if failed:
|
||||
logger.exception(
|
||||
"failed to fetch data for {}".format(", ".join(failed))
|
||||
)
|
||||
|
||||
return num_downloads, len(failed)
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
from . import locked
|
||||
|
||||
|
||||
@locked
|
||||
def _set(repo, target, frozen):
|
||||
from dvc.utils import parse_target
|
||||
|
||||
path, name = parse_target(target)
|
||||
stage = repo.get_stage(path, name)
|
||||
stage.frozen = frozen
|
||||
stage.dvcfile.dump(stage, update_pipeline=True)
|
||||
|
||||
return stage
|
||||
|
||||
|
||||
def freeze(repo, target):
|
||||
return _set(repo, target, True)
|
||||
|
||||
|
||||
def unfreeze(repo, target):
|
||||
return _set(repo, target, False)
|
||||
+12
-9
@@ -8,8 +8,10 @@ from . import locked
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _do_gc(typ, func, clist, jobs=None):
|
||||
removed = func(clist, jobs=jobs)
|
||||
def _do_gc(typ, remote, clist, jobs=None):
|
||||
from dvc.remote.base import Remote
|
||||
|
||||
removed = Remote.gc(clist, remote, jobs=jobs)
|
||||
if not removed:
|
||||
logger.info(f"No unused '{typ}' cache to remove.")
|
||||
|
||||
@@ -74,22 +76,23 @@ def gc(
|
||||
)
|
||||
)
|
||||
|
||||
_do_gc("local", self.cache.local.gc, used, jobs)
|
||||
# treat caches as remotes for garbage collection
|
||||
_do_gc("local", self.cache.local, used, jobs)
|
||||
|
||||
if self.cache.s3:
|
||||
_do_gc("s3", self.cache.s3.gc, used, jobs)
|
||||
_do_gc("s3", self.cache.s3, used, jobs)
|
||||
|
||||
if self.cache.gs:
|
||||
_do_gc("gs", self.cache.gs.gc, used, jobs)
|
||||
_do_gc("gs", self.cache.gs, used, jobs)
|
||||
|
||||
if self.cache.ssh:
|
||||
_do_gc("ssh", self.cache.ssh.gc, used, jobs)
|
||||
_do_gc("ssh", self.cache.ssh, used, jobs)
|
||||
|
||||
if self.cache.hdfs:
|
||||
_do_gc("hdfs", self.cache.hdfs.gc, used, jobs)
|
||||
_do_gc("hdfs", self.cache.hdfs, used, jobs)
|
||||
|
||||
if self.cache.azure:
|
||||
_do_gc("azure", self.cache.azure.gc, used, jobs)
|
||||
_do_gc("azure", self.cache.azure, used, jobs)
|
||||
|
||||
if cloud:
|
||||
_do_gc("remote", self.cloud.get_remote(remote, "gc -c").gc, used, jobs)
|
||||
_do_gc("remote", self.cloud.get_remote(remote, "gc -c"), used, jobs)
|
||||
|
||||
+1
-3
@@ -4,7 +4,6 @@ import os
|
||||
import shortuuid
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.utils import resolve_output
|
||||
from dvc.utils.fs import remove
|
||||
|
||||
@@ -51,7 +50,6 @@ def get(url, path, out=None, rev=None):
|
||||
# Also, we can't use theoretical "move" link type here, because
|
||||
# the same cache file might be used a few times in a directory.
|
||||
repo.cache.local.cache_types = ["reflink", "hardlink", "copy"]
|
||||
|
||||
repo.pull_to(path, PathInfo(out))
|
||||
repo.get_external(path, out)
|
||||
finally:
|
||||
remove(tmp_dir)
|
||||
|
||||
@@ -16,4 +16,6 @@ def get_url(url, out=None):
|
||||
|
||||
(dep,) = dependency.loads_from(None, [url])
|
||||
(out,) = output.loads_from(None, [out], use_cache=False)
|
||||
dep.save()
|
||||
dep.download(out)
|
||||
out.save()
|
||||
|
||||
+1
-1
@@ -3,4 +3,4 @@ def imp(self, url, path, out=None, rev=None):
|
||||
if rev is not None:
|
||||
erepo["rev"] = rev
|
||||
|
||||
return self.imp_url(path, out=out, erepo=erepo, locked=True)
|
||||
return self.imp_url(path, out=out, erepo=erepo, frozen=True)
|
||||
|
||||
+5
-5
@@ -5,12 +5,12 @@ from dvc.utils import relpath, resolve_output, resolve_paths
|
||||
from dvc.utils.fs import path_isin
|
||||
|
||||
from ..exceptions import OutputDuplicationError
|
||||
from . import locked as locked_repo
|
||||
from . import locked
|
||||
|
||||
|
||||
@locked_repo
|
||||
@locked
|
||||
@scm_context
|
||||
def imp_url(self, url, out=None, fname=None, erepo=None, locked=True):
|
||||
def imp_url(self, url, out=None, fname=None, erepo=None, frozen=True):
|
||||
from dvc.dvcfile import Dvcfile
|
||||
from dvc.stage import Stage, create_stage
|
||||
|
||||
@@ -39,7 +39,7 @@ def imp_url(self, url, out=None, fname=None, erepo=None, locked=True):
|
||||
return None
|
||||
|
||||
dvcfile = Dvcfile(self, stage.path)
|
||||
dvcfile.remove_with_prompt(force=True)
|
||||
dvcfile.remove()
|
||||
|
||||
try:
|
||||
self.check_modified_graph([stage])
|
||||
@@ -48,7 +48,7 @@ def imp_url(self, url, out=None, fname=None, erepo=None, locked=True):
|
||||
|
||||
stage.run()
|
||||
|
||||
stage.locked = locked
|
||||
stage.frozen = frozen
|
||||
|
||||
dvcfile.dump(stage)
|
||||
|
||||
|
||||
@@ -1,15 +0,0 @@
|
||||
from . import locked
|
||||
|
||||
|
||||
@locked
|
||||
def lock(self, target, unlock=False):
|
||||
from .. import dvcfile
|
||||
from dvc.utils import parse_target
|
||||
|
||||
path, name = parse_target(target)
|
||||
dvcfile = dvcfile.Dvcfile(self, path)
|
||||
stage = dvcfile.stages[name]
|
||||
stage.locked = False if unlock else True
|
||||
dvcfile.dump(stage, update_pipeline=True)
|
||||
|
||||
return stage
|
||||
+37
-40
@@ -4,7 +4,7 @@ from dvc.path_info import PathInfo
|
||||
|
||||
@staticmethod
|
||||
def ls(
|
||||
url, path=None, rev=None, recursive=None, outs_only=False,
|
||||
url, path=None, rev=None, recursive=None, dvc_only=False,
|
||||
):
|
||||
"""Methods for getting files and outputs for the repo.
|
||||
|
||||
@@ -13,7 +13,7 @@ def ls(
|
||||
path (str, optional): relative path into the repo
|
||||
rev (str, optional): SHA commit, branch or tag name
|
||||
recursive (bool, optional): recursively walk the repo
|
||||
outs_only (bool, optional): show only DVC-artifacts
|
||||
dvc_only (bool, optional): show only DVC-artifacts
|
||||
|
||||
Returns:
|
||||
list of `entry`
|
||||
@@ -28,25 +28,16 @@ def ls(
|
||||
}
|
||||
"""
|
||||
from dvc.external_repo import external_repo
|
||||
from dvc.repo import Repo
|
||||
|
||||
with external_repo(url, rev) as repo:
|
||||
path_info = PathInfo(repo.root_dir)
|
||||
if path:
|
||||
path_info /= path
|
||||
|
||||
ret = {}
|
||||
if isinstance(repo, Repo):
|
||||
ret = _ls(repo, path_info, recursive, True)
|
||||
|
||||
nondvc = {}
|
||||
if not outs_only:
|
||||
nondvc = _ls(repo, path_info, recursive, False)
|
||||
|
||||
ret.update(nondvc)
|
||||
ret = _ls(repo, path_info, recursive, dvc_only)
|
||||
|
||||
if path and not ret:
|
||||
raise PathMissingError(path, repo, output_only=outs_only)
|
||||
raise PathMissingError(path, repo, dvc_only=dvc_only)
|
||||
|
||||
ret_list = []
|
||||
for path, info in ret.items():
|
||||
@@ -56,46 +47,52 @@ def ls(
|
||||
return ret_list
|
||||
|
||||
|
||||
def _ls(repo, path_info, recursive=None, dvc=False):
|
||||
from dvc.ignore import CleanTree
|
||||
from dvc.repo.tree import DvcTree
|
||||
from dvc.scm.tree import WorkingTree
|
||||
def _ls(repo, path_info, recursive=None, dvc_only=False):
|
||||
from dvc.repo.tree import RepoTree
|
||||
|
||||
if dvc:
|
||||
tree = DvcTree(repo)
|
||||
else:
|
||||
tree = CleanTree(WorkingTree(repo.root_dir))
|
||||
# use our own RepoTree instance instead of repo.repo_tree since we do not
|
||||
# want fetch/stream enabled for ls
|
||||
tree = RepoTree(repo)
|
||||
|
||||
ret = {}
|
||||
try:
|
||||
for root, dirs, files in tree.walk(path_info.fspath):
|
||||
for root, dirs, files in tree.walk(path_info.fspath, dvcfiles=True):
|
||||
for fname in files:
|
||||
info = PathInfo(root) / fname
|
||||
path = str(info.relative_to(path_info))
|
||||
ret[path] = {
|
||||
"isout": dvc,
|
||||
"isdir": False,
|
||||
"isexec": False if dvc else tree.isexec(info.fspath),
|
||||
}
|
||||
dvc = tree.isdvc(info)
|
||||
if dvc or not dvc_only:
|
||||
path = str(info.relative_to(path_info))
|
||||
ret[path] = {
|
||||
"isout": dvc,
|
||||
"isdir": False,
|
||||
"isexec": False if dvc else tree.isexec(info),
|
||||
}
|
||||
|
||||
if not recursive:
|
||||
for dname in dirs:
|
||||
info = PathInfo(root) / dname
|
||||
path = str(info.relative_to(path_info))
|
||||
ret[path] = {
|
||||
"isout": tree.isdvc(info.fspath) if dvc else False,
|
||||
"isdir": True,
|
||||
"isexec": False if dvc else tree.isexec(info.fspath),
|
||||
}
|
||||
if not dvc_only or (
|
||||
tree.dvctree and tree.dvctree.exists(info)
|
||||
):
|
||||
dvc = tree.isdvc(info)
|
||||
path = str(info.relative_to(path_info))
|
||||
ret[path] = {
|
||||
"isout": dvc,
|
||||
"isdir": True,
|
||||
"isexec": False if dvc else tree.isexec(info),
|
||||
}
|
||||
break
|
||||
except NotADirectoryError:
|
||||
return {
|
||||
path_info.name: {
|
||||
"isout": dvc,
|
||||
"isdir": False,
|
||||
"isexec": False if dvc else tree.isexec(path_info.fspath),
|
||||
dvc = tree.isdvc(path_info)
|
||||
if dvc or not dvc_only:
|
||||
return {
|
||||
path_info.name: {
|
||||
"isout": dvc,
|
||||
"isdir": False,
|
||||
"isexec": False if dvc else tree.isexec(path_info),
|
||||
}
|
||||
}
|
||||
}
|
||||
return {}
|
||||
except FileNotFoundError:
|
||||
return {}
|
||||
|
||||
|
||||
@@ -2,21 +2,11 @@ class Metrics:
|
||||
def __init__(self, repo):
|
||||
self.repo = repo
|
||||
|
||||
def add(self, *args, **kwargs):
|
||||
from dvc.repo.metrics.add import add
|
||||
|
||||
return add(self.repo, *args, **kwargs)
|
||||
|
||||
def show(self, *args, **kwargs):
|
||||
from dvc.repo.metrics.show import show
|
||||
|
||||
return show(self.repo, *args, **kwargs)
|
||||
|
||||
def remove(self, *args, **kwargs):
|
||||
from dvc.repo.metrics.remove import remove
|
||||
|
||||
return remove(self.repo, *args, **kwargs)
|
||||
|
||||
def diff(self, *args, **kwargs):
|
||||
from .diff import diff
|
||||
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
from dvc.repo.metrics.modify import modify
|
||||
|
||||
|
||||
def add(repo, path):
|
||||
modify(repo, path)
|
||||
@@ -1,22 +0,0 @@
|
||||
from dvc.dvcfile import Dvcfile
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.repo import locked
|
||||
|
||||
|
||||
@locked
|
||||
def modify(repo, path, delete=False):
|
||||
outs = repo.find_outs_by_path(path)
|
||||
assert len(outs) == 1
|
||||
out = outs[0]
|
||||
|
||||
if out.scheme != "local":
|
||||
msg = "output '{}' scheme '{}' is not supported for metrics"
|
||||
raise DvcException(msg.format(out.path, out.path_info.scheme))
|
||||
|
||||
if delete:
|
||||
out.metric = None
|
||||
|
||||
out.verify_metric()
|
||||
|
||||
dvcfile = Dvcfile(repo, out.stage.path)
|
||||
dvcfile.dump(out.stage)
|
||||
@@ -1,5 +0,0 @@
|
||||
from dvc.repo.metrics.modify import modify
|
||||
|
||||
|
||||
def remove(repo, path):
|
||||
modify(repo, path, delete=True)
|
||||
@@ -104,13 +104,13 @@ def show(
|
||||
if not res:
|
||||
raise NoMetricsError()
|
||||
|
||||
# Hide working tree metrics if they are the same as in the active branch
|
||||
# Hide workspace metrics if they are the same as in the active branch
|
||||
try:
|
||||
active_branch = repo.scm.active_branch()
|
||||
except TypeError:
|
||||
pass # Detached head
|
||||
else:
|
||||
if res.get("working tree") == res.get(active_branch):
|
||||
res.pop("working tree", None)
|
||||
if res.get("workspace") == res.get(active_branch):
|
||||
res.pop("workspace", None)
|
||||
|
||||
return res
|
||||
|
||||
@@ -58,13 +58,13 @@ def show(repo, revs=None):
|
||||
if not res:
|
||||
raise NoParamsError("no parameter configs files in this repository")
|
||||
|
||||
# Hide working tree params if they are the same as in the active branch
|
||||
# Hide workspace params if they are the same as in the active branch
|
||||
try:
|
||||
active_branch = repo.scm.active_branch()
|
||||
except TypeError:
|
||||
pass # Detached head
|
||||
else:
|
||||
if res.get("working tree") == res.get(active_branch):
|
||||
res.pop("working tree", None)
|
||||
if res.get("workspace") == res.get(active_branch):
|
||||
res.pop("workspace", None)
|
||||
|
||||
return res
|
||||
|
||||
+237
-3
@@ -1,13 +1,247 @@
|
||||
import logging
|
||||
|
||||
from funcy import first, project
|
||||
|
||||
from dvc.exceptions import DvcException, NoPlotsError, OutputNotFoundError
|
||||
from dvc.repo.tree import RepoTree
|
||||
from dvc.schema import PLOT_PROPS
|
||||
from dvc.utils import relpath
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class NotAPlotError(DvcException):
|
||||
def __init__(self, out):
|
||||
super().__init__(
|
||||
f"'{out}' is not a plot. Use `dvc plots modify` to change that."
|
||||
)
|
||||
|
||||
|
||||
class PropsNotFoundError(DvcException):
|
||||
pass
|
||||
|
||||
|
||||
class Plots:
|
||||
def __init__(self, repo):
|
||||
self.repo = repo
|
||||
|
||||
def show(self, *args, **kwargs):
|
||||
from .show import show
|
||||
def collect(self, targets=None, revs=None):
|
||||
"""Collects all props and data for plots.
|
||||
|
||||
return show(self.repo, *args, **kwargs)
|
||||
Returns a structure like:
|
||||
{rev: {plots.csv: {
|
||||
props: {x: ..., "header": ..., ...},
|
||||
data: "...data as a string...",
|
||||
}}}
|
||||
Data parsing is postponed, since it's affected by props.
|
||||
"""
|
||||
targets = [targets] if isinstance(targets, str) else targets or []
|
||||
data = {}
|
||||
for rev in self.repo.brancher(revs=revs):
|
||||
# .brancher() adds unwanted workspace
|
||||
if revs is not None and rev not in revs:
|
||||
continue
|
||||
rev = rev or "workspace"
|
||||
|
||||
tree = RepoTree(self.repo)
|
||||
plots = _collect_plots(self.repo, targets, rev)
|
||||
for path_info, props in plots.items():
|
||||
datafile = relpath(path_info, self.repo.root_dir)
|
||||
if rev not in data:
|
||||
data[rev] = {}
|
||||
data[rev].update({datafile: {"props": props}})
|
||||
|
||||
# Load data from git or dvc cache
|
||||
try:
|
||||
with tree.open(path_info) as fd:
|
||||
data[rev][datafile]["data"] = fd.read()
|
||||
except FileNotFoundError as e:
|
||||
# This might happen simply because cache is absent
|
||||
print(e)
|
||||
pass
|
||||
|
||||
return data
|
||||
|
||||
@staticmethod
|
||||
def render(data, revs=None, props=None, templates=None):
|
||||
"""Renders plots"""
|
||||
props = props or {}
|
||||
|
||||
# Merge data by plot file and apply overriding props
|
||||
plots = _prepare_plots(data, revs, props)
|
||||
|
||||
return {
|
||||
datafile: _render(datafile, desc["data"], desc["props"], templates)
|
||||
for datafile, desc in plots.items()
|
||||
}
|
||||
|
||||
def show(self, targets=None, revs=None, props=None):
|
||||
from .data import NoMetricInHistoryError
|
||||
|
||||
data = self.collect(targets, revs)
|
||||
|
||||
# If any mentioned plot doesn't have any data then that's an error
|
||||
targets = [targets] if isinstance(targets, str) else targets or []
|
||||
for target in targets:
|
||||
if not any("data" in d[target] for d in data.values()):
|
||||
raise NoMetricInHistoryError(target)
|
||||
|
||||
# No data at all is a special error with a special message
|
||||
if not data:
|
||||
raise NoPlotsError()
|
||||
|
||||
return self.render(data, revs, props, self.repo.plot_templates)
|
||||
|
||||
def diff(self, *args, **kwargs):
|
||||
from .diff import diff
|
||||
|
||||
return diff(self.repo, *args, **kwargs)
|
||||
|
||||
@staticmethod
|
||||
def _unset(out, props):
|
||||
missing = list(set(props) - set(out.plot.keys()))
|
||||
if missing:
|
||||
raise PropsNotFoundError(
|
||||
f"display properties {missing} not found in plot '{out}'"
|
||||
)
|
||||
|
||||
for prop in props:
|
||||
out.plot.pop(prop)
|
||||
|
||||
def modify(self, path, props=None, unset=None):
|
||||
from dvc.dvcfile import Dvcfile
|
||||
|
||||
props = props or {}
|
||||
template = props.get("template")
|
||||
if template:
|
||||
self.repo.plot_templates.get_template(template)
|
||||
|
||||
(out,) = self.repo.find_outs_by_path(path)
|
||||
if not out.plot and unset is not None:
|
||||
raise NotAPlotError(out)
|
||||
|
||||
# This out will become a plot unless it is one already
|
||||
if not isinstance(out.plot, dict):
|
||||
out.plot = {}
|
||||
|
||||
if unset:
|
||||
self._unset(out, unset)
|
||||
|
||||
out.plot.update(props)
|
||||
|
||||
# Empty dict will move it to non-plots
|
||||
if not out.plot:
|
||||
out.plot = True
|
||||
|
||||
out.verify_metric()
|
||||
|
||||
dvcfile = Dvcfile(self.repo, out.stage.path)
|
||||
dvcfile.dump(out.stage, update_pipeline=True, no_lock=True)
|
||||
|
||||
|
||||
def _collect_plots(repo, targets=None, rev=None):
|
||||
def _targets_to_outs(targets):
|
||||
for t in targets:
|
||||
try:
|
||||
(out,) = repo.find_outs_by_path(t)
|
||||
yield out
|
||||
except OutputNotFoundError:
|
||||
logger.warning(
|
||||
"File '{}' was not found at: '{}'. It will not be "
|
||||
"plotted.".format(t, rev)
|
||||
)
|
||||
|
||||
if targets:
|
||||
outs = _targets_to_outs(targets)
|
||||
else:
|
||||
outs = (out for stage in repo.stages for out in stage.outs if out.plot)
|
||||
|
||||
return {out.path_info: _plot_props(out) for out in outs}
|
||||
|
||||
|
||||
def _plot_props(out):
|
||||
if not out.plot:
|
||||
raise NotAPlotError(out)
|
||||
if isinstance(out.plot, list):
|
||||
raise DvcException("Multiple plots per data file not supported.")
|
||||
if isinstance(out.plot, bool):
|
||||
return {}
|
||||
|
||||
return project(out.plot, PLOT_PROPS)
|
||||
|
||||
|
||||
def _prepare_plots(data, revs, props):
|
||||
"""Groups data by plot file.
|
||||
|
||||
Also resolves props conflicts between revs and applies global props.
|
||||
"""
|
||||
# we go in order revs are supplied on props conflict first ones win.
|
||||
revs = iter(data) if revs is None else revs
|
||||
|
||||
plots, props_revs = {}, {}
|
||||
for rev in revs:
|
||||
# Asked for revision without data
|
||||
if rev not in data:
|
||||
continue
|
||||
|
||||
for datafile, desc in data[rev].items():
|
||||
# props from command line overwrite plot props from out definition
|
||||
full_props = {**desc["props"], **props}
|
||||
|
||||
if datafile in plots:
|
||||
saved = plots[datafile]
|
||||
if saved["props"] != full_props:
|
||||
logger.warning(
|
||||
f"Inconsistent plot props for '{datafile}' in "
|
||||
f"'{props_revs[datafile]}' and '{rev}'. "
|
||||
f"Going to use ones from '{props_revs[datafile]}'"
|
||||
)
|
||||
|
||||
saved["data"][rev] = desc["data"]
|
||||
else:
|
||||
plots[datafile] = {
|
||||
"props": full_props,
|
||||
"data": {rev: desc["data"]},
|
||||
}
|
||||
# Save rev we got props from
|
||||
props_revs[datafile] = rev
|
||||
|
||||
return plots
|
||||
|
||||
|
||||
def _render(datafile, datas, props, templates):
|
||||
from .data import plot_data, PlotData
|
||||
|
||||
# Copy it to not modify a passed value
|
||||
props = props.copy()
|
||||
|
||||
# Add x and y to fields if set
|
||||
fields = props.get("fields")
|
||||
if fields is not None:
|
||||
fields = {*fields, props.get("x"), props.get("y")} - {None}
|
||||
|
||||
template = templates.load(props.get("template") or "default")
|
||||
|
||||
# If x is not set add index field
|
||||
if not props.get("x") and template.has_anchor("x"):
|
||||
props["append_index"] = True
|
||||
props["x"] = PlotData.INDEX_FIELD
|
||||
|
||||
# Parse all data, preprocess it and collect as a list of dicts
|
||||
data = []
|
||||
for rev, datablob in datas.items():
|
||||
rev_data = plot_data(datafile, rev, datablob).to_datapoints(
|
||||
fields=fields,
|
||||
path=props.get("path"),
|
||||
header=props.get("header", True),
|
||||
append_index=props.get("append_index", False),
|
||||
)
|
||||
data.extend(rev_data)
|
||||
|
||||
# If y is not set then use last field not used yet
|
||||
if not props.get("y") and template.has_anchor("y"):
|
||||
fields = list(first(data))
|
||||
skip = (PlotData.REVISION_FIELD, props.get("x"))
|
||||
props["y"] = first(f for f in reversed(fields) if f not in skip)
|
||||
|
||||
return template.render(data, props=props)
|
||||
|
||||
+7
-65
@@ -1,7 +1,6 @@
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from collections import OrderedDict
|
||||
from copy import copy
|
||||
@@ -10,9 +9,7 @@ import yaml
|
||||
from funcy import first
|
||||
from yaml import SafeLoader
|
||||
|
||||
from dvc.exceptions import DvcException, PathMissingError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
|
||||
class PlotMetricTypeError(DvcException):
|
||||
@@ -31,21 +28,6 @@ class PlotDataStructureError(DvcException):
|
||||
)
|
||||
|
||||
|
||||
class JsonParsingError(DvcException):
|
||||
def __init__(self, file):
|
||||
super().__init__(
|
||||
"Failed to infer data structure from '{}'. Did you forget "
|
||||
"to specify JSONpath?".format(file)
|
||||
)
|
||||
|
||||
|
||||
class NoMetricOnRevisionError(DvcException):
|
||||
def __init__(self, path, revision):
|
||||
self.path = path
|
||||
self.revision = revision
|
||||
super().__init__(f"Could not find '{path}' on revision '{revision}'")
|
||||
|
||||
|
||||
class NoMetricInHistoryError(DvcException):
|
||||
def __init__(self, path):
|
||||
super().__init__(f"Could not find '{path}'.")
|
||||
@@ -77,7 +59,7 @@ def _filter_fields(data_points, filename, revision, fields=None, **kwargs):
|
||||
if keys & fields != fields:
|
||||
raise DvcException(
|
||||
"Could not find fields: '{}' for '{}' at '{}'.".format(
|
||||
", " "".join(fields), filename, revision
|
||||
", ".join(fields), filename, revision
|
||||
)
|
||||
)
|
||||
|
||||
@@ -199,10 +181,10 @@ class CSVPlotData(PlotData):
|
||||
super().__init__(filename, revision, content)
|
||||
self.delimiter = delimiter
|
||||
|
||||
def raw(self, csv_header=True, **kwargs):
|
||||
def raw(self, header=True, **kwargs):
|
||||
first_row = first(csv.reader(io.StringIO(self.content)))
|
||||
|
||||
if csv_header:
|
||||
if header:
|
||||
reader = csv.DictReader(
|
||||
io.StringIO(self.content), delimiter=self.delimiter,
|
||||
)
|
||||
@@ -237,46 +219,6 @@ class YAMLPlotData(PlotData):
|
||||
|
||||
return yaml.load(self.content, OrderedLoader)
|
||||
|
||||
|
||||
def _load_from_revision(repo, datafile, revision):
|
||||
from dvc.repo.tree import RepoTree
|
||||
|
||||
tree = RepoTree(repo)
|
||||
|
||||
try:
|
||||
with tree.open(datafile) as fobj:
|
||||
datafile_content = fobj.read()
|
||||
|
||||
except (FileNotFoundError, PathMissingError):
|
||||
raise NoMetricOnRevisionError(datafile, revision)
|
||||
|
||||
return plot_data(datafile, revision, datafile_content)
|
||||
|
||||
|
||||
def _load_from_revisions(repo, datafile, revisions):
|
||||
data = []
|
||||
exceptions = []
|
||||
|
||||
for rev in repo.brancher(revs=revisions):
|
||||
if rev == "working tree" and rev not in revisions:
|
||||
continue
|
||||
|
||||
try:
|
||||
data.append(_load_from_revision(repo, datafile, rev))
|
||||
except NoMetricOnRevisionError as e:
|
||||
exceptions.append(e)
|
||||
except PlotMetricTypeError:
|
||||
raise
|
||||
except (yaml.error.YAMLError, json.decoder.JSONDecodeError, csv.Error):
|
||||
logger.error(f"Failed to parse '{datafile}' at '{rev}'.")
|
||||
raise
|
||||
|
||||
if not data and exceptions:
|
||||
raise NoMetricInHistoryError(datafile)
|
||||
else:
|
||||
for e in exceptions:
|
||||
logger.warning(
|
||||
"File '{}' was not found at: '{}'. It will not be "
|
||||
"plotted.".format(e.path, e.revision)
|
||||
)
|
||||
return data
|
||||
def _processors(self):
|
||||
parent_processors = super()._processors()
|
||||
return [_find_data] + parent_processors
|
||||
|
||||
@@ -3,7 +3,7 @@ def _revisions(revs, is_dirty):
|
||||
if len(revisions) <= 1:
|
||||
if len(revisions) == 0 and is_dirty:
|
||||
revisions.append("HEAD")
|
||||
revisions.append("working tree")
|
||||
revisions.append("workspace")
|
||||
return revisions
|
||||
|
||||
|
||||
|
||||
@@ -1,214 +0,0 @@
|
||||
import logging
|
||||
import os
|
||||
from collections import defaultdict
|
||||
|
||||
from funcy import first, last
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.repo import locked
|
||||
|
||||
from .data import PlotData
|
||||
from .template import NoDataForTemplateError, Template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TooManyDataSourcesError(DvcException):
|
||||
def __init__(self, datafile, template_datafiles):
|
||||
super().__init__(
|
||||
"Unable to infer which of possible data sources: '{}' "
|
||||
"should be replaced with '{}'.".format(
|
||||
", ".join(template_datafiles), datafile
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class NoDataOrTemplateProvided(DvcException):
|
||||
def __init__(self):
|
||||
super().__init__("Datafile or template is not specified.")
|
||||
|
||||
|
||||
class TooManyTemplatesError(DvcException):
|
||||
pass
|
||||
|
||||
|
||||
def _evaluate_templatepath(repo, template=None):
|
||||
if not template:
|
||||
return repo.plot_templates.default_template
|
||||
|
||||
if os.path.exists(template):
|
||||
return template
|
||||
return repo.plot_templates.get_template(template)
|
||||
|
||||
|
||||
@locked
|
||||
def fill_template(
|
||||
repo,
|
||||
datafile,
|
||||
template_path,
|
||||
revisions,
|
||||
fields=None,
|
||||
path=None,
|
||||
csv_header=True,
|
||||
x_field=None,
|
||||
y_field=None,
|
||||
**kwargs,
|
||||
):
|
||||
if x_field and fields:
|
||||
fields.add(x_field)
|
||||
|
||||
if y_field and fields:
|
||||
fields.add(y_field)
|
||||
|
||||
template_datafiles, x_anchor, y_anchor = _parse_template(
|
||||
template_path, datafile
|
||||
)
|
||||
append_index = x_anchor and not x_field
|
||||
if append_index:
|
||||
x_field = PlotData.INDEX_FIELD
|
||||
|
||||
template_data = {}
|
||||
for template_datafile in template_datafiles:
|
||||
from .data import _load_from_revisions
|
||||
|
||||
plot_datas = _load_from_revisions(repo, template_datafile, revisions)
|
||||
tmp_data = []
|
||||
for pd in plot_datas:
|
||||
rev_data_points = pd.to_datapoints(
|
||||
fields=fields,
|
||||
path=path,
|
||||
csv_header=csv_header,
|
||||
append_index=append_index,
|
||||
)
|
||||
|
||||
if y_anchor and not y_field:
|
||||
y_field = _infer_y_field(rev_data_points, x_field)
|
||||
tmp_data.extend(rev_data_points)
|
||||
|
||||
template_data[template_datafile] = tmp_data
|
||||
|
||||
if len(template_data) == 0:
|
||||
raise NoDataForTemplateError(template_path)
|
||||
|
||||
content = Template.fill(
|
||||
template_path,
|
||||
template_data,
|
||||
priority_datafile=datafile,
|
||||
x_field=x_field,
|
||||
y_field=y_field,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
path = datafile or ",".join(template_datafiles)
|
||||
|
||||
return path, content
|
||||
|
||||
|
||||
def _infer_y_field(rev_data_points, x_field):
|
||||
all_fields = list(first(rev_data_points).keys())
|
||||
all_fields.remove(PlotData.REVISION_FIELD)
|
||||
if x_field and x_field in all_fields:
|
||||
all_fields.remove(x_field)
|
||||
y_field = last(all_fields)
|
||||
return y_field
|
||||
|
||||
|
||||
def _show(repo, datafile=None, template=None, revs=None, **kwargs):
|
||||
if revs is None:
|
||||
revs = ["working tree"]
|
||||
|
||||
if not datafile and not template:
|
||||
raise NoDataOrTemplateProvided()
|
||||
|
||||
template_path = _evaluate_templatepath(repo, template)
|
||||
|
||||
plot_datafile, plot_content = fill_template(
|
||||
repo, datafile, template_path, revs, **kwargs
|
||||
)
|
||||
|
||||
return plot_datafile, plot_content
|
||||
|
||||
|
||||
def _parse_template(template_path, priority_datafile):
|
||||
with open(template_path) as fobj:
|
||||
tempalte_content = fobj.read()
|
||||
|
||||
template_datafiles = Template.parse_data_anchors(tempalte_content)
|
||||
if priority_datafile:
|
||||
if len(template_datafiles) > 1:
|
||||
raise TooManyDataSourcesError(
|
||||
priority_datafile, template_datafiles
|
||||
)
|
||||
template_datafiles = {priority_datafile}
|
||||
|
||||
return (
|
||||
template_datafiles,
|
||||
Template.X_ANCHOR in tempalte_content,
|
||||
Template.Y_ANCHOR in tempalte_content,
|
||||
)
|
||||
|
||||
|
||||
def _collect_plots(repo):
|
||||
plots = defaultdict(set)
|
||||
|
||||
for stage in repo.stages:
|
||||
for out in stage.outs:
|
||||
if not out.plot:
|
||||
continue
|
||||
|
||||
if isinstance(out.plot, dict):
|
||||
template = out.plot[out.PARAM_PLOT_TEMPLATE]
|
||||
else:
|
||||
template = None
|
||||
|
||||
plots[str(out)].add(template)
|
||||
|
||||
return plots
|
||||
|
||||
|
||||
def show(repo, targets=None, template=None, revs=None, **kwargs) -> dict:
|
||||
if isinstance(targets, str):
|
||||
targets = [targets]
|
||||
|
||||
if targets:
|
||||
for target in targets:
|
||||
return {
|
||||
target: _show(
|
||||
repo,
|
||||
datafile=target,
|
||||
template=template,
|
||||
revs=revs,
|
||||
**kwargs,
|
||||
)[1]
|
||||
}
|
||||
|
||||
if not revs:
|
||||
plots = _collect_plots(repo)
|
||||
else:
|
||||
plots = defaultdict(set)
|
||||
for rev in repo.brancher(revs=revs):
|
||||
for plot, templates in _collect_plots(repo).items():
|
||||
plots[plot].update(templates)
|
||||
|
||||
if not plots:
|
||||
datafile, plot = _show(
|
||||
repo, datafile=None, template=template, revs=revs, **kwargs
|
||||
)
|
||||
return {datafile: plot}
|
||||
|
||||
ret = {}
|
||||
for plot, templates in plots.items():
|
||||
tmplt = template
|
||||
if len(templates) == 1:
|
||||
tmplt = list(templates)[0]
|
||||
elif not template:
|
||||
raise TooManyTemplatesError(
|
||||
f"'{plot}' uses multiple templates '{templates}'. "
|
||||
"Use `-t|--template` to specify the template to use. "
|
||||
)
|
||||
|
||||
ret[plot] = _show(
|
||||
repo, datafile=plot, template=tmplt, revs=revs, **kwargs
|
||||
)[1]
|
||||
|
||||
return ret
|
||||
+113
-138
@@ -1,26 +1,19 @@
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
|
||||
from funcy import cached_property
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.utils.fs import makedirs
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TemplateNotFoundError(DvcException):
|
||||
def __init__(self, path):
|
||||
super().__init__(f"Template '{path}' not found.")
|
||||
|
||||
|
||||
class NoDataForTemplateError(DvcException):
|
||||
def __init__(self, template_path):
|
||||
super().__init__(
|
||||
"No data provided for '{}'.".format(os.path.relpath(template_path))
|
||||
)
|
||||
class BadTemplateError(DvcException):
|
||||
pass
|
||||
|
||||
|
||||
class NoFieldInDataError(DvcException):
|
||||
@@ -34,148 +27,74 @@ class Template:
|
||||
INDENT = 4
|
||||
SEPARATORS = (",", ": ")
|
||||
EXTENSION = ".json"
|
||||
ANCHOR = '"<DVC_METRIC_{}>"'
|
||||
METRIC_DATA_ANCHOR = "<DVC_METRIC_DATA>"
|
||||
X_ANCHOR = "<DVC_METRIC_X>"
|
||||
Y_ANCHOR = "<DVC_METRIC_Y>"
|
||||
TITLE_ANCHOR = "<DVC_METRIC_TITLE>"
|
||||
X_TITLE_ANCHOR = "<DVC_METRIC_X_TITLE>"
|
||||
Y_TITLE_ANCHOR = "<DVC_METRIC_Y_TITLE>"
|
||||
X_LABEL_ANCHOR = "<DVC_METRIC_X_LABEL>"
|
||||
Y_LABEL_ANCHOR = "<DVC_METRIC_Y_LABEL>"
|
||||
|
||||
def __init__(self, templates_dir):
|
||||
self.plot_templates_dir = templates_dir
|
||||
def __init__(self, content=None, name=None):
|
||||
self.content = self.DEFAULT_CONTENT if content is None else content
|
||||
self.name = name or self.DEFAULT_NAME
|
||||
self.filename = self.name + self.EXTENSION
|
||||
|
||||
def dump(self):
|
||||
makedirs(self.plot_templates_dir, exist_ok=True)
|
||||
def render(self, data, props=None):
|
||||
props = props or {}
|
||||
|
||||
with open(
|
||||
os.path.join(
|
||||
self.plot_templates_dir, self.TEMPLATE_NAME + self.EXTENSION
|
||||
),
|
||||
"w",
|
||||
) as fobj:
|
||||
json.dump(
|
||||
self.DEFAULT_CONTENT,
|
||||
fobj,
|
||||
indent=self.INDENT,
|
||||
separators=self.SEPARATORS,
|
||||
if self.METRIC_DATA_ANCHOR not in self.content:
|
||||
raise BadTemplateError(
|
||||
f"Template '{self.filename}' is not using "
|
||||
f"'{self.METRIC_DATA_ANCHOR}' anchor"
|
||||
)
|
||||
fobj.write("\n")
|
||||
|
||||
@staticmethod
|
||||
def get_data_anchor(template_content):
|
||||
regex = re.compile('"<DVC_METRIC_DATA[^>"]*>"')
|
||||
return regex.findall(template_content)
|
||||
if props.get("x"):
|
||||
Template._check_field_exists(data, props.get("x"))
|
||||
if props.get("y"):
|
||||
Template._check_field_exists(data, props.get("y"))
|
||||
|
||||
@staticmethod
|
||||
def parse_data_anchors(template_content):
|
||||
data_files = {
|
||||
Template.get_datafile(m)
|
||||
for m in Template.get_data_anchor(template_content)
|
||||
}
|
||||
return {df for df in data_files if df}
|
||||
content = self._fill_anchor(self.content, "data", data)
|
||||
content = self._fill_metadata(content, props)
|
||||
|
||||
@staticmethod
|
||||
def get_datafile(anchor_string):
|
||||
return (
|
||||
anchor_string.replace("<", "")
|
||||
.replace(">", "")
|
||||
.replace('"', "")
|
||||
.replace("DVC_METRIC_DATA", "")
|
||||
.replace(",", "")
|
||||
return content
|
||||
|
||||
def has_anchor(self, name):
|
||||
return self._anchor(name) in self.content
|
||||
|
||||
@classmethod
|
||||
def _fill_anchor(cls, content, name, value):
|
||||
value_str = json.dumps(
|
||||
value, indent=cls.INDENT, separators=cls.SEPARATORS, sort_keys=True
|
||||
)
|
||||
return content.replace(cls._anchor(name), value_str)
|
||||
|
||||
@staticmethod
|
||||
def fill(
|
||||
template_path,
|
||||
data,
|
||||
priority_datafile=None,
|
||||
x_field=None,
|
||||
y_field=None,
|
||||
title=None,
|
||||
x_title=None,
|
||||
y_title=None,
|
||||
):
|
||||
with open(template_path) as fobj:
|
||||
result_content = fobj.read()
|
||||
@classmethod
|
||||
def _anchor(cls, name):
|
||||
return cls.ANCHOR.format(name.upper())
|
||||
|
||||
if x_field:
|
||||
Template._check_field_exists(data, x_field)
|
||||
if y_field:
|
||||
Template._check_field_exists(data, y_field)
|
||||
@classmethod
|
||||
def _fill_metadata(cls, content, props):
|
||||
props.setdefault("title", "")
|
||||
props.setdefault("x_label", props.get("x"))
|
||||
props.setdefault("y_label", props.get("y"))
|
||||
|
||||
result_content = Template._replace_data_anchors(
|
||||
result_content, data, priority_datafile
|
||||
)
|
||||
names = ["title", "x", "y", "x_label", "y_label"]
|
||||
for name in names:
|
||||
value = props.get(name)
|
||||
if value is not None:
|
||||
content = cls._fill_anchor(content, name, value)
|
||||
|
||||
result_content = Template._replace_metadata_anchors(
|
||||
result_content, title, x_field, x_title, y_field, y_title
|
||||
)
|
||||
|
||||
return result_content
|
||||
return content
|
||||
|
||||
@staticmethod
|
||||
def _check_field_exists(data, field):
|
||||
for file, data_points in data.items():
|
||||
if not any(
|
||||
field in data_point.keys() for data_point in data_points
|
||||
):
|
||||
raise NoFieldInDataError(field)
|
||||
|
||||
@staticmethod
|
||||
def _replace_metadata_anchors(
|
||||
result_content, title, x_field, x_title, y_field, y_title
|
||||
):
|
||||
if Template.TITLE_ANCHOR in result_content:
|
||||
if title:
|
||||
result_content = result_content.replace(
|
||||
Template.TITLE_ANCHOR, title
|
||||
)
|
||||
else:
|
||||
result_content = result_content.replace(
|
||||
Template.TITLE_ANCHOR, ""
|
||||
)
|
||||
if Template.X_ANCHOR in result_content and x_field:
|
||||
result_content = result_content.replace(Template.X_ANCHOR, x_field)
|
||||
if Template.Y_ANCHOR in result_content and y_field:
|
||||
result_content = result_content.replace(Template.Y_ANCHOR, y_field)
|
||||
if Template.X_TITLE_ANCHOR in result_content:
|
||||
if not x_title and x_field:
|
||||
x_title = x_field
|
||||
result_content = result_content.replace(
|
||||
Template.X_TITLE_ANCHOR, x_title
|
||||
)
|
||||
if Template.Y_TITLE_ANCHOR in result_content:
|
||||
if not y_title and y_field:
|
||||
y_title = y_field
|
||||
result_content = result_content.replace(
|
||||
Template.Y_TITLE_ANCHOR, y_title
|
||||
)
|
||||
return result_content
|
||||
|
||||
@staticmethod
|
||||
def _replace_data_anchors(result_content, data, priority_datafile):
|
||||
for anchor in Template.get_data_anchor(result_content):
|
||||
file = Template.get_datafile(anchor)
|
||||
|
||||
if not file or priority_datafile:
|
||||
key = priority_datafile
|
||||
else:
|
||||
key = file
|
||||
|
||||
result_content = result_content.replace(
|
||||
anchor,
|
||||
json.dumps(
|
||||
data[key],
|
||||
indent=Template.INDENT,
|
||||
separators=Template.SEPARATORS,
|
||||
sort_keys=True,
|
||||
),
|
||||
)
|
||||
return result_content
|
||||
if not any(field in row for row in data):
|
||||
raise NoFieldInDataError(field)
|
||||
|
||||
|
||||
class DefaultLinearTemplate(Template):
|
||||
TEMPLATE_NAME = "default"
|
||||
DEFAULT_NAME = "default"
|
||||
|
||||
DEFAULT_CONTENT = {
|
||||
"$schema": "https://vega.github.io/schema/vega-lite/v4.json",
|
||||
@@ -186,12 +105,12 @@ class DefaultLinearTemplate(Template):
|
||||
"x": {
|
||||
"field": Template.X_ANCHOR,
|
||||
"type": "quantitative",
|
||||
"title": Template.X_TITLE_ANCHOR,
|
||||
"title": Template.X_LABEL_ANCHOR,
|
||||
},
|
||||
"y": {
|
||||
"field": Template.Y_ANCHOR,
|
||||
"type": "quantitative",
|
||||
"title": Template.Y_TITLE_ANCHOR,
|
||||
"title": Template.Y_LABEL_ANCHOR,
|
||||
"scale": {"zero": False},
|
||||
},
|
||||
"color": {"field": "rev", "type": "nominal"},
|
||||
@@ -200,7 +119,7 @@ class DefaultLinearTemplate(Template):
|
||||
|
||||
|
||||
class DefaultConfusionTemplate(Template):
|
||||
TEMPLATE_NAME = "confusion"
|
||||
DEFAULT_NAME = "confusion"
|
||||
DEFAULT_CONTENT = {
|
||||
"$schema": "https://vega.github.io/schema/vega-lite/v4.json",
|
||||
"data": {"values": Template.METRIC_DATA_ANCHOR},
|
||||
@@ -211,13 +130,13 @@ class DefaultConfusionTemplate(Template):
|
||||
"field": Template.X_ANCHOR,
|
||||
"type": "nominal",
|
||||
"sort": "ascending",
|
||||
"title": Template.X_TITLE_ANCHOR,
|
||||
"title": Template.X_LABEL_ANCHOR,
|
||||
},
|
||||
"y": {
|
||||
"field": Template.Y_ANCHOR,
|
||||
"type": "nominal",
|
||||
"sort": "ascending",
|
||||
"title": Template.Y_TITLE_ANCHOR,
|
||||
"title": Template.Y_LABEL_ANCHOR,
|
||||
},
|
||||
"color": {"aggregate": "count", "type": "quantitative"},
|
||||
"facet": {"field": "rev", "type": "nominal"},
|
||||
@@ -226,7 +145,7 @@ class DefaultConfusionTemplate(Template):
|
||||
|
||||
|
||||
class DefaultScatterTemplate(Template):
|
||||
TEMPLATE_NAME = "scatter"
|
||||
DEFAULT_NAME = "scatter"
|
||||
DEFAULT_CONTENT = {
|
||||
"$schema": "https://vega.github.io/schema/vega-lite/v4.json",
|
||||
"data": {"values": Template.METRIC_DATA_ANCHOR},
|
||||
@@ -236,12 +155,12 @@ class DefaultScatterTemplate(Template):
|
||||
"x": {
|
||||
"field": Template.X_ANCHOR,
|
||||
"type": "quantitative",
|
||||
"title": Template.X_TITLE_ANCHOR,
|
||||
"title": Template.X_LABEL_ANCHOR,
|
||||
},
|
||||
"y": {
|
||||
"field": Template.Y_ANCHOR,
|
||||
"type": "quantitative",
|
||||
"title": Template.Y_TITLE_ANCHOR,
|
||||
"title": Template.Y_LABEL_ANCHOR,
|
||||
"scale": {"zero": False},
|
||||
},
|
||||
"color": {"field": "rev", "type": "nominal"},
|
||||
@@ -249,12 +168,46 @@ class DefaultScatterTemplate(Template):
|
||||
}
|
||||
|
||||
|
||||
class SmoothLinearTemplate(Template):
|
||||
DEFAULT_NAME = "smooth"
|
||||
|
||||
DEFAULT_CONTENT = {
|
||||
"$schema": "https://vega.github.io/schema/vega-lite/v4.json",
|
||||
"data": {"values": Template.METRIC_DATA_ANCHOR},
|
||||
"title": Template.TITLE_ANCHOR,
|
||||
"mark": {"type": "line"},
|
||||
"encoding": {
|
||||
"x": {
|
||||
"field": Template.X_ANCHOR,
|
||||
"type": "quantitative",
|
||||
"title": Template.X_LABEL_ANCHOR,
|
||||
},
|
||||
"y": {
|
||||
"field": Template.Y_ANCHOR,
|
||||
"type": "quantitative",
|
||||
"title": Template.Y_LABEL_ANCHOR,
|
||||
"scale": {"zero": False},
|
||||
},
|
||||
"color": {"field": "rev", "type": "nominal"},
|
||||
},
|
||||
"transform": [
|
||||
{
|
||||
"loess": Template.Y_ANCHOR,
|
||||
"on": Template.X_ANCHOR,
|
||||
"groupby": ["rev"],
|
||||
"bandwidth": 0.3,
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
class PlotTemplates:
|
||||
TEMPLATES_DIR = "plots"
|
||||
TEMPLATES = [
|
||||
DefaultLinearTemplate,
|
||||
DefaultConfusionTemplate,
|
||||
DefaultScatterTemplate,
|
||||
SmoothLinearTemplate,
|
||||
]
|
||||
|
||||
@cached_property
|
||||
@@ -269,6 +222,9 @@ class PlotTemplates:
|
||||
return default_plot_path
|
||||
|
||||
def get_template(self, path):
|
||||
if os.path.exists(path):
|
||||
return path
|
||||
|
||||
t_path = os.path.join(self.templates_dir, path)
|
||||
if os.path.exists(t_path):
|
||||
return t_path
|
||||
@@ -295,4 +251,23 @@ class PlotTemplates:
|
||||
if not os.path.exists(self.templates_dir):
|
||||
makedirs(self.templates_dir, exist_ok=True)
|
||||
for t in self.TEMPLATES:
|
||||
t(self.templates_dir).dump()
|
||||
self.dump(t())
|
||||
|
||||
def dump(self, template):
|
||||
path = os.path.join(self.templates_dir, template.filename)
|
||||
with open(path, "w") as fd:
|
||||
json.dump(
|
||||
template.content,
|
||||
fd,
|
||||
indent=template.INDENT,
|
||||
separators=template.SEPARATORS,
|
||||
)
|
||||
fd.write("\n")
|
||||
|
||||
def load(self, name):
|
||||
path = self.get_template(name)
|
||||
|
||||
with open(path) as fd:
|
||||
content = fd.read()
|
||||
|
||||
return Template(content, name=name)
|
||||
|
||||
+1
-1
@@ -32,4 +32,4 @@ def push(
|
||||
used_run_cache=used_run_cache,
|
||||
)
|
||||
|
||||
return self.cloud.push(used, jobs, remote=remote)
|
||||
return len(used_run_cache) + self.cloud.push(used, jobs, remote=remote)
|
||||
|
||||
+4
-9
@@ -7,16 +7,11 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@locked
|
||||
def remove(self, target, outs_only=False):
|
||||
from ..dvcfile import Dvcfile
|
||||
|
||||
def remove(self, target, outs=False):
|
||||
path, name = parse_target(target)
|
||||
dvcfile = Dvcfile(self, path)
|
||||
stages = list(dvcfile.stages.filter(name).values())
|
||||
for stage in stages:
|
||||
stage.remove_outs(force=True)
|
||||
stages = self.get_stages(path, name)
|
||||
|
||||
if not outs_only:
|
||||
dvcfile.remove()
|
||||
for stage in stages:
|
||||
stage.remove(remove_outs=outs, force=outs)
|
||||
|
||||
return stages
|
||||
|
||||
+10
-6
@@ -10,9 +10,9 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _reproduce_stage(stage, **kwargs):
|
||||
if stage.locked and not stage.is_import:
|
||||
if stage.frozen and not stage.is_import:
|
||||
logger.warning(
|
||||
"{} is locked. Its dependencies are"
|
||||
"{} is frozen. Its dependencies are"
|
||||
" not going to be reproduced.".format(stage)
|
||||
)
|
||||
|
||||
@@ -34,7 +34,7 @@ def _get_active_graph(G):
|
||||
|
||||
active = G.copy()
|
||||
for stage in G:
|
||||
if not stage.locked:
|
||||
if not stage.frozen:
|
||||
continue
|
||||
active.remove_edges_from(G.out_edges(stage))
|
||||
for edge in G.out_edges(stage):
|
||||
@@ -61,9 +61,9 @@ def reproduce(
|
||||
all_pipelines=False,
|
||||
**kwargs
|
||||
):
|
||||
from ..dvcfile import Dvcfile
|
||||
from dvc.utils import parse_target
|
||||
|
||||
assert target is None or isinstance(target, str)
|
||||
if not target and not all_pipelines:
|
||||
raise InvalidArgumentError(
|
||||
"Neither `target` nor `--all-pipelines` are specified."
|
||||
@@ -81,8 +81,7 @@ def reproduce(
|
||||
if all_pipelines:
|
||||
pipelines = active_pipelines
|
||||
else:
|
||||
dvcfile = Dvcfile(self, path)
|
||||
stage = dvcfile.stages[name]
|
||||
stage = self.get_stage(path, name)
|
||||
pipelines = [get_pipeline(active_pipelines, stage)]
|
||||
|
||||
targets = []
|
||||
@@ -164,7 +163,12 @@ def _reproduce_stages(
|
||||
|
||||
force_downstream = kwargs.pop("force_downstream", False)
|
||||
result = []
|
||||
# `ret` is used to add a cosmetic newline.
|
||||
ret = []
|
||||
for stage in pipeline:
|
||||
if ret:
|
||||
logger.info("")
|
||||
|
||||
try:
|
||||
ret = _reproduce_stage(stage, **kwargs)
|
||||
|
||||
|
||||
+51
-13
@@ -1,9 +1,15 @@
|
||||
import os
|
||||
from contextlib import suppress
|
||||
|
||||
from funcy import concat, first
|
||||
from funcy import concat, first, lfilter
|
||||
|
||||
from dvc.exceptions import InvalidArgumentError
|
||||
from dvc.stage.exceptions import DuplicateStageName, InvalidStageName
|
||||
from dvc.stage import PipelineStage
|
||||
from dvc.stage.exceptions import (
|
||||
DuplicateStageName,
|
||||
InvalidStageName,
|
||||
StageFileAlreadyExistsError,
|
||||
)
|
||||
|
||||
from ..exceptions import OutputDuplicationError
|
||||
from . import locked
|
||||
@@ -16,6 +22,19 @@ def is_valid_name(name: str):
|
||||
return not INVALID_STAGENAME_CHARS & set(name)
|
||||
|
||||
|
||||
def parse_params(path_params):
|
||||
ret = []
|
||||
for path_param in path_params:
|
||||
path, _, params_str = path_param.rpartition(":")
|
||||
# remove empty strings from params, on condition such as `-p "file1:"`
|
||||
params = lfilter(bool, params_str.split(","))
|
||||
if not path:
|
||||
ret.extend(params)
|
||||
else:
|
||||
ret.append({path: params})
|
||||
return ret
|
||||
|
||||
|
||||
def _get_file_path(kwargs):
|
||||
from dvc.dvcfile import DVC_FILE_SUFFIX, DVC_FILE
|
||||
|
||||
@@ -39,12 +58,30 @@ def _get_file_path(kwargs):
|
||||
)
|
||||
|
||||
|
||||
def _check_stage_exists(dvcfile, stage):
|
||||
if not dvcfile.exists():
|
||||
return
|
||||
|
||||
hint = "Use '--force' to overwrite."
|
||||
if stage.__class__ != PipelineStage:
|
||||
raise StageFileAlreadyExistsError(
|
||||
f"'{stage.relpath}' already exists. {hint}"
|
||||
)
|
||||
elif stage.name and stage.name in dvcfile.stages:
|
||||
raise DuplicateStageName(
|
||||
f"Stage '{stage.name}' already exists in '{stage.relpath}'. {hint}"
|
||||
)
|
||||
|
||||
|
||||
@locked
|
||||
@scm_context
|
||||
def run(self, fname=None, no_exec=False, single_stage=False, **kwargs):
|
||||
from dvc.stage import PipelineStage, Stage, create_stage
|
||||
from dvc.stage import Stage, create_stage
|
||||
from dvc.dvcfile import Dvcfile, PIPELINE_FILE
|
||||
|
||||
if not kwargs.get("cmd"):
|
||||
raise InvalidArgumentError("command is not specified")
|
||||
|
||||
stage_cls = PipelineStage
|
||||
path = PIPELINE_FILE
|
||||
stage_name = kwargs.get("name")
|
||||
@@ -65,30 +102,31 @@ def run(self, fname=None, no_exec=False, single_stage=False, **kwargs):
|
||||
if not is_valid_name(stage_name):
|
||||
raise InvalidStageName
|
||||
|
||||
stage = create_stage(stage_cls, repo=self, path=path, **kwargs)
|
||||
params = parse_params(kwargs.pop("params", []))
|
||||
stage = create_stage(
|
||||
stage_cls, repo=self, path=path, params=params, **kwargs
|
||||
)
|
||||
if stage is None:
|
||||
return None
|
||||
|
||||
dvcfile = Dvcfile(self, stage.path)
|
||||
if dvcfile.exists():
|
||||
if stage_name and stage_name in dvcfile.stages:
|
||||
raise DuplicateStageName(stage_name, dvcfile)
|
||||
if stage_cls != PipelineStage:
|
||||
dvcfile.remove_with_prompt(force=kwargs.get("overwrite", True))
|
||||
|
||||
try:
|
||||
if kwargs.get("force", True):
|
||||
with suppress(ValueError):
|
||||
self.stages.remove(stage)
|
||||
else:
|
||||
_check_stage_exists(dvcfile, stage)
|
||||
self.check_modified_graph([stage])
|
||||
except OutputDuplicationError as exc:
|
||||
raise OutputDuplicationError(exc.output, set(exc.stages) - {stage})
|
||||
|
||||
if no_exec:
|
||||
for out in stage.outs:
|
||||
out.ignore()
|
||||
stage.ignore_outs()
|
||||
else:
|
||||
stage.run(
|
||||
no_commit=kwargs.get("no_commit", False),
|
||||
run_cache=kwargs.get("run_cache", True),
|
||||
)
|
||||
|
||||
dvcfile.dump(stage, update_pipeline=True)
|
||||
dvcfile.dump(stage, update_pipeline=True, no_lock=no_exec)
|
||||
return stage
|
||||
|
||||
+2
-2
@@ -14,9 +14,9 @@ def _joint_status(stages):
|
||||
status = {}
|
||||
|
||||
for stage in stages:
|
||||
if stage.locked and not stage.is_repo_import:
|
||||
if stage.frozen and not stage.is_repo_import:
|
||||
logger.warning(
|
||||
"{} is locked. Its dependencies are"
|
||||
"{} is frozen. Its dependencies are"
|
||||
" not going to be shown in the status output.".format(stage)
|
||||
)
|
||||
|
||||
|
||||
+180
-58
@@ -1,10 +1,13 @@
|
||||
import logging
|
||||
import os
|
||||
|
||||
from dvc.dvcfile import is_valid_filename
|
||||
from dvc.exceptions import OutputNotFoundError
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.remote.base import RemoteActionNotImplemented
|
||||
from dvc.scm.tree import BaseTree, WorkingTree
|
||||
from dvc.scm.tree import BaseTree
|
||||
from dvc.utils import file_md5
|
||||
from dvc.utils.fs import copy_fobj_to_file, makedirs
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -40,43 +43,60 @@ class DvcTree(BaseTree):
|
||||
|
||||
return outs
|
||||
|
||||
def _get_granular_checksum(self, path, out, remote=None):
|
||||
if not self.fetch and not self.stream:
|
||||
raise FileNotFoundError
|
||||
dir_cache = out.get_dir_cache(remote=remote)
|
||||
for entry in dir_cache:
|
||||
entry_relpath = entry[out.remote.tree.PARAM_RELPATH]
|
||||
if os.name == "nt":
|
||||
entry_relpath = entry_relpath.replace("/", os.sep)
|
||||
if path == out.path_info / entry_relpath:
|
||||
return entry[out.remote.tree.PARAM_CHECKSUM]
|
||||
raise FileNotFoundError
|
||||
|
||||
def open(self, path, mode="r", encoding="utf-8", remote=None):
|
||||
try:
|
||||
outs = self._find_outs(path, strict=False)
|
||||
except OutputNotFoundError as exc:
|
||||
raise FileNotFoundError from exc
|
||||
|
||||
if len(outs) != 1 or outs[0].is_dir_checksum:
|
||||
# NOTE: this handles both dirty and checkout-ed out at the same time
|
||||
if self.repo.tree.exists(path):
|
||||
return self.repo.tree.open(path, mode=mode, encoding=encoding)
|
||||
|
||||
if len(outs) != 1 or (
|
||||
outs[0].is_dir_checksum and path == outs[0].path_info
|
||||
):
|
||||
raise IsADirectoryError
|
||||
|
||||
out = outs[0]
|
||||
# temporary hack to make cache use WorkingTree and not GitTree, because
|
||||
# cache dir doesn't exist in the latter.
|
||||
saved_tree = self.repo.tree
|
||||
self.repo.tree = WorkingTree(self.repo.root_dir)
|
||||
try:
|
||||
if out.changed_cache():
|
||||
if not self.fetch and not self.stream:
|
||||
raise FileNotFoundError
|
||||
if out.changed_cache(filter_info=path):
|
||||
if not self.fetch and not self.stream:
|
||||
raise FileNotFoundError
|
||||
|
||||
remote_obj = self.repo.cloud.get_remote(remote)
|
||||
if self.stream:
|
||||
try:
|
||||
remote_info = remote_obj.checksum_to_path_info(
|
||||
out.checksum
|
||||
)
|
||||
return remote_obj.open(
|
||||
remote_info, mode=mode, encoding=encoding
|
||||
)
|
||||
except RemoteActionNotImplemented:
|
||||
pass
|
||||
with self.repo.state:
|
||||
cache_info = out.get_used_cache(remote=remote)
|
||||
self.repo.cloud.pull(cache_info, remote=remote)
|
||||
finally:
|
||||
self.repo.tree = saved_tree
|
||||
remote_obj = self.repo.cloud.get_remote(remote)
|
||||
if self.stream:
|
||||
if out.is_dir_checksum:
|
||||
checksum = self._get_granular_checksum(path, out)
|
||||
else:
|
||||
checksum = out.checksum
|
||||
try:
|
||||
remote_info = remote_obj.hash_to_path_info(checksum)
|
||||
return remote_obj.open(
|
||||
remote_info, mode=mode, encoding=encoding
|
||||
)
|
||||
except RemoteActionNotImplemented:
|
||||
pass
|
||||
cache_info = out.get_used_cache(filter_info=path, remote=remote)
|
||||
self.repo.cloud.pull(cache_info, remote=remote)
|
||||
|
||||
return open(out.cache_path, mode=mode, encoding=encoding)
|
||||
if out.is_dir_checksum:
|
||||
checksum = self._get_granular_checksum(path, out)
|
||||
cache_path = out.cache.hash_to_path_info(checksum).url
|
||||
else:
|
||||
cache_path = out.cache_path
|
||||
return open(cache_path, mode=mode, encoding=encoding)
|
||||
|
||||
def exists(self, path):
|
||||
try:
|
||||
@@ -91,10 +111,25 @@ class DvcTree(BaseTree):
|
||||
|
||||
path_info = PathInfo(os.path.abspath(path))
|
||||
outs = self._find_outs(path, strict=False, recursive=True)
|
||||
if len(outs) != 1 or outs[0].path_info != path_info:
|
||||
if len(outs) != 1:
|
||||
return True
|
||||
|
||||
return outs[0].is_dir_checksum
|
||||
out = outs[0]
|
||||
if not out.is_dir_checksum:
|
||||
if out.path_info != path_info:
|
||||
return True
|
||||
return False
|
||||
|
||||
if out.path_info == path_info:
|
||||
return True
|
||||
|
||||
# for dir checksum, we need to check if this is a file inside the
|
||||
# directory
|
||||
try:
|
||||
self._get_granular_checksum(path, out)
|
||||
return False
|
||||
except FileNotFoundError:
|
||||
return True
|
||||
|
||||
def isfile(self, path):
|
||||
if not self.exists(path):
|
||||
@@ -127,7 +162,7 @@ class DvcTree(BaseTree):
|
||||
else:
|
||||
assert False
|
||||
|
||||
def walk(self, top, topdown=True):
|
||||
def walk(self, top, topdown=True, download_callback=None, **kwargs):
|
||||
from pygtrie import Trie
|
||||
|
||||
assert topdown
|
||||
@@ -147,19 +182,29 @@ class DvcTree(BaseTree):
|
||||
trie[out.path_info.parts] = out
|
||||
|
||||
if out.is_dir_checksum and (self.fetch or self.stream):
|
||||
# will pull dir cache if needed
|
||||
with self.repo.state:
|
||||
cache = out.collect_used_dir_cache()
|
||||
for _, names in cache.scheme_names(out.scheme):
|
||||
for name in names:
|
||||
path_info = out.path_info.parent / name
|
||||
trie[path_info.parts] = None
|
||||
# pull dir cache if needed
|
||||
dir_cache = out.get_dir_cache(**kwargs)
|
||||
|
||||
# pull dir contents if needed
|
||||
if self.fetch:
|
||||
if out.changed_cache(filter_info=top):
|
||||
used_cache = out.get_used_cache(filter_info=top)
|
||||
downloaded = self.repo.cloud.pull(used_cache, **kwargs)
|
||||
if download_callback:
|
||||
download_callback(downloaded)
|
||||
|
||||
for entry in dir_cache:
|
||||
entry_relpath = entry[out.remote.tree.PARAM_RELPATH]
|
||||
if os.name == "nt":
|
||||
entry_relpath = entry_relpath.replace("/", os.sep)
|
||||
path_info = out.path_info / entry_relpath
|
||||
trie[path_info.parts] = None
|
||||
|
||||
yield from self._walk(root, trie, topdown=topdown)
|
||||
|
||||
def isdvc(self, path):
|
||||
def isdvc(self, path, **kwargs):
|
||||
try:
|
||||
return len(self._find_outs(path)) == 1
|
||||
return len(self._find_outs(path, **kwargs)) == 1
|
||||
except OutputNotFoundError:
|
||||
pass
|
||||
return False
|
||||
@@ -167,6 +212,15 @@ class DvcTree(BaseTree):
|
||||
def isexec(self, path):
|
||||
return False
|
||||
|
||||
def get_file_hash(self, path_info):
|
||||
outs = self._find_outs(path_info, strict=False)
|
||||
if len(outs) != 1:
|
||||
raise OutputNotFoundError
|
||||
out = outs[0]
|
||||
if out.is_dir_checksum:
|
||||
return self._get_granular_checksum(path_info, out)
|
||||
return out.checksum
|
||||
|
||||
|
||||
class RepoTree(BaseTree):
|
||||
"""DVC + git-tracked files tree.
|
||||
@@ -185,15 +239,26 @@ class RepoTree(BaseTree):
|
||||
# git-only erepo's do not need dvctree
|
||||
self.dvctree = None
|
||||
|
||||
@property
|
||||
def fetch(self):
|
||||
if self.dvctree:
|
||||
return self.dvctree.fetch
|
||||
return False
|
||||
|
||||
@property
|
||||
def stream(self):
|
||||
if self.dvctree:
|
||||
return self.dvctree.stream
|
||||
return False
|
||||
|
||||
def open(self, path, mode="r", encoding="utf-8", **kwargs):
|
||||
if "b" in mode:
|
||||
encoding = None
|
||||
|
||||
if self.dvctree and self.dvctree.exists(path):
|
||||
try:
|
||||
return self.dvctree.open(
|
||||
path, mode=mode, encoding=encoding, **kwargs
|
||||
)
|
||||
except FileNotFoundError:
|
||||
if self.isdvc(path):
|
||||
raise
|
||||
return self.dvctree.open(
|
||||
path, mode=mode, encoding=encoding, **kwargs
|
||||
)
|
||||
return self.repo.tree.open(path, mode=mode, encoding=encoding)
|
||||
|
||||
def exists(self, path):
|
||||
@@ -206,8 +271,8 @@ class RepoTree(BaseTree):
|
||||
self.dvctree and self.dvctree.isdir(path)
|
||||
)
|
||||
|
||||
def isdvc(self, path):
|
||||
return self.dvctree is not None and self.dvctree.isdvc(path)
|
||||
def isdvc(self, path, **kwargs):
|
||||
return self.dvctree is not None and self.dvctree.isdvc(path, **kwargs)
|
||||
|
||||
def isfile(self, path):
|
||||
return self.repo.tree.isfile(path) or (
|
||||
@@ -219,6 +284,9 @@ class RepoTree(BaseTree):
|
||||
return self.dvctree.isexec(path)
|
||||
return self.repo.tree.isexec(path)
|
||||
|
||||
def stat(self, path):
|
||||
return self.repo.tree.stat(path)
|
||||
|
||||
def _walk_one(self, walk):
|
||||
try:
|
||||
root, dirs, files = next(walk)
|
||||
@@ -228,7 +296,7 @@ class RepoTree(BaseTree):
|
||||
for _ in dirs:
|
||||
yield from self._walk_one(walk)
|
||||
|
||||
def _walk(self, dvc_walk, repo_walk):
|
||||
def _walk(self, dvc_walk, repo_walk, dvcfiles=False):
|
||||
try:
|
||||
_, dvc_dirs, dvc_fnames = next(dvc_walk)
|
||||
repo_root, repo_dirs, repo_fnames = next(repo_walk)
|
||||
@@ -244,9 +312,11 @@ class RepoTree(BaseTree):
|
||||
dirs = shared + dvc_only + repo_only
|
||||
|
||||
# merge file lists
|
||||
files = set(dvc_fnames)
|
||||
for filename in repo_fnames:
|
||||
files.add(filename)
|
||||
files = {
|
||||
fname
|
||||
for fname in dvc_fnames + repo_fnames
|
||||
if dvcfiles or not is_valid_filename(fname)
|
||||
}
|
||||
|
||||
yield repo_root, dirs, list(files)
|
||||
|
||||
@@ -257,14 +327,24 @@ class RepoTree(BaseTree):
|
||||
|
||||
for dirname in dirs:
|
||||
if dirname in shared:
|
||||
yield from self._walk(dvc_walk, repo_walk)
|
||||
yield from self._walk(dvc_walk, repo_walk, dvcfiles=dvcfiles)
|
||||
elif dirname in dvc_set:
|
||||
yield from self._walk_one(dvc_walk)
|
||||
elif dirname in repo_set:
|
||||
yield from self._walk_one(repo_walk)
|
||||
|
||||
def walk(self, top, topdown=True):
|
||||
"""Walk and merge both DVC and repo trees."""
|
||||
def walk(self, top, topdown=True, dvcfiles=False, **kwargs):
|
||||
"""Walk and merge both DVC and repo trees.
|
||||
|
||||
Args:
|
||||
top: path to walk from
|
||||
topdown: if True, tree will be walked from top down.
|
||||
dvcfiles: if True, dvcfiles will be included in the files list
|
||||
for walked directories.
|
||||
|
||||
Any kwargs will be passed into methods used for fetching and/or
|
||||
streaming DVC outs from remotes.
|
||||
"""
|
||||
assert topdown
|
||||
|
||||
if not self.exists(top):
|
||||
@@ -276,7 +356,7 @@ class RepoTree(BaseTree):
|
||||
dvc_exists = self.dvctree and self.dvctree.exists(top)
|
||||
repo_exists = self.repo.tree.exists(top)
|
||||
if dvc_exists and not repo_exists:
|
||||
yield from self.dvctree.walk(top, topdown=topdown)
|
||||
yield from self.dvctree.walk(top, topdown=topdown, **kwargs)
|
||||
return
|
||||
if repo_exists and not dvc_exists:
|
||||
yield from self.repo.tree.walk(top, topdown=topdown)
|
||||
@@ -284,6 +364,48 @@ class RepoTree(BaseTree):
|
||||
if not dvc_exists and not repo_exists:
|
||||
raise FileNotFoundError
|
||||
|
||||
dvc_walk = self.dvctree.walk(top, topdown=topdown)
|
||||
dvc_walk = self.dvctree.walk(top, topdown=topdown, **kwargs)
|
||||
repo_walk = self.repo.tree.walk(top, topdown=topdown)
|
||||
yield from self._walk(dvc_walk, repo_walk)
|
||||
yield from self._walk(dvc_walk, repo_walk, dvcfiles=dvcfiles)
|
||||
|
||||
def walk_files(self, top, **kwargs):
|
||||
for root, _, files in self.walk(top, **kwargs):
|
||||
for fname in files:
|
||||
yield PathInfo(root) / fname
|
||||
|
||||
def get_file_hash(self, path_info):
|
||||
"""Return file checksum for specified path.
|
||||
|
||||
If path_info is a DVC out, the pre-computed checksum for the file
|
||||
will be used. If path_info is a git file, MD5 will be computed for
|
||||
the git object.
|
||||
"""
|
||||
if not self.exists(path_info):
|
||||
raise FileNotFoundError
|
||||
if self.dvctree and self.dvctree.exists(path_info):
|
||||
try:
|
||||
return self.dvctree.get_file_hash(path_info)
|
||||
except OutputNotFoundError:
|
||||
pass
|
||||
return file_md5(path_info, self)[0]
|
||||
|
||||
def copytree(self, top, dest):
|
||||
top = PathInfo(top)
|
||||
dest = PathInfo(dest)
|
||||
|
||||
if not self.exists(top):
|
||||
raise FileNotFoundError
|
||||
|
||||
if self.isfile(top):
|
||||
makedirs(dest.parent, exist_ok=True)
|
||||
with self.open(top, mode="rb") as fobj:
|
||||
copy_fobj_to_file(fobj, dest)
|
||||
return
|
||||
|
||||
for root, _, files in self.walk(top):
|
||||
root = PathInfo(root)
|
||||
makedirs(dest, exist_ok=True)
|
||||
for fname in files:
|
||||
src = root / fname
|
||||
with self.open(src, mode="rb") as fobj:
|
||||
copy_fobj_to_file(fobj, dest / fname)
|
||||
|
||||
+17
-4
@@ -11,7 +11,8 @@ SINGLE_STAGE_SCHEMA = {
|
||||
StageParams.PARAM_WDIR: Any(str, None),
|
||||
StageParams.PARAM_DEPS: Any([dependency.SCHEMA], None),
|
||||
StageParams.PARAM_OUTS: Any([output.SCHEMA], None),
|
||||
StageParams.PARAM_LOCKED: bool,
|
||||
StageParams.PARAM_LOCKED: bool, # backard compatibility
|
||||
StageParams.PARAM_FROZEN: bool,
|
||||
StageParams.PARAM_META: object,
|
||||
StageParams.PARAM_ALWAYS_CHANGED: bool,
|
||||
}
|
||||
@@ -29,8 +30,20 @@ OUT_PSTAGE_DETAILED_SCHEMA = {
|
||||
str: {BaseOutput.PARAM_CACHE: bool, BaseOutput.PARAM_PERSIST: bool}
|
||||
}
|
||||
|
||||
PLOT_PSTAGE_SCHEMA = OUT_PSTAGE_DETAILED_SCHEMA.copy()
|
||||
PLOT_PSTAGE_SCHEMA[str][BaseOutput.PARAM_PLOT_TEMPLATE] = str
|
||||
PLOT_PROPS = {
|
||||
BaseOutput.PARAM_PLOT_TEMPLATE: str,
|
||||
BaseOutput.PARAM_PLOT_X: str,
|
||||
BaseOutput.PARAM_PLOT_Y: str,
|
||||
BaseOutput.PARAM_PLOT_X_LABEL: str,
|
||||
BaseOutput.PARAM_PLOT_Y_LABEL: str,
|
||||
BaseOutput.PARAM_PLOT_TITLE: str,
|
||||
BaseOutput.PARAM_PLOT_HEADER: bool,
|
||||
}
|
||||
PLOT_PROPS_SCHEMA = {
|
||||
**OUT_PSTAGE_DETAILED_SCHEMA[str],
|
||||
**PLOT_PROPS,
|
||||
}
|
||||
PLOT_PSTAGE_SCHEMA = {str: Any(PLOT_PROPS_SCHEMA, [PLOT_PROPS_SCHEMA])}
|
||||
|
||||
PARAM_PSTAGE_NON_DEFAULT_SCHEMA = {str: [str]}
|
||||
|
||||
@@ -42,7 +55,7 @@ SINGLE_PIPELINE_STAGE_SCHEMA = {
|
||||
Optional(StageParams.PARAM_PARAMS): [
|
||||
Any(str, PARAM_PSTAGE_NON_DEFAULT_SCHEMA)
|
||||
],
|
||||
Optional(StageParams.PARAM_LOCKED): bool,
|
||||
Optional(StageParams.PARAM_FROZEN): bool,
|
||||
Optional(StageParams.PARAM_META): object,
|
||||
Optional(StageParams.PARAM_ALWAYS_CHANGED): bool,
|
||||
Optional(StageParams.PARAM_OUTS): [
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user