mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-18 23:10:14 +01:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
63972c3d12 | ||
|
|
7027c631c5 | ||
|
|
e8f6cd7f5b | ||
|
|
10f2bb93fa | ||
|
|
2fecc66cb9 | ||
|
|
b47998463f | ||
|
|
d53b740f35 | ||
|
|
9198fa7459 | ||
|
|
6954efc6e6 | ||
|
|
d77af8196e | ||
|
|
c5920b00ab | ||
|
|
ce8036e535 | ||
|
|
b7ad2dc841 | ||
|
|
5ab948f8b6 | ||
|
|
f9fbe97509 | ||
|
|
2fba2e030a | ||
|
|
3e951162b8 | ||
|
|
d74358eecd | ||
|
|
58c9ff0de2 | ||
|
|
f0e991790e | ||
|
|
b7cd785eba | ||
|
|
21a3efe2c7 | ||
|
|
865d355554 | ||
|
|
84712c4acc | ||
|
|
0f16e15453 | ||
|
|
61f30ed5ba | ||
|
|
cde15e3d3e | ||
|
|
7672e324d2 | ||
|
|
130b76c7a5 | ||
|
|
040458a726 | ||
|
|
75e795c491 | ||
|
|
4dac2092c9 | ||
|
|
19eaced304 | ||
|
|
184028f176 | ||
|
|
5f63d538d2 | ||
|
|
324ff4be59 | ||
|
|
3f846a1794 | ||
|
|
b52c925d5f | ||
|
|
4a4f07968c | ||
|
|
09136f2378 | ||
|
|
aaca1401ea | ||
|
|
2aaf065ea3 | ||
|
|
d4ec871a76 | ||
|
|
f44ddcf64e | ||
|
|
b88b3b81c6 | ||
|
|
81d04f887a | ||
|
|
afbb3a0add | ||
|
|
b8aa8ad951 | ||
|
|
9897f7c470 | ||
|
|
cf7f125c14 | ||
|
|
cec4b4e003 | ||
|
|
ff4fcc937b | ||
|
|
1676a7a1cd | ||
|
|
974cf097dd | ||
|
|
597ab98d0e | ||
|
|
833e82a7ea | ||
|
|
3916d4fa07 | ||
|
|
d81b56978f | ||
|
|
1e7efffe42 | ||
|
|
cc16b28a26 | ||
|
|
451c7aac04 | ||
|
|
7b4466c982 | ||
|
|
7ca3a1119c | ||
|
|
2c026066db | ||
|
|
7f17c6aa03 | ||
|
|
9f015fac1d | ||
|
|
666e203bbb | ||
|
|
ce946d1160 | ||
|
|
76b31ac282 | ||
|
|
6093a1fe97 | ||
|
|
d62a54c7b0 | ||
|
|
c922da4f3f | ||
|
|
2a16b021ad | ||
|
|
fc42ca721c | ||
|
|
785dcfeef0 | ||
|
|
9714b14456 | ||
|
|
6a406abde8 | ||
|
|
7ca055e868 | ||
|
|
6a64f86178 | ||
|
|
227dbf44d9 | ||
|
|
2bb7257a06 | ||
|
|
e3831066de | ||
|
|
aa99cc9934 | ||
|
|
26e203242f | ||
|
|
d00bd4dfe5 | ||
|
|
795f001929 | ||
|
|
93d7b9173b | ||
|
|
bc7f177c7a | ||
|
|
6f73a67c60 | ||
|
|
68a0c39ad1 | ||
|
|
c186a8ea8f | ||
|
|
af8ac006ad | ||
|
|
b2ea279c9e | ||
|
|
2dda540db6 | ||
|
|
76e6ba24f7 | ||
|
|
505b88afae | ||
|
|
415a85c6be | ||
|
|
4e9b38970a | ||
|
|
3e1eb6d1e7 | ||
|
|
c873787d73 | ||
|
|
e553511085 | ||
|
|
827c994e74 | ||
|
|
22c60dd608 | ||
|
|
f9088e1c67 | ||
|
|
71d156ac49 | ||
|
|
d998c0a536 | ||
|
|
af12f0d06c | ||
|
|
18e8f07572 | ||
|
|
8aefbac223 | ||
|
|
40c6b567e3 | ||
|
|
907853b985 | ||
|
|
31fe77848f | ||
|
|
7edee84306 | ||
|
|
1e65098819 | ||
|
|
4fdf7ca7fd | ||
|
|
a7b77672e6 | ||
|
|
6d8499ea6d | ||
|
|
6d698eab08 | ||
|
|
1937527249 | ||
|
|
e03128a8de | ||
|
|
fefbcc0b4c | ||
|
|
b312895e77 | ||
|
|
e2981d2743 | ||
|
|
ae9bc914d6 | ||
|
|
6167702345 | ||
|
|
7e9ff62d89 | ||
|
|
a8b7e33010 | ||
|
|
a2de48bdfb | ||
|
|
d1640cf10f | ||
|
|
45944ddde7 | ||
|
|
52369bdf47 | ||
|
|
254cf83310 | ||
|
|
bca5edf108 | ||
|
|
b04964f4b7 | ||
|
|
63c466d821 | ||
|
|
224b0fc2ce | ||
|
|
14b213adec | ||
|
|
f0c81bcef5 | ||
|
|
128fa7f887 | ||
|
|
11e56dbdf9 | ||
|
|
c60f364774 | ||
|
|
45c2c72b60 | ||
|
|
d3d471c0a5 | ||
|
|
8455b7bece | ||
|
|
7275843c08 | ||
|
|
65854c4946 | ||
|
|
211f56c043 | ||
|
|
fc6a10f486 | ||
|
|
63fe2893a8 | ||
|
|
43dcc97512 | ||
|
|
ce2481bbe8 | ||
|
|
e05157f8e0 | ||
|
|
0e854f9534 | ||
|
|
cdef709e8a | ||
|
|
c1ac6bb06b | ||
|
|
290a9fd434 | ||
|
|
26980ae45b | ||
|
|
f325921315 | ||
|
|
dfe184f149 | ||
|
|
ca60c8e0c6 | ||
|
|
6ab06bbf9e | ||
|
|
1b8e0725d8 | ||
|
|
7b1dbd6109 | ||
|
|
7bd473371c | ||
|
|
b9d560c584 | ||
|
|
f4bec650ed | ||
|
|
8e675a9d42 | ||
|
|
a338fad036 | ||
|
|
ee0066fae4 | ||
|
|
b02babee9e | ||
|
|
f4d91b35d7 | ||
|
|
60ace79305 | ||
|
|
d2ea180a9a | ||
|
|
55644f376b | ||
|
|
38e818bb1d | ||
|
|
d276ba4acc | ||
|
|
f6077d06c4 | ||
|
|
0c2e4d0b72 | ||
|
|
617014ffbb | ||
|
|
eeb6b60fe6 | ||
|
|
1ff4706f3a | ||
|
|
acdc8763c6 | ||
|
|
af6c4291b9 | ||
|
|
3190480c12 | ||
|
|
8a4827017e | ||
|
|
d8ac973362 | ||
|
|
a7aecf7c82 | ||
|
|
eee3d7674f | ||
|
|
f69cb7a902 | ||
|
|
02f92b3ac8 | ||
|
|
974f81791f | ||
|
|
912a98e05e | ||
|
|
42e3b8aa25 | ||
|
|
9de8d1e0fa | ||
|
|
0bf6fdae14 | ||
|
|
9b38aa4680 | ||
|
|
28748ff345 | ||
|
|
1ddab1077b | ||
|
|
1455e9551e | ||
|
|
9fd7d747c1 | ||
|
|
a4094f0a7f | ||
|
|
ee26afe65e | ||
|
|
9e7740572c | ||
|
|
205a605887 | ||
|
|
8f546f253b | ||
|
|
65d22c7dbd | ||
|
|
caf5b931b9 | ||
|
|
84760dc738 | ||
|
|
93246f0658 | ||
|
|
4489f98dc3 | ||
|
|
de19d15332 | ||
|
|
a1fe6c6f44 | ||
|
|
7edee656da | ||
|
|
498ee509da | ||
|
|
55f86df5b1 | ||
|
|
eae432818a | ||
|
|
aef3dfba13 | ||
|
|
42c8fab47f | ||
|
|
f19d800f43 | ||
|
|
137373c165 | ||
|
|
6559b9d222 | ||
|
|
4b5d4d759a | ||
|
|
e1334448ea |
@@ -1,7 +0,0 @@
|
||||
[flake8]
|
||||
ignore =
|
||||
E203, # Whitespace before ':'
|
||||
E266, # Too many leading '#' for block comment
|
||||
W503, # Line break occurred before a binary operator
|
||||
max-line-length = 79
|
||||
select = B,C,E,F,W,T4,B9
|
||||
@@ -0,0 +1,3 @@
|
||||
# These are supported funding model platforms
|
||||
|
||||
patreon: DVCorg
|
||||
@@ -1,7 +1,7 @@
|
||||
* [ ] ❗ Have you followed the guidelines in the [Contributing to DVC](https://dvc.org/doc/user-guide/contributing/core) list?
|
||||
* [ ] ❗ I have followed the [Contributing to DVC](https://dvc.org/doc/user-guide/contributing/core) checklist.
|
||||
|
||||
* [ ] 📖 Check this box if this PR **does not** require [documentation](https://dvc.org/doc) updates, or if it does **and** you have created a separate PR in [dvc.org](https://github.com/iterative/dvc.org) with such updates (or at least opened an issue about it in that repo). Please link below to your PR (or issue) in the [dvc.org](https://github.com/iterative/dvc.org) repo.
|
||||
* [ ] 📖 If this PR requires [documentation](https://dvc.org/doc) updates, I have created a separate PR (or issue, at least) in [dvc.org](https://github.com/iterative/dvc.org) and linked it here. If the CLI API is changed, I have updated [tab completion scripts](https://github.com/iterative/dvc/tree/master/scripts/completion).
|
||||
|
||||
* [ ] ❌ Have you checked DeepSource, CodeClimate, and other sanity checks below? We consider their findings recommendatory and don't expect everything to be addressed. Please review them carefully and fix those that actually improve code or fix bugs.
|
||||
* [ ] ❌ I will check DeepSource, CodeClimate, and other sanity checks below. (We consider them recommendatory and don't expect everything to be addressed. Please fix things that actually improve code or fix bugs.)
|
||||
|
||||
Thank you for the contribution - we'll try to review it as soon as possible. 🙏
|
||||
|
||||
+10
-1
@@ -4,11 +4,20 @@ repos:
|
||||
language_version: python3
|
||||
repo: https://github.com/ambv/black
|
||||
rev: 19.10b0
|
||||
- repo: https://github.com/asottile/seed-isort-config
|
||||
rev: v2.1.1
|
||||
hooks:
|
||||
- id: seed-isort-config
|
||||
- hooks:
|
||||
- id: isort
|
||||
language_version: python3
|
||||
repo: https://github.com/timothycrosley/isort
|
||||
rev: 4.3.21
|
||||
- hooks:
|
||||
- id: flake8
|
||||
language_version: python3
|
||||
repo: https://gitlab.com/pycqa/flake8
|
||||
rev: master
|
||||
rev: 3.7.9
|
||||
- hooks:
|
||||
- args:
|
||||
- -i
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
name: DVC pre-commit
|
||||
stages:
|
||||
- commit
|
||||
verbose: true
|
||||
- args:
|
||||
- git-hook
|
||||
- pre-push
|
||||
|
||||
@@ -9,3 +9,12 @@ restylers:
|
||||
- "**/*.py"
|
||||
interpreters:
|
||||
- python
|
||||
- name: isort
|
||||
image: restyled/restyler-isort:v4.3.21
|
||||
command:
|
||||
- isort
|
||||
arguments: []
|
||||
include:
|
||||
- "**/*.py"
|
||||
interpreters:
|
||||
- python
|
||||
|
||||
+3
-6
@@ -46,9 +46,6 @@ jobs:
|
||||
env:
|
||||
- PYTHON_VERSION=3.8.0
|
||||
- PATH=/c/Python38:/c/Python38/Scripts:$PATH
|
||||
- name: "3.5 on Linux"
|
||||
language: python
|
||||
python: 3.5
|
||||
- name: "3.6 on Linux"
|
||||
language: python
|
||||
python: 3.6
|
||||
@@ -106,6 +103,7 @@ jobs:
|
||||
- sudo /snap/bin/lxd.migrate -yes
|
||||
- sudo /snap/bin/lxd waitready
|
||||
- sudo /snap/bin/lxd init --auto
|
||||
- git fetch --tags
|
||||
script:
|
||||
- ./scripts/build_snap.sh
|
||||
after_failure:
|
||||
@@ -151,10 +149,9 @@ deploy:
|
||||
tags: true
|
||||
repo: iterative/dvc
|
||||
stage: build
|
||||
- provider: snap
|
||||
snap: dvc_*.snap
|
||||
channel: $SNAP_CHANNEL
|
||||
- provider: script
|
||||
skip_cleanup: true
|
||||
script: "(echo $SNAP_TOKEN | snapcraft login --with -) && timeout 600 snapcraft push dvc_*.snap --release $SNAP_CHANNEL || echo timed out"
|
||||
on:
|
||||
all_branches: true
|
||||
condition: "$(./scripts/ci/deploy_condition.sh dvc_*.snap) && ($TRAVIS_BRANCH = master || -n $TRAVIS_TAG)"
|
||||
|
||||
+5
-4
@@ -101,7 +101,8 @@ Snap (Snapcraft/Linux)
|
||||
snap install dvc --classic
|
||||
|
||||
This corresponds to the latest tagged release.
|
||||
Add ``--edge`` for the latest ``master`` version.
|
||||
Add ``--beta`` for the latest tagged release candidate,
|
||||
or ``--edge`` for the latest ``master`` version.
|
||||
|
||||
Choco (Chocolatey/Windows)
|
||||
--------------------------
|
||||
@@ -255,7 +256,7 @@ Citation
|
||||
Iterative, *DVC: Data Version Control - Git for Data & Models* (2020)
|
||||
`DOI:10.5281/zenodo.012345 <https://doi.org/10.5281/zenodo.3677553>`_.
|
||||
|
||||
.. |Banner| image:: https://dvc.org/static/img/logo-github-readme.png
|
||||
.. |Banner| image:: https://dvc.org/img/logo-github-readme.png
|
||||
:target: https://dvc.org
|
||||
:alt: DVC logo
|
||||
|
||||
@@ -307,6 +308,6 @@ Iterative, *DVC: Data Version Control - Git for Data & Models* (2020)
|
||||
:target: https://doi.org/10.5281/zenodo.3677553
|
||||
:alt: DOI
|
||||
|
||||
.. |Flowchart| image:: https://dvc.org/static/img/flow.gif
|
||||
:target: https://dvc.org/static/img/flow.gif
|
||||
.. |Flowchart| image:: https://dvc.org/img/flow.gif
|
||||
:target: https://dvc.org/img/flow.gif
|
||||
:alt: how_dvc_works
|
||||
|
||||
@@ -3,17 +3,7 @@ DVC
|
||||
----
|
||||
Make your data science projects reproducible and shareable.
|
||||
"""
|
||||
import warnings
|
||||
|
||||
import dvc.logger
|
||||
from dvc.version import __version__ # noqa: F401
|
||||
|
||||
|
||||
dvc.logger.setup()
|
||||
|
||||
# Ignore numpy's runtime warnings: https://github.com/numpy/numpy/pull/432.
|
||||
# We don't directly import numpy, but our dependency networkx does, causing
|
||||
# these warnings in some environments. Luckily these warnings are benign and
|
||||
# we can simply ignore them so that they don't show up when you are using dvc.
|
||||
warnings.filterwarnings("ignore", message="numpy.dtype size changed")
|
||||
warnings.filterwarnings("ignore", message="numpy.ufunc size changed")
|
||||
|
||||
+15
-14
@@ -2,12 +2,12 @@ import json
|
||||
import logging
|
||||
import os
|
||||
import platform
|
||||
import requests
|
||||
import sys
|
||||
import tempfile
|
||||
import uuid
|
||||
|
||||
import distro
|
||||
import requests
|
||||
|
||||
from dvc import __version__
|
||||
from dvc.config import Config, to_bool
|
||||
@@ -20,7 +20,6 @@ from dvc.scm.base import SCMError
|
||||
from dvc.utils import env2bool, is_binary
|
||||
from dvc.utils.fs import makedirs
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -36,7 +35,7 @@ def collect_and_send_report(args=None, return_code=None):
|
||||
report as a JSON, where the _collector_ generates it and the _sender_
|
||||
removes it after sending it.
|
||||
"""
|
||||
report = _runtime_info()
|
||||
report = {}
|
||||
|
||||
# Include command execution information on the report only when available.
|
||||
if args and hasattr(args, "func"):
|
||||
@@ -62,7 +61,7 @@ def is_enabled():
|
||||
return enabled
|
||||
|
||||
|
||||
def send(report):
|
||||
def send(path):
|
||||
"""
|
||||
Side effect: Removes the report after sending it.
|
||||
|
||||
@@ -73,13 +72,17 @@ def send(report):
|
||||
url = "https://analytics.dvc.org"
|
||||
headers = {"content-type": "application/json"}
|
||||
|
||||
with open(report, "rb") as fobj:
|
||||
try:
|
||||
requests.post(url, data=fobj, headers=headers, timeout=5)
|
||||
except requests.exceptions.RequestException:
|
||||
logger.debug("failed to send analytics report", exc_info=True)
|
||||
with open(path) as fobj:
|
||||
report = json.load(fobj)
|
||||
|
||||
os.remove(report)
|
||||
report.update(_runtime_info())
|
||||
|
||||
try:
|
||||
requests.post(url, json=report, headers=headers, timeout=5)
|
||||
except requests.exceptions.RequestException:
|
||||
logger.debug("failed to send analytics report", exc_info=True)
|
||||
|
||||
os.remove(path)
|
||||
|
||||
|
||||
def _scm_in_use():
|
||||
@@ -155,7 +158,7 @@ def _find_or_create_user_id():
|
||||
try:
|
||||
with Lock(lockfile):
|
||||
try:
|
||||
with open(fname, "r") as fobj:
|
||||
with open(fname) as fobj:
|
||||
user_id = json.load(fobj)["user_id"]
|
||||
|
||||
except (FileNotFoundError, ValueError, KeyError):
|
||||
@@ -167,6 +170,4 @@ def _find_or_create_user_id():
|
||||
return user_id
|
||||
|
||||
except LockError:
|
||||
logger.debug(
|
||||
"Failed to acquire '{lockfile}'".format(lockfile=lockfile)
|
||||
)
|
||||
logger.debug(f"Failed to acquire '{lockfile}'")
|
||||
|
||||
+4
-3
@@ -1,16 +1,17 @@
|
||||
import os
|
||||
from contextlib import contextmanager, _GeneratorContextManager as GCM
|
||||
from contextlib import _GeneratorContextManager as GCM
|
||||
from contextlib import contextmanager
|
||||
|
||||
from dvc.repo import Repo
|
||||
from dvc.exceptions import DvcException, NotDvcRepoError
|
||||
from dvc.external_repo import external_repo
|
||||
from dvc.repo import Repo
|
||||
|
||||
|
||||
class UrlNotDvcRepoError(DvcException):
|
||||
"""Thrown if the given URL is not a DVC repository."""
|
||||
|
||||
def __init__(self, url):
|
||||
super().__init__("'{}' is not a DVC repository.".format(url))
|
||||
super().__init__(f"'{url}' is not a DVC repository.")
|
||||
|
||||
|
||||
def get_url(path, repo=None, rev=None, remote=None):
|
||||
|
||||
+71
-9
@@ -40,7 +40,7 @@ def _make_remote_property(name):
|
||||
return cached_property(getter)
|
||||
|
||||
|
||||
class Cache(object):
|
||||
class Cache:
|
||||
"""Class that manages cache locations of a DVC repo.
|
||||
|
||||
Args:
|
||||
@@ -71,9 +71,39 @@ class Cache(object):
|
||||
azure = _make_remote_property("azure")
|
||||
|
||||
|
||||
class NamedCache(object):
|
||||
class NamedCacheItem:
|
||||
def __init__(self):
|
||||
self._items = defaultdict(lambda: defaultdict(set))
|
||||
self.names = set()
|
||||
self.children = defaultdict(NamedCacheItem)
|
||||
|
||||
def __eq__(self, other):
|
||||
return self.names == other.names and self.children == other.children
|
||||
|
||||
def child_keys(self):
|
||||
for key, child in self.children.items():
|
||||
yield key
|
||||
yield from child.child_keys()
|
||||
|
||||
def child_names(self):
|
||||
for key, child in self.children.items():
|
||||
yield key, child.names
|
||||
yield from child.child_names()
|
||||
|
||||
def add(self, checksum, item):
|
||||
self.children[checksum].update(item)
|
||||
|
||||
def update(self, item, suffix=""):
|
||||
if suffix:
|
||||
self.names.update(n + suffix for n in item.names)
|
||||
else:
|
||||
self.names.update(item.names)
|
||||
for checksum, child_item in item.children.items():
|
||||
self.children[checksum].update(child_item)
|
||||
|
||||
|
||||
class NamedCache:
|
||||
def __init__(self):
|
||||
self._items = defaultdict(lambda: defaultdict(NamedCacheItem))
|
||||
self.external = defaultdict(set)
|
||||
|
||||
@classmethod
|
||||
@@ -86,7 +116,18 @@ class NamedCache(object):
|
||||
return self._items[key]
|
||||
|
||||
def add(self, scheme, checksum, name):
|
||||
self._items[scheme][checksum].add(name)
|
||||
"""Add a mapped name for the specified checksum."""
|
||||
self._items[scheme][checksum].names.add(name)
|
||||
|
||||
def add_child_cache(self, checksum, cache, suffix=""):
|
||||
"""Add/update child cache for the specified checksum."""
|
||||
for scheme, src in cache._items.items():
|
||||
dst = self._items[scheme][checksum].children
|
||||
for child_checksum, item in src.items():
|
||||
dst[child_checksum].update(item, suffix=suffix)
|
||||
|
||||
for repo_pair, files in cache.external.items():
|
||||
self.external[repo_pair].update(files)
|
||||
|
||||
def add_external(self, url, rev, path):
|
||||
self.external[url, rev].add(path)
|
||||
@@ -94,11 +135,32 @@ class NamedCache(object):
|
||||
def update(self, cache, suffix=""):
|
||||
for scheme, src in cache._items.items():
|
||||
dst = self._items[scheme]
|
||||
for checksum, names in src.items():
|
||||
if suffix:
|
||||
dst[checksum].update(n + suffix for n in names)
|
||||
else:
|
||||
dst[checksum].update(names)
|
||||
for checksum, item in src.items():
|
||||
dst[checksum].update(item, suffix=suffix)
|
||||
|
||||
for repo_pair, files in cache.external.items():
|
||||
self.external[repo_pair].update(files)
|
||||
|
||||
def scheme_keys(self, scheme):
|
||||
"""Iterate over a flat list of all keys for the specified scheme,
|
||||
including children.
|
||||
"""
|
||||
for key, item in self._items[scheme].items():
|
||||
yield key
|
||||
yield from item.child_keys()
|
||||
|
||||
def scheme_names(self, scheme):
|
||||
"""Iterate over a flat list of checksum, names items for the specified
|
||||
scheme, including children.
|
||||
"""
|
||||
for key, item in self._items[scheme].items():
|
||||
yield key, item.names
|
||||
yield from item.child_names()
|
||||
|
||||
def dir_keys(self, scheme):
|
||||
return (
|
||||
key for key, item in self._items[scheme].items() if item.children
|
||||
)
|
||||
|
||||
def child_keys(self, scheme, checksum):
|
||||
return self._items[scheme][checksum].child_keys()
|
||||
|
||||
+8
-11
@@ -16,30 +16,30 @@ from .command import (
|
||||
gc,
|
||||
get,
|
||||
get_url,
|
||||
git_hook,
|
||||
imp,
|
||||
imp_url,
|
||||
init,
|
||||
install,
|
||||
ls,
|
||||
lock,
|
||||
ls,
|
||||
metrics,
|
||||
move,
|
||||
params,
|
||||
pipeline,
|
||||
plots,
|
||||
remote,
|
||||
remove,
|
||||
repro,
|
||||
root,
|
||||
run,
|
||||
tag,
|
||||
unprotect,
|
||||
update,
|
||||
version,
|
||||
git_hook,
|
||||
)
|
||||
from .command.base import fix_subparsers
|
||||
from .exceptions import DvcParserError
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
COMMANDS = [
|
||||
@@ -62,6 +62,7 @@ COMMANDS = [
|
||||
remote,
|
||||
cache,
|
||||
metrics,
|
||||
params,
|
||||
install,
|
||||
root,
|
||||
ls,
|
||||
@@ -69,11 +70,11 @@ COMMANDS = [
|
||||
pipeline,
|
||||
daemon,
|
||||
commit,
|
||||
tag,
|
||||
diff,
|
||||
version,
|
||||
update,
|
||||
git_hook,
|
||||
plots,
|
||||
]
|
||||
|
||||
|
||||
@@ -115,14 +116,10 @@ def get_parent_parser():
|
||||
|
||||
log_level_group = parent_parser.add_mutually_exclusive_group()
|
||||
log_level_group.add_argument(
|
||||
"-q", "--quiet", action="store_true", default=False, help="Be quiet."
|
||||
"-q", "--quiet", action="count", default=0, help="Be quiet."
|
||||
)
|
||||
log_level_group.add_argument(
|
||||
"-v",
|
||||
"--verbose",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Be verbose.",
|
||||
"-v", "--verbose", action="count", default=0, help="Be verbose."
|
||||
)
|
||||
|
||||
return parent_parser
|
||||
|
||||
+2
-1
@@ -1,7 +1,7 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command.base import append_doc_link, CmdBase
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException, RecursiveAddingWhileUsingFilename
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -53,6 +53,7 @@ def add_parser(subparsers, parent_parser):
|
||||
"-f",
|
||||
"--file",
|
||||
help="Specify name of the DVC-file this command will generate.",
|
||||
metavar="<filename>",
|
||||
)
|
||||
parser.add_argument(
|
||||
"targets", nargs="+", help="Input files/directories to add."
|
||||
|
||||
+6
-8
@@ -1,6 +1,5 @@
|
||||
import logging
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -27,7 +26,7 @@ def append_doc_link(help_message, path):
|
||||
)
|
||||
|
||||
|
||||
class CmdBase(object):
|
||||
class CmdBase:
|
||||
def __init__(self, args):
|
||||
from dvc.repo import Repo
|
||||
from dvc.updater import Updater
|
||||
@@ -36,17 +35,16 @@ class CmdBase(object):
|
||||
self.config = self.repo.config
|
||||
self.args = args
|
||||
hardlink_lock = self.config["core"].get("hardlink_lock", False)
|
||||
updater = Updater(self.repo.dvc_dir, hardlink_lock=hardlink_lock)
|
||||
updater = Updater(self.repo.tmp_dir, hardlink_lock=hardlink_lock)
|
||||
updater.check()
|
||||
|
||||
@property
|
||||
def default_targets(self):
|
||||
"""Default targets for `dvc repro` and `dvc pipeline`."""
|
||||
from dvc.stage import Stage
|
||||
"""Default targets for `dvc repro`."""
|
||||
from dvc.dvcfile import PIPELINE_FILE
|
||||
|
||||
msg = "assuming default target '{}'.".format(Stage.STAGE_FILE)
|
||||
logger.warning(msg)
|
||||
return [Stage.STAGE_FILE]
|
||||
logger.debug(f"assuming default target '{PIPELINE_FILE}'.")
|
||||
return [PIPELINE_FILE]
|
||||
|
||||
# Abstract methods that have to be implemented by any inheritance class
|
||||
def run(self):
|
||||
|
||||
+8
-27
@@ -1,39 +1,16 @@
|
||||
import argparse
|
||||
import logging
|
||||
import operator
|
||||
|
||||
import colorama
|
||||
|
||||
from dvc.command.base import append_doc_link
|
||||
from dvc.command.base import CmdBase
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import CheckoutError
|
||||
from dvc.utils.humanize import get_summary
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _human_join(words):
|
||||
words = list(words)
|
||||
if not words:
|
||||
return ""
|
||||
|
||||
return (
|
||||
"{before} and {after}".format(
|
||||
before=", ".join(words[:-1]), after=words[-1],
|
||||
)
|
||||
if len(words) > 1
|
||||
else words[0]
|
||||
)
|
||||
|
||||
|
||||
def log_summary(stats):
|
||||
states = ("added", "deleted", "modified")
|
||||
summary = (
|
||||
"{} {}".format(len(stats[state]), state)
|
||||
for state in states
|
||||
if stats.get(state)
|
||||
)
|
||||
logger.info(_human_join(summary))
|
||||
|
||||
|
||||
def log_changes(stats):
|
||||
colors = [
|
||||
("modified", colorama.Fore.YELLOW,),
|
||||
@@ -75,7 +52,11 @@ class CmdCheckout(CmdBase):
|
||||
stats = exc.stats
|
||||
|
||||
if self.args.summary:
|
||||
log_summary(stats)
|
||||
default_message = "No changes."
|
||||
msg = get_summary(
|
||||
sorted(stats.items(), key=operator.itemgetter(0))
|
||||
)
|
||||
logger.info(msg or default_message)
|
||||
else:
|
||||
log_changes(stats)
|
||||
|
||||
|
||||
@@ -1,11 +1,9 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command.base import append_doc_link
|
||||
from dvc.command.base import CmdBase
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command.base import append_doc_link, CmdBaseNoRepo
|
||||
from dvc.command.base import CmdBaseNoRepo, append_doc_link
|
||||
from dvc.config import Config, ConfigError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
from dvc.command.base import CmdBaseNoRepo
|
||||
from dvc.command.base import fix_subparsers
|
||||
from dvc.command.base import CmdBaseNoRepo, fix_subparsers
|
||||
|
||||
|
||||
class CmdDaemonBase(CmdBaseNoRepo):
|
||||
@@ -15,9 +14,10 @@ class CmdDaemonUpdater(CmdDaemonBase):
|
||||
|
||||
root_dir = Repo.find_root()
|
||||
dvc_dir = os.path.join(root_dir, Repo.DVC_DIR)
|
||||
tmp_dir = os.path.join(dvc_dir, "tmp")
|
||||
config = Config(dvc_dir, validate=False)
|
||||
hardlink_lock = config.get("core", {}).get("hardlink_lock", False)
|
||||
updater = Updater(dvc_dir, hardlink_lock=hardlink_lock)
|
||||
updater = Updater(tmp_dir, hardlink_lock=hardlink_lock)
|
||||
updater.fetch(detach=False)
|
||||
|
||||
return 0
|
||||
|
||||
+98
-29
@@ -1,23 +1,25 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command.base import append_doc_link
|
||||
from dvc.command.base import CmdBase
|
||||
from dvc.command.checkout import log_summary
|
||||
from dvc.exceptions import DvcException, CheckoutError
|
||||
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.command.checkout import log_changes
|
||||
from dvc.exceptions import CheckoutError, DvcException
|
||||
from dvc.utils.humanize import get_summary
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CmdDataBase(CmdBase):
|
||||
@classmethod
|
||||
def check_up_to_date(cls, processed_files_count):
|
||||
if processed_files_count == 0:
|
||||
logger.info("Everything is up to date.")
|
||||
def log_summary(self, stats):
|
||||
default_msg = "Everything is up to date."
|
||||
logger.info(get_summary(stats.items()) or default_msg)
|
||||
|
||||
|
||||
class CmdDataPull(CmdDataBase):
|
||||
def log_summary(self, stats):
|
||||
log_changes(stats)
|
||||
super().log_summary(stats)
|
||||
|
||||
def run(self):
|
||||
try:
|
||||
stats = self.repo.pull(
|
||||
@@ -26,17 +28,18 @@ class CmdDataPull(CmdDataBase):
|
||||
remote=self.args.remote,
|
||||
all_branches=self.args.all_branches,
|
||||
all_tags=self.args.all_tags,
|
||||
all_commits=self.args.all_commits,
|
||||
with_deps=self.args.with_deps,
|
||||
force=self.args.force,
|
||||
recursive=self.args.recursive,
|
||||
run_cache=self.args.run_cache,
|
||||
)
|
||||
log_summary(stats)
|
||||
self.log_summary(stats)
|
||||
except (CheckoutError, DvcException) as exc:
|
||||
log_summary(getattr(exc, "stats", {}))
|
||||
self.log_summary(getattr(exc, "stats", {}))
|
||||
logger.exception("failed to pull data from the cloud")
|
||||
return 1
|
||||
|
||||
self.check_up_to_date(stats["downloaded"])
|
||||
return 0
|
||||
|
||||
|
||||
@@ -49,13 +52,15 @@ class CmdDataPush(CmdDataBase):
|
||||
remote=self.args.remote,
|
||||
all_branches=self.args.all_branches,
|
||||
all_tags=self.args.all_tags,
|
||||
all_commits=self.args.all_commits,
|
||||
with_deps=self.args.with_deps,
|
||||
recursive=self.args.recursive,
|
||||
run_cache=self.args.run_cache,
|
||||
)
|
||||
self.log_summary({"pushed": processed_files_count})
|
||||
except DvcException:
|
||||
logger.exception("failed to push data to the cloud")
|
||||
return 1
|
||||
self.check_up_to_date(processed_files_count)
|
||||
return 0
|
||||
|
||||
|
||||
@@ -68,13 +73,15 @@ class CmdDataFetch(CmdDataBase):
|
||||
remote=self.args.remote,
|
||||
all_branches=self.args.all_branches,
|
||||
all_tags=self.args.all_tags,
|
||||
all_commits=self.args.all_commits,
|
||||
with_deps=self.args.with_deps,
|
||||
recursive=self.args.recursive,
|
||||
run_cache=self.args.run_cache,
|
||||
)
|
||||
self.log_summary({"fetched": processed_files_count})
|
||||
except DvcException:
|
||||
logger.exception("failed to fetch data from the cloud")
|
||||
return 1
|
||||
self.check_up_to_date(processed_files_count)
|
||||
return 0
|
||||
|
||||
|
||||
@@ -82,27 +89,31 @@ def shared_parent_parser():
|
||||
from dvc.cli import get_parent_parser
|
||||
|
||||
# Parent parser used in pull/push/status
|
||||
shared_parent_parser = argparse.ArgumentParser(
|
||||
parent_parser = argparse.ArgumentParser(
|
||||
add_help=False, parents=[get_parent_parser()]
|
||||
)
|
||||
shared_parent_parser.add_argument(
|
||||
"-j", "--jobs", type=int, help="Number of jobs to run simultaneously."
|
||||
parent_parser.add_argument(
|
||||
"-j",
|
||||
"--jobs",
|
||||
type=int,
|
||||
help="Number of jobs to run simultaneously.",
|
||||
metavar="<number>",
|
||||
)
|
||||
shared_parent_parser.add_argument(
|
||||
parent_parser.add_argument(
|
||||
"targets",
|
||||
nargs="*",
|
||||
help="Limit command scope to these DVC-files. "
|
||||
"Using -R, directories to search DVC-files in can also be given.",
|
||||
)
|
||||
|
||||
return shared_parent_parser
|
||||
return parent_parser
|
||||
|
||||
|
||||
def add_parser(subparsers, _parent_parser):
|
||||
from dvc.command.status import CmdDataStatus
|
||||
|
||||
# Pull
|
||||
PULL_HELP = "Pull data files from a DVC remote storage."
|
||||
PULL_HELP = "Download tracked files or directories from remote storage."
|
||||
|
||||
pull_parser = subparsers.add_parser(
|
||||
"pull",
|
||||
@@ -112,7 +123,7 @@ def add_parser(subparsers, _parent_parser):
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
pull_parser.add_argument(
|
||||
"-r", "--remote", help="Remote repository to pull from."
|
||||
"-r", "--remote", help="Remote storage to pull from", metavar="<name>",
|
||||
)
|
||||
pull_parser.add_argument(
|
||||
"-a",
|
||||
@@ -128,6 +139,12 @@ def add_parser(subparsers, _parent_parser):
|
||||
default=False,
|
||||
help="Fetch cache for all tags.",
|
||||
)
|
||||
pull_parser.add_argument(
|
||||
"--all-commits",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Fetch cache for all commits.",
|
||||
)
|
||||
pull_parser.add_argument(
|
||||
"-f",
|
||||
"--force",
|
||||
@@ -149,11 +166,16 @@ def add_parser(subparsers, _parent_parser):
|
||||
default=False,
|
||||
help="Pull cache for subdirectories of the specified directory.",
|
||||
)
|
||||
pull_parser.add_argument(
|
||||
"--run-cache",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help=argparse.SUPPRESS,
|
||||
)
|
||||
pull_parser.set_defaults(func=CmdDataPull)
|
||||
|
||||
# Push
|
||||
PUSH_HELP = "Uploads files or directories tracked by DVC"
|
||||
" to remote storage."
|
||||
PUSH_HELP = "Upload tracked files or directories to remote storage."
|
||||
|
||||
push_parser = subparsers.add_parser(
|
||||
"push",
|
||||
@@ -163,7 +185,7 @@ def add_parser(subparsers, _parent_parser):
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
push_parser.add_argument(
|
||||
"-r", "--remote", help="Remote repository to push to."
|
||||
"-r", "--remote", help="Remote storage to push to", metavar="<name>",
|
||||
)
|
||||
push_parser.add_argument(
|
||||
"-a",
|
||||
@@ -179,6 +201,12 @@ def add_parser(subparsers, _parent_parser):
|
||||
default=False,
|
||||
help="Push cache for all tags.",
|
||||
)
|
||||
push_parser.add_argument(
|
||||
"--all-commits",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Push cache for all commits.",
|
||||
)
|
||||
push_parser.add_argument(
|
||||
"-d",
|
||||
"--with-deps",
|
||||
@@ -191,13 +219,21 @@ def add_parser(subparsers, _parent_parser):
|
||||
"--recursive",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Push cache from subdirectories of specified directory.",
|
||||
help="Push cache for subdirectories of specified directory.",
|
||||
)
|
||||
push_parser.add_argument(
|
||||
"--run-cache",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help=argparse.SUPPRESS,
|
||||
)
|
||||
push_parser.set_defaults(func=CmdDataPush)
|
||||
|
||||
# Fetch
|
||||
FETCH_HELP = "Get files or directories tracked by DVC from remote storage"
|
||||
" into the cache."
|
||||
FETCH_HELP = (
|
||||
"Get tracked files or directories from remote storage"
|
||||
" into the cache."
|
||||
)
|
||||
|
||||
fetch_parser = subparsers.add_parser(
|
||||
"fetch",
|
||||
@@ -207,7 +243,10 @@ def add_parser(subparsers, _parent_parser):
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
fetch_parser.add_argument(
|
||||
"-r", "--remote", help="Remote repository to fetch from."
|
||||
"-r",
|
||||
"--remote",
|
||||
help="Remote storage to fetch from",
|
||||
metavar="<name>",
|
||||
)
|
||||
fetch_parser.add_argument(
|
||||
"-a",
|
||||
@@ -223,6 +262,12 @@ def add_parser(subparsers, _parent_parser):
|
||||
default=False,
|
||||
help="Fetch cache for all tags.",
|
||||
)
|
||||
fetch_parser.add_argument(
|
||||
"--all-commits",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Fetch cache for all commits.",
|
||||
)
|
||||
fetch_parser.add_argument(
|
||||
"-d",
|
||||
"--with-deps",
|
||||
@@ -237,6 +282,12 @@ def add_parser(subparsers, _parent_parser):
|
||||
default=False,
|
||||
help="Fetch cache for subdirectories of specified directory.",
|
||||
)
|
||||
fetch_parser.add_argument(
|
||||
"--run-cache",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help=argparse.SUPPRESS,
|
||||
)
|
||||
fetch_parser.set_defaults(func=CmdDataFetch)
|
||||
|
||||
# Status
|
||||
@@ -270,7 +321,10 @@ def add_parser(subparsers, _parent_parser):
|
||||
help="Show status of a local cache compared to a remote repository.",
|
||||
)
|
||||
status_parser.add_argument(
|
||||
"-r", "--remote", help="Remote repository to compare local cache to."
|
||||
"-r",
|
||||
"--remote",
|
||||
help="Remote storage to compare local cache to",
|
||||
metavar="<name>",
|
||||
)
|
||||
status_parser.add_argument(
|
||||
"-a",
|
||||
@@ -288,6 +342,13 @@ def add_parser(subparsers, _parent_parser):
|
||||
help="Show status of a local cache compared to a remote repository "
|
||||
"for all tags.",
|
||||
)
|
||||
status_parser.add_argument(
|
||||
"--all-commits",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Show status of a local cache compared to a remote repository "
|
||||
"for all commits.",
|
||||
)
|
||||
status_parser.add_argument(
|
||||
"-d",
|
||||
"--with-deps",
|
||||
@@ -295,4 +356,12 @@ def add_parser(subparsers, _parent_parser):
|
||||
default=False,
|
||||
help="Show status for all dependencies of the specified target.",
|
||||
)
|
||||
status_parser.add_argument(
|
||||
"-R",
|
||||
"--recursive",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Show status of all stages in the specified directory.",
|
||||
)
|
||||
|
||||
status_parser.set_defaults(func=CmdDataStatus)
|
||||
|
||||
@@ -2,11 +2,9 @@ import argparse
|
||||
import logging
|
||||
|
||||
import dvc.prompt as prompt
|
||||
from dvc.command.base import append_doc_link
|
||||
from dvc.command.base import CmdBase
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
||||
+28
-12
@@ -8,10 +8,24 @@ import colorama
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _show_md(diff):
|
||||
from dvc.utils.diff import table
|
||||
|
||||
rows = []
|
||||
for status in ["added", "deleted", "modified"]:
|
||||
entries = diff.get(status, [])
|
||||
if not entries:
|
||||
continue
|
||||
paths = sorted([entry["path"] for entry in entries])
|
||||
for path in paths:
|
||||
rows.append([status, path])
|
||||
|
||||
return table(["Status", "Path"], rows, True)
|
||||
|
||||
|
||||
class CmdDiff(CmdBase):
|
||||
@staticmethod
|
||||
def _format(diff):
|
||||
@@ -97,20 +111,17 @@ class CmdDiff(CmdBase):
|
||||
try:
|
||||
diff = self.repo.diff(self.args.a_rev, self.args.b_rev)
|
||||
|
||||
if not any(diff.values()):
|
||||
return 0
|
||||
|
||||
if not self.args.show_hash:
|
||||
for _, entries in diff.items():
|
||||
for entry in entries:
|
||||
del entry["hash"]
|
||||
|
||||
if self.args.show_json:
|
||||
res = json.dumps(diff)
|
||||
else:
|
||||
res = self._format(diff)
|
||||
|
||||
logger.info(res)
|
||||
logger.info(json.dumps(diff))
|
||||
elif self.args.show_md:
|
||||
logger.info(_show_md(diff))
|
||||
elif diff:
|
||||
logger.info(self._format(diff))
|
||||
|
||||
except DvcException:
|
||||
logger.exception("failed to get diff")
|
||||
@@ -120,9 +131,8 @@ class CmdDiff(CmdBase):
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
DIFF_DESCRIPTION = (
|
||||
"Show changes between commits in the DVC repository,"
|
||||
" or between a commit and the workspace."
|
||||
"Prints the list of paths added, modified, or deleted."
|
||||
"Show added, modified, or deleted data between commits in the DVC"
|
||||
" repository, or between a commit and the workspace."
|
||||
)
|
||||
diff_parser = subparsers.add_parser(
|
||||
"diff",
|
||||
@@ -154,4 +164,10 @@ def add_parser(subparsers, parent_parser):
|
||||
action="store_true",
|
||||
default=False,
|
||||
)
|
||||
diff_parser.add_argument(
|
||||
"--show-md",
|
||||
help="Show tabulated output in the Markdown format (GFM).",
|
||||
action="store_true",
|
||||
default=False,
|
||||
)
|
||||
diff_parser.set_defaults(func=CmdDiff)
|
||||
|
||||
+11
-6
@@ -3,9 +3,7 @@ import logging
|
||||
import os
|
||||
|
||||
import dvc.prompt as prompt
|
||||
from dvc.command.base import append_doc_link
|
||||
from dvc.command.base import CmdBase
|
||||
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -19,7 +17,6 @@ class CmdGC(CmdBase):
|
||||
all_tags=self.args.all_tags,
|
||||
all_commits=self.args.all_commits,
|
||||
workspace=self.args.workspace,
|
||||
cloud=self.args.cloud,
|
||||
)
|
||||
|
||||
msg = "This will remove all cache except items used in "
|
||||
@@ -110,7 +107,10 @@ def add_parser(subparsers, parent_parser):
|
||||
help="Collect garbage in remote repository.",
|
||||
)
|
||||
gc_parser.add_argument(
|
||||
"-r", "--remote", help="Remote storage to collect garbage in."
|
||||
"-r",
|
||||
"--remote",
|
||||
help="Remote storage to collect garbage in",
|
||||
metavar="<name>",
|
||||
)
|
||||
gc_parser.add_argument(
|
||||
"-f",
|
||||
@@ -120,7 +120,11 @@ def add_parser(subparsers, parent_parser):
|
||||
help="Force garbage collection - automatically agree to all prompts.",
|
||||
)
|
||||
gc_parser.add_argument(
|
||||
"-j", "--jobs", type=int, help="Number of jobs to run simultaneously."
|
||||
"-j",
|
||||
"--jobs",
|
||||
type=int,
|
||||
help="Number of jobs to run simultaneously.",
|
||||
metavar="<number>",
|
||||
)
|
||||
gc_parser.add_argument(
|
||||
"-p",
|
||||
@@ -131,5 +135,6 @@ def add_parser(subparsers, parent_parser):
|
||||
help="Keep data files required by these projects "
|
||||
"in addition to the current one. "
|
||||
"Useful if you share a single cache across repos.",
|
||||
metavar="<paths>",
|
||||
)
|
||||
gc_parser.set_defaults(func=CmdGC)
|
||||
|
||||
+10
-4
@@ -1,10 +1,9 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from .base import append_doc_link
|
||||
from .base import CmdBaseNoRepo
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
from .base import CmdBaseNoRepo, append_doc_link
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -66,10 +65,17 @@ def add_parser(subparsers, parent_parser):
|
||||
"path", help="Path to a file or directory within the repository"
|
||||
)
|
||||
get_parser.add_argument(
|
||||
"-o", "--out", nargs="?", help="Destination path to download files to"
|
||||
"-o",
|
||||
"--out",
|
||||
nargs="?",
|
||||
help="Destination path to download files to",
|
||||
metavar="<path>",
|
||||
)
|
||||
get_parser.add_argument(
|
||||
"--rev", nargs="?", help="Git revision (e.g. SHA, branch, tag)"
|
||||
"--rev",
|
||||
nargs="?",
|
||||
help="Git revision (e.g. SHA, branch, tag)",
|
||||
metavar="<commit>",
|
||||
)
|
||||
get_parser.add_argument(
|
||||
"--show-url",
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from .base import append_doc_link
|
||||
from .base import CmdBaseNoRepo
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
from .base import CmdBaseNoRepo, append_doc_link
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -17,7 +16,7 @@ class CmdGetUrl(CmdBaseNoRepo):
|
||||
Repo.get_url(self.args.url, out=self.args.out)
|
||||
return 0
|
||||
except DvcException:
|
||||
logger.exception("failed to get '{}'".format(self.args.url))
|
||||
logger.exception(f"failed to get '{self.args.url}'")
|
||||
return 1
|
||||
|
||||
|
||||
|
||||
+10
-5
@@ -1,11 +1,9 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command.base import append_doc_link
|
||||
from dvc.command.base import CmdBase
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -48,9 +46,16 @@ def add_parser(subparsers, parent_parser):
|
||||
"path", help="Path to a file or directory within the repository"
|
||||
)
|
||||
import_parser.add_argument(
|
||||
"-o", "--out", nargs="?", help="Destination path to download files to"
|
||||
"-o",
|
||||
"--out",
|
||||
nargs="?",
|
||||
help="Destination path to download files to",
|
||||
metavar="<path>",
|
||||
)
|
||||
import_parser.add_argument(
|
||||
"--rev", nargs="?", help="Git revision (e.g. SHA, branch, tag)"
|
||||
"--rev",
|
||||
nargs="?",
|
||||
help="Git revision (e.g. SHA, branch, tag)",
|
||||
metavar="<commit>",
|
||||
)
|
||||
import_parser.set_defaults(func=CmdImport)
|
||||
|
||||
@@ -1,11 +1,9 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command.base import append_doc_link
|
||||
from dvc.command.base import CmdBase
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -59,6 +57,9 @@ def add_parser(subparsers, parent_parser):
|
||||
"out", nargs="?", help="Destination path to put files to."
|
||||
)
|
||||
import_parser.add_argument(
|
||||
"-f", "--file", help="Specify name of the DVC-file it generates."
|
||||
"-f",
|
||||
"--file",
|
||||
help="Specify name of the DVC-file this command will generate.",
|
||||
metavar="<filename>",
|
||||
)
|
||||
import_parser.set_defaults(func=CmdImportUrl)
|
||||
|
||||
+2
-4
@@ -1,9 +1,7 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command.base import append_doc_link
|
||||
from dvc.command.base import CmdBaseNoRepo
|
||||
|
||||
from dvc.command.base import CmdBaseNoRepo, append_doc_link
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -47,7 +45,7 @@ def add_parser(subparsers, parent_parser):
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Initiate DVC in directory that is "
|
||||
"not tracked by any scm tool (e.g. git).",
|
||||
"not tracked by any SCM tool (e.g. Git).",
|
||||
)
|
||||
init_parser.add_argument(
|
||||
"-f",
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command.base import append_doc_link
|
||||
from dvc.command.base import CmdBase
|
||||
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
+1
-3
@@ -1,11 +1,9 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command.base import append_doc_link
|
||||
from dvc.command.base import CmdBase
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
||||
@@ -2,12 +2,10 @@ import argparse
|
||||
import logging
|
||||
import sys
|
||||
|
||||
from dvc.command.base import append_doc_link
|
||||
from dvc.command.base import CmdBaseNoRepo
|
||||
from dvc.command.base import CmdBaseNoRepo, append_doc_link
|
||||
from dvc.command.ls.ls_colors import LsColors
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -33,14 +31,14 @@ class CmdList(CmdBaseNoRepo):
|
||||
self.args.path,
|
||||
rev=self.args.rev,
|
||||
recursive=self.args.recursive,
|
||||
outs_only=self.args.outs_only,
|
||||
dvc_only=self.args.dvc_only,
|
||||
)
|
||||
if entries:
|
||||
entries = _prettify(entries, sys.stdout.isatty())
|
||||
logger.info("\n".join(entries))
|
||||
return 0
|
||||
except DvcException:
|
||||
logger.exception("failed to list '{}'".format(self.args.url))
|
||||
logger.exception(f"failed to list '{self.args.url}'")
|
||||
return 1
|
||||
|
||||
|
||||
@@ -64,10 +62,13 @@ def add_parser(subparsers, parent_parser):
|
||||
help="Recursively list files.",
|
||||
)
|
||||
list_parser.add_argument(
|
||||
"--outs-only", action="store_true", help="Show only DVC outputs."
|
||||
"--dvc-only", action="store_true", help="Show only DVC outputs."
|
||||
)
|
||||
list_parser.add_argument(
|
||||
"--rev", nargs="?", help="Git revision (e.g. branch, tag, SHA)"
|
||||
"--rev",
|
||||
nargs="?",
|
||||
help="Git revision (e.g. SHA, branch, tag)",
|
||||
metavar="<commit>",
|
||||
)
|
||||
list_parser.add_argument(
|
||||
"path",
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import os
|
||||
|
||||
|
||||
class LsColors(object):
|
||||
class LsColors:
|
||||
default = "rs=0:di=01;34:ex=01;32"
|
||||
|
||||
def __init__(self, lscolors=None):
|
||||
@@ -45,4 +45,4 @@ class LsColors(object):
|
||||
if not val:
|
||||
return text
|
||||
rs = self._codes.get("rs", 0)
|
||||
return "\033[{}m{}\033[{}m".format(val, text, rs)
|
||||
return f"\033[{val}m{text}\033[{rs}m"
|
||||
|
||||
+121
-145
@@ -1,46 +1,34 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command.base import append_doc_link
|
||||
from dvc.command.base import CmdBase
|
||||
from dvc.command.base import fix_subparsers
|
||||
from dvc.exceptions import BadMetricError
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
|
||||
from dvc.exceptions import BadMetricError, DvcException
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def show_metrics(metrics, all_branches=False, all_tags=False):
|
||||
"""
|
||||
Args:
|
||||
metrics (list): Where each element is either a `list`
|
||||
if an xpath was specified, otherwise a `str`
|
||||
"""
|
||||
def show_metrics(
|
||||
metrics, all_branches=False, all_tags=False, all_commits=False
|
||||
):
|
||||
from flatten_json import flatten
|
||||
from dvc.utils.diff import format_dict
|
||||
|
||||
# When `metrics` contains a `None` key, it means that some files
|
||||
# specified as `targets` in `repo.metrics.show` didn't contain any metrics.
|
||||
missing = metrics.pop(None, None)
|
||||
|
||||
for branch, val in metrics.items():
|
||||
if all_branches or all_tags:
|
||||
logger.info("{branch}:".format(branch=branch))
|
||||
if all_branches or all_tags or all_commits:
|
||||
logger.info(f"{branch}:")
|
||||
|
||||
for fname, metric in val.items():
|
||||
if isinstance(metric, dict):
|
||||
lines = list(metric.values())
|
||||
elif isinstance(metric, list):
|
||||
lines = metric
|
||||
else:
|
||||
lines = metric.splitlines()
|
||||
if not isinstance(metric, dict):
|
||||
logger.info("\t{}: {}".format(fname, str(metric)))
|
||||
continue
|
||||
|
||||
if len(lines) > 1:
|
||||
logger.info("\t{fname}:".format(fname=fname))
|
||||
|
||||
for line in lines:
|
||||
logger.info("\t\t{content}".format(content=line))
|
||||
|
||||
else:
|
||||
logger.info("\t{}: {}".format(fname, metric))
|
||||
logger.info(f"\t{fname}:")
|
||||
for key, value in flatten(format_dict(metric), ".").items():
|
||||
logger.info(f"\t\t{key}: {value}")
|
||||
|
||||
if missing:
|
||||
raise BadMetricError(missing)
|
||||
@@ -51,14 +39,23 @@ class CmdMetricsShow(CmdBase):
|
||||
try:
|
||||
metrics = self.repo.metrics.show(
|
||||
self.args.targets,
|
||||
typ=self.args.type,
|
||||
xpath=self.args.xpath,
|
||||
all_branches=self.args.all_branches,
|
||||
all_tags=self.args.all_tags,
|
||||
all_commits=self.args.all_commits,
|
||||
recursive=self.args.recursive,
|
||||
)
|
||||
|
||||
show_metrics(metrics, self.args.all_branches, self.args.all_tags)
|
||||
if self.args.show_json:
|
||||
import json
|
||||
|
||||
logger.info(json.dumps(metrics))
|
||||
else:
|
||||
show_metrics(
|
||||
metrics,
|
||||
self.args.all_branches,
|
||||
self.args.all_tags,
|
||||
self.args.all_commits,
|
||||
)
|
||||
except DvcException:
|
||||
logger.exception("failed to show metrics")
|
||||
return 1
|
||||
@@ -66,27 +63,12 @@ class CmdMetricsShow(CmdBase):
|
||||
return 0
|
||||
|
||||
|
||||
class CmdMetricsModify(CmdBase):
|
||||
def run(self):
|
||||
try:
|
||||
self.repo.metrics.modify(
|
||||
self.args.path, typ=self.args.type, xpath=self.args.xpath
|
||||
)
|
||||
except DvcException:
|
||||
logger.exception("failed to modify metric file settings")
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
class CmdMetricsAdd(CmdBase):
|
||||
def run(self):
|
||||
try:
|
||||
self.repo.metrics.add(
|
||||
self.args.path, self.args.type, self.args.xpath
|
||||
)
|
||||
self.repo.metrics.add(self.args.path)
|
||||
except DvcException:
|
||||
msg = "failed to add metric file '{}'".format(self.args.path)
|
||||
msg = f"failed to add metric file '{self.args.path}'"
|
||||
logger.exception(msg)
|
||||
return 1
|
||||
|
||||
@@ -98,38 +80,39 @@ class CmdMetricsRemove(CmdBase):
|
||||
try:
|
||||
self.repo.metrics.remove(self.args.path)
|
||||
except DvcException:
|
||||
msg = "failed to remove metric file '{}'".format(self.args.path)
|
||||
msg = f"failed to remove metric file '{self.args.path}'"
|
||||
logger.exception(msg)
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def _show_diff(diff):
|
||||
from texttable import Texttable
|
||||
def _show_diff(diff, markdown=False, no_path=False, old=False):
|
||||
from collections import OrderedDict
|
||||
|
||||
if not diff:
|
||||
return "No changes."
|
||||
from dvc.utils.diff import table
|
||||
|
||||
table = Texttable()
|
||||
|
||||
# remove borders to make it easier for users to copy stuff
|
||||
table.set_chars(("", "", "", ""))
|
||||
table.set_deco(0)
|
||||
|
||||
rows = [["Path", "Metric", "Value", "Change"]]
|
||||
rows = []
|
||||
for fname, mdiff in diff.items():
|
||||
for metric, change in mdiff.items():
|
||||
rows.append(
|
||||
[
|
||||
fname,
|
||||
metric,
|
||||
change["new"],
|
||||
change.get("diff", "diff not supported"),
|
||||
]
|
||||
)
|
||||
table.add_rows(rows)
|
||||
return table.draw()
|
||||
sorted_mdiff = OrderedDict(sorted(mdiff.items()))
|
||||
for metric, change in sorted_mdiff.items():
|
||||
row = [] if no_path else [fname]
|
||||
row.append(metric)
|
||||
if old:
|
||||
row.append(change.get("old"))
|
||||
row.append(change["new"])
|
||||
row.append(change.get("diff", "diff not supported"))
|
||||
rows.append(row)
|
||||
|
||||
header = [] if no_path else ["Path"]
|
||||
header.append("Metric")
|
||||
if old:
|
||||
header.extend(["Old", "New"])
|
||||
else:
|
||||
header.append("Value")
|
||||
header.append("Change")
|
||||
|
||||
return table(header, rows, markdown)
|
||||
|
||||
|
||||
class CmdMetricsDiff(CmdBase):
|
||||
@@ -139,9 +122,8 @@ class CmdMetricsDiff(CmdBase):
|
||||
a_rev=self.args.a_rev,
|
||||
b_rev=self.args.b_rev,
|
||||
targets=self.args.targets,
|
||||
typ=self.args.type,
|
||||
xpath=self.args.xpath,
|
||||
recursive=self.args.recursive,
|
||||
all=self.args.all,
|
||||
)
|
||||
|
||||
if self.args.show_json:
|
||||
@@ -149,7 +131,11 @@ class CmdMetricsDiff(CmdBase):
|
||||
|
||||
logger.info(json.dumps(diff))
|
||||
else:
|
||||
logger.info(_show_diff(diff))
|
||||
table = _show_diff(
|
||||
diff, self.args.show_md, self.args.no_path, self.args.old
|
||||
)
|
||||
if table:
|
||||
logger.info(table)
|
||||
|
||||
except DvcException:
|
||||
logger.exception("failed to show metrics diff")
|
||||
@@ -159,7 +145,7 @@ class CmdMetricsDiff(CmdBase):
|
||||
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
METRICS_HELP = "Commands to add, manage, collect and display metrics."
|
||||
METRICS_HELP = "Commands to add, manage, collect, and display metrics."
|
||||
|
||||
metrics_parser = subparsers.add_parser(
|
||||
"metrics",
|
||||
@@ -176,7 +162,18 @@ def add_parser(subparsers, parent_parser):
|
||||
|
||||
fix_subparsers(metrics_subparsers)
|
||||
|
||||
METRICS_SHOW_HELP = "Output metric values."
|
||||
METRICS_ADD_HELP = "Mark a DVC-tracked file as a metric."
|
||||
metrics_add_parser = metrics_subparsers.add_parser(
|
||||
"add",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(METRICS_ADD_HELP, "metrics/add"),
|
||||
help=METRICS_ADD_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
metrics_add_parser.add_argument("path", help="Path to a metric file.")
|
||||
metrics_add_parser.set_defaults(func=CmdMetricsAdd)
|
||||
|
||||
METRICS_SHOW_HELP = "Print metrics, with optional formatting."
|
||||
metrics_show_parser = metrics_subparsers.add_parser(
|
||||
"show",
|
||||
parents=[parent_parser],
|
||||
@@ -189,18 +186,6 @@ def add_parser(subparsers, parent_parser):
|
||||
nargs="*",
|
||||
help="Metric files or directories (see -R) to show",
|
||||
)
|
||||
metrics_show_parser.add_argument(
|
||||
"-t",
|
||||
"--type",
|
||||
help=(
|
||||
"Type of metrics (json/tsv/htsv/csv/hcsv). "
|
||||
"It can be detected by the file extension automatically. "
|
||||
"Unsupported types will be treated as raw."
|
||||
),
|
||||
)
|
||||
metrics_show_parser.add_argument(
|
||||
"-x", "--xpath", help="json/tsv/htsv/csv/hcsv path."
|
||||
)
|
||||
metrics_show_parser.add_argument(
|
||||
"-a",
|
||||
"--all-branches",
|
||||
@@ -215,6 +200,12 @@ def add_parser(subparsers, parent_parser):
|
||||
default=False,
|
||||
help="Show metrics for all tags.",
|
||||
)
|
||||
metrics_show_parser.add_argument(
|
||||
"--all-commits",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Show metrics for all commits.",
|
||||
)
|
||||
metrics_show_parser.add_argument(
|
||||
"-R",
|
||||
"--recursive",
|
||||
@@ -225,53 +216,14 @@ def add_parser(subparsers, parent_parser):
|
||||
"metric files."
|
||||
),
|
||||
)
|
||||
metrics_show_parser.add_argument(
|
||||
"--show-json",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Show output in JSON format.",
|
||||
)
|
||||
metrics_show_parser.set_defaults(func=CmdMetricsShow)
|
||||
|
||||
METRICS_ADD_HELP = "Tag file as a metric file."
|
||||
metrics_add_parser = metrics_subparsers.add_parser(
|
||||
"add",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(METRICS_ADD_HELP, "metrics/add"),
|
||||
help=METRICS_ADD_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
metrics_add_parser.add_argument(
|
||||
"-t", "--type", help="Type of metrics (raw/json/tsv/htsv/csv/hcsv)."
|
||||
)
|
||||
metrics_add_parser.add_argument(
|
||||
"-x", "--xpath", help="json/tsv/htsv/csv/hcsv path."
|
||||
)
|
||||
metrics_add_parser.add_argument("path", help="Path to a metric file.")
|
||||
metrics_add_parser.set_defaults(func=CmdMetricsAdd)
|
||||
|
||||
METRICS_MODIFY_HELP = "Modify metric file options."
|
||||
metrics_modify_parser = metrics_subparsers.add_parser(
|
||||
"modify",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(METRICS_MODIFY_HELP, "metrics/modify"),
|
||||
help=METRICS_MODIFY_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
metrics_modify_parser.add_argument(
|
||||
"-t", "--type", help="Type of metrics (raw/json/tsv/htsv/csv/hcsv)."
|
||||
)
|
||||
metrics_modify_parser.add_argument(
|
||||
"-x", "--xpath", help="json/tsv/htsv/csv/hcsv path."
|
||||
)
|
||||
metrics_modify_parser.add_argument("path", help="Path to a metric file.")
|
||||
metrics_modify_parser.set_defaults(func=CmdMetricsModify)
|
||||
|
||||
METRICS_REMOVE_HELP = "Remove files's metric tag."
|
||||
metrics_remove_parser = metrics_subparsers.add_parser(
|
||||
"remove",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(METRICS_REMOVE_HELP, "metrics/remove"),
|
||||
help=METRICS_REMOVE_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
metrics_remove_parser.add_argument("path", help="Path to a metric file.")
|
||||
metrics_remove_parser.set_defaults(func=CmdMetricsRemove)
|
||||
|
||||
METRICS_DIFF_HELP = "Show changes in metrics between commits"
|
||||
" in the DVC repository, or between a commit and the workspace."
|
||||
metrics_diff_parser = metrics_subparsers.add_parser(
|
||||
@@ -296,18 +248,7 @@ def add_parser(subparsers, parent_parser):
|
||||
"Metric files or directories (see -R) to show diff for. "
|
||||
"Shows diff for all metric files by default."
|
||||
),
|
||||
)
|
||||
metrics_diff_parser.add_argument(
|
||||
"-t",
|
||||
"--type",
|
||||
help=(
|
||||
"Type of metrics (json/tsv/htsv/csv/hcsv). "
|
||||
"It can be detected by the file extension automatically. "
|
||||
"Unsupported types will be treated as raw."
|
||||
),
|
||||
)
|
||||
metrics_diff_parser.add_argument(
|
||||
"-x", "--xpath", help="json/tsv/htsv/csv/hcsv path."
|
||||
metavar="<paths>",
|
||||
)
|
||||
metrics_diff_parser.add_argument(
|
||||
"-R",
|
||||
@@ -319,10 +260,45 @@ def add_parser(subparsers, parent_parser):
|
||||
"metric files."
|
||||
),
|
||||
)
|
||||
metrics_diff_parser.add_argument(
|
||||
"--all",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Show unchanged metrics as well.",
|
||||
)
|
||||
metrics_diff_parser.add_argument(
|
||||
"--show-json",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Show output in JSON format.",
|
||||
)
|
||||
metrics_diff_parser.add_argument(
|
||||
"--show-md",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Show tabulated output in the Markdown format (GFM).",
|
||||
)
|
||||
metrics_diff_parser.add_argument(
|
||||
"--no-path",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Don't show metric path.",
|
||||
)
|
||||
metrics_diff_parser.add_argument(
|
||||
"--old",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Show old metric value.",
|
||||
)
|
||||
metrics_diff_parser.set_defaults(func=CmdMetricsDiff)
|
||||
|
||||
METRICS_REMOVE_HELP = "Remove metric mark on a DVC-tracked file."
|
||||
metrics_remove_parser = metrics_subparsers.add_parser(
|
||||
"remove",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(METRICS_REMOVE_HELP, "metrics/remove"),
|
||||
help=METRICS_REMOVE_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
metrics_remove_parser.add_argument("path", help="Path to a metric file.")
|
||||
metrics_remove_parser.set_defaults(func=CmdMetricsRemove)
|
||||
|
||||
+1
-3
@@ -1,11 +1,9 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command.base import append_doc_link
|
||||
from dvc.command.base import CmdBase
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
import argparse
|
||||
import logging
|
||||
from collections import OrderedDict
|
||||
|
||||
from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _show_diff(diff, markdown=False, no_path=False):
|
||||
from dvc.utils.diff import table
|
||||
|
||||
rows = []
|
||||
for fname, pdiff in diff.items():
|
||||
sorted_pdiff = OrderedDict(sorted(pdiff.items()))
|
||||
for param, change in sorted_pdiff.items():
|
||||
row = [] if no_path else [fname]
|
||||
row.append(param)
|
||||
row.append(change["old"])
|
||||
row.append(change["new"])
|
||||
rows.append(row)
|
||||
|
||||
header = [] if no_path else ["Path"]
|
||||
header.append("Param")
|
||||
header.append("Old")
|
||||
header.append("New")
|
||||
|
||||
return table(header, rows, markdown)
|
||||
|
||||
|
||||
class CmdParamsDiff(CmdBase):
|
||||
def run(self):
|
||||
try:
|
||||
diff = self.repo.params.diff(
|
||||
a_rev=self.args.a_rev,
|
||||
b_rev=self.args.b_rev,
|
||||
all=self.args.all,
|
||||
)
|
||||
|
||||
if self.args.show_json:
|
||||
import json
|
||||
|
||||
logger.info(json.dumps(diff))
|
||||
else:
|
||||
table = _show_diff(diff, self.args.show_md, self.args.no_path)
|
||||
if table:
|
||||
logger.info(table)
|
||||
|
||||
except DvcException:
|
||||
logger.exception("failed to show params diff")
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
PARAMS_HELP = "Commands to display params."
|
||||
|
||||
params_parser = subparsers.add_parser(
|
||||
"params",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(PARAMS_HELP, "params"),
|
||||
help=PARAMS_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
|
||||
params_subparsers = params_parser.add_subparsers(
|
||||
dest="cmd",
|
||||
help="Use `dvc params CMD --help` to display command-specific help.",
|
||||
)
|
||||
|
||||
fix_subparsers(params_subparsers)
|
||||
|
||||
PARAMS_DIFF_HELP = (
|
||||
"Show changes in params between commits in the DVC repository, or "
|
||||
"between a commit and the workspace."
|
||||
)
|
||||
params_diff_parser = params_subparsers.add_parser(
|
||||
"diff",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(PARAMS_DIFF_HELP, "params/diff"),
|
||||
help=PARAMS_DIFF_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
params_diff_parser.add_argument(
|
||||
"a_rev", nargs="?", help="Old Git commit to compare (defaults to HEAD)"
|
||||
)
|
||||
params_diff_parser.add_argument(
|
||||
"b_rev",
|
||||
nargs="?",
|
||||
help=("New Git commit to compare (defaults to the current workspace)"),
|
||||
)
|
||||
params_diff_parser.add_argument(
|
||||
"--all",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Show unchanged params as well.",
|
||||
)
|
||||
params_diff_parser.add_argument(
|
||||
"--show-json",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Show output in JSON format.",
|
||||
)
|
||||
params_diff_parser.add_argument(
|
||||
"--show-md",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Show tabulated output in the Markdown format (GFM).",
|
||||
)
|
||||
params_diff_parser.add_argument(
|
||||
"--no-path",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Don't show params path.",
|
||||
)
|
||||
params_diff_parser.set_defaults(func=CmdParamsDiff)
|
||||
+60
-42
@@ -4,19 +4,18 @@ import logging
|
||||
from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CmdPipelineShow(CmdBase):
|
||||
def _show(self, target, commands, outs, locked):
|
||||
import networkx
|
||||
from dvc.stage import Stage
|
||||
from dvc.utils import parse_target
|
||||
|
||||
stage = Stage.load(self.repo, target)
|
||||
path, name = parse_target(target)
|
||||
stage = self.repo.get_stage(path, name)
|
||||
G = self.repo.graph
|
||||
stages = networkx.dfs_postorder_nodes(G, stage)
|
||||
|
||||
if locked:
|
||||
stages = [s for s in stages if s.locked]
|
||||
|
||||
@@ -29,14 +28,40 @@ class CmdPipelineShow(CmdBase):
|
||||
for out in stage.outs:
|
||||
logger.info(str(out))
|
||||
else:
|
||||
logger.info(stage.path_in_repo)
|
||||
logger.info(stage.addressing)
|
||||
|
||||
def _build_graph(self, target, commands, outs):
|
||||
@staticmethod
|
||||
def _build_output_graph(G, target_stage):
|
||||
import networkx
|
||||
from dvc.stage import Stage
|
||||
from dvc.repo.graph import get_pipeline
|
||||
from itertools import product
|
||||
|
||||
target_stage = Stage.load(self.repo, target)
|
||||
nodes = {str(out) for out in target_stage.outs}
|
||||
edges = []
|
||||
|
||||
for from_stage, to_stage in networkx.edge_dfs(G, target_stage):
|
||||
from_stage_deps = {dep.path_info.parts for dep in from_stage.deps}
|
||||
to_outs = {
|
||||
to_out
|
||||
for to_out in to_stage.outs
|
||||
if to_out.path_info.parts in from_stage_deps
|
||||
}
|
||||
from_outs = {
|
||||
from_out
|
||||
for from_out in from_stage.outs
|
||||
if str(from_out) in nodes
|
||||
}
|
||||
nodes |= {str(to_out) for to_out in to_outs}
|
||||
for from_out, to_out in product(from_outs, to_outs):
|
||||
edges.append((str(from_out), str(to_out)))
|
||||
return nodes, edges
|
||||
|
||||
def _build_graph(self, target, commands=False, outs=False):
|
||||
import networkx
|
||||
from dvc.repo.graph import get_pipeline
|
||||
from dvc.utils import parse_target
|
||||
|
||||
path, name = parse_target(target)
|
||||
target_stage = self.repo.get_stage(path, name)
|
||||
G = get_pipeline(self.repo.pipelines, target_stage)
|
||||
|
||||
nodes = set()
|
||||
@@ -45,29 +70,20 @@ class CmdPipelineShow(CmdBase):
|
||||
if stage.cmd is None:
|
||||
continue
|
||||
nodes.add(stage.cmd)
|
||||
elif outs:
|
||||
for out in stage.outs:
|
||||
nodes.add(str(out))
|
||||
for dep in stage.deps:
|
||||
nodes.add(str(dep))
|
||||
else:
|
||||
nodes.add(stage.relpath)
|
||||
elif not outs:
|
||||
nodes.add(stage.addressing)
|
||||
|
||||
edges = []
|
||||
for from_stage, to_stage in networkx.edge_dfs(G, target_stage):
|
||||
if commands:
|
||||
if to_stage.cmd is None:
|
||||
continue
|
||||
edges.append((from_stage.cmd, to_stage.cmd))
|
||||
elif not outs:
|
||||
edges.append((from_stage.addressing, to_stage.addressing))
|
||||
|
||||
if outs:
|
||||
for stage in networkx.dfs_preorder_nodes(G, target_stage):
|
||||
for dep in stage.deps:
|
||||
for out in stage.outs:
|
||||
edges.append((str(out), str(dep)))
|
||||
else:
|
||||
for from_stage, to_stage in networkx.edge_dfs(G, target_stage):
|
||||
if commands:
|
||||
if to_stage.cmd is None:
|
||||
continue
|
||||
edges.append((from_stage.cmd, to_stage.cmd))
|
||||
else:
|
||||
edges.append((from_stage.relpath, to_stage.relpath))
|
||||
nodes, edges = self._build_output_graph(G, target_stage)
|
||||
|
||||
return list(nodes), edges, networkx.is_tree(G)
|
||||
|
||||
@@ -121,8 +137,10 @@ class CmdPipelineShow(CmdBase):
|
||||
logger.info(dot_file.getvalue())
|
||||
|
||||
def run(self):
|
||||
from dvc.dvcfile import DVC_FILE
|
||||
|
||||
if not self.args.targets:
|
||||
self.args.targets = self.default_targets
|
||||
self.args.targets = [DVC_FILE]
|
||||
|
||||
for target in self.args.targets:
|
||||
try:
|
||||
@@ -144,7 +162,7 @@ class CmdPipelineShow(CmdBase):
|
||||
self.args.locked,
|
||||
)
|
||||
except DvcException:
|
||||
msg = "failed to show pipeline for '{}'".format(target)
|
||||
msg = f"failed to show pipeline for '{target}'"
|
||||
logger.exception(msg)
|
||||
return 1
|
||||
return 0
|
||||
@@ -155,7 +173,7 @@ class CmdPipelineList(CmdBase):
|
||||
pipelines = self.repo.pipelines
|
||||
for pipeline in pipelines:
|
||||
for stage in pipeline:
|
||||
logger.info(stage.relpath)
|
||||
logger.info(stage.addressing)
|
||||
if len(pipeline) != 0:
|
||||
logger.info("=" * 80)
|
||||
logger.info("{} pipelines total".format(len(pipelines)))
|
||||
@@ -180,7 +198,17 @@ def add_parser(subparsers, parent_parser):
|
||||
|
||||
fix_subparsers(pipeline_subparsers)
|
||||
|
||||
PIPELINE_SHOW_HELP = "Show pipelines."
|
||||
PIPELINE_LIST_HELP = "List connected groups of stages (pipelines)."
|
||||
pipeline_list_parser = pipeline_subparsers.add_parser(
|
||||
"list",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(PIPELINE_LIST_HELP, "pipeline/list"),
|
||||
help=PIPELINE_LIST_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
pipeline_list_parser.set_defaults(func=CmdPipelineList)
|
||||
|
||||
PIPELINE_SHOW_HELP = "Show stages in a pipeline."
|
||||
pipeline_show_parser = pipeline_subparsers.add_parser(
|
||||
"show",
|
||||
parents=[parent_parser],
|
||||
@@ -235,13 +263,3 @@ def add_parser(subparsers, parent_parser):
|
||||
"(Finds all DVC-files in the workspace by default.)",
|
||||
)
|
||||
pipeline_show_parser.set_defaults(func=CmdPipelineShow)
|
||||
|
||||
PIPELINE_LIST_HELP = "List pipelines."
|
||||
pipeline_list_parser = pipeline_subparsers.add_parser(
|
||||
"list",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(PIPELINE_LIST_HELP, "pipeline/list"),
|
||||
help=PIPELINE_LIST_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
pipeline_list_parser.set_defaults(func=CmdPipelineList)
|
||||
|
||||
@@ -0,0 +1,219 @@
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
|
||||
from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.utils import format_link
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
PAGE_HTML = """<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>DVC Plot</title>
|
||||
<script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
|
||||
</head>
|
||||
<body>
|
||||
{divs}
|
||||
</body>
|
||||
</html>"""
|
||||
|
||||
DIV_HTML = """<div id = "{id}"></div>
|
||||
<script type = "text/javascript">
|
||||
var spec = {vega_json};
|
||||
vegaEmbed('#{id}', spec);
|
||||
</script>"""
|
||||
|
||||
|
||||
class CmdPlots(CmdBase):
|
||||
def _func(self, *args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
def run(self):
|
||||
try:
|
||||
plots = self._func(
|
||||
targets=self.args.targets,
|
||||
template=self.args.template,
|
||||
x_field=self.args.x,
|
||||
y_field=self.args.y,
|
||||
csv_header=not self.args.no_csv_header,
|
||||
title=self.args.title,
|
||||
x_title=self.args.xlab,
|
||||
y_title=self.args.ylab,
|
||||
)
|
||||
|
||||
if self.args.show_json:
|
||||
import json
|
||||
|
||||
logger.info(json.dumps(plots))
|
||||
return 0
|
||||
|
||||
divs = [
|
||||
DIV_HTML.format(id=f"plot{i}", vega_json=plot)
|
||||
for i, plot in enumerate(plots.values())
|
||||
]
|
||||
html = PAGE_HTML.format(divs="\n".join(divs))
|
||||
path = self.args.out or "plots.html"
|
||||
|
||||
with open(path, "w") as fobj:
|
||||
fobj.write(html)
|
||||
|
||||
logger.info(
|
||||
"file://{}".format(os.path.join(self.repo.root_dir, path))
|
||||
)
|
||||
|
||||
except DvcException:
|
||||
logger.exception("")
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
class CmdPlotsShow(CmdPlots):
|
||||
def _func(self, *args, **kwargs):
|
||||
return self.repo.plots.show(*args, **kwargs)
|
||||
|
||||
|
||||
class CmdPlotsDiff(CmdPlots):
|
||||
def _func(self, *args, **kwargs):
|
||||
return self.repo.plots.diff(*args, revs=self.args.revisions, **kwargs)
|
||||
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
PLOTS_HELP = (
|
||||
"Generating plots for metrics stored in structured files "
|
||||
"(JSON, CSV, TSV)."
|
||||
)
|
||||
|
||||
plots_parser = subparsers.add_parser(
|
||||
"plots",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(PLOTS_HELP, "plots"),
|
||||
help=PLOTS_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
plots_subparsers = plots_parser.add_subparsers(
|
||||
dest="cmd",
|
||||
help="Use `dvc plots CMD --help` to display command-specific help.",
|
||||
)
|
||||
|
||||
fix_subparsers(plots_subparsers)
|
||||
|
||||
SHOW_HELP = "Generate a plots image file from a metrics file."
|
||||
plots_show_parser = plots_subparsers.add_parser(
|
||||
"show",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(SHOW_HELP, "plots/show"),
|
||||
help=SHOW_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
plots_show_parser.add_argument(
|
||||
"-t",
|
||||
"--template",
|
||||
nargs="?",
|
||||
default=None,
|
||||
help=(
|
||||
"Special JSON or HTML schema file to inject with the data. "
|
||||
"See {}".format(
|
||||
format_link("https://man.dvc.org/plots#plot-templates")
|
||||
)
|
||||
),
|
||||
)
|
||||
plots_show_parser.add_argument(
|
||||
"-o", "--out", default=None, help="Destination path to save plots to.",
|
||||
)
|
||||
plots_show_parser.add_argument(
|
||||
"-x", default=None, help="Field name for x axis."
|
||||
)
|
||||
plots_show_parser.add_argument(
|
||||
"-y", default=None, help="Field name for y axis."
|
||||
)
|
||||
plots_show_parser.add_argument(
|
||||
"--no-csv-header",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Required when CSV or TSV datafile does not have a header.",
|
||||
)
|
||||
plots_show_parser.add_argument(
|
||||
"--show-json",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Show output in JSON format.",
|
||||
)
|
||||
plots_show_parser.add_argument("--title", default=None, help="Plot title.")
|
||||
plots_show_parser.add_argument(
|
||||
"--xlab", default=None, help="X axis title."
|
||||
)
|
||||
plots_show_parser.add_argument(
|
||||
"--ylab", default=None, help="Y axis title."
|
||||
)
|
||||
plots_show_parser.add_argument(
|
||||
"targets",
|
||||
nargs="*",
|
||||
help="Metrics files to visualize. Shows all plots by default.",
|
||||
)
|
||||
plots_show_parser.set_defaults(func=CmdPlotsShow)
|
||||
|
||||
PLOTS_DIFF_HELP = (
|
||||
"Plot differences in metrics between commits in the DVC "
|
||||
"repository, or between the last commit and the workspace."
|
||||
)
|
||||
plots_diff_parser = plots_subparsers.add_parser(
|
||||
"diff",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(PLOTS_DIFF_HELP, "plots/diff"),
|
||||
help=PLOTS_DIFF_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"-t",
|
||||
"--template",
|
||||
nargs="?",
|
||||
default=None,
|
||||
help=(
|
||||
"Special JSON or HTML schema file to inject with the data. "
|
||||
"See {}".format(
|
||||
format_link("https://man.dvc.org/plots#plot-templates")
|
||||
)
|
||||
),
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"--targets",
|
||||
nargs="*",
|
||||
help="Metrics file to visualize. Shows all plots by default.",
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"-o", "--out", default=None, help="Destination path to save plots to.",
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"-x", default=None, help="Field name for x axis."
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"-y", default=None, help="Field name for y axis."
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"--no-csv-header",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Provided CSV ot TSV datafile does not have a header.",
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"--show-json",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Show output in JSON format.",
|
||||
)
|
||||
plots_diff_parser.add_argument("--title", default=None, help="Plot title.")
|
||||
plots_diff_parser.add_argument(
|
||||
"--xlab", default=None, help="X axis title."
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"--ylab", default=None, help="Y axis title."
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"revisions", nargs="*", default=None, help="Git commits to plot from",
|
||||
)
|
||||
plots_diff_parser.set_defaults(func=CmdPlotsDiff)
|
||||
+73
-26
@@ -1,9 +1,9 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.config import ConfigError
|
||||
from dvc.command.base import append_doc_link, fix_subparsers
|
||||
from dvc.command.config import CmdConfig
|
||||
from dvc.config import ConfigError
|
||||
from dvc.utils import format_link
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -18,17 +18,13 @@ class CmdRemote(CmdConfig):
|
||||
|
||||
def _check_exists(self, conf):
|
||||
if self.args.name not in conf["remote"]:
|
||||
raise ConfigError(
|
||||
"remote '{}' doesn't exists.".format(self.args.name)
|
||||
)
|
||||
raise ConfigError(f"remote '{self.args.name}' doesn't exists.")
|
||||
|
||||
|
||||
class CmdRemoteAdd(CmdRemote):
|
||||
def run(self):
|
||||
if self.args.default:
|
||||
logger.info(
|
||||
"Setting '{}' as a default remote.".format(self.args.name)
|
||||
)
|
||||
logger.info(f"Setting '{self.args.name}' as a default remote.")
|
||||
|
||||
with self.config.edit(self.args.level) as conf:
|
||||
if self.args.name in conf["remote"] and not self.args.force:
|
||||
@@ -85,7 +81,18 @@ class CmdRemoteDefault(CmdRemote):
|
||||
if self.args.unset:
|
||||
conf["core"].pop("remote", None)
|
||||
else:
|
||||
conf["core"]["remote"] = self.args.name
|
||||
merged_conf = self.config.load_config_to_level(
|
||||
self.args.level
|
||||
)
|
||||
if (
|
||||
self.args.name in conf["remote"]
|
||||
or self.args.name in merged_conf["remote"]
|
||||
):
|
||||
conf["core"]["remote"] = self.args.name
|
||||
else:
|
||||
raise ConfigError(
|
||||
"default remote must be present in remote list."
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
@@ -97,10 +104,39 @@ class CmdRemoteList(CmdRemote):
|
||||
return 0
|
||||
|
||||
|
||||
class CmdRemoteRename(CmdRemote):
|
||||
def _rename_default(self, conf):
|
||||
if conf["core"].get("remote") == self.args.name:
|
||||
conf["core"]["remote"] = self.args.new
|
||||
|
||||
def run(self):
|
||||
all_config = self.config.load_config_to_level(None)
|
||||
if self.args.new in all_config.get("remote", {}):
|
||||
raise ConfigError(
|
||||
"Rename failed. Remote name '{}' already exists.".format(
|
||||
{self.args.new}
|
||||
)
|
||||
)
|
||||
|
||||
with self.config.edit(self.args.level) as conf:
|
||||
self._check_exists(conf)
|
||||
conf["remote"][self.args.new] = conf["remote"][self.args.name]
|
||||
del conf["remote"][self.args.name]
|
||||
self._rename_default(conf)
|
||||
|
||||
for level in reversed(self.config.LEVELS):
|
||||
if level == self.args.level:
|
||||
break
|
||||
with self.config.edit(level) as level_conf:
|
||||
self._rename_default(level_conf)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
from dvc.command.config import parent_config_parser
|
||||
|
||||
REMOTE_HELP = "Manage remote storage configuration."
|
||||
REMOTE_HELP = "Set up and manage data remotes."
|
||||
remote_parser = subparsers.add_parser(
|
||||
"remote",
|
||||
parents=[parent_parser],
|
||||
@@ -116,7 +152,7 @@ def add_parser(subparsers, parent_parser):
|
||||
|
||||
fix_subparsers(remote_subparsers)
|
||||
|
||||
REMOTE_ADD_HELP = "Add remote."
|
||||
REMOTE_ADD_HELP = "Add a new data remote."
|
||||
remote_add_parser = remote_subparsers.add_parser(
|
||||
"add",
|
||||
parents=[parent_config_parser, parent_parser],
|
||||
@@ -147,7 +183,7 @@ def add_parser(subparsers, parent_parser):
|
||||
)
|
||||
remote_add_parser.set_defaults(func=CmdRemoteAdd)
|
||||
|
||||
REMOTE_DEFAULT_HELP = "Set/unset default remote."
|
||||
REMOTE_DEFAULT_HELP = "Set/unset the default data remote."
|
||||
remote_default_parser = remote_subparsers.add_parser(
|
||||
"default",
|
||||
parents=[parent_config_parser, parent_parser],
|
||||
@@ -167,20 +203,7 @@ def add_parser(subparsers, parent_parser):
|
||||
)
|
||||
remote_default_parser.set_defaults(func=CmdRemoteDefault)
|
||||
|
||||
REMOTE_REMOVE_HELP = "Remove remote."
|
||||
remote_remove_parser = remote_subparsers.add_parser(
|
||||
"remove",
|
||||
parents=[parent_config_parser, parent_parser],
|
||||
description=append_doc_link(REMOTE_REMOVE_HELP, "remote/remove"),
|
||||
help=REMOTE_REMOVE_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
remote_remove_parser.add_argument(
|
||||
"name", help="Name of the remote to remove."
|
||||
)
|
||||
remote_remove_parser.set_defaults(func=CmdRemoteRemove)
|
||||
|
||||
REMOTE_MODIFY_HELP = "Modify remote."
|
||||
REMOTE_MODIFY_HELP = "Modify the configuration of a data remote."
|
||||
remote_modify_parser = remote_subparsers.add_parser(
|
||||
"modify",
|
||||
parents=[parent_config_parser, parent_parser],
|
||||
@@ -204,7 +227,7 @@ def add_parser(subparsers, parent_parser):
|
||||
)
|
||||
remote_modify_parser.set_defaults(func=CmdRemoteModify)
|
||||
|
||||
REMOTE_LIST_HELP = "List available remotes."
|
||||
REMOTE_LIST_HELP = "List all available data remotes."
|
||||
remote_list_parser = remote_subparsers.add_parser(
|
||||
"list",
|
||||
parents=[parent_config_parser, parent_parser],
|
||||
@@ -213,3 +236,27 @@ def add_parser(subparsers, parent_parser):
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
remote_list_parser.set_defaults(func=CmdRemoteList)
|
||||
|
||||
REMOTE_REMOVE_HELP = "Remove a data remote."
|
||||
remote_remove_parser = remote_subparsers.add_parser(
|
||||
"remove",
|
||||
parents=[parent_config_parser, parent_parser],
|
||||
description=append_doc_link(REMOTE_REMOVE_HELP, "remote/remove"),
|
||||
help=REMOTE_REMOVE_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
remote_remove_parser.add_argument(
|
||||
"name", help="Name of the remote to remove."
|
||||
)
|
||||
remote_remove_parser.set_defaults(func=CmdRemoteRemove)
|
||||
REMOTE_RENAME_HELP = "Rename a DVC remote"
|
||||
remote_rename_parser = remote_subparsers.add_parser(
|
||||
"rename",
|
||||
parents=[parent_config_parser, parent_parser],
|
||||
description=append_doc_link(REMOTE_RENAME_HELP, "remote/rename"),
|
||||
help=REMOTE_RENAME_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
remote_rename_parser.add_argument("name", help="Remote to be renamed")
|
||||
remote_rename_parser.add_argument("new", help="New name of the remote")
|
||||
remote_rename_parser.set_defaults(func=CmdRemoteRename)
|
||||
|
||||
@@ -2,16 +2,14 @@ import argparse
|
||||
import logging
|
||||
|
||||
import dvc.prompt as prompt
|
||||
from dvc.command.base import append_doc_link
|
||||
from dvc.command.base import CmdBase
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CmdRemove(CmdBase):
|
||||
def _is_outs_only(self, target):
|
||||
def _is_dvc_only(self, target):
|
||||
if not self.args.purge:
|
||||
return True
|
||||
|
||||
@@ -33,16 +31,16 @@ class CmdRemove(CmdBase):
|
||||
def run(self):
|
||||
for target in self.args.targets:
|
||||
try:
|
||||
outs_only = self._is_outs_only(target)
|
||||
self.repo.remove(target, outs_only=outs_only)
|
||||
dvc_only = self._is_dvc_only(target)
|
||||
self.repo.remove(target, dvc_only=dvc_only)
|
||||
except DvcException:
|
||||
logger.exception("failed to remove '{}'".format(target))
|
||||
logger.exception(f"failed to remove '{target}'")
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
REMOVE_HELP = "Remove DVC-file outputs."
|
||||
REMOVE_HELP = "Remove DVC-tracked files or directories."
|
||||
remove_parser = subparsers.add_parser(
|
||||
"remove",
|
||||
parents=[parent_parser],
|
||||
|
||||
+19
-7
@@ -2,13 +2,12 @@ import argparse
|
||||
import logging
|
||||
import os
|
||||
|
||||
from dvc.command.base import append_doc_link
|
||||
from dvc.command.base import CmdBase
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.command.metrics import show_metrics
|
||||
from dvc.command.status import CmdDataStatus
|
||||
from dvc.dvcfile import PIPELINE_FILE
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -35,10 +34,11 @@ class CmdRepro(CmdBase):
|
||||
interactive=self.args.interactive,
|
||||
pipeline=self.args.pipeline,
|
||||
all_pipelines=self.args.all_pipelines,
|
||||
ignore_build_cache=self.args.ignore_build_cache,
|
||||
run_cache=not self.args.no_run_cache,
|
||||
no_commit=self.args.no_commit,
|
||||
downstream=self.args.downstream,
|
||||
recursive=self.args.recursive,
|
||||
force_downstream=self.args.force_downstream,
|
||||
)
|
||||
|
||||
if len(stages) == 0:
|
||||
@@ -57,7 +57,9 @@ class CmdRepro(CmdBase):
|
||||
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
REPRO_HELP = "Check for changes and reproduce stages and dependencies."
|
||||
REPRO_HELP = (
|
||||
"Reproduce complete or partial pipelines by executing their stages."
|
||||
)
|
||||
repro_parser = subparsers.add_parser(
|
||||
"repro",
|
||||
parents=[parent_parser],
|
||||
@@ -68,7 +70,7 @@ def add_parser(subparsers, parent_parser):
|
||||
repro_parser.add_argument(
|
||||
"targets",
|
||||
nargs="*",
|
||||
help="DVC-file to reproduce. 'Dvcfile' by default.",
|
||||
help=f"Stages to reproduce. '{PIPELINE_FILE}' by default.",
|
||||
)
|
||||
repro_parser.add_argument(
|
||||
"-f",
|
||||
@@ -90,6 +92,7 @@ def add_parser(subparsers, parent_parser):
|
||||
"--cwd",
|
||||
default=os.path.curdir,
|
||||
help="Directory within your repo to reproduce from.",
|
||||
metavar="<path>",
|
||||
)
|
||||
repro_parser.add_argument(
|
||||
"-m",
|
||||
@@ -135,7 +138,16 @@ def add_parser(subparsers, parent_parser):
|
||||
help="Reproduce all stages in the specified directory.",
|
||||
)
|
||||
repro_parser.add_argument(
|
||||
"--ignore-build-cache",
|
||||
"--no-run-cache",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help=(
|
||||
"Execute stage commands even if they have already been run with "
|
||||
"the same command/dependencies/outputs/etc before."
|
||||
),
|
||||
)
|
||||
repro_parser.add_argument(
|
||||
"--force-downstream",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Reproduce all descendants of a changed stage even if their "
|
||||
|
||||
+2
-3
@@ -1,8 +1,7 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command.base import append_doc_link
|
||||
from dvc.command.base import CmdBaseNoRepo
|
||||
from dvc.command.base import CmdBaseNoRepo, append_doc_link
|
||||
from dvc.utils import relpath
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -17,7 +16,7 @@ class CmdRoot(CmdBaseNoRepo):
|
||||
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
ROOT_HELP = "Relative path to the repository's directory."
|
||||
ROOT_HELP = "Return the relative path to the root of the DVC project."
|
||||
root_parser = subparsers.add_parser(
|
||||
"root",
|
||||
parents=[parent_parser],
|
||||
|
||||
+61
-31
@@ -1,18 +1,14 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command.base import append_doc_link
|
||||
from dvc.command.base import CmdBase
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CmdRun(CmdBase):
|
||||
def run(self):
|
||||
overwrite = self.args.yes or self.args.overwrite_dvcfile
|
||||
|
||||
if not any(
|
||||
[
|
||||
self.args.deps,
|
||||
@@ -20,15 +16,18 @@ class CmdRun(CmdBase):
|
||||
self.args.outs_no_cache,
|
||||
self.args.metrics,
|
||||
self.args.metrics_no_cache,
|
||||
self.args.plots,
|
||||
self.args.plots_no_cache,
|
||||
self.args.outs_persist,
|
||||
self.args.outs_persist_no_cache,
|
||||
self.args.params,
|
||||
self.args.command,
|
||||
]
|
||||
): # pragma: no cover
|
||||
logger.error(
|
||||
"too few arguments. Specify at least one: `-d`, `-o`, `-O`, "
|
||||
"`-m`, `-M`, `--outs-persist`, `--outs-persist-no-cache`, "
|
||||
"`command`."
|
||||
"`-m`, `-M`, `-p`, `--plots`, `--plots-no-cache`, "
|
||||
"`--outs-persist`, `--outs-persist-no-cache`, `command`."
|
||||
)
|
||||
return 1
|
||||
|
||||
@@ -39,21 +38,24 @@ class CmdRun(CmdBase):
|
||||
outs_no_cache=self.args.outs_no_cache,
|
||||
metrics=self.args.metrics,
|
||||
metrics_no_cache=self.args.metrics_no_cache,
|
||||
plots=self.args.plots,
|
||||
plots_no_cache=self.args.plots_no_cache,
|
||||
deps=self.args.deps,
|
||||
params=self.args.params,
|
||||
fname=self.args.file,
|
||||
cwd=self.args.cwd,
|
||||
wdir=self.args.wdir,
|
||||
no_exec=self.args.no_exec,
|
||||
overwrite=overwrite,
|
||||
ignore_build_cache=self.args.ignore_build_cache,
|
||||
remove_outs=self.args.remove_outs,
|
||||
overwrite=self.args.overwrite_dvcfile,
|
||||
run_cache=not self.args.no_run_cache,
|
||||
no_commit=self.args.no_commit,
|
||||
outs_persist=self.args.outs_persist,
|
||||
outs_persist_no_cache=self.args.outs_persist_no_cache,
|
||||
always_changed=self.args.always_changed,
|
||||
name=self.args.name,
|
||||
single_stage=self.args.single_stage,
|
||||
)
|
||||
except DvcException:
|
||||
logger.exception("failed to run command")
|
||||
logger.exception("")
|
||||
return 1
|
||||
|
||||
return 0
|
||||
@@ -77,7 +79,7 @@ class CmdRun(CmdBase):
|
||||
if " " not in argument or '"' in argument:
|
||||
return argument
|
||||
|
||||
return '"{}"'.format(argument)
|
||||
return f'"{argument}"'
|
||||
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
@@ -95,6 +97,10 @@ def add_parser(subparsers, parent_parser):
|
||||
action="append",
|
||||
default=[],
|
||||
help="Declare dependencies for reproducible cmd.",
|
||||
metavar="<path>",
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"-n", "--name", help="Stage name.",
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"-o",
|
||||
@@ -102,6 +108,7 @@ def add_parser(subparsers, parent_parser):
|
||||
action="append",
|
||||
default=[],
|
||||
help="Declare output file or directory.",
|
||||
metavar="<filename>",
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"-O",
|
||||
@@ -110,6 +117,15 @@ def add_parser(subparsers, parent_parser):
|
||||
default=[],
|
||||
help="Declare output file or directory "
|
||||
"(do not put into DVC cache).",
|
||||
metavar="<filename>",
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"-p",
|
||||
"--params",
|
||||
action="append",
|
||||
default=[],
|
||||
help="Declare parameter to use as additional dependency.",
|
||||
metavar="[<filename>:]<params_list>",
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"-m",
|
||||
@@ -117,6 +133,7 @@ def add_parser(subparsers, parent_parser):
|
||||
action="append",
|
||||
default=[],
|
||||
help="Declare output metric file or directory.",
|
||||
metavar="<path>",
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"-M",
|
||||
@@ -125,17 +142,33 @@ def add_parser(subparsers, parent_parser):
|
||||
default=[],
|
||||
help="Declare output metric file or directory "
|
||||
"(do not put into DVC cache).",
|
||||
metavar="<path>",
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"-f", "--file", help="Specify name of the DVC-file it generates."
|
||||
"--plots",
|
||||
action="append",
|
||||
default=[],
|
||||
help="Declare output plot file.",
|
||||
metavar="<path>",
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"-c", "--cwd", help="Deprecated, use -w and -f instead."
|
||||
"--plots-no-cache",
|
||||
action="append",
|
||||
default=[],
|
||||
help="Declare output plot file (do not put into DVC cache).",
|
||||
metavar="<path>",
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"-f",
|
||||
"--file",
|
||||
help="Specify name of the DVC-file this command will generate.",
|
||||
metavar="<filename>",
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"-w",
|
||||
"--wdir",
|
||||
help="Directory within your repo to run your command in.",
|
||||
metavar="<path>",
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"--no-exec",
|
||||
@@ -143,13 +176,6 @@ def add_parser(subparsers, parent_parser):
|
||||
default=False,
|
||||
help="Only create stage file without actually running it.",
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"-y",
|
||||
"--yes",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Deprecated, use --overwrite-dvcfile instead",
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"--overwrite-dvcfile",
|
||||
action="store_true",
|
||||
@@ -157,17 +183,13 @@ def add_parser(subparsers, parent_parser):
|
||||
help="Overwrite existing DVC-file without asking for confirmation.",
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"--ignore-build-cache",
|
||||
"--no-run-cache",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Run this stage even if it has been already ran with the same "
|
||||
"command/dependencies/outputs/etc before.",
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"--remove-outs",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Deprecated, this is now the default behavior",
|
||||
help=(
|
||||
"Execute the command even if this stage has already been run "
|
||||
"with the same command/dependencies/outputs/etc before."
|
||||
),
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"--no-commit",
|
||||
@@ -181,6 +203,7 @@ def add_parser(subparsers, parent_parser):
|
||||
default=[],
|
||||
help="Declare output file or directory that will not be "
|
||||
"removed upon repro.",
|
||||
metavar="<filename>",
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"--outs-persist-no-cache",
|
||||
@@ -188,6 +211,7 @@ def add_parser(subparsers, parent_parser):
|
||||
default=[],
|
||||
help="Declare output file or directory that will not be "
|
||||
"removed upon repro (do not put into DVC cache).",
|
||||
metavar="<filename>",
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"--always-changed",
|
||||
@@ -195,6 +219,12 @@ def add_parser(subparsers, parent_parser):
|
||||
default=False,
|
||||
help="Always consider this DVC-file as changed.",
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"--single-stage",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help=argparse.SUPPRESS,
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"command", nargs=argparse.REMAINDER, help="Command to execute."
|
||||
)
|
||||
|
||||
@@ -2,7 +2,6 @@ import logging
|
||||
|
||||
from dvc.command.data_sync import CmdDataBase
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -20,7 +19,7 @@ class CmdDataStatus(CmdDataBase):
|
||||
ind = indent * self.STATUS_INDENT
|
||||
|
||||
if isinstance(status, str):
|
||||
logger.info("{}{}".format(ind, status))
|
||||
logger.info(f"{ind}{status}")
|
||||
return
|
||||
|
||||
if isinstance(status, list):
|
||||
@@ -34,7 +33,7 @@ class CmdDataStatus(CmdDataBase):
|
||||
if isinstance(value, str):
|
||||
logger.info("{}{}{}".format(ind, self._normalize(value), key))
|
||||
elif value:
|
||||
logger.info("{}{}:".format(ind, key))
|
||||
logger.info(f"{ind}{key}:")
|
||||
self._show(value, indent + 1)
|
||||
|
||||
def run(self):
|
||||
@@ -47,7 +46,9 @@ class CmdDataStatus(CmdDataBase):
|
||||
remote=self.args.remote,
|
||||
all_branches=self.args.all_branches,
|
||||
all_tags=self.args.all_tags,
|
||||
all_commits=self.args.all_commits,
|
||||
with_deps=self.args.with_deps,
|
||||
recursive=self.args.recursive,
|
||||
)
|
||||
if st:
|
||||
if self.args.quiet:
|
||||
|
||||
@@ -1,156 +0,0 @@
|
||||
import logging
|
||||
|
||||
from dvc.command.base import append_doc_link
|
||||
from dvc.command.base import CmdBase
|
||||
from dvc.command.base import fix_subparsers
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.utils import to_yaml_string
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CmdTagAdd(CmdBase):
|
||||
def run(self):
|
||||
for target in self.args.targets:
|
||||
try:
|
||||
self.repo.tag.add(
|
||||
self.args.tag,
|
||||
target=target,
|
||||
with_deps=self.args.with_deps,
|
||||
recursive=self.args.recursive,
|
||||
)
|
||||
except DvcException:
|
||||
logger.exception("failed to add tag")
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
class CmdTagRemove(CmdBase):
|
||||
def run(self):
|
||||
for target in self.args.targets:
|
||||
try:
|
||||
self.repo.tag.remove(
|
||||
self.args.tag,
|
||||
target=target,
|
||||
with_deps=self.args.with_deps,
|
||||
recursive=self.args.recursive,
|
||||
)
|
||||
except DvcException:
|
||||
logger.exception("failed to remove tag")
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
class CmdTagList(CmdBase):
|
||||
def run(self):
|
||||
for target in self.args.targets:
|
||||
try:
|
||||
tags = self.repo.tag.list(
|
||||
target,
|
||||
with_deps=self.args.with_deps,
|
||||
recursive=self.args.recursive,
|
||||
)
|
||||
if tags:
|
||||
logger.info(to_yaml_string(tags))
|
||||
except DvcException:
|
||||
logger.exception("failed list tags")
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
TAG_HELP = "A set of commands to manage DVC tags."
|
||||
tag_parser = subparsers.add_parser(
|
||||
"tag",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(TAG_HELP, "tag"),
|
||||
add_help=False,
|
||||
)
|
||||
|
||||
tag_subparsers = tag_parser.add_subparsers(
|
||||
dest="cmd",
|
||||
help="Use DVC tag CMD --help to display command-specific help.",
|
||||
)
|
||||
|
||||
fix_subparsers(tag_subparsers)
|
||||
|
||||
TAG_ADD_HELP = "Add DVC tag."
|
||||
tag_add_parser = tag_subparsers.add_parser(
|
||||
"add",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(TAG_ADD_HELP, "tag-add"),
|
||||
help=TAG_ADD_HELP,
|
||||
)
|
||||
tag_add_parser.add_argument("tag", help="Dvc tag.")
|
||||
tag_add_parser.add_argument(
|
||||
"targets", nargs="*", default=[None], help="DVC-files."
|
||||
)
|
||||
tag_add_parser.add_argument(
|
||||
"-d",
|
||||
"--with-deps",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Add tag for all dependencies of the specified DVC-file.",
|
||||
)
|
||||
tag_add_parser.add_argument(
|
||||
"-R",
|
||||
"--recursive",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Add tag for subdirectories of the specified directory.",
|
||||
)
|
||||
tag_add_parser.set_defaults(func=CmdTagAdd)
|
||||
|
||||
TAG_REMOVE_HELP = "Remove DVC tag."
|
||||
tag_remove_parser = tag_subparsers.add_parser(
|
||||
"remove",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(TAG_REMOVE_HELP, "tag-remove"),
|
||||
help=TAG_REMOVE_HELP,
|
||||
)
|
||||
tag_remove_parser.add_argument("tag", help="Dvc tag.")
|
||||
tag_remove_parser.add_argument(
|
||||
"targets", nargs="*", default=[None], help="DVC-files."
|
||||
)
|
||||
tag_remove_parser.add_argument(
|
||||
"-d",
|
||||
"--with-deps",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Remove tag for all dependencies of the specified DVC-file.",
|
||||
)
|
||||
tag_remove_parser.add_argument(
|
||||
"-R",
|
||||
"--recursive",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Remove tag for subdirectories of the specified directory.",
|
||||
)
|
||||
tag_remove_parser.set_defaults(func=CmdTagRemove)
|
||||
|
||||
TAG_LIST_HELP = "List DVC tags."
|
||||
tag_list_parser = tag_subparsers.add_parser(
|
||||
"list",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(TAG_LIST_HELP, "tag-list"),
|
||||
help=TAG_LIST_HELP,
|
||||
)
|
||||
tag_list_parser.add_argument(
|
||||
"targets", nargs="*", default=[None], help="DVC-files."
|
||||
)
|
||||
tag_list_parser.add_argument(
|
||||
"-d",
|
||||
"--with-deps",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="List tags for all dependencies of the specified DVC-file.",
|
||||
)
|
||||
tag_list_parser.add_argument(
|
||||
"-R",
|
||||
"--recursive",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="List tags for subdirectories of the specified directory.",
|
||||
)
|
||||
tag_list_parser.set_defaults(func=CmdTagList)
|
||||
@@ -1,11 +1,9 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command.base import append_doc_link
|
||||
from dvc.command.base import CmdBase
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -15,14 +13,17 @@ class CmdUnprotect(CmdBase):
|
||||
try:
|
||||
self.repo.unprotect(target)
|
||||
except DvcException:
|
||||
msg = "failed to unprotect '{}'".format(target)
|
||||
msg = f"failed to unprotect '{target}'"
|
||||
logger.exception(msg)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
UNPROTECT_HELP = "Unprotect data files or directories."
|
||||
UNPROTECT_HELP = (
|
||||
"Unprotect tracked files or directories (when hardlinks or symlinks "
|
||||
"have been enabled with `dvc config cache.type`)"
|
||||
)
|
||||
unprotect_parser = subparsers.add_parser(
|
||||
"unprotect",
|
||||
parents=[parent_parser],
|
||||
|
||||
+21
-10
@@ -1,23 +1,24 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command.base import append_doc_link
|
||||
from dvc.command.base import CmdBase
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CmdUpdate(CmdBase):
|
||||
def run(self):
|
||||
ret = 0
|
||||
for target in self.args.targets:
|
||||
try:
|
||||
self.repo.update(target, self.args.rev)
|
||||
except DvcException:
|
||||
logger.exception("failed to update '{}'.".format(target))
|
||||
ret = 1
|
||||
try:
|
||||
self.repo.update(
|
||||
targets=self.args.targets,
|
||||
rev=self.args.rev,
|
||||
recursive=self.args.recursive,
|
||||
)
|
||||
except DvcException:
|
||||
logger.exception("failed update data")
|
||||
ret = 1
|
||||
return ret
|
||||
|
||||
|
||||
@@ -34,6 +35,16 @@ def add_parser(subparsers, parent_parser):
|
||||
"targets", nargs="+", help="DVC-files to update."
|
||||
)
|
||||
update_parser.add_argument(
|
||||
"--rev", nargs="?", help="Git revision (e.g. SHA, branch, tag)"
|
||||
"--rev",
|
||||
nargs="?",
|
||||
help="Git revision (e.g. SHA, branch, tag)",
|
||||
metavar="<commit>",
|
||||
)
|
||||
update_parser.add_argument(
|
||||
"-R",
|
||||
"--recursive",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Update all stages in the specified directory.",
|
||||
)
|
||||
update_parser.set_defaults(func=CmdUpdate)
|
||||
|
||||
+47
-17
@@ -2,21 +2,23 @@ import argparse
|
||||
import itertools
|
||||
import logging
|
||||
import os
|
||||
import pathlib
|
||||
import platform
|
||||
import uuid
|
||||
import pathlib
|
||||
|
||||
from dvc.command.base import CmdBaseNoRepo, append_doc_link
|
||||
from dvc.exceptions import DvcException, NotDvcRepoError
|
||||
from dvc.scm.base import SCMError
|
||||
from dvc.system import System
|
||||
from dvc.utils import is_binary, relpath
|
||||
from dvc.utils.pkg import PKG
|
||||
from dvc.version import __version__
|
||||
|
||||
try:
|
||||
import psutil
|
||||
except ImportError:
|
||||
psutil = None
|
||||
|
||||
from dvc.utils import is_binary, relpath
|
||||
from dvc.command.base import CmdBaseNoRepo, append_doc_link
|
||||
from dvc.version import __version__
|
||||
from dvc.exceptions import DvcException, NotDvcRepoError
|
||||
from dvc.system import System
|
||||
from dvc.utils.pkg import PKG
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -26,11 +28,12 @@ class CmdVersion(CmdBaseNoRepo):
|
||||
from dvc.repo import Repo
|
||||
|
||||
info = [
|
||||
"DVC version: {}".format(__version__),
|
||||
"Python version: {}".format(platform.python_version()),
|
||||
"Platform: {}".format(platform.platform()),
|
||||
"Binary: {}".format(is_binary()),
|
||||
"Package: {}".format(PKG),
|
||||
f"DVC version: {__version__}",
|
||||
f"Python version: {platform.python_version()}",
|
||||
f"Platform: {platform.platform()}",
|
||||
f"Binary: {is_binary()}",
|
||||
f"Package: {PKG}",
|
||||
f"Supported remotes: {self.get_supported_remotes()}",
|
||||
]
|
||||
|
||||
try:
|
||||
@@ -48,7 +51,7 @@ class CmdVersion(CmdBaseNoRepo):
|
||||
if psutil:
|
||||
fs_type = self.get_fs_type(repo.cache.local.cache_dir)
|
||||
info.append(
|
||||
"Filesystem type (cache directory): {}".format(fs_type)
|
||||
f"Filesystem type (cache directory): {fs_type}"
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
@@ -61,10 +64,15 @@ class CmdVersion(CmdBaseNoRepo):
|
||||
|
||||
except NotDvcRepoError:
|
||||
root_directory = os.getcwd()
|
||||
except SCMError:
|
||||
root_directory = os.getcwd()
|
||||
info.append("Repo: dvc, git (broken)")
|
||||
else:
|
||||
info.append("Repo: {}".format(_get_dvc_repo_info(repo)))
|
||||
|
||||
if psutil:
|
||||
fs_root = self.get_fs_type(os.path.abspath(root_directory))
|
||||
info.append("Filesystem type (workspace): {}".format(fs_root))
|
||||
info.append(f"Filesystem type (workspace): {fs_root}")
|
||||
|
||||
logger.info("\n".join(info))
|
||||
return 0
|
||||
@@ -107,15 +115,37 @@ class CmdVersion(CmdBaseNoRepo):
|
||||
os.unlink(dst)
|
||||
except DvcException:
|
||||
status = "not supported"
|
||||
cache.append("{name} - {status}".format(name=name, status=status))
|
||||
cache.append(f"{name} - {status}")
|
||||
os.remove(src)
|
||||
|
||||
return ", ".join(cache)
|
||||
|
||||
@staticmethod
|
||||
def get_supported_remotes():
|
||||
from dvc.remote import REMOTES
|
||||
|
||||
supported_remotes = []
|
||||
for remote in REMOTES:
|
||||
if not remote.get_missing_deps():
|
||||
supported_remotes.append(remote.scheme)
|
||||
|
||||
return ", ".join(supported_remotes)
|
||||
|
||||
|
||||
def _get_dvc_repo_info(repo):
|
||||
if repo.config.get("core", {}).get("no_scm", False):
|
||||
return "dvc (no_scm)"
|
||||
|
||||
if repo.root_dir != repo.scm.root_dir:
|
||||
return "dvc (subdir), git"
|
||||
|
||||
return "dvc, git"
|
||||
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
VERSION_HELP = "Show DVC version and system/environment information."
|
||||
|
||||
VERSION_HELP = (
|
||||
"Display the DVC version and system/environment information."
|
||||
)
|
||||
version_parser = subparsers.add_parser(
|
||||
"version",
|
||||
parents=[parent_parser],
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
"""Python version compatibility code"""
|
||||
|
||||
# Backport os.fspath() from Python 3.6
|
||||
try:
|
||||
from os import fspath # noqa: F821
|
||||
|
||||
fspath_py35 = lambda s: s # noqa: E731
|
||||
except ImportError:
|
||||
|
||||
def fspath(path):
|
||||
"""Return the path representation of a path-like object.
|
||||
|
||||
If str or bytes is passed in, it is returned unchanged. Otherwise the
|
||||
os.PathLike interface is used to get the path representation. If the
|
||||
path representation is not str or bytes, TypeError is raised. If the
|
||||
provided path is not str, bytes, or os.PathLike, TypeError is raised.
|
||||
"""
|
||||
if isinstance(path, (str, bytes)):
|
||||
return path
|
||||
|
||||
# Work from the object's type to match method resolution of other magic
|
||||
# methods.
|
||||
path_type = type(path)
|
||||
try:
|
||||
path_repr = path_type.__fspath__(path)
|
||||
except AttributeError:
|
||||
if hasattr(path_type, "__fspath__"):
|
||||
raise
|
||||
else:
|
||||
raise TypeError(
|
||||
"expected str, bytes or os.PathLike object, "
|
||||
"not " + path_type.__name__
|
||||
)
|
||||
if isinstance(path_repr, (str, bytes)):
|
||||
return path_repr
|
||||
else:
|
||||
raise TypeError(
|
||||
"expected {}.__fspath__() to return str or bytes, "
|
||||
"not {}".format(path_type.__name__, type(path_repr).__name__)
|
||||
)
|
||||
|
||||
fspath_py35 = fspath
|
||||
+56
-24
@@ -1,16 +1,27 @@
|
||||
"""DVC config objects."""
|
||||
from contextlib import contextmanager
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from contextlib import contextmanager
|
||||
from functools import partial
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from funcy import cached_property, re_find, walk_values, compact
|
||||
import configobj
|
||||
from voluptuous import Schema, Optional, Invalid, ALLOW_EXTRA
|
||||
from voluptuous import All, Any, Lower, Range, Coerce
|
||||
from funcy import cached_property, compact, re_find, walk_values
|
||||
from voluptuous import (
|
||||
ALLOW_EXTRA,
|
||||
All,
|
||||
Any,
|
||||
Coerce,
|
||||
Invalid,
|
||||
Lower,
|
||||
Optional,
|
||||
Range,
|
||||
Schema,
|
||||
)
|
||||
|
||||
from dvc.exceptions import DvcException, NotDvcRepoError
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.utils import relpath
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -20,7 +31,7 @@ class ConfigError(DvcException):
|
||||
"""DVC config exception."""
|
||||
|
||||
def __init__(self, msg):
|
||||
super().__init__("config file error: {}".format(msg))
|
||||
super().__init__(f"config file error: {msg}")
|
||||
|
||||
|
||||
class NoRemoteError(ConfigError):
|
||||
@@ -78,7 +89,7 @@ def ByUrl(mapping):
|
||||
if os.name == "nt" and len(parsed.scheme) == 1 and parsed.netloc == "":
|
||||
return schemas[""](data)
|
||||
if parsed.scheme not in schemas:
|
||||
raise Invalid("Unsupported URL type {}://".format(parsed.scheme))
|
||||
raise Invalid(f"Unsupported URL type {parsed.scheme}://")
|
||||
|
||||
return schemas[parsed.scheme](data)
|
||||
|
||||
@@ -92,12 +103,12 @@ class RelPath(str):
|
||||
REMOTE_COMMON = {
|
||||
"url": str,
|
||||
"checksum_jobs": All(Coerce(int), Range(1)),
|
||||
"no_traverse": Bool,
|
||||
Optional("no_traverse"): Bool, # obsoleted
|
||||
"verify": Bool,
|
||||
}
|
||||
LOCAL_COMMON = {
|
||||
"type": supported_cache_type,
|
||||
Optional("protected", default=False): Bool,
|
||||
Optional("protected", default=False): Bool, # obsoleted
|
||||
"shared": All(Lower, Choices("group")),
|
||||
Optional("slow_link_warning", default=True): Bool,
|
||||
}
|
||||
@@ -140,6 +151,7 @@ SCHEMA = {
|
||||
Optional("listobjects", default=False): Bool,
|
||||
Optional("use_ssl", default=True): Bool,
|
||||
"sse": str,
|
||||
"sse_kms_key_id": str,
|
||||
"acl": str,
|
||||
"grant_read": str,
|
||||
"grant_read_acp": str,
|
||||
@@ -179,6 +191,7 @@ SCHEMA = {
|
||||
"gdrive_service_account_email": str,
|
||||
"gdrive_service_account_user_email": str,
|
||||
"gdrive_service_account_p12_file_path": str,
|
||||
Optional("gdrive_trash_only", default=False): Bool,
|
||||
**REMOTE_COMMON,
|
||||
},
|
||||
"http": {**HTTP_COMMON, **REMOTE_COMMON},
|
||||
@@ -277,10 +290,7 @@ class Config(dict):
|
||||
Raises:
|
||||
ConfigError: thrown if config has an invalid format.
|
||||
"""
|
||||
conf = {}
|
||||
for level in self.LEVELS:
|
||||
if level in self.files:
|
||||
_merge(conf, self.load_one(level))
|
||||
conf = self.load_config_to_level()
|
||||
|
||||
if validate:
|
||||
conf = self.validate(conf)
|
||||
@@ -296,7 +306,7 @@ class Config(dict):
|
||||
conf = _load_config(self.files[level])
|
||||
conf = self._load_paths(conf, self.files[level])
|
||||
|
||||
# Autovivify sections
|
||||
# Auto-verify sections
|
||||
for key in COMPILED_SCHEMA.schema:
|
||||
conf.setdefault(key, {})
|
||||
|
||||
@@ -313,25 +323,42 @@ class Config(dict):
|
||||
|
||||
return Config._map_dirs(conf, resolve)
|
||||
|
||||
@staticmethod
|
||||
def _to_relpath(conf_dir, path):
|
||||
if re.match(r"\w+://", path):
|
||||
return path
|
||||
|
||||
if isinstance(path, RelPath) or not os.path.isabs(path):
|
||||
path = relpath(path, conf_dir)
|
||||
|
||||
return PathInfo(path).as_posix()
|
||||
|
||||
@staticmethod
|
||||
def _save_paths(conf, filename):
|
||||
conf_dir = os.path.dirname(filename)
|
||||
|
||||
def rel(path):
|
||||
if re.match(r"\w+://", path):
|
||||
return path
|
||||
|
||||
if isinstance(path, RelPath) or not os.path.isabs(path):
|
||||
return relpath(path, conf_dir)
|
||||
return path
|
||||
rel = partial(Config._to_relpath, conf_dir)
|
||||
|
||||
return Config._map_dirs(conf, rel)
|
||||
|
||||
@staticmethod
|
||||
def _map_dirs(conf, func):
|
||||
dirs_schema = {"cache": {"dir": func}, "remote": {str: {"url": func}}}
|
||||
dirs_schema = {
|
||||
"cache": {"dir": func},
|
||||
"remote": {
|
||||
str: {"url": func, "gdrive_user_credentials_file": func}
|
||||
},
|
||||
}
|
||||
return Schema(dirs_schema, extra=ALLOW_EXTRA)(conf)
|
||||
|
||||
def load_config_to_level(self, level=None):
|
||||
merged_conf = {}
|
||||
for merge_level in self.LEVELS:
|
||||
if merge_level == level:
|
||||
break
|
||||
if merge_level in self.files:
|
||||
_merge(merged_conf, self.load_one(merge_level))
|
||||
return merged_conf
|
||||
|
||||
@contextmanager
|
||||
def edit(self, level="repo"):
|
||||
if level in {"repo", "local"} and self.dvc_dir is None:
|
||||
@@ -341,6 +368,11 @@ class Config(dict):
|
||||
yield conf
|
||||
|
||||
conf = self._save_paths(conf, self.files[level])
|
||||
|
||||
merged_conf = self.load_config_to_level(level)
|
||||
_merge(merged_conf, conf)
|
||||
self.validate(merged_conf)
|
||||
|
||||
_save_config(self.files[level], conf)
|
||||
self.load()
|
||||
|
||||
@@ -358,7 +390,7 @@ def _load_config(filename):
|
||||
|
||||
|
||||
def _save_config(filename, conf_dict):
|
||||
logger.debug("Writing '{}'.".format(filename))
|
||||
logger.debug(f"Writing '{filename}'.")
|
||||
os.makedirs(os.path.dirname(filename), exist_ok=True)
|
||||
|
||||
config = configobj.ConfigObj(_pack_remotes(conf_dict))
|
||||
@@ -385,7 +417,7 @@ def _pack_remotes(conf):
|
||||
|
||||
# Transform remote.name -> 'remote "name"'
|
||||
for name, val in conf["remote"].items():
|
||||
result['remote "{}"'.format(name)] = val
|
||||
result[f'remote "{name}"'] = val
|
||||
result.pop("remote", None)
|
||||
|
||||
return result
|
||||
|
||||
+3
-5
@@ -7,9 +7,7 @@ import sys
|
||||
from subprocess import Popen
|
||||
|
||||
from dvc.env import DVC_DAEMON
|
||||
from dvc.utils import fix_env
|
||||
from dvc.utils import is_binary
|
||||
|
||||
from dvc.utils import fix_env, is_binary
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -68,7 +66,7 @@ def _spawn_posix(cmd, env):
|
||||
|
||||
|
||||
def _spawn(cmd, env):
|
||||
logger.debug("Trying to spawn '{}'".format(cmd))
|
||||
logger.debug(f"Trying to spawn '{cmd}'")
|
||||
|
||||
if os.name == "nt":
|
||||
_spawn_windows(cmd, env)
|
||||
@@ -77,7 +75,7 @@ def _spawn(cmd, env):
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
logger.debug("Spawned '{}'".format(cmd))
|
||||
logger.debug(f"Spawned '{cmd}'")
|
||||
|
||||
|
||||
def daemon(args):
|
||||
|
||||
+10
-13
@@ -6,12 +6,9 @@ import os
|
||||
import pydoc
|
||||
import sys
|
||||
|
||||
from grandalf.graphs import Edge
|
||||
from grandalf.graphs import Graph
|
||||
from grandalf.graphs import Vertex
|
||||
from grandalf.graphs import Edge, Graph, Vertex
|
||||
from grandalf.layouts import SugiyamaLayout
|
||||
from grandalf.routing import EdgeViewer
|
||||
from grandalf.routing import route_with_lines
|
||||
from grandalf.routing import EdgeViewer, route_with_lines
|
||||
|
||||
from dvc.env import DVC_PAGER
|
||||
from dvc.utils import format_link
|
||||
@@ -40,7 +37,7 @@ def find_pager():
|
||||
if env_pager:
|
||||
return make_pager(env_pager)
|
||||
|
||||
if os.system("({}) 2>{}".format(DEFAULT_PAGER, os.devnull)) == 0:
|
||||
if os.system(f"({DEFAULT_PAGER}) 2>{os.devnull}") == 0:
|
||||
return make_pager(DEFAULT_PAGER_FORMATTED)
|
||||
|
||||
logger.warning(
|
||||
@@ -52,7 +49,7 @@ def find_pager():
|
||||
return pydoc.plainpager
|
||||
|
||||
|
||||
class VertexViewer(object):
|
||||
class VertexViewer:
|
||||
"""Class to define vertex box boundaries that will be accounted for during
|
||||
graph building by grandalf.
|
||||
|
||||
@@ -78,7 +75,7 @@ class VertexViewer(object):
|
||||
return self._w
|
||||
|
||||
|
||||
class AsciiCanvas(object):
|
||||
class AsciiCanvas:
|
||||
"""Class for drawing in ASCII.
|
||||
|
||||
Args:
|
||||
@@ -95,7 +92,7 @@ class AsciiCanvas(object):
|
||||
self.cols = cols
|
||||
self.lines = lines
|
||||
|
||||
self.canvas = [[" "] * cols for l in range(lines)]
|
||||
self.canvas = [[" "] * cols for line in range(lines)]
|
||||
|
||||
def draw(self):
|
||||
"""Draws ASCII canvas on the screen."""
|
||||
@@ -148,21 +145,21 @@ class AsciiCanvas(object):
|
||||
if dx == 0:
|
||||
y = y0
|
||||
else:
|
||||
y = y0 + int(round((x - x0) * dy / float((dx))))
|
||||
y = y0 + int(round((x - x0) * dy / float(dx)))
|
||||
self.point(x, y, char)
|
||||
elif y0 < y1:
|
||||
for y in range(y0, y1 + 1):
|
||||
if dy == 0:
|
||||
x = x0
|
||||
else:
|
||||
x = x0 + int(round((y - y0) * dx / float((dy))))
|
||||
x = x0 + int(round((y - y0) * dx / float(dy)))
|
||||
self.point(x, y, char)
|
||||
else:
|
||||
for y in range(y1, y0 + 1):
|
||||
if dy == 0:
|
||||
x = x0
|
||||
else:
|
||||
x = x1 + int(round((y - y1) * dx / float((dy))))
|
||||
x = x1 + int(round((y - y1) * dx / float(dy)))
|
||||
self.point(x, y, char)
|
||||
|
||||
def text(self, x, y, text):
|
||||
@@ -216,7 +213,7 @@ def _build_sugiyama_layout(vertexes, edges):
|
||||
# Y
|
||||
#
|
||||
|
||||
vertexes = {v: Vertex(" {} ".format(v)) for v in vertexes}
|
||||
vertexes = {v: Vertex(f" {v} ") for v in vertexes}
|
||||
# NOTE: reverting edges to correctly orientate the graph
|
||||
edges = [Edge(vertexes[e], vertexes[s]) for s, e in edges]
|
||||
vertexes = vertexes.values()
|
||||
|
||||
+26
-22
@@ -5,11 +5,10 @@ import logging
|
||||
from dvc.config import NoRemoteError
|
||||
from dvc.remote import Remote
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DataCloud(object):
|
||||
class DataCloud:
|
||||
"""Class that manages dvc remotes.
|
||||
|
||||
Args:
|
||||
@@ -23,12 +22,12 @@ class DataCloud(object):
|
||||
def __init__(self, repo):
|
||||
self.repo = repo
|
||||
|
||||
def get_remote(self, remote=None, command="<command>"):
|
||||
if not remote:
|
||||
remote = self.repo.config["core"].get("remote")
|
||||
def get_remote(self, name=None, command="<command>"):
|
||||
if not name:
|
||||
name = self.repo.config["core"].get("remote")
|
||||
|
||||
if remote:
|
||||
return self._init_remote(remote)
|
||||
if name:
|
||||
return self._init_remote(name)
|
||||
|
||||
if bool(self.repo.config["remote"]):
|
||||
error_msg = (
|
||||
@@ -45,39 +44,43 @@ class DataCloud(object):
|
||||
|
||||
raise NoRemoteError(error_msg)
|
||||
|
||||
def _init_remote(self, remote):
|
||||
return Remote(self.repo, name=remote)
|
||||
def _init_remote(self, name):
|
||||
return Remote(self.repo, name=name)
|
||||
|
||||
def push(self, cache, jobs=None, remote=None, show_checksums=False):
|
||||
def push(
|
||||
self, cache, jobs=None, remote=None, show_checksums=False,
|
||||
):
|
||||
"""Push data items in a cloud-agnostic way.
|
||||
|
||||
Args:
|
||||
cache (NamedCache): named checksums to push to the cloud.
|
||||
jobs (int): number of jobs that can be running simultaneously.
|
||||
remote (dvc.remote.base.RemoteBASE): optional remote to push to.
|
||||
remote (dvc.remote.base.BaseRemote): optional remote to push to.
|
||||
By default remote from core.remote config option is used.
|
||||
show_checksums (bool): show checksums instead of file names in
|
||||
information messages.
|
||||
"""
|
||||
remote = self.get_remote(remote, "push")
|
||||
|
||||
return self.repo.cache.local.push(
|
||||
cache,
|
||||
jobs=jobs,
|
||||
remote=self.get_remote(remote, "push"),
|
||||
show_checksums=show_checksums,
|
||||
cache, jobs=jobs, remote=remote, show_checksums=show_checksums,
|
||||
)
|
||||
|
||||
def pull(self, cache, jobs=None, remote=None, show_checksums=False):
|
||||
def pull(
|
||||
self, cache, jobs=None, remote=None, show_checksums=False,
|
||||
):
|
||||
"""Pull data items in a cloud-agnostic way.
|
||||
|
||||
Args:
|
||||
cache (NamedCache): named checksums to pull from the cloud.
|
||||
jobs (int): number of jobs that can be running simultaneously.
|
||||
remote (dvc.remote.base.RemoteBASE): optional remote to pull from.
|
||||
remote (dvc.remote.base.BaseRemote): optional remote to pull from.
|
||||
By default remote from core.remote config option is used.
|
||||
show_checksums (bool): show checksums instead of file names in
|
||||
information messages.
|
||||
"""
|
||||
remote = self.get_remote(remote, "pull")
|
||||
|
||||
downloaded_items_num = self.repo.cache.local.pull(
|
||||
cache, jobs=jobs, remote=remote, show_checksums=show_checksums
|
||||
)
|
||||
@@ -88,12 +91,13 @@ class DataCloud(object):
|
||||
return downloaded_items_num
|
||||
|
||||
def _save_pulled_checksums(self, cache):
|
||||
for checksum in cache["local"].keys():
|
||||
for checksum in cache.scheme_keys("local"):
|
||||
cache_file = self.repo.cache.local.checksum_to_path_info(checksum)
|
||||
if self.repo.cache.local.exists(cache_file):
|
||||
# We can safely save here, as existing corrupted files will be
|
||||
# removed upon status, while files corrupted during download
|
||||
# will not be moved from tmp_file (see `RemoteBASE.download()`)
|
||||
# We can safely save here, as existing corrupted files will
|
||||
# be removed upon status, while files corrupted during
|
||||
# download will not be moved from tmp_file
|
||||
# (see `RemoteBASE.download()`)
|
||||
self.repo.state.save(cache_file, checksum)
|
||||
|
||||
def status(self, cache, jobs=None, remote=None, show_checksums=False):
|
||||
@@ -102,7 +106,7 @@ class DataCloud(object):
|
||||
Args:
|
||||
cache (NamedCache): named checksums to check status for.
|
||||
jobs (int): number of jobs that can be running simultaneously.
|
||||
remote (dvc.remote.base.RemoteBASE): optional remote to compare
|
||||
remote (dvc.remote.base.BaseRemote): optional remote to compare
|
||||
cache to. By default remote from core.remote config option
|
||||
is used.
|
||||
show_checksums (bool): show checksums instead of file names in
|
||||
|
||||
+69
-35
@@ -1,37 +1,42 @@
|
||||
from collections import defaultdict
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import dvc.output as output
|
||||
from dvc.dependency.gs import DependencyGS
|
||||
from dvc.dependency.hdfs import DependencyHDFS
|
||||
from dvc.dependency.http import DependencyHTTP
|
||||
from dvc.dependency.https import DependencyHTTPS
|
||||
from dvc.dependency.local import DependencyLOCAL
|
||||
from dvc.dependency.s3 import DependencyS3
|
||||
from dvc.dependency.ssh import DependencySSH
|
||||
from dvc.output.base import OutputBase
|
||||
from dvc.dependency.azure import AzureDependency
|
||||
from dvc.dependency.gs import GSDependency
|
||||
from dvc.dependency.hdfs import HDFSDependency
|
||||
from dvc.dependency.http import HTTPDependency
|
||||
from dvc.dependency.https import HTTPSDependency
|
||||
from dvc.dependency.local import LocalDependency
|
||||
from dvc.dependency.param import ParamsDependency
|
||||
from dvc.dependency.s3 import S3Dependency
|
||||
from dvc.dependency.ssh import SSHDependency
|
||||
from dvc.output.base import BaseOutput
|
||||
from dvc.remote import Remote
|
||||
from dvc.scheme import Schemes
|
||||
from .repo import DependencyREPO
|
||||
|
||||
from .repo import RepoDependency
|
||||
|
||||
DEPS = [
|
||||
DependencyGS,
|
||||
DependencyHDFS,
|
||||
DependencyHTTP,
|
||||
DependencyHTTPS,
|
||||
DependencyS3,
|
||||
DependencySSH,
|
||||
# NOTE: DependencyLOCAL is the default choice
|
||||
AzureDependency,
|
||||
GSDependency,
|
||||
HDFSDependency,
|
||||
HTTPDependency,
|
||||
HTTPSDependency,
|
||||
S3Dependency,
|
||||
SSHDependency,
|
||||
# NOTE: LocalDependency is the default choice
|
||||
]
|
||||
|
||||
DEP_MAP = {
|
||||
Schemes.LOCAL: DependencyLOCAL,
|
||||
Schemes.SSH: DependencySSH,
|
||||
Schemes.S3: DependencyS3,
|
||||
Schemes.GS: DependencyGS,
|
||||
Schemes.HDFS: DependencyHDFS,
|
||||
Schemes.HTTP: DependencyHTTP,
|
||||
Schemes.HTTPS: DependencyHTTPS,
|
||||
Schemes.LOCAL: LocalDependency,
|
||||
Schemes.SSH: SSHDependency,
|
||||
Schemes.S3: S3Dependency,
|
||||
Schemes.AZURE: AzureDependency,
|
||||
Schemes.GS: GSDependency,
|
||||
Schemes.HDFS: HDFSDependency,
|
||||
Schemes.HTTP: HTTPDependency,
|
||||
Schemes.HTTPS: HTTPSDependency,
|
||||
}
|
||||
|
||||
|
||||
@@ -40,32 +45,36 @@ DEP_MAP = {
|
||||
# cached, see -o and -O flags for `dvc run`) and 'metric' (whether or not
|
||||
# output is a metric file and how to parse it, see `-M` flag for `dvc run`).
|
||||
SCHEMA = output.SCHEMA.copy()
|
||||
del SCHEMA[OutputBase.PARAM_CACHE]
|
||||
del SCHEMA[OutputBase.PARAM_METRIC]
|
||||
SCHEMA[DependencyREPO.PARAM_REPO] = DependencyREPO.REPO_SCHEMA
|
||||
del SCHEMA[BaseOutput.PARAM_CACHE]
|
||||
del SCHEMA[BaseOutput.PARAM_METRIC]
|
||||
SCHEMA.update(RepoDependency.REPO_SCHEMA)
|
||||
SCHEMA.update(ParamsDependency.PARAM_SCHEMA)
|
||||
|
||||
|
||||
def _get(stage, p, info):
|
||||
parsed = urlparse(p)
|
||||
|
||||
if parsed.scheme == "remote":
|
||||
parsed = urlparse(p) if p else None
|
||||
if parsed and parsed.scheme == "remote":
|
||||
remote = Remote(stage.repo, name=parsed.netloc)
|
||||
return DEP_MAP[remote.scheme](stage, p, info, remote=remote)
|
||||
|
||||
if info and info.get(DependencyREPO.PARAM_REPO):
|
||||
repo = info.pop(DependencyREPO.PARAM_REPO)
|
||||
return DependencyREPO(repo, stage, p, info)
|
||||
if info and info.get(RepoDependency.PARAM_REPO):
|
||||
repo = info.pop(RepoDependency.PARAM_REPO)
|
||||
return RepoDependency(repo, stage, p, info)
|
||||
|
||||
if info and info.get(ParamsDependency.PARAM_PARAMS):
|
||||
params = info.pop(ParamsDependency.PARAM_PARAMS)
|
||||
return ParamsDependency(stage, p, params)
|
||||
|
||||
for d in DEPS:
|
||||
if d.supported(p):
|
||||
return d(stage, p, info)
|
||||
return DependencyLOCAL(stage, p, info)
|
||||
return LocalDependency(stage, p, info)
|
||||
|
||||
|
||||
def loadd_from(stage, d_list):
|
||||
ret = []
|
||||
for d in d_list:
|
||||
p = d.pop(OutputBase.PARAM_PATH)
|
||||
p = d.pop(BaseOutput.PARAM_PATH, None)
|
||||
ret.append(_get(stage, p, d))
|
||||
return ret
|
||||
|
||||
@@ -73,6 +82,31 @@ def loadd_from(stage, d_list):
|
||||
def loads_from(stage, s_list, erepo=None):
|
||||
ret = []
|
||||
for s in s_list:
|
||||
info = {DependencyREPO.PARAM_REPO: erepo} if erepo else {}
|
||||
info = {RepoDependency.PARAM_REPO: erepo} if erepo else {}
|
||||
ret.append(_get(stage, s, info))
|
||||
return ret
|
||||
|
||||
|
||||
def _parse_params(path_params):
|
||||
path, _, params_str = path_params.rpartition(":")
|
||||
params = params_str.split(",")
|
||||
return path, params
|
||||
|
||||
|
||||
def loads_params(stage, s_list):
|
||||
# Creates an object for each unique file that is referenced in the list
|
||||
params_by_path = defaultdict(list)
|
||||
for s in s_list:
|
||||
path, params = _parse_params(s)
|
||||
params_by_path[path].extend(params)
|
||||
|
||||
d_list = []
|
||||
for path, params in params_by_path.items():
|
||||
d_list.append(
|
||||
{
|
||||
BaseOutput.PARAM_PATH: path,
|
||||
ParamsDependency.PARAM_PARAMS: params,
|
||||
}
|
||||
)
|
||||
|
||||
return loadd_from(stage, d_list)
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
from dvc.dependency.base import BaseDependency
|
||||
from dvc.output.base import BaseOutput
|
||||
from dvc.remote.azure import AzureRemote
|
||||
|
||||
|
||||
class AzureDependency(BaseDependency, BaseOutput):
|
||||
REMOTE = AzureRemote
|
||||
@@ -3,24 +3,22 @@ from dvc.exceptions import DvcException
|
||||
|
||||
class DependencyDoesNotExistError(DvcException):
|
||||
def __init__(self, path):
|
||||
msg = "dependency '{}' does not exist".format(path)
|
||||
msg = f"dependency '{path}' does not exist"
|
||||
super().__init__(msg)
|
||||
|
||||
|
||||
class DependencyIsNotFileOrDirError(DvcException):
|
||||
def __init__(self, path):
|
||||
msg = "dependency '{}' is not a file or directory".format(path)
|
||||
msg = f"dependency '{path}' is not a file or directory"
|
||||
super().__init__(msg)
|
||||
|
||||
|
||||
class DependencyIsStageFileError(DvcException):
|
||||
def __init__(self, path):
|
||||
super().__init__(
|
||||
"Stage file '{}' cannot be a dependency.".format(path)
|
||||
)
|
||||
super().__init__(f"Stage file '{path}' cannot be a dependency.")
|
||||
|
||||
|
||||
class DependencyBase(object):
|
||||
class BaseDependency:
|
||||
IS_DEPENDENCY = True
|
||||
|
||||
DoesNotExistError = DependencyDoesNotExistError
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from dvc.dependency.base import DependencyBase
|
||||
from dvc.output.gs import OutputGS
|
||||
from dvc.dependency.base import BaseDependency
|
||||
from dvc.output.gs import GSOutput
|
||||
|
||||
|
||||
class DependencyGS(DependencyBase, OutputGS):
|
||||
class GSDependency(BaseDependency, GSOutput):
|
||||
pass
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from dvc.dependency.base import DependencyBase
|
||||
from dvc.output.hdfs import OutputHDFS
|
||||
from dvc.dependency.base import BaseDependency
|
||||
from dvc.output.hdfs import HDFSOutput
|
||||
|
||||
|
||||
class DependencyHDFS(DependencyBase, OutputHDFS):
|
||||
class HDFSDependency(BaseDependency, HDFSOutput):
|
||||
pass
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from dvc.dependency.base import DependencyBase
|
||||
from dvc.output.base import OutputBase
|
||||
from dvc.remote.http import RemoteHTTP
|
||||
from dvc.dependency.base import BaseDependency
|
||||
from dvc.output.base import BaseOutput
|
||||
from dvc.remote.http import HTTPRemote
|
||||
|
||||
|
||||
class DependencyHTTP(DependencyBase, OutputBase):
|
||||
REMOTE = RemoteHTTP
|
||||
class HTTPDependency(BaseDependency, BaseOutput):
|
||||
REMOTE = HTTPRemote
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from .http import DependencyHTTP
|
||||
from dvc.remote.https import RemoteHTTPS
|
||||
from dvc.remote.https import HTTPSRemote
|
||||
|
||||
from .http import HTTPDependency
|
||||
|
||||
|
||||
class DependencyHTTPS(DependencyHTTP):
|
||||
REMOTE = RemoteHTTPS
|
||||
class HTTPSDependency(HTTPDependency):
|
||||
REMOTE = HTTPSRemote
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from dvc.dependency.base import DependencyBase
|
||||
from dvc.output.local import OutputLOCAL
|
||||
from dvc.dependency.base import BaseDependency
|
||||
from dvc.output.local import LocalOutput
|
||||
|
||||
|
||||
class DependencyLOCAL(DependencyBase, OutputLOCAL):
|
||||
class LocalDependency(BaseDependency, LocalOutput):
|
||||
pass
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
import os
|
||||
from collections import defaultdict
|
||||
|
||||
import dpath.util
|
||||
import yaml
|
||||
from voluptuous import Any
|
||||
|
||||
from dvc.dependency.local import LocalDependency
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
|
||||
class MissingParamsError(DvcException):
|
||||
pass
|
||||
|
||||
|
||||
class BadParamFileError(DvcException):
|
||||
pass
|
||||
|
||||
|
||||
class ParamsDependency(LocalDependency):
|
||||
PARAM_PARAMS = "params"
|
||||
PARAM_SCHEMA = {PARAM_PARAMS: Any(dict, list, None)}
|
||||
DEFAULT_PARAMS_FILE = "params.yaml"
|
||||
|
||||
def __init__(self, stage, path, params):
|
||||
info = {}
|
||||
self.params = []
|
||||
if params:
|
||||
if isinstance(params, list):
|
||||
self.params = params
|
||||
else:
|
||||
assert isinstance(params, dict)
|
||||
self.params = list(params.keys())
|
||||
info = params
|
||||
|
||||
super().__init__(
|
||||
stage,
|
||||
path
|
||||
or os.path.join(stage.repo.root_dir, self.DEFAULT_PARAMS_FILE),
|
||||
info=info,
|
||||
)
|
||||
|
||||
def _dyn_load(self, values=None):
|
||||
"""Load params values dynamically."""
|
||||
if not values:
|
||||
return
|
||||
for param in self.params:
|
||||
value = values.get(param)
|
||||
if value:
|
||||
self.info[param] = value
|
||||
|
||||
def save(self):
|
||||
super().save()
|
||||
self.info = self.save_info()
|
||||
|
||||
def status(self):
|
||||
status = super().status()
|
||||
|
||||
if status[str(self)] == "deleted":
|
||||
return status
|
||||
|
||||
status = defaultdict(dict)
|
||||
info = self.read_params()
|
||||
for param in self.params:
|
||||
if param not in info.keys():
|
||||
st = "deleted"
|
||||
elif param not in self.info:
|
||||
st = "new"
|
||||
elif info[param] != self.info[param]:
|
||||
st = "modified"
|
||||
else:
|
||||
assert info[param] == self.info[param]
|
||||
continue
|
||||
|
||||
status[str(self)][param] = st
|
||||
|
||||
return status
|
||||
|
||||
def dumpd(self):
|
||||
return {
|
||||
self.PARAM_PATH: self.def_path,
|
||||
self.PARAM_PARAMS: self.info or self.params,
|
||||
}
|
||||
|
||||
def read_params(self):
|
||||
if not self.exists:
|
||||
return {}
|
||||
|
||||
with self.repo.tree.open(self.path_info, "r") as fobj:
|
||||
try:
|
||||
config = yaml.safe_load(fobj)
|
||||
except yaml.YAMLError as exc:
|
||||
raise BadParamFileError(
|
||||
f"Unable to read parameters from '{self}'"
|
||||
) from exc
|
||||
|
||||
ret = {}
|
||||
for param in self.params:
|
||||
try:
|
||||
ret[param] = dpath.util.get(config, param, separator=".")
|
||||
except KeyError:
|
||||
pass
|
||||
return ret
|
||||
|
||||
def save_info(self):
|
||||
info = self.read_params()
|
||||
|
||||
missing_params = set(self.params) - set(info.keys())
|
||||
if missing_params:
|
||||
raise MissingParamsError(
|
||||
"Parameters '{}' are missing from '{}'.".format(
|
||||
", ".join(missing_params), self,
|
||||
)
|
||||
)
|
||||
|
||||
return info
|
||||
+26
-12
@@ -2,21 +2,24 @@ import os
|
||||
|
||||
from voluptuous import Required
|
||||
|
||||
from .local import DependencyLOCAL
|
||||
from dvc.exceptions import OutputNotFoundError
|
||||
from dvc.path_info import PathInfo
|
||||
|
||||
from .local import LocalDependency
|
||||
|
||||
class DependencyREPO(DependencyLOCAL):
|
||||
|
||||
class RepoDependency(LocalDependency):
|
||||
PARAM_REPO = "repo"
|
||||
PARAM_URL = "url"
|
||||
PARAM_REV = "rev"
|
||||
PARAM_REV_LOCK = "rev_lock"
|
||||
|
||||
REPO_SCHEMA = {
|
||||
Required(PARAM_URL): str,
|
||||
PARAM_REV: str,
|
||||
PARAM_REV_LOCK: str,
|
||||
PARAM_REPO: {
|
||||
Required(PARAM_URL): str,
|
||||
PARAM_REV: str,
|
||||
PARAM_REV_LOCK: str,
|
||||
}
|
||||
}
|
||||
|
||||
def __init__(self, def_repo, stage, *args, **kwargs):
|
||||
@@ -47,13 +50,24 @@ class DependencyREPO(DependencyLOCAL):
|
||||
return external_repo(d["url"], rev=rev)
|
||||
|
||||
def _get_checksum(self, locked=True):
|
||||
from dvc.repo.tree import RepoTree
|
||||
|
||||
with self._make_repo(locked=locked) as repo:
|
||||
try:
|
||||
return repo.find_out_by_relpath(self.def_path).info["md5"]
|
||||
except OutputNotFoundError:
|
||||
path = PathInfo(os.path.join(repo.root_dir, self.def_path))
|
||||
|
||||
# we want stream but not fetch, so DVC out directories are
|
||||
# walked, but dir contents is not fetched
|
||||
tree = RepoTree(repo, stream=True)
|
||||
|
||||
# We are polluting our repo cache with some dir listing here
|
||||
return self.repo.cache.local.get_checksum(path)
|
||||
if tree.isdir(path):
|
||||
return self.repo.cache.local.get_dir_checksum(
|
||||
path, tree=tree
|
||||
)
|
||||
return tree.get_file_checksum(path)
|
||||
|
||||
def status(self):
|
||||
current_checksum = self._get_checksum(locked=True)
|
||||
@@ -73,16 +87,16 @@ class DependencyREPO(DependencyLOCAL):
|
||||
def download(self, to):
|
||||
with self._make_repo() as repo:
|
||||
if self.def_repo.get(self.PARAM_REV_LOCK) is None:
|
||||
self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()
|
||||
self.def_repo[self.PARAM_REV_LOCK] = repo.get_rev()
|
||||
|
||||
if hasattr(repo, "cache"):
|
||||
repo.cache.local.cache_dir = self.repo.cache.local.cache_dir
|
||||
|
||||
repo.pull_to(self.def_path, to.path_info)
|
||||
cache = self.repo.cache.local
|
||||
with repo.use_cache(cache):
|
||||
_, _, cache_infos = repo.fetch_external([self.def_path])
|
||||
cache.checkout(to.path_info, cache_infos[0])
|
||||
|
||||
def update(self, rev=None):
|
||||
if rev:
|
||||
self.def_repo[self.PARAM_REV] = rev
|
||||
|
||||
with self._make_repo(locked=False) as repo:
|
||||
self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()
|
||||
self.def_repo[self.PARAM_REV_LOCK] = repo.get_rev()
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from dvc.dependency.base import DependencyBase
|
||||
from dvc.output.s3 import OutputS3
|
||||
from dvc.dependency.base import BaseDependency
|
||||
from dvc.output.s3 import S3Output
|
||||
|
||||
|
||||
class DependencyS3(DependencyBase, OutputS3):
|
||||
class S3Dependency(BaseDependency, S3Output):
|
||||
pass
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from dvc.dependency.base import DependencyBase
|
||||
from dvc.output.ssh import OutputSSH
|
||||
from dvc.dependency.base import BaseDependency
|
||||
from dvc.output.ssh import SSHOutput
|
||||
|
||||
|
||||
class DependencySSH(DependencyBase, OutputSSH):
|
||||
class SSHDependency(BaseDependency, SSHOutput):
|
||||
pass
|
||||
|
||||
+284
@@ -0,0 +1,284 @@
|
||||
import contextlib
|
||||
import logging
|
||||
import os
|
||||
|
||||
from voluptuous import MultipleInvalid
|
||||
|
||||
import dvc.prompt as prompt
|
||||
from dvc import serialize
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.stage.exceptions import (
|
||||
StageFileAlreadyExistsError,
|
||||
StageFileBadNameError,
|
||||
StageFileDoesNotExistError,
|
||||
StageFileFormatError,
|
||||
StageFileIsNotDvcFileError,
|
||||
)
|
||||
from dvc.stage.loader import SingleStageLoader, StageLoader
|
||||
from dvc.utils import relpath
|
||||
from dvc.utils.collections import apply_diff
|
||||
from dvc.utils.stage import (
|
||||
dump_stage_file,
|
||||
parse_stage,
|
||||
parse_stage_for_update,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DVC_FILE = "Dvcfile"
|
||||
DVC_FILE_SUFFIX = ".dvc"
|
||||
PIPELINE_FILE = "dvc.yaml"
|
||||
PIPELINE_LOCK = "dvc.lock"
|
||||
|
||||
|
||||
class LockfileCorruptedError(DvcException):
|
||||
pass
|
||||
|
||||
|
||||
def is_valid_filename(path):
|
||||
return path.endswith(DVC_FILE_SUFFIX) or os.path.basename(path) in [
|
||||
DVC_FILE,
|
||||
PIPELINE_FILE,
|
||||
]
|
||||
|
||||
|
||||
def is_dvc_file(path):
|
||||
return os.path.isfile(path) and (
|
||||
is_valid_filename(path) or os.path.basename(path) == PIPELINE_LOCK
|
||||
)
|
||||
|
||||
|
||||
def check_dvc_filename(path):
|
||||
if not is_valid_filename(path):
|
||||
raise StageFileBadNameError(
|
||||
"bad DVC-file name '{}'. DVC-files should be named "
|
||||
"'Dvcfile' or have a '.dvc' suffix (e.g. '{}.dvc').".format(
|
||||
relpath(path), os.path.basename(path)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class FileMixin:
|
||||
SCHEMA = None
|
||||
|
||||
def __init__(self, repo, path, **kwargs):
|
||||
self.repo = repo
|
||||
self.path = path
|
||||
|
||||
def __repr__(self):
|
||||
return "{}: {}".format(
|
||||
self.__class__.__name__, relpath(self.path, self.repo.root_dir)
|
||||
)
|
||||
|
||||
def __hash__(self):
|
||||
return hash(self.path)
|
||||
|
||||
def __eq__(self, other):
|
||||
return self.repo == other.repo and os.path.abspath(
|
||||
self.path
|
||||
) == os.path.abspath(other.path)
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.__class__.__name__}: {self.relpath}"
|
||||
|
||||
@property
|
||||
def relpath(self):
|
||||
return relpath(self.path)
|
||||
|
||||
def exists(self):
|
||||
return self.repo.tree.exists(self.path)
|
||||
|
||||
def _load(self):
|
||||
# it raises the proper exceptions by priority:
|
||||
# 1. when the file doesn't exists
|
||||
# 2. filename is not a DVC-file
|
||||
# 3. path doesn't represent a regular file
|
||||
if not self.exists():
|
||||
raise StageFileDoesNotExistError(self.path)
|
||||
check_dvc_filename(self.path)
|
||||
if not self.repo.tree.isfile(self.path):
|
||||
raise StageFileIsNotDvcFileError(self.path)
|
||||
|
||||
with self.repo.tree.open(self.path) as fd:
|
||||
stage_text = fd.read()
|
||||
d = parse_stage(stage_text, self.path)
|
||||
self.validate(d, self.relpath)
|
||||
return d, stage_text
|
||||
|
||||
@classmethod
|
||||
def validate(cls, d, fname=None):
|
||||
assert cls.SCHEMA
|
||||
try:
|
||||
cls.SCHEMA(d)
|
||||
except MultipleInvalid as exc:
|
||||
raise StageFileFormatError(f"'{fname}' format error: {exc}")
|
||||
|
||||
def remove_with_prompt(self, force=False):
|
||||
raise NotImplementedError
|
||||
|
||||
def remove(self, force=False):
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
os.unlink(self.path)
|
||||
|
||||
def dump(self, stage, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class SingleStageFile(FileMixin):
|
||||
from dvc.schema import COMPILED_SINGLE_STAGE_SCHEMA as SCHEMA
|
||||
|
||||
def __init__(self, repo, path):
|
||||
super().__init__(repo, path)
|
||||
|
||||
@property
|
||||
def stage(self):
|
||||
data, raw = self._load()
|
||||
return SingleStageLoader.load_stage(self, data, raw)
|
||||
|
||||
@property
|
||||
def stages(self):
|
||||
data, raw = self._load()
|
||||
return SingleStageLoader(self, data, raw)
|
||||
|
||||
def dump(self, stage, **kwargs):
|
||||
"""Dumps given stage appropriately in the dvcfile."""
|
||||
from dvc.stage import PipelineStage
|
||||
|
||||
assert not isinstance(stage, PipelineStage)
|
||||
check_dvc_filename(self.path)
|
||||
logger.debug(
|
||||
"Saving information to '{file}'.".format(file=relpath(self.path))
|
||||
)
|
||||
dump_stage_file(self.path, serialize.to_single_stage_file(stage))
|
||||
self.repo.scm.track_file(relpath(self.path))
|
||||
|
||||
def remove_with_prompt(self, force=False):
|
||||
if not self.exists():
|
||||
return
|
||||
|
||||
msg = (
|
||||
"'{}' already exists. Do you wish to run the command and "
|
||||
"overwrite it?".format(relpath(self.path))
|
||||
)
|
||||
if not (force or prompt.confirm(msg)):
|
||||
raise StageFileAlreadyExistsError(self.path)
|
||||
|
||||
self.remove()
|
||||
|
||||
|
||||
class PipelineFile(FileMixin):
|
||||
"""Abstraction for pipelines file, .yaml + .lock combined."""
|
||||
|
||||
from dvc.schema import COMPILED_MULTI_STAGE_SCHEMA as SCHEMA
|
||||
|
||||
@property
|
||||
def _lockfile(self):
|
||||
return Lockfile(self.repo, os.path.splitext(self.path)[0] + ".lock")
|
||||
|
||||
def dump(self, stage, update_pipeline=False, no_lock=False):
|
||||
"""Dumps given stage appropriately in the dvcfile."""
|
||||
from dvc.stage import PipelineStage
|
||||
|
||||
assert isinstance(stage, PipelineStage)
|
||||
check_dvc_filename(self.path)
|
||||
|
||||
if update_pipeline and not stage.is_data_source:
|
||||
self._dump_pipeline_file(stage)
|
||||
|
||||
if not no_lock:
|
||||
self._dump_lockfile(stage)
|
||||
|
||||
def _dump_lockfile(self, stage):
|
||||
self._lockfile.dump(stage)
|
||||
|
||||
def _dump_pipeline_file(self, stage):
|
||||
data = {}
|
||||
if self.exists():
|
||||
with open(self.path) as fd:
|
||||
data = parse_stage_for_update(fd.read(), self.path)
|
||||
else:
|
||||
logger.info("Creating '%s'", self.relpath)
|
||||
open(self.path, "w+").close()
|
||||
|
||||
data["stages"] = data.get("stages", {})
|
||||
stage_data = serialize.to_pipeline_file(stage)
|
||||
if data["stages"].get(stage.name):
|
||||
orig_stage_data = data["stages"][stage.name]
|
||||
apply_diff(stage_data[stage.name], orig_stage_data)
|
||||
else:
|
||||
data["stages"].update(stage_data)
|
||||
|
||||
logger.info(
|
||||
"Adding stage '%s' to '%s'", stage.name, self.relpath,
|
||||
)
|
||||
dump_stage_file(self.path, data)
|
||||
self.repo.scm.track_file(relpath(self.path))
|
||||
|
||||
@property
|
||||
def stage(self):
|
||||
raise DvcException(
|
||||
"PipelineFile has multiple stages. Please specify it's name."
|
||||
)
|
||||
|
||||
@property
|
||||
def stages(self):
|
||||
data, _ = self._load()
|
||||
lockfile_data = self._lockfile.load()
|
||||
return StageLoader(self, data.get("stages", {}), lockfile_data)
|
||||
|
||||
def remove(self, force=False):
|
||||
if not force:
|
||||
logger.warning("Cannot remove pipeline file.")
|
||||
return
|
||||
|
||||
super().remove()
|
||||
self._lockfile.remove()
|
||||
|
||||
|
||||
class Lockfile(FileMixin):
|
||||
from dvc.schema import COMPILED_LOCKFILE_SCHEMA as SCHEMA
|
||||
|
||||
def load(self):
|
||||
if not self.exists():
|
||||
return {}
|
||||
with self.repo.tree.open(self.path) as fd:
|
||||
data = parse_stage(fd.read(), self.path)
|
||||
try:
|
||||
self.validate(data, fname=self.relpath)
|
||||
except StageFileFormatError:
|
||||
raise LockfileCorruptedError(
|
||||
f"Lockfile '{self.relpath}' is corrupted."
|
||||
)
|
||||
return data
|
||||
|
||||
def dump(self, stage, **kwargs):
|
||||
stage_data = serialize.to_lockfile(stage)
|
||||
if not self.exists():
|
||||
modified = True
|
||||
logger.info("Generating lock file '%s'", self.relpath)
|
||||
data = stage_data
|
||||
open(self.path, "w+").close()
|
||||
else:
|
||||
with self.repo.tree.open(self.path, "r") as fd:
|
||||
data = parse_stage_for_update(fd.read(), self.path)
|
||||
modified = data.get(stage.name, {}) != stage_data.get(
|
||||
stage.name, {}
|
||||
)
|
||||
if modified:
|
||||
logger.info("Updating lock file '%s'", self.relpath)
|
||||
data.update(stage_data)
|
||||
dump_stage_file(self.path, data)
|
||||
if modified:
|
||||
self.repo.scm.track_file(self.relpath)
|
||||
|
||||
|
||||
class Dvcfile:
|
||||
def __new__(cls, repo, path, **kwargs):
|
||||
assert path
|
||||
assert repo
|
||||
|
||||
_, ext = os.path.splitext(path)
|
||||
if ext in [".yaml", ".yml"]:
|
||||
return PipelineFile(repo, path, **kwargs)
|
||||
# fallback to single stage file for better error messages
|
||||
return SingleStageFile(repo, path, **kwargs)
|
||||
+47
-42
@@ -1,11 +1,16 @@
|
||||
"""Exceptions raised by the dvc."""
|
||||
from funcy import first
|
||||
|
||||
from dvc.utils import relpath, format_link
|
||||
from dvc.utils import format_link, relpath
|
||||
|
||||
|
||||
class DvcException(Exception):
|
||||
"""Base class for all dvc exceptions."""
|
||||
|
||||
def __init__(self, msg, *args):
|
||||
assert msg
|
||||
super().__init__(msg, *args)
|
||||
|
||||
|
||||
class InvalidArgumentError(ValueError, DvcException):
|
||||
"""Thrown if arguments are invalid."""
|
||||
@@ -23,12 +28,17 @@ class OutputDuplicationError(DvcException):
|
||||
def __init__(self, output, stages):
|
||||
assert isinstance(output, str)
|
||||
assert all(hasattr(stage, "relpath") for stage in stages)
|
||||
msg = (
|
||||
"file/directory '{}' is specified as an output in more than one "
|
||||
"stage: {}\n"
|
||||
"This is not allowed. Consider using a different output name."
|
||||
).format(output, "\n ".join(s.relpath for s in stages))
|
||||
if len(stages) == 1:
|
||||
msg = "output '{}' is already specified in {}.".format(
|
||||
output, first(stages)
|
||||
)
|
||||
else:
|
||||
msg = "output '{}' is already specified in stages:\n{}".format(
|
||||
output, "\n".join(f"\t- {s.addressing}" for s in stages),
|
||||
)
|
||||
super().__init__(msg)
|
||||
self.stages = stages
|
||||
self.output = output
|
||||
|
||||
|
||||
class OutputNotFoundError(DvcException):
|
||||
@@ -60,8 +70,8 @@ class StagePathAsOutputError(DvcException):
|
||||
def __init__(self, stage, output):
|
||||
assert isinstance(output, str)
|
||||
super().__init__(
|
||||
"'{stage}' is within an output '{output}' of another stage".format(
|
||||
stage=stage.relpath, output=output
|
||||
"{stage} is within an output '{output}' of another stage".format(
|
||||
stage=stage, output=output
|
||||
)
|
||||
)
|
||||
|
||||
@@ -77,10 +87,7 @@ class CircularDependencyError(DvcException):
|
||||
def __init__(self, dependency):
|
||||
assert isinstance(dependency, str)
|
||||
|
||||
msg = (
|
||||
"file/directory '{}' is specified as an output and as a "
|
||||
"dependency."
|
||||
)
|
||||
msg = "'{}' is specified as an output and as a dependency."
|
||||
super().__init__(msg.format(dependency))
|
||||
|
||||
|
||||
@@ -94,7 +101,7 @@ class ArgumentDuplicationError(DvcException):
|
||||
|
||||
def __init__(self, path):
|
||||
assert isinstance(path, str)
|
||||
super().__init__("file '{}' is specified more than once.".format(path))
|
||||
super().__init__(f"file '{path}' is specified more than once.")
|
||||
|
||||
|
||||
class MoveNotDataSourceError(DvcException):
|
||||
@@ -129,12 +136,8 @@ class DvcParserError(DvcException):
|
||||
class CyclicGraphError(DvcException):
|
||||
def __init__(self, stages):
|
||||
assert isinstance(stages, list)
|
||||
stages = "\n".join("\t- {}".format(stage.relpath) for stage in stages)
|
||||
msg = (
|
||||
"you've introduced a cycle in your pipeline that involves "
|
||||
"the following stages:"
|
||||
"\n"
|
||||
"{stages}".format(stages=stages)
|
||||
msg = "Pipeline has a cycle involving: {}.".format(
|
||||
", ".join(s.addressing for s in stages)
|
||||
)
|
||||
super().__init__(msg)
|
||||
|
||||
@@ -142,7 +145,7 @@ class CyclicGraphError(DvcException):
|
||||
class ConfirmRemoveError(DvcException):
|
||||
def __init__(self, path):
|
||||
super().__init__(
|
||||
"unable to remove '{}' without a confirmation from the user. Use "
|
||||
"unable to remove '{}' without a confirmation. Use "
|
||||
"`-f` to force.".format(path)
|
||||
)
|
||||
|
||||
@@ -154,7 +157,7 @@ class InitError(DvcException):
|
||||
class ReproductionError(DvcException):
|
||||
def __init__(self, dvc_file_name):
|
||||
self.path = dvc_file_name
|
||||
super().__init__("failed to reproduce '{}'".format(dvc_file_name))
|
||||
super().__init__(f"failed to reproduce '{dvc_file_name}'")
|
||||
|
||||
|
||||
class BadMetricError(DvcException):
|
||||
@@ -162,7 +165,7 @@ class BadMetricError(DvcException):
|
||||
super().__init__(
|
||||
"the following metrics do not exist, "
|
||||
"are not metric files or are malformed: {paths}".format(
|
||||
paths=", ".join("'{}'".format(path) for path in paths)
|
||||
paths=", ".join(f"'{path}'" for path in paths)
|
||||
)
|
||||
)
|
||||
|
||||
@@ -200,7 +203,7 @@ class OverlappingOutputPathsError(DvcException):
|
||||
|
||||
class CheckoutErrorSuggestGit(DvcException):
|
||||
def __init__(self, target):
|
||||
super().__init__("Did you mean `git checkout {}`?".format(target))
|
||||
super().__init__(f"Did you mean `git checkout {target}`?")
|
||||
|
||||
|
||||
class ETagMismatchError(DvcException):
|
||||
@@ -214,9 +217,7 @@ class ETagMismatchError(DvcException):
|
||||
class FileMissingError(DvcException):
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
super().__init__(
|
||||
"Can't find '{}' neither locally nor on remote".format(path)
|
||||
)
|
||||
super().__init__(f"Can't find '{path}' neither locally nor on remote")
|
||||
|
||||
|
||||
class DvcIgnoreInCollectedDirError(DvcException):
|
||||
@@ -241,18 +242,14 @@ class DownloadError(DvcException):
|
||||
def __init__(self, amount):
|
||||
self.amount = amount
|
||||
|
||||
super().__init__(
|
||||
"{amount} files failed to download".format(amount=amount)
|
||||
)
|
||||
super().__init__(f"{amount} files failed to download")
|
||||
|
||||
|
||||
class UploadError(DvcException):
|
||||
def __init__(self, amount):
|
||||
self.amount = amount
|
||||
|
||||
super().__init__(
|
||||
"{amount} files failed to upload".format(amount=amount)
|
||||
)
|
||||
super().__init__(f"{amount} files failed to upload")
|
||||
|
||||
|
||||
class CheckoutError(DvcException):
|
||||
@@ -274,7 +271,7 @@ class CollectCacheError(DvcException):
|
||||
class NoRemoteInExternalRepoError(DvcException):
|
||||
def __init__(self, url):
|
||||
super().__init__(
|
||||
"No DVC remote is specified in target repository '{}'.".format(url)
|
||||
f"No DVC remote is specified in target repository '{url}'."
|
||||
)
|
||||
|
||||
|
||||
@@ -289,7 +286,7 @@ class NoOutputInExternalRepoError(DvcException):
|
||||
|
||||
class HTTPError(DvcException):
|
||||
def __init__(self, code, reason):
|
||||
super().__init__("'{} {}'".format(code, reason))
|
||||
super().__init__(f"'{code} {reason}'")
|
||||
|
||||
|
||||
class PathMissingError(DvcException):
|
||||
@@ -297,19 +294,15 @@ class PathMissingError(DvcException):
|
||||
"The path '{}' does not exist in the target repository '{}'"
|
||||
" neither as a DVC output nor as a Git-tracked file."
|
||||
)
|
||||
default_msg_output_only = (
|
||||
default_msg_dvc_only = (
|
||||
"The path '{}' does not exist in the target repository '{}'"
|
||||
" as an DVC output."
|
||||
)
|
||||
|
||||
def __init__(self, path, repo, output_only=False):
|
||||
msg = (
|
||||
self.default_msg
|
||||
if not output_only
|
||||
else self.default_msg_output_only
|
||||
)
|
||||
def __init__(self, path, repo, dvc_only=False):
|
||||
msg = self.default_msg if not dvc_only else self.default_msg_dvc_only
|
||||
super().__init__(msg.format(path, repo))
|
||||
self.output_only = output_only
|
||||
self.dvc_only = dvc_only
|
||||
|
||||
|
||||
class RemoteCacheRequiredError(DvcException):
|
||||
@@ -329,3 +322,15 @@ class RemoteCacheRequiredError(DvcException):
|
||||
|
||||
class IsADirectoryError(DvcException):
|
||||
"""Raised when a file operation is requested on a directory."""
|
||||
|
||||
|
||||
class NoOutputOrStageError(DvcException):
|
||||
"""
|
||||
Raised when the target is neither an output nor a stage name in dvc.yaml
|
||||
"""
|
||||
|
||||
def __init__(self, target, file):
|
||||
super().__init__(
|
||||
f"'{target}' "
|
||||
f"does not exist as an output or a stage name in '{file}'"
|
||||
)
|
||||
|
||||
+136
-90
@@ -1,23 +1,27 @@
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
from contextlib import contextmanager
|
||||
from distutils.dir_util import copy_tree
|
||||
import threading
|
||||
from typing import Iterable
|
||||
|
||||
from funcy import retry, suppress, wrap_with, cached_property
|
||||
from funcy import cached_property, retry, wrap_with
|
||||
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.compat import fspath
|
||||
from dvc.repo import Repo
|
||||
from dvc.config import NoRemoteError, NotDvcRepoError
|
||||
from dvc.exceptions import NoRemoteInExternalRepoError, CheckoutError
|
||||
from dvc.exceptions import OutputNotFoundError, NoOutputInExternalRepoError
|
||||
from dvc.exceptions import FileMissingError, PathMissingError
|
||||
from dvc.utils.fs import remove, fs_copy, move
|
||||
from dvc.utils import tmp_fname
|
||||
from dvc.exceptions import (
|
||||
FileMissingError,
|
||||
NoOutputInExternalRepoError,
|
||||
NoRemoteInExternalRepoError,
|
||||
OutputNotFoundError,
|
||||
PathMissingError,
|
||||
)
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.repo import Repo
|
||||
from dvc.repo.tree import RepoTree
|
||||
from dvc.scm.git import Git
|
||||
|
||||
from dvc.scm.tree import is_working_tree
|
||||
from dvc.utils.fs import remove
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -26,10 +30,12 @@ logger = logging.getLogger(__name__)
|
||||
def external_repo(url, rev=None, for_write=False):
|
||||
logger.debug("Creating external repo %s@%s", url, rev)
|
||||
path = _cached_clone(url, rev, for_write=for_write)
|
||||
if not rev:
|
||||
rev = "HEAD"
|
||||
try:
|
||||
repo = ExternalRepo(path, url)
|
||||
repo = ExternalRepo(path, url, rev, for_write=for_write)
|
||||
except NotDvcRepoError:
|
||||
repo = ExternalGitRepo(path, url)
|
||||
repo = ExternalGitRepo(path, url, rev)
|
||||
|
||||
try:
|
||||
yield repo
|
||||
@@ -61,59 +67,99 @@ def clean_repos():
|
||||
_remove(path)
|
||||
|
||||
|
||||
class ExternalRepo(Repo):
|
||||
def __init__(self, root_dir, url):
|
||||
super().__init__(root_dir)
|
||||
class BaseExternalRepo:
|
||||
|
||||
_local_cache = None
|
||||
|
||||
@property
|
||||
def local_cache(self):
|
||||
if hasattr(self, "cache"):
|
||||
return self.cache.local
|
||||
return self._local_cache
|
||||
|
||||
@contextmanager
|
||||
def use_cache(self, cache):
|
||||
"""Use the specified cache in place of default tmpdir cache for
|
||||
download operations.
|
||||
"""
|
||||
if hasattr(self, "cache"):
|
||||
save_cache = self.cache.local
|
||||
self.cache.local = cache
|
||||
self._local_cache = cache
|
||||
|
||||
yield
|
||||
|
||||
if hasattr(self, "cache"):
|
||||
self.cache.local = save_cache
|
||||
self._local_cache = None
|
||||
|
||||
@cached_property
|
||||
def repo_tree(self):
|
||||
return RepoTree(self, fetch=True)
|
||||
|
||||
def get_rev(self):
|
||||
if is_working_tree(self.tree):
|
||||
return self.scm.get_rev()
|
||||
if hasattr(self.tree, "tree"):
|
||||
return self.tree.tree.rev
|
||||
return self.tree.rev
|
||||
|
||||
def fetch_external(self, paths: Iterable, **kwargs):
|
||||
"""Fetch specified external repo paths into cache.
|
||||
|
||||
Returns 3-tuple in the form
|
||||
(downloaded, failed, list(cache_infos))
|
||||
where cache_infos can be used as checkout targets for the
|
||||
fetched paths.
|
||||
"""
|
||||
download_results = []
|
||||
failed = 0
|
||||
|
||||
paths = [PathInfo(self.root_dir) / path for path in paths]
|
||||
|
||||
def download_update(result):
|
||||
download_results.append(result)
|
||||
|
||||
save_infos = []
|
||||
for path in paths:
|
||||
if not self.repo_tree.exists(path):
|
||||
raise PathMissingError(path, self.url)
|
||||
save_info = self.local_cache.save(
|
||||
path,
|
||||
None,
|
||||
tree=self.repo_tree,
|
||||
download_callback=download_update,
|
||||
)
|
||||
save_infos.append(save_info)
|
||||
|
||||
return sum(download_results), failed, save_infos
|
||||
|
||||
def get_external(self, path, dest):
|
||||
"""Convenience wrapper for fetch_external and checkout."""
|
||||
if self.local_cache:
|
||||
# fetch DVC and git files to tmpdir cache, then checkout
|
||||
_, _, save_infos = self.fetch_external([path])
|
||||
self.local_cache.checkout(PathInfo(dest), save_infos[0])
|
||||
else:
|
||||
# git-only erepo with no cache, just copy files directly
|
||||
# to dest
|
||||
path = PathInfo(self.root_dir) / path
|
||||
if not self.repo_tree.exists(path):
|
||||
raise PathMissingError(path, self.url)
|
||||
self.repo_tree.copytree(path, dest)
|
||||
|
||||
|
||||
class ExternalRepo(Repo, BaseExternalRepo):
|
||||
def __init__(self, root_dir, url, rev, for_write=False):
|
||||
if for_write:
|
||||
super().__init__(root_dir)
|
||||
else:
|
||||
root_dir = os.path.realpath(root_dir)
|
||||
super().__init__(root_dir, scm=Git(root_dir), rev=rev)
|
||||
self.url = url
|
||||
self._set_cache_dir()
|
||||
self._fix_upstream()
|
||||
|
||||
def pull_to(self, path, to_info):
|
||||
"""
|
||||
Pull the corresponding file or directory specified by `path` and
|
||||
checkout it into `to_info`.
|
||||
|
||||
It works with files tracked by Git and DVC, and also local files
|
||||
outside the repository.
|
||||
"""
|
||||
out = None
|
||||
path_info = PathInfo(self.root_dir) / path
|
||||
|
||||
with suppress(OutputNotFoundError):
|
||||
(out,) = self.find_outs_by_path(fspath(path_info), strict=False)
|
||||
|
||||
try:
|
||||
if out and out.use_cache:
|
||||
self._pull_cached(out, path_info, to_info)
|
||||
return
|
||||
|
||||
# Check if it is handled by Git (it can't have an absolute path)
|
||||
if os.path.isabs(path):
|
||||
raise FileNotFoundError
|
||||
|
||||
fs_copy(fspath(path_info), fspath(to_info))
|
||||
except FileNotFoundError:
|
||||
raise PathMissingError(path, self.url)
|
||||
|
||||
def _pull_cached(self, out, path_info, dest):
|
||||
with self.state:
|
||||
tmp = PathInfo(tmp_fname(dest))
|
||||
src = tmp / path_info.relative_to(out.path_info)
|
||||
|
||||
out.path_info = tmp
|
||||
|
||||
# Only pull unless all needed cache is present
|
||||
if out.changed_cache(filter_info=src):
|
||||
self.cloud.pull(out.get_used_cache(filter_info=src))
|
||||
|
||||
try:
|
||||
out.checkout(filter_info=src)
|
||||
except CheckoutError:
|
||||
raise FileNotFoundError
|
||||
|
||||
move(src, dest)
|
||||
remove(tmp)
|
||||
|
||||
@wrap_with(threading.Lock())
|
||||
def _set_cache_dir(self):
|
||||
try:
|
||||
@@ -122,14 +168,23 @@ class ExternalRepo(Repo):
|
||||
cache_dir = CACHE_DIRS[self.url] = tempfile.mkdtemp("dvc-cache")
|
||||
|
||||
self.cache.local.cache_dir = cache_dir
|
||||
self._local_cache = self.cache.local
|
||||
|
||||
def _fix_upstream(self):
|
||||
if not os.path.isdir(self.url):
|
||||
return
|
||||
|
||||
remote_name = self.config["core"].get("remote")
|
||||
src_repo = Repo(self.url)
|
||||
try:
|
||||
src_repo = Repo(self.url)
|
||||
except NotDvcRepoError:
|
||||
# If ExternalRepo does not throw NotDvcRepoError and Repo does,
|
||||
# the self.url might be a bare git repo.
|
||||
# NOTE: This will fail to resolve remote with relative path,
|
||||
# same as if it was a remote DVC repo.
|
||||
return
|
||||
|
||||
try:
|
||||
remote_name = self.config["core"].get("remote")
|
||||
if remote_name:
|
||||
self._fix_local_remote(src_repo, remote_name)
|
||||
else:
|
||||
@@ -154,10 +209,11 @@ class ExternalRepo(Repo):
|
||||
self.config["core"]["remote"] = "auto-generated-upstream"
|
||||
|
||||
|
||||
class ExternalGitRepo:
|
||||
def __init__(self, root_dir, url):
|
||||
self.root_dir = root_dir
|
||||
class ExternalGitRepo(BaseExternalRepo):
|
||||
def __init__(self, root_dir, url, rev):
|
||||
self.root_dir = os.path.realpath(root_dir)
|
||||
self.url = url
|
||||
self.tree = self.scm.get_tree(rev)
|
||||
|
||||
@cached_property
|
||||
def scm(self):
|
||||
@@ -170,22 +226,15 @@ class ExternalGitRepo:
|
||||
def find_out_by_relpath(self, path):
|
||||
raise OutputNotFoundError(path, self)
|
||||
|
||||
def pull_to(self, path, to_info):
|
||||
try:
|
||||
# Git handled files can't have absolute path
|
||||
if os.path.isabs(path):
|
||||
raise FileNotFoundError
|
||||
|
||||
fs_copy(os.path.join(self.root_dir, path), fspath(to_info))
|
||||
except FileNotFoundError:
|
||||
raise PathMissingError(path, self.url)
|
||||
|
||||
@contextmanager
|
||||
def open_by_relpath(self, path, mode="r", encoding=None, **kwargs):
|
||||
"""Opens a specified resource as a file object."""
|
||||
tree = RepoTree(self)
|
||||
try:
|
||||
abs_path = os.path.join(self.root_dir, path)
|
||||
with open(abs_path, mode, encoding=encoding) as fd:
|
||||
yield fd
|
||||
with tree.open(
|
||||
path, mode=mode, encoding=encoding, **kwargs
|
||||
) as fobj:
|
||||
yield fobj
|
||||
except FileNotFoundError:
|
||||
raise PathMissingError(path, self.url)
|
||||
|
||||
@@ -197,14 +246,12 @@ def _cached_clone(url, rev, for_write=False):
|
||||
revision checked out. If for_write is set prevents reusing this dir via
|
||||
cache.
|
||||
"""
|
||||
if not for_write and Git.is_sha(rev) and (url, rev) in CLONES:
|
||||
return CLONES[url, rev]
|
||||
|
||||
# even if we have already cloned this repo, we may need to
|
||||
# fetch/fast-forward to get specified rev
|
||||
clone_path = _clone_default_branch(url, rev)
|
||||
rev_sha = Git(clone_path).resolve_rev(rev or "HEAD")
|
||||
|
||||
if not for_write and (url, rev_sha) in CLONES:
|
||||
return CLONES[url, rev_sha]
|
||||
if not for_write and (url) in CLONES:
|
||||
return CLONES[url]
|
||||
|
||||
# Copy to a new dir to keep the clone clean
|
||||
repo_path = tempfile.mkdtemp("dvc-erepo")
|
||||
@@ -212,11 +259,10 @@ def _cached_clone(url, rev, for_write=False):
|
||||
copy_tree(clone_path, repo_path)
|
||||
|
||||
# Check out the specified revision
|
||||
if rev is not None:
|
||||
if for_write:
|
||||
_git_checkout(repo_path, rev)
|
||||
|
||||
if not for_write:
|
||||
CLONES[url, rev_sha] = repo_path
|
||||
else:
|
||||
CLONES[url] = repo_path
|
||||
return repo_path
|
||||
|
||||
|
||||
|
||||
+90
-17
@@ -5,13 +5,14 @@ from funcy import cached_property
|
||||
from pathspec import PathSpec
|
||||
from pathspec.patterns import GitWildMatchPattern
|
||||
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.scm.tree import BaseTree
|
||||
from dvc.utils import relpath
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DvcIgnore(object):
|
||||
class DvcIgnore:
|
||||
DVCIGNORE_FILE = ".dvcignore"
|
||||
|
||||
def __call__(self, root, dirs, files):
|
||||
@@ -35,12 +36,19 @@ class DvcIgnorePatterns(DvcIgnore):
|
||||
return dirs, files
|
||||
|
||||
def matches(self, dirname, basename):
|
||||
abs_path = os.path.join(dirname, basename)
|
||||
rel_path = relpath(abs_path, self.dirname)
|
||||
|
||||
if os.pardir + os.sep in rel_path:
|
||||
# NOTE: `relpath` is too slow, so we have to assume that both
|
||||
# `dirname` and `self.dirname` are relative or absolute together.
|
||||
prefix = self.dirname + os.sep
|
||||
if dirname == self.dirname:
|
||||
path = basename
|
||||
elif dirname.startswith(prefix):
|
||||
rel = dirname[len(prefix) :]
|
||||
# NOTE: `os.path.join` is ~x5.5 slower
|
||||
path = f"{rel}{os.sep}{basename}"
|
||||
else:
|
||||
return False
|
||||
return self.ignore_spec.match_file(rel_path)
|
||||
|
||||
return self.ignore_spec.match_file(path)
|
||||
|
||||
def __hash__(self):
|
||||
return hash(self.ignore_file_path)
|
||||
@@ -76,14 +84,14 @@ class DvcIgnoreRepo(DvcIgnore):
|
||||
def is_dvc_repo(directory):
|
||||
from dvc.repo import Repo
|
||||
|
||||
return os.path.isdir(os.path.join(directory, Repo.DVC_DIR))
|
||||
return os.path.isdir(os.path.join(root, directory, Repo.DVC_DIR))
|
||||
|
||||
dirs = [d for d in dirs if not is_dvc_repo(d)]
|
||||
|
||||
return dirs, files
|
||||
|
||||
|
||||
class DvcIgnoreFilter(object):
|
||||
class DvcIgnoreFilter:
|
||||
def __init__(self, tree):
|
||||
self.tree = tree
|
||||
self.ignores = {
|
||||
@@ -119,24 +127,89 @@ class CleanTree(BaseTree):
|
||||
return self.tree.tree_root
|
||||
|
||||
def open(self, path, mode="r", encoding="utf-8"):
|
||||
return self.tree.open(path, mode, encoding)
|
||||
if self.isfile(path):
|
||||
return self.tree.open(path, mode, encoding)
|
||||
raise FileNotFoundError
|
||||
|
||||
def exists(self, path):
|
||||
return self.tree.exists(path)
|
||||
if self.tree.exists(path) and self._parents_exist(path):
|
||||
if self.tree.isdir(path):
|
||||
return self._valid_dirname(path)
|
||||
return self._valid_filename(path)
|
||||
return False
|
||||
|
||||
def isdir(self, path):
|
||||
return self.tree.isdir(path)
|
||||
return (
|
||||
self.tree.isdir(path)
|
||||
and self._parents_exist(path)
|
||||
and self._valid_dirname(path)
|
||||
)
|
||||
|
||||
def _valid_dirname(self, path):
|
||||
dirname, basename = os.path.split(os.path.normpath(path))
|
||||
dirs, _ = self.dvcignore(os.path.abspath(dirname), [basename], [])
|
||||
if dirs:
|
||||
return True
|
||||
return False
|
||||
|
||||
def isfile(self, path):
|
||||
return self.tree.isfile(path)
|
||||
return (
|
||||
self.tree.isfile(path)
|
||||
and self._parents_exist(path)
|
||||
and self._valid_filename(path)
|
||||
)
|
||||
|
||||
def _valid_filename(self, path):
|
||||
dirname, basename = os.path.split(os.path.normpath(path))
|
||||
_, files = self.dvcignore(os.path.abspath(dirname), [], [basename])
|
||||
if files:
|
||||
return True
|
||||
return False
|
||||
|
||||
def isexec(self, path):
|
||||
return self.exists(path) and self.tree.isexec(path)
|
||||
|
||||
def _parents_exist(self, path):
|
||||
from dvc.repo import Repo
|
||||
|
||||
path = PathInfo(path)
|
||||
|
||||
# if parent is tree_root or inside a .dvc dir we can skip this check
|
||||
if path.parent == self.tree_root or Repo.DVC_DIR in path.parts:
|
||||
return True
|
||||
|
||||
# if path is outside of tree, assume this is a local remote/local cache
|
||||
# link/move operation where we do not need to filter ignores
|
||||
path = relpath(path, self.tree_root)
|
||||
if path.startswith("..") or (
|
||||
os.name == "nt"
|
||||
and not os.path.commonprefix(
|
||||
[os.path.abspath(path), self.tree_root]
|
||||
)
|
||||
):
|
||||
return True
|
||||
|
||||
# check if parent directories are in our ignores, starting from
|
||||
# tree_root
|
||||
for parent_dir in reversed(PathInfo(path).parents):
|
||||
dirname, basename = os.path.split(parent_dir)
|
||||
if basename == ".":
|
||||
# parent_dir == tree_root
|
||||
continue
|
||||
dirs, _ = self.dvcignore(os.path.abspath(dirname), [basename], [])
|
||||
if not dirs:
|
||||
return False
|
||||
return True
|
||||
|
||||
def walk(self, top, topdown=True):
|
||||
for root, dirs, files in self.tree.walk(top, topdown):
|
||||
dirs[:], files[:] = self.dvcignore(root, dirs, files)
|
||||
dirs[:], files[:] = self.dvcignore(
|
||||
os.path.abspath(root), dirs, files
|
||||
)
|
||||
|
||||
yield root, dirs, files
|
||||
|
||||
def walk_files(self, top):
|
||||
for root, _, files in self.walk(top):
|
||||
for file in files:
|
||||
yield os.path.join(root, file)
|
||||
def stat(self, path):
|
||||
if self.exists(path):
|
||||
return self.tree.stat(path)
|
||||
raise FileNotFoundError
|
||||
|
||||
+6
-2
@@ -7,13 +7,17 @@
|
||||
TEXT_CHARS = bytes(range(32, 127)) + b"\n\r\t\f\b"
|
||||
|
||||
|
||||
def istextfile(fname, blocksize=512):
|
||||
def istextfile(fname, blocksize=512, tree=None):
|
||||
""" Uses heuristics to guess whether the given file is text or binary,
|
||||
by reading a single block of bytes from the file.
|
||||
If more than 30% of the chars in the block are non-text, or there
|
||||
are NUL ('\x00') bytes in the block, assume this is a binary file.
|
||||
"""
|
||||
with open(fname, "rb") as fobj:
|
||||
if tree:
|
||||
open_func = tree.open
|
||||
else:
|
||||
open_func = open
|
||||
with open_func(fname, "rb") as fobj:
|
||||
block = fobj.read(blocksize)
|
||||
|
||||
if not block:
|
||||
|
||||
+5
-5
@@ -5,19 +5,19 @@ import os
|
||||
import time
|
||||
from datetime import timedelta
|
||||
|
||||
import zc.lockfile
|
||||
import flufl.lock
|
||||
import zc.lockfile
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.utils import format_link
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.utils import format_link
|
||||
|
||||
DEFAULT_TIMEOUT = 5
|
||||
|
||||
FAILED_TO_LOCK_MESSAGE = (
|
||||
"cannot perform the command because another DVC process seems to be "
|
||||
"running on this project. If that is not the case, manually remove "
|
||||
"`.dvc/lock` and try again."
|
||||
"`.dvc/tmp/lock` and try again."
|
||||
)
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ class LockError(DvcException):
|
||||
"""Thrown when unable to acquire the lock for DVC repo."""
|
||||
|
||||
|
||||
class Lock(object):
|
||||
class Lock:
|
||||
"""Class for DVC repo lock.
|
||||
|
||||
Uses zc.lockfile as backend.
|
||||
@@ -47,7 +47,7 @@ class Lock(object):
|
||||
disable=not self._friendly,
|
||||
desc=(
|
||||
"If DVC froze, see `hardlink_lock` in {}".format(
|
||||
format_link("man.dvc.org/config#core")
|
||||
format_link("https://man.dvc.org/config#core")
|
||||
)
|
||||
),
|
||||
):
|
||||
|
||||
+51
-11
@@ -7,7 +7,6 @@ import colorama
|
||||
|
||||
from dvc.progress import Tqdm
|
||||
|
||||
|
||||
FOOTER = (
|
||||
"\n{yellow}Having any troubles?{nc}"
|
||||
" Hit us up at {blue}https://dvc.org/support{nc},"
|
||||
@@ -19,20 +18,45 @@ FOOTER = (
|
||||
)
|
||||
|
||||
|
||||
def addLoggingLevel(levelName, levelNum, methodName=None):
|
||||
"""
|
||||
Adds a new logging level to the `logging` module and the
|
||||
currently configured logging class.
|
||||
|
||||
Based on https://stackoverflow.com/questions/2183233
|
||||
"""
|
||||
if methodName is None:
|
||||
methodName = levelName.lower()
|
||||
|
||||
assert not hasattr(logging, levelName)
|
||||
assert not hasattr(logging, methodName)
|
||||
assert not hasattr(logging.getLoggerClass(), methodName)
|
||||
|
||||
def logForLevel(self, message, *args, **kwargs):
|
||||
if self.isEnabledFor(levelNum):
|
||||
self._log(levelNum, message, args, **kwargs)
|
||||
|
||||
def logToRoot(message, *args, **kwargs):
|
||||
logging.log(levelNum, message, *args, **kwargs)
|
||||
|
||||
logging.addLevelName(levelNum, levelName)
|
||||
setattr(logging, levelName, levelNum)
|
||||
setattr(logging.getLoggerClass(), methodName, logForLevel)
|
||||
setattr(logging, methodName, logToRoot)
|
||||
|
||||
|
||||
class LoggingException(Exception):
|
||||
def __init__(self, record):
|
||||
msg = "failed to log {}".format(str(record))
|
||||
super().__init__(msg)
|
||||
|
||||
|
||||
class ExcludeErrorsFilter(logging.Filter):
|
||||
def filter(self, record):
|
||||
return record.levelno < logging.WARNING
|
||||
def excludeFilter(level):
|
||||
class ExcludeLevelFilter(logging.Filter):
|
||||
def filter(self, record):
|
||||
return record.levelno < level
|
||||
|
||||
|
||||
class ExcludeInfoFilter(logging.Filter):
|
||||
def filter(self, record):
|
||||
return record.levelno < logging.INFO
|
||||
return ExcludeLevelFilter
|
||||
|
||||
|
||||
class ColorFormatter(logging.Formatter):
|
||||
@@ -48,6 +72,7 @@ class ColorFormatter(logging.Formatter):
|
||||
"""
|
||||
|
||||
color_code = {
|
||||
"TRACE": colorama.Fore.GREEN,
|
||||
"DEBUG": colorama.Fore.BLUE,
|
||||
"WARNING": colorama.Fore.YELLOW,
|
||||
"ERROR": colorama.Fore.RED,
|
||||
@@ -117,7 +142,11 @@ class LoggerHandler(logging.StreamHandler):
|
||||
|
||||
|
||||
def _is_verbose():
|
||||
return logging.getLogger("dvc").getEffectiveLevel() == logging.DEBUG
|
||||
return (
|
||||
logging.NOTSET
|
||||
< logging.getLogger("dvc").getEffectiveLevel()
|
||||
<= logging.DEBUG
|
||||
)
|
||||
|
||||
|
||||
def _iter_causes(exc):
|
||||
@@ -143,6 +172,7 @@ def _stack_trace(exc_info):
|
||||
|
||||
|
||||
def disable_other_loggers():
|
||||
logging.captureWarnings(True)
|
||||
root = logging.root
|
||||
for (logger_name, logger) in root.manager.loggerDict.items():
|
||||
if logger_name != "dvc" and not logger_name.startswith("dvc."):
|
||||
@@ -152,12 +182,14 @@ def disable_other_loggers():
|
||||
def setup(level=logging.INFO):
|
||||
colorama.init()
|
||||
|
||||
addLoggingLevel("TRACE", logging.DEBUG - 5)
|
||||
logging.config.dictConfig(
|
||||
{
|
||||
"version": 1,
|
||||
"filters": {
|
||||
"exclude_errors": {"()": ExcludeErrorsFilter},
|
||||
"exclude_info": {"()": ExcludeInfoFilter},
|
||||
"exclude_errors": {"()": excludeFilter(logging.WARNING)},
|
||||
"exclude_info": {"()": excludeFilter(logging.INFO)},
|
||||
"exclude_debug": {"()": excludeFilter(logging.DEBUG)},
|
||||
},
|
||||
"formatters": {"color": {"()": ColorFormatter}},
|
||||
"handlers": {
|
||||
@@ -175,6 +207,13 @@ def setup(level=logging.INFO):
|
||||
"stream": "ext://sys.stdout",
|
||||
"filters": ["exclude_info"],
|
||||
},
|
||||
"console_trace": {
|
||||
"class": "dvc.logger.LoggerHandler",
|
||||
"level": "TRACE",
|
||||
"formatter": "color",
|
||||
"stream": "ext://sys.stdout",
|
||||
"filters": ["exclude_debug"],
|
||||
},
|
||||
"console_errors": {
|
||||
"class": "dvc.logger.LoggerHandler",
|
||||
"level": "WARNING",
|
||||
@@ -188,6 +227,7 @@ def setup(level=logging.INFO):
|
||||
"handlers": [
|
||||
"console_info",
|
||||
"console_debug",
|
||||
"console_trace",
|
||||
"console_errors",
|
||||
],
|
||||
},
|
||||
|
||||
+17
-11
@@ -6,12 +6,11 @@ import logging
|
||||
from dvc import analytics
|
||||
from dvc.cli import parse_args
|
||||
from dvc.config import ConfigError
|
||||
from dvc.exceptions import DvcParserError
|
||||
from dvc.exceptions import NotDvcRepoError
|
||||
from dvc.exceptions import DvcException, DvcParserError, NotDvcRepoError
|
||||
from dvc.external_repo import clean_repos
|
||||
from dvc.logger import disable_other_loggers, FOOTER
|
||||
from dvc.utils import format_link
|
||||
from dvc.logger import FOOTER, disable_other_loggers
|
||||
from dvc.remote.pool import close_pools
|
||||
from dvc.utils import format_link
|
||||
|
||||
# Workaround for CPython bug. See [1] and [2] for more info.
|
||||
# [1] https://github.com/aws/aws-cli/blob/1.16.277/awscli/clidriver.py#L55
|
||||
@@ -19,7 +18,6 @@ from dvc.remote.pool import close_pools
|
||||
|
||||
"".encode("idna")
|
||||
|
||||
|
||||
logger = logging.getLogger("dvc")
|
||||
|
||||
|
||||
@@ -33,18 +31,23 @@ def main(argv=None):
|
||||
int: command's return code.
|
||||
"""
|
||||
args = None
|
||||
cmd = None
|
||||
disable_other_loggers()
|
||||
|
||||
outerLogLevel = logger.level
|
||||
try:
|
||||
args = parse_args(argv)
|
||||
|
||||
if args.quiet:
|
||||
logger.setLevel(logging.CRITICAL)
|
||||
|
||||
elif args.verbose:
|
||||
logger.setLevel(logging.DEBUG)
|
||||
verbosity = args.verbose - args.quiet
|
||||
if verbosity:
|
||||
logger.setLevel(
|
||||
{
|
||||
-2: logging.CRITICAL,
|
||||
-1: logging.ERROR,
|
||||
1: logging.DEBUG,
|
||||
2: logging.TRACE,
|
||||
}[max(-2, min(verbosity, 2))]
|
||||
)
|
||||
logger.trace(args)
|
||||
|
||||
cmd = args.func(args)
|
||||
ret = cmd.run()
|
||||
@@ -59,6 +62,9 @@ def main(argv=None):
|
||||
ret = 253
|
||||
except DvcParserError:
|
||||
ret = 254
|
||||
except DvcException:
|
||||
ret = 255
|
||||
logger.exception("")
|
||||
except Exception as exc: # pylint: disable=broad-except
|
||||
if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
|
||||
logger.exception(
|
||||
|
||||
+54
-54
@@ -1,32 +1,33 @@
|
||||
from urllib.parse import urlparse
|
||||
from voluptuous import Any, Required, Lower, Length, Coerce, And, SetTo
|
||||
|
||||
from dvc.output.base import OutputBase
|
||||
from dvc.output.gs import OutputGS
|
||||
from dvc.output.hdfs import OutputHDFS
|
||||
from dvc.output.local import OutputLOCAL
|
||||
from dvc.output.s3 import OutputS3
|
||||
from dvc.output.ssh import OutputSSH
|
||||
from voluptuous import And, Any, Coerce, Length, Lower, Required, SetTo
|
||||
|
||||
from dvc.output.base import BaseOutput
|
||||
from dvc.output.gs import GSOutput
|
||||
from dvc.output.hdfs import HDFSOutput
|
||||
from dvc.output.local import LocalOutput
|
||||
from dvc.output.s3 import S3Output
|
||||
from dvc.output.ssh import SSHOutput
|
||||
from dvc.remote import Remote
|
||||
from dvc.remote.hdfs import RemoteHDFS
|
||||
from dvc.remote.local import RemoteLOCAL
|
||||
from dvc.remote.s3 import RemoteS3
|
||||
from dvc.remote.hdfs import HDFSRemote
|
||||
from dvc.remote.local import LocalRemote
|
||||
from dvc.remote.s3 import S3Remote
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
OUTS = [
|
||||
OutputHDFS,
|
||||
OutputS3,
|
||||
OutputGS,
|
||||
OutputSSH,
|
||||
# NOTE: OutputLOCAL is the default choice
|
||||
HDFSOutput,
|
||||
S3Output,
|
||||
GSOutput,
|
||||
SSHOutput,
|
||||
# NOTE: LocalOutput is the default choice
|
||||
]
|
||||
|
||||
OUTS_MAP = {
|
||||
Schemes.HDFS: OutputHDFS,
|
||||
Schemes.S3: OutputS3,
|
||||
Schemes.GS: OutputGS,
|
||||
Schemes.SSH: OutputSSH,
|
||||
Schemes.LOCAL: OutputLOCAL,
|
||||
Schemes.HDFS: HDFSOutput,
|
||||
Schemes.S3: S3Output,
|
||||
Schemes.GS: GSOutput,
|
||||
Schemes.SSH: SSHOutput,
|
||||
Schemes.LOCAL: LocalOutput,
|
||||
}
|
||||
|
||||
CHECKSUM_SCHEMA = Any(
|
||||
@@ -44,22 +45,20 @@ CHECKSUM_SCHEMA = Any(
|
||||
# so when a few types of outputs share the same name, we only need
|
||||
# specify it once.
|
||||
CHECKSUMS_SCHEMA = {
|
||||
RemoteLOCAL.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
|
||||
RemoteS3.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
|
||||
RemoteHDFS.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
|
||||
LocalRemote.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
|
||||
S3Remote.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
|
||||
HDFSRemote.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
|
||||
}
|
||||
|
||||
TAGS_SCHEMA = {str: CHECKSUMS_SCHEMA}
|
||||
|
||||
SCHEMA = CHECKSUMS_SCHEMA.copy()
|
||||
SCHEMA[Required(OutputBase.PARAM_PATH)] = str
|
||||
SCHEMA[OutputBase.PARAM_CACHE] = bool
|
||||
SCHEMA[OutputBase.PARAM_METRIC] = OutputBase.METRIC_SCHEMA
|
||||
SCHEMA[OutputBase.PARAM_TAGS] = TAGS_SCHEMA
|
||||
SCHEMA[OutputBase.PARAM_PERSIST] = bool
|
||||
SCHEMA[Required(BaseOutput.PARAM_PATH)] = str
|
||||
SCHEMA[BaseOutput.PARAM_CACHE] = bool
|
||||
SCHEMA[BaseOutput.PARAM_METRIC] = BaseOutput.METRIC_SCHEMA
|
||||
SCHEMA[BaseOutput.PARAM_PLOT] = bool
|
||||
SCHEMA[BaseOutput.PARAM_PERSIST] = bool
|
||||
|
||||
|
||||
def _get(stage, p, info, cache, metric, persist=False, tags=None):
|
||||
def _get(stage, p, info, cache, metric, plot=False, persist=False):
|
||||
parsed = urlparse(p)
|
||||
|
||||
if parsed.scheme == "remote":
|
||||
@@ -71,8 +70,8 @@ def _get(stage, p, info, cache, metric, persist=False, tags=None):
|
||||
cache=cache,
|
||||
remote=remote,
|
||||
metric=metric,
|
||||
plot=plot,
|
||||
persist=persist,
|
||||
tags=tags,
|
||||
)
|
||||
|
||||
for o in OUTS:
|
||||
@@ -84,29 +83,29 @@ def _get(stage, p, info, cache, metric, persist=False, tags=None):
|
||||
cache=cache,
|
||||
remote=None,
|
||||
metric=metric,
|
||||
plot=plot,
|
||||
persist=persist,
|
||||
tags=tags,
|
||||
)
|
||||
return OutputLOCAL(
|
||||
return LocalOutput(
|
||||
stage,
|
||||
p,
|
||||
info,
|
||||
cache=cache,
|
||||
remote=None,
|
||||
metric=metric,
|
||||
plot=plot,
|
||||
persist=persist,
|
||||
tags=tags,
|
||||
)
|
||||
|
||||
|
||||
def loadd_from(stage, d_list):
|
||||
ret = []
|
||||
for d in d_list:
|
||||
p = d.pop(OutputBase.PARAM_PATH)
|
||||
cache = d.pop(OutputBase.PARAM_CACHE, True)
|
||||
metric = d.pop(OutputBase.PARAM_METRIC, False)
|
||||
persist = d.pop(OutputBase.PARAM_PERSIST, False)
|
||||
tags = d.pop(OutputBase.PARAM_TAGS, None)
|
||||
p = d.pop(BaseOutput.PARAM_PATH)
|
||||
cache = d.pop(BaseOutput.PARAM_CACHE, True)
|
||||
metric = d.pop(BaseOutput.PARAM_METRIC, False)
|
||||
plot = d.pop(BaseOutput.PARAM_PLOT, False)
|
||||
persist = d.pop(BaseOutput.PARAM_PERSIST, False)
|
||||
ret.append(
|
||||
_get(
|
||||
stage,
|
||||
@@ -114,24 +113,25 @@ def loadd_from(stage, d_list):
|
||||
info=d,
|
||||
cache=cache,
|
||||
metric=metric,
|
||||
plot=plot,
|
||||
persist=persist,
|
||||
tags=tags,
|
||||
)
|
||||
)
|
||||
return ret
|
||||
|
||||
|
||||
def loads_from(stage, s_list, use_cache=True, metric=False, persist=False):
|
||||
ret = []
|
||||
for s in s_list:
|
||||
ret.append(
|
||||
_get(
|
||||
stage,
|
||||
s,
|
||||
info={},
|
||||
cache=use_cache,
|
||||
metric=metric,
|
||||
persist=persist,
|
||||
)
|
||||
def loads_from(
|
||||
stage, s_list, use_cache=True, metric=False, plot=False, persist=False
|
||||
):
|
||||
return [
|
||||
_get(
|
||||
stage,
|
||||
s,
|
||||
info={},
|
||||
cache=use_cache,
|
||||
metric=metric,
|
||||
plot=plot,
|
||||
persist=persist,
|
||||
)
|
||||
return ret
|
||||
for s in s_list
|
||||
]
|
||||
|
||||
+102
-67
@@ -1,52 +1,61 @@
|
||||
import logging
|
||||
from urllib.parse import urlparse
|
||||
import os
|
||||
from copy import copy
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from voluptuous import Any
|
||||
|
||||
import dvc.prompt as prompt
|
||||
from dvc.cache import NamedCache
|
||||
from dvc.exceptions import CollectCacheError, RemoteCacheRequiredError
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.remote.base import RemoteBASE
|
||||
|
||||
from dvc.exceptions import (
|
||||
CollectCacheError,
|
||||
DvcException,
|
||||
RemoteCacheRequiredError,
|
||||
)
|
||||
from dvc.remote.base import BaseRemote
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class OutputDoesNotExistError(DvcException):
|
||||
def __init__(self, path):
|
||||
msg = "output '{}' does not exist".format(path)
|
||||
msg = f"output '{path}' does not exist"
|
||||
super().__init__(msg)
|
||||
|
||||
|
||||
class OutputIsNotFileOrDirError(DvcException):
|
||||
def __init__(self, path):
|
||||
msg = "output '{}' is not a file or directory".format(path)
|
||||
msg = f"output '{path}' is not a file or directory"
|
||||
super().__init__(msg)
|
||||
|
||||
|
||||
class OutputAlreadyTrackedError(DvcException):
|
||||
def __init__(self, path):
|
||||
msg = "output '{}' is already tracked by scm (e.g. git)".format(path)
|
||||
msg = f""" output '{path}' is already tracked by SCM (e.g. Git).
|
||||
You can remove it from Git, then add to DVC.
|
||||
To stop tracking from Git:
|
||||
git rm -r --cached '{path}'
|
||||
git commit -m "stop tracking {path}" """
|
||||
super().__init__(msg)
|
||||
|
||||
|
||||
class OutputIsStageFileError(DvcException):
|
||||
def __init__(self, path):
|
||||
super().__init__("Stage file '{}' cannot be an output.".format(path))
|
||||
super().__init__(f"Stage file '{path}' cannot be an output.")
|
||||
|
||||
|
||||
class OutputBase(object):
|
||||
class BaseOutput:
|
||||
IS_DEPENDENCY = False
|
||||
|
||||
REMOTE = RemoteBASE
|
||||
REMOTE = BaseRemote
|
||||
|
||||
PARAM_PATH = "path"
|
||||
PARAM_CACHE = "cache"
|
||||
PARAM_METRIC = "metric"
|
||||
PARAM_METRIC_TYPE = "type"
|
||||
PARAM_METRIC_XPATH = "xpath"
|
||||
PARAM_PLOT = "plot"
|
||||
PARAM_PLOT_TEMPLATE = "template"
|
||||
PARAM_PERSIST = "persist"
|
||||
|
||||
METRIC_SCHEMA = Any(
|
||||
@@ -58,8 +67,6 @@ class OutputBase(object):
|
||||
},
|
||||
)
|
||||
|
||||
PARAM_TAGS = "tags"
|
||||
|
||||
DoesNotExistError = OutputDoesNotExistError
|
||||
IsNotFileOrDirError = OutputIsNotFileOrDirError
|
||||
IsStageFileError = OutputIsStageFileError
|
||||
@@ -74,8 +81,8 @@ class OutputBase(object):
|
||||
remote=None,
|
||||
cache=True,
|
||||
metric=False,
|
||||
plot=False,
|
||||
persist=False,
|
||||
tags=None,
|
||||
):
|
||||
self._validate_output_path(path)
|
||||
# This output (and dependency) objects have too many paths/urls
|
||||
@@ -95,8 +102,8 @@ class OutputBase(object):
|
||||
self.remote = remote or self.REMOTE(self.repo, {})
|
||||
self.use_cache = False if self.IS_DEPENDENCY else cache
|
||||
self.metric = False if self.IS_DEPENDENCY else metric
|
||||
self.plot = False if self.IS_DEPENDENCY else plot
|
||||
self.persist = persist
|
||||
self.tags = None if self.IS_DEPENDENCY else (tags or {})
|
||||
|
||||
self.path_info = self._parse_path(remote, path)
|
||||
if self.use_cache and self.cache is None:
|
||||
@@ -147,10 +154,21 @@ class OutputBase(object):
|
||||
def cache_path(self):
|
||||
return self.cache.checksum_to_path_info(self.checksum).url
|
||||
|
||||
@property
|
||||
def checksum_type(self):
|
||||
return self.remote.PARAM_CHECKSUM
|
||||
|
||||
@property
|
||||
def checksum(self):
|
||||
return self.info.get(self.remote.PARAM_CHECKSUM)
|
||||
|
||||
@checksum.setter
|
||||
def checksum(self, checksum):
|
||||
self.info[self.remote.PARAM_CHECKSUM] = checksum
|
||||
|
||||
def get_checksum(self):
|
||||
return self.remote.get_checksum(self.path_info)
|
||||
|
||||
@property
|
||||
def is_dir_checksum(self):
|
||||
return self.remote.is_dir_checksum(self.checksum)
|
||||
@@ -159,8 +177,11 @@ class OutputBase(object):
|
||||
def exists(self):
|
||||
return self.remote.exists(self.path_info)
|
||||
|
||||
def save_info(self):
|
||||
return self.remote.save_info(self.path_info)
|
||||
|
||||
def changed_checksum(self):
|
||||
return self.checksum != self.remote.get_checksum(self.path_info)
|
||||
return self.checksum != self.get_checksum()
|
||||
|
||||
def changed_cache(self, filter_info=None):
|
||||
if not self.use_cache or not self.checksum:
|
||||
@@ -198,6 +219,15 @@ class OutputBase(object):
|
||||
def isfile(self):
|
||||
return self.remote.isfile(self.path_info)
|
||||
|
||||
def ignore(self):
|
||||
if not self.use_scm_ignore:
|
||||
return
|
||||
|
||||
if self.repo.scm.is_tracked(self.fspath):
|
||||
raise OutputAlreadyTrackedError(self)
|
||||
|
||||
self.repo.scm.ignore(self.fspath)
|
||||
|
||||
def save(self):
|
||||
if not self.exists:
|
||||
raise self.DoesNotExistError(self)
|
||||
@@ -206,35 +236,27 @@ class OutputBase(object):
|
||||
raise self.IsNotFileOrDirError(self)
|
||||
|
||||
if self.is_empty:
|
||||
logger.warning("'{}' is empty.".format(self))
|
||||
logger.warning(f"'{self}' is empty.")
|
||||
|
||||
if self.use_scm_ignore:
|
||||
if self.repo.scm.is_tracked(self.fspath):
|
||||
raise OutputAlreadyTrackedError(self)
|
||||
|
||||
self.repo.scm.ignore(self.fspath)
|
||||
self.ignore()
|
||||
|
||||
if not self.use_cache:
|
||||
self.info = self.remote.save_info(self.path_info)
|
||||
if self.metric:
|
||||
self.info = self.save_info()
|
||||
if self.metric or self.plot:
|
||||
self.verify_metric()
|
||||
if not self.IS_DEPENDENCY:
|
||||
logger.info(
|
||||
"Output '{}' doesn't use cache. Skipping saving.".format(
|
||||
self
|
||||
)
|
||||
logger.debug(
|
||||
"Output '%s' doesn't use cache. Skipping saving.", self
|
||||
)
|
||||
return
|
||||
|
||||
assert not self.IS_DEPENDENCY
|
||||
|
||||
if not self.changed():
|
||||
logger.info(
|
||||
"Output '{}' didn't change. Skipping saving.".format(self)
|
||||
)
|
||||
logger.debug("Output '%s' didn't change. Skipping saving.", self)
|
||||
return
|
||||
|
||||
self.info = self.remote.save_info(self.path_info)
|
||||
self.info = self.save_info()
|
||||
|
||||
def commit(self):
|
||||
if self.use_cache:
|
||||
@@ -247,7 +269,8 @@ class OutputBase(object):
|
||||
if self.IS_DEPENDENCY:
|
||||
return ret
|
||||
|
||||
ret[self.PARAM_CACHE] = self.use_cache
|
||||
if not self.use_cache:
|
||||
ret[self.PARAM_CACHE] = self.use_cache
|
||||
|
||||
if isinstance(self.metric, dict):
|
||||
if (
|
||||
@@ -256,18 +279,19 @@ class OutputBase(object):
|
||||
):
|
||||
del self.metric[self.PARAM_METRIC_XPATH]
|
||||
|
||||
ret[self.PARAM_METRIC] = self.metric
|
||||
ret[self.PARAM_PERSIST] = self.persist
|
||||
if self.metric:
|
||||
ret[self.PARAM_METRIC] = self.metric
|
||||
|
||||
if self.tags:
|
||||
ret[self.PARAM_TAGS] = self.tags
|
||||
if self.plot:
|
||||
ret[self.PARAM_PLOT] = self.plot
|
||||
|
||||
if self.persist:
|
||||
ret[self.PARAM_PERSIST] = self.persist
|
||||
|
||||
return ret
|
||||
|
||||
def verify_metric(self):
|
||||
raise DvcException(
|
||||
"verify metric is not supported for {}".format(self.scheme)
|
||||
)
|
||||
raise DvcException(f"verify metric is not supported for {self.scheme}")
|
||||
|
||||
def download(self, to):
|
||||
self.remote.download(self.path_info, to.path_info)
|
||||
@@ -276,7 +300,6 @@ class OutputBase(object):
|
||||
self,
|
||||
force=False,
|
||||
progress_callback=None,
|
||||
tag=None,
|
||||
relink=False,
|
||||
filter_info=None,
|
||||
):
|
||||
@@ -287,14 +310,9 @@ class OutputBase(object):
|
||||
)
|
||||
return None
|
||||
|
||||
if tag:
|
||||
info = self.tags[tag]
|
||||
else:
|
||||
info = self.info
|
||||
|
||||
return self.cache.checkout(
|
||||
self.path_info,
|
||||
info,
|
||||
self.info,
|
||||
force=force,
|
||||
progress_callback=progress_callback,
|
||||
relink=relink,
|
||||
@@ -334,7 +352,18 @@ class OutputBase(object):
|
||||
if self.exists:
|
||||
self.remote.unprotect(self.path_info)
|
||||
|
||||
def _collect_used_dir_cache(
|
||||
def get_dir_cache(self, **kwargs):
|
||||
if not self.is_dir_checksum:
|
||||
raise DvcException("cannot get dir cache for file checksum")
|
||||
if self.cache.changed_cache_file(self.checksum):
|
||||
self.repo.cloud.pull(
|
||||
NamedCache.make("local", self.checksum, str(self)),
|
||||
show_checksums=False,
|
||||
**kwargs,
|
||||
)
|
||||
return self.dir_cache
|
||||
|
||||
def collect_used_dir_cache(
|
||||
self, remote=None, force=False, jobs=None, filter_info=None
|
||||
):
|
||||
"""Get a list of `info`s related to the given directory.
|
||||
@@ -358,16 +387,10 @@ class OutputBase(object):
|
||||
|
||||
cache = NamedCache()
|
||||
|
||||
if self.cache.changed_cache_file(self.checksum):
|
||||
try:
|
||||
self.repo.cloud.pull(
|
||||
NamedCache.make("local", self.checksum, str(self)),
|
||||
jobs=jobs,
|
||||
remote=remote,
|
||||
show_checksums=False,
|
||||
)
|
||||
except DvcException:
|
||||
logger.debug("failed to pull cache for '{}'".format(self))
|
||||
try:
|
||||
self.get_dir_cache(jobs=jobs, remote=remote)
|
||||
except DvcException:
|
||||
logger.debug(f"failed to pull cache for '{self}'")
|
||||
|
||||
if self.cache.changed_cache_file(self.checksum):
|
||||
msg = (
|
||||
@@ -383,11 +406,21 @@ class OutputBase(object):
|
||||
else:
|
||||
return cache
|
||||
|
||||
path = str(self.path_info)
|
||||
filter_path = str(filter_info) if filter_info else None
|
||||
is_win = os.name == "nt"
|
||||
for entry in self.dir_cache:
|
||||
checksum = entry[self.remote.PARAM_CHECKSUM]
|
||||
info = self.path_info / entry[self.remote.PARAM_RELPATH]
|
||||
if not filter_info or info.isin_or_eq(filter_info):
|
||||
cache.add(self.scheme, checksum, str(info))
|
||||
entry_relpath = entry[self.remote.PARAM_RELPATH]
|
||||
if is_win:
|
||||
entry_relpath = entry_relpath.replace("/", os.sep)
|
||||
entry_path = os.path.join(path, entry_relpath)
|
||||
if (
|
||||
not filter_path
|
||||
or entry_path == filter_path
|
||||
or entry_path.startswith(filter_path + os.sep)
|
||||
):
|
||||
cache.add(self.scheme, checksum, entry_path)
|
||||
|
||||
return cache
|
||||
|
||||
@@ -420,9 +453,9 @@ class OutputBase(object):
|
||||
if self.exists:
|
||||
msg += (
|
||||
"\n"
|
||||
"You can also use `dvc commit {stage}` to associate "
|
||||
"existing '{out}' with '{stage}'.".format(
|
||||
out=self, stage=self.stage.relpath
|
||||
"You can also use `dvc commit {stage.addressing}` "
|
||||
"to associate existing '{out}' with {stage}.".format(
|
||||
out=self, stage=self.stage
|
||||
)
|
||||
)
|
||||
logger.warning(msg)
|
||||
@@ -433,13 +466,15 @@ class OutputBase(object):
|
||||
if not self.is_dir_checksum:
|
||||
return ret
|
||||
|
||||
ret.update(self._collect_used_dir_cache(**kwargs))
|
||||
ret.add_child_cache(
|
||||
self.checksum, self.collect_used_dir_cache(**kwargs),
|
||||
)
|
||||
|
||||
return ret
|
||||
|
||||
@classmethod
|
||||
def _validate_output_path(cls, path):
|
||||
from dvc.stage import Stage
|
||||
from dvc.dvcfile import is_valid_filename
|
||||
|
||||
if Stage.is_valid_filename(path):
|
||||
if is_valid_filename(path):
|
||||
raise cls.IsStageFileError(path)
|
||||
|
||||
+4
-4
@@ -1,6 +1,6 @@
|
||||
from dvc.output.s3 import OutputS3
|
||||
from dvc.remote.gs import RemoteGS
|
||||
from dvc.output.s3 import S3Output
|
||||
from dvc.remote.gs import GSRemote
|
||||
|
||||
|
||||
class OutputGS(OutputS3):
|
||||
REMOTE = RemoteGS
|
||||
class GSOutput(S3Output):
|
||||
REMOTE = GSRemote
|
||||
|
||||
+4
-4
@@ -1,6 +1,6 @@
|
||||
from dvc.output.base import OutputBase
|
||||
from dvc.remote.hdfs import RemoteHDFS
|
||||
from dvc.output.base import BaseOutput
|
||||
from dvc.remote.hdfs import HDFSRemote
|
||||
|
||||
|
||||
class OutputHDFS(OutputBase):
|
||||
REMOTE = RemoteHDFS
|
||||
class HDFSOutput(BaseOutput):
|
||||
REMOTE = HDFSRemote
|
||||
|
||||
+12
-13
@@ -4,18 +4,16 @@ from urllib.parse import urlparse
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.istextfile import istextfile
|
||||
from dvc.output.base import OutputBase
|
||||
from dvc.remote.local import RemoteLOCAL
|
||||
from dvc.output.base import BaseOutput
|
||||
from dvc.remote.local import LocalRemote
|
||||
from dvc.utils import relpath
|
||||
from dvc.compat import fspath_py35
|
||||
from dvc.utils.fs import path_isin
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class OutputLOCAL(OutputBase):
|
||||
REMOTE = RemoteLOCAL
|
||||
class LocalOutput(BaseOutput):
|
||||
REMOTE = LocalRemote
|
||||
sep = os.sep
|
||||
|
||||
def __init__(self, stage, path, *args, **kwargs):
|
||||
@@ -39,7 +37,7 @@ class OutputLOCAL(OutputBase):
|
||||
if not p.is_absolute():
|
||||
p = self.stage.wdir / p
|
||||
|
||||
abs_p = os.path.abspath(os.path.normpath(fspath_py35(p)))
|
||||
abs_p = os.path.abspath(os.path.normpath(p))
|
||||
return self.REMOTE.path_cls(abs_p)
|
||||
|
||||
def __str__(self):
|
||||
@@ -73,17 +71,18 @@ class OutputLOCAL(OutputBase):
|
||||
return ret
|
||||
|
||||
def verify_metric(self):
|
||||
if not self.metric:
|
||||
if not self.metric or self.plot:
|
||||
return
|
||||
|
||||
path = fspath_py35(self.path_info)
|
||||
path = os.fspath(self.path_info)
|
||||
if not os.path.exists(path):
|
||||
return
|
||||
|
||||
name = "metrics" if self.metric else "plot"
|
||||
if os.path.isdir(path):
|
||||
msg = "directory '{}' cannot be used as metrics."
|
||||
raise DvcException(msg.format(self.path_info))
|
||||
msg = "directory '{}' cannot be used as {}."
|
||||
raise DvcException(msg.format(self.path_info, name))
|
||||
|
||||
if not istextfile(path):
|
||||
msg = "binary file '{}' cannot be used as metrics."
|
||||
raise DvcException(msg.format(self.path_info))
|
||||
msg = "binary file '{}' cannot be used as {}."
|
||||
raise DvcException(msg.format(self.path_info, name))
|
||||
|
||||
+4
-4
@@ -1,6 +1,6 @@
|
||||
from dvc.output.base import OutputBase
|
||||
from dvc.remote.s3 import RemoteS3
|
||||
from dvc.output.base import BaseOutput
|
||||
from dvc.remote.s3 import S3Remote
|
||||
|
||||
|
||||
class OutputS3(OutputBase):
|
||||
REMOTE = RemoteS3
|
||||
class S3Output(BaseOutput):
|
||||
REMOTE = S3Remote
|
||||
|
||||
+4
-4
@@ -1,6 +1,6 @@
|
||||
from dvc.output.base import OutputBase
|
||||
from dvc.remote.ssh import RemoteSSH
|
||||
from dvc.output.base import BaseOutput
|
||||
from dvc.remote.ssh import SSHRemote
|
||||
|
||||
|
||||
class OutputSSH(OutputBase):
|
||||
REMOTE = RemoteSSH
|
||||
class SSHOutput(BaseOutput):
|
||||
REMOTE = SSHRemote
|
||||
|
||||
+86
-16
@@ -1,6 +1,6 @@
|
||||
import os
|
||||
import posixpath
|
||||
import pathlib
|
||||
import posixpath
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from funcy import cached_property
|
||||
@@ -8,7 +8,7 @@ from funcy import cached_property
|
||||
from dvc.utils import relpath
|
||||
|
||||
|
||||
class _BasePath(object):
|
||||
class _BasePath:
|
||||
def overlaps(self, other):
|
||||
if isinstance(other, (str, bytes)):
|
||||
other = self.__class__(other)
|
||||
@@ -33,10 +33,6 @@ class PathInfo(pathlib.PurePath, _BasePath):
|
||||
cls = WindowsPathInfo if os.name == "nt" else PosixPathInfo
|
||||
return cls._from_parts(args)
|
||||
|
||||
@classmethod
|
||||
def from_posix(cls, s):
|
||||
return cls(PosixPathInfo(s))
|
||||
|
||||
def as_posix(self):
|
||||
f = self._flavour
|
||||
# Unlike original implementation [1] that uses `str()` we actually need
|
||||
@@ -54,7 +50,6 @@ class PathInfo(pathlib.PurePath, _BasePath):
|
||||
return "{}: '{}'".format(type(self).__name__, self)
|
||||
|
||||
# This permits passing it to file utils directly in Python 3.6+
|
||||
# With Python 2.7, Python 3.5+ we are stuck with path_info.fspath for now
|
||||
def __fspath__(self):
|
||||
return pathlib.PurePath.__str__(self)
|
||||
|
||||
@@ -92,7 +87,7 @@ class _URLPathInfo(PosixPathInfo):
|
||||
__unicode__ = __str__
|
||||
|
||||
|
||||
class _URLPathParents(object):
|
||||
class _URLPathParents:
|
||||
def __init__(self, src):
|
||||
self.src = src
|
||||
self._parents = self.src._path.parents
|
||||
@@ -104,7 +99,7 @@ class _URLPathParents(object):
|
||||
return self.src.replace(path=self._parents[idx])
|
||||
|
||||
def __repr__(self):
|
||||
return "<{}.parents>".format(self.src)
|
||||
return f"<{self.src}.parents>"
|
||||
|
||||
|
||||
class URLInfo(_BasePath):
|
||||
@@ -115,7 +110,7 @@ class URLInfo(_BasePath):
|
||||
assert not p.query and not p.params and not p.fragment
|
||||
assert p.password is None
|
||||
|
||||
self.fill_parts(p.scheme, p.hostname, p.username, p.port, p.path)
|
||||
self._fill_parts(p.scheme, p.hostname, p.username, p.port, p.path)
|
||||
|
||||
@classmethod
|
||||
def from_parts(
|
||||
@@ -124,13 +119,13 @@ class URLInfo(_BasePath):
|
||||
assert bool(host) ^ bool(netloc)
|
||||
|
||||
if netloc is not None:
|
||||
return cls("{}://{}{}".format(scheme, netloc, path))
|
||||
return cls(f"{scheme}://{netloc}{path}")
|
||||
|
||||
obj = cls.__new__(cls)
|
||||
obj.fill_parts(scheme, host, user, port, path)
|
||||
obj._fill_parts(scheme, host, user, port, path)
|
||||
return obj
|
||||
|
||||
def fill_parts(self, scheme, host, user, port, path):
|
||||
def _fill_parts(self, scheme, host, user, port, path):
|
||||
assert scheme != "remote"
|
||||
assert isinstance(path, (str, bytes, _URLPathInfo))
|
||||
|
||||
@@ -158,7 +153,7 @@ class URLInfo(_BasePath):
|
||||
|
||||
@cached_property
|
||||
def url(self):
|
||||
return "{}://{}{}".format(self.scheme, self.netloc, self._spath)
|
||||
return f"{self.scheme}://{self.netloc}{self._spath}"
|
||||
|
||||
def __str__(self):
|
||||
return self.url
|
||||
@@ -220,10 +215,10 @@ class URLInfo(_BasePath):
|
||||
if isinstance(other, (str, bytes)):
|
||||
other = self.__class__(other)
|
||||
if self.__class__ != other.__class__:
|
||||
msg = "'{}' has incompatible class with '{}'".format(self, other)
|
||||
msg = f"'{self}' has incompatible class with '{other}'"
|
||||
raise ValueError(msg)
|
||||
if self._base_parts != other._base_parts:
|
||||
msg = "'{}' does not start with '{}'".format(self, other)
|
||||
msg = f"'{self}' does not start with '{other}'"
|
||||
raise ValueError(msg)
|
||||
return self._path.relative_to(other._path)
|
||||
|
||||
@@ -241,3 +236,78 @@ class CloudURLInfo(URLInfo):
|
||||
@property
|
||||
def path(self):
|
||||
return self._spath.lstrip("/")
|
||||
|
||||
|
||||
class HTTPURLInfo(URLInfo):
|
||||
__hash__ = URLInfo.__hash__
|
||||
|
||||
def __init__(self, url):
|
||||
p = urlparse(url)
|
||||
stripped = p._replace(params=None, query=None, fragment=None)
|
||||
super().__init__(stripped.geturl())
|
||||
self.params = p.params
|
||||
self.query = p.query
|
||||
self.fragment = p.fragment
|
||||
|
||||
@classmethod
|
||||
def from_parts(
|
||||
cls,
|
||||
scheme=None,
|
||||
host=None,
|
||||
user=None,
|
||||
port=None,
|
||||
path="",
|
||||
netloc=None,
|
||||
params=None,
|
||||
query=None,
|
||||
fragment=None,
|
||||
):
|
||||
assert bool(host) ^ bool(netloc)
|
||||
|
||||
if netloc is not None:
|
||||
return cls(
|
||||
"{}://{}{}{}{}{}".format(
|
||||
scheme,
|
||||
netloc,
|
||||
path,
|
||||
(";" + params) if params else "",
|
||||
("?" + query) if query else "",
|
||||
("#" + fragment) if fragment else "",
|
||||
)
|
||||
)
|
||||
|
||||
obj = cls.__new__(cls)
|
||||
obj._fill_parts(scheme, host, user, port, path)
|
||||
obj.params = params
|
||||
obj.query = query
|
||||
obj.fragment = fragment
|
||||
return obj
|
||||
|
||||
@property
|
||||
def _extra_parts(self):
|
||||
return (self.params, self.query, self.fragment)
|
||||
|
||||
@property
|
||||
def parts(self):
|
||||
return self._base_parts + self._path.parts + self._extra_parts
|
||||
|
||||
@cached_property
|
||||
def url(self):
|
||||
return "{}://{}{}{}{}{}".format(
|
||||
self.scheme,
|
||||
self.netloc,
|
||||
self._spath,
|
||||
(";" + self.params) if self.params else "",
|
||||
("?" + self.query) if self.query else "",
|
||||
("#" + self.fragment) if self.fragment else "",
|
||||
)
|
||||
|
||||
def __eq__(self, other):
|
||||
if isinstance(other, (str, bytes)):
|
||||
other = self.__class__(other)
|
||||
return (
|
||||
self.__class__ == other.__class__
|
||||
and self._base_parts == other._base_parts
|
||||
and self._path == other._path
|
||||
and self._extra_parts == other._extra_parts
|
||||
)
|
||||
|
||||
+38
-21
@@ -19,18 +19,17 @@ class Tqdm(tqdm):
|
||||
|
||||
BAR_FMT_DEFAULT = (
|
||||
"{percentage:3.0f}% {desc}|{bar}|"
|
||||
"{n_fmt}/{total_fmt}"
|
||||
" [{elapsed}<{remaining}, {rate_fmt:>11}{postfix}]"
|
||||
"{postfix[info]}{n_fmt}/{total_fmt}"
|
||||
" [{elapsed}<{remaining}, {rate_fmt:>11}]"
|
||||
)
|
||||
# nested bars should have fixed bar widths to align nicely
|
||||
BAR_FMT_DEFAULT_NESTED = (
|
||||
"{percentage:3.0f}%|{bar:10}|{desc:{ncols_desc}.{ncols_desc}}"
|
||||
"{n_fmt}/{total_fmt}"
|
||||
" [{elapsed}<{remaining}, {rate_fmt:>11}{postfix}]"
|
||||
"{postfix[info]}{n_fmt}/{total_fmt}"
|
||||
" [{elapsed}<{remaining}, {rate_fmt:>11}]"
|
||||
)
|
||||
BAR_FMT_NOTOTAL = (
|
||||
"{desc:{ncols_desc}.{ncols_desc}}{n_fmt}"
|
||||
" [{elapsed}, {rate_fmt:>11}{postfix}]"
|
||||
"{desc}{bar:b}|{postfix[info]}{n_fmt} [{elapsed}, {rate_fmt:>11}]"
|
||||
)
|
||||
BYTES_DEFAULTS = dict(
|
||||
unit="B", unit_scale=True, unit_divisor=1024, miniters=1
|
||||
@@ -47,6 +46,7 @@ class Tqdm(tqdm):
|
||||
bytes=False, # pylint: disable=W0622
|
||||
file=None,
|
||||
total=None,
|
||||
postfix=None,
|
||||
**kwargs
|
||||
):
|
||||
"""
|
||||
@@ -68,7 +68,6 @@ class Tqdm(tqdm):
|
||||
kwargs.setdefault("unit_scale", total > 999 if total else True)
|
||||
if file is None:
|
||||
file = sys.stderr
|
||||
self.desc_persist = desc
|
||||
# auto-disable based on `logger.level`
|
||||
if not disable:
|
||||
disable = logger.getEffectiveLevel() > level
|
||||
@@ -89,6 +88,7 @@ class Tqdm(tqdm):
|
||||
total=total,
|
||||
**kwargs
|
||||
)
|
||||
self.postfix = postfix or {"info": ""}
|
||||
if bar_format is None:
|
||||
if self.__len__():
|
||||
self.bar_format = (
|
||||
@@ -102,11 +102,11 @@ class Tqdm(tqdm):
|
||||
self.bar_format = bar_format
|
||||
self.refresh()
|
||||
|
||||
def update_desc(self, desc, n=1):
|
||||
def update_msg(self, msg, n=1):
|
||||
"""
|
||||
Calls `set_description_str(desc)` and `update(n)`
|
||||
Sets `msg` as a postfix and calls `update(n)`.
|
||||
"""
|
||||
self.set_description_str(desc, refresh=False)
|
||||
self.postfix["info"] = " %s |" % msg
|
||||
self.update(n)
|
||||
|
||||
def update_to(self, current, total=None):
|
||||
@@ -114,13 +114,27 @@ class Tqdm(tqdm):
|
||||
self.total = total # pylint: disable=W0613,W0201
|
||||
self.update(current - self.n)
|
||||
|
||||
def wrap_fn(self, fn, callback=None):
|
||||
"""
|
||||
Returns a wrapped `fn` which calls `callback()` on each call.
|
||||
`callback` is `self.update` by default.
|
||||
"""
|
||||
if callback is None:
|
||||
callback = self.update
|
||||
|
||||
def wrapped(*args, **kwargs):
|
||||
res = fn(*args, **kwargs)
|
||||
callback()
|
||||
return res
|
||||
|
||||
return wrapped
|
||||
|
||||
def close(self):
|
||||
if self.desc_persist is not None:
|
||||
self.set_description_str(self.desc_persist, refresh=False)
|
||||
# unknown/zero ETA
|
||||
self.bar_format = self.bar_format.replace("<{remaining}", "")
|
||||
# remove completed bar
|
||||
self.bar_format = self.bar_format.replace("|{bar:10}|", " ")
|
||||
self.postfix["info"] = ""
|
||||
# remove ETA (either unknown or zero); remove completed bar
|
||||
self.bar_format = self.bar_format.replace("<{remaining}", "").replace(
|
||||
"|{bar:10}|", " "
|
||||
)
|
||||
super().close()
|
||||
|
||||
@property
|
||||
@@ -128,12 +142,15 @@ class Tqdm(tqdm):
|
||||
"""inject `ncols_desc` to fill the display width (`ncols`)"""
|
||||
d = super().format_dict
|
||||
ncols = d["ncols"] or 80
|
||||
ncols_desc = ncols - len(self.format_meter(ncols_desc=1, **d)) + 1
|
||||
ncols_desc = max(ncols_desc, 0)
|
||||
if ncols_desc:
|
||||
d["ncols_desc"] = ncols_desc
|
||||
# assumes `bar_format` has max one of ("ncols_desc" & "ncols_info")
|
||||
ncols_left = (
|
||||
ncols - len(self.format_meter(ncols_desc=1, ncols_info=1, **d)) + 1
|
||||
)
|
||||
ncols_left = max(ncols_left, 0)
|
||||
if ncols_left:
|
||||
d["ncols_desc"] = d["ncols_info"] = ncols_left
|
||||
else:
|
||||
# work-around for zero-width description
|
||||
d["ncols_desc"] = 1
|
||||
d["ncols_desc"] = d["ncols_info"] = 1
|
||||
d["prefix"] = ""
|
||||
return d
|
||||
|
||||
+2
-2
@@ -38,7 +38,7 @@ def confirm(statement):
|
||||
Returns:
|
||||
bool: whether or not specified statement was confirmed.
|
||||
"""
|
||||
prompt = "{statement} [y/n]".format(statement=statement)
|
||||
prompt = f"{statement} [y/n]"
|
||||
answer = _ask(prompt, limited_to=["yes", "no", "y", "n"])
|
||||
return answer and answer.startswith("y")
|
||||
|
||||
@@ -52,5 +52,5 @@ def password(statement):
|
||||
Returns:
|
||||
str: password entered by the user.
|
||||
"""
|
||||
logger.info("{statement}: ".format(statement=statement))
|
||||
logger.info(f"{statement}: ")
|
||||
return getpass("")
|
||||
|
||||
+21
-22
@@ -1,29 +1,28 @@
|
||||
import posixpath
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from dvc.remote.azure import RemoteAZURE
|
||||
from dvc.remote.gdrive import RemoteGDrive
|
||||
from dvc.remote.gs import RemoteGS
|
||||
from dvc.remote.hdfs import RemoteHDFS
|
||||
from dvc.remote.http import RemoteHTTP
|
||||
from dvc.remote.https import RemoteHTTPS
|
||||
from dvc.remote.local import RemoteLOCAL
|
||||
from dvc.remote.oss import RemoteOSS
|
||||
from dvc.remote.s3 import RemoteS3
|
||||
from dvc.remote.ssh import RemoteSSH
|
||||
|
||||
from dvc.remote.azure import AzureRemote
|
||||
from dvc.remote.gdrive import GDriveRemote
|
||||
from dvc.remote.gs import GSRemote
|
||||
from dvc.remote.hdfs import HDFSRemote
|
||||
from dvc.remote.http import HTTPRemote
|
||||
from dvc.remote.https import HTTPSRemote
|
||||
from dvc.remote.local import LocalRemote
|
||||
from dvc.remote.oss import OSSRemote
|
||||
from dvc.remote.s3 import S3Remote
|
||||
from dvc.remote.ssh import SSHRemote
|
||||
|
||||
REMOTES = [
|
||||
RemoteAZURE,
|
||||
RemoteGDrive,
|
||||
RemoteGS,
|
||||
RemoteHDFS,
|
||||
RemoteHTTP,
|
||||
RemoteHTTPS,
|
||||
RemoteS3,
|
||||
RemoteSSH,
|
||||
RemoteOSS,
|
||||
# NOTE: RemoteLOCAL is the default
|
||||
AzureRemote,
|
||||
GDriveRemote,
|
||||
GSRemote,
|
||||
HDFSRemote,
|
||||
HTTPRemote,
|
||||
HTTPSRemote,
|
||||
S3Remote,
|
||||
SSHRemote,
|
||||
OSSRemote,
|
||||
# NOTE: LocalRemote is the default
|
||||
]
|
||||
|
||||
|
||||
@@ -31,7 +30,7 @@ def _get(remote_conf):
|
||||
for remote in REMOTES:
|
||||
if remote.supported(remote_conf):
|
||||
return remote
|
||||
return RemoteLOCAL
|
||||
return LocalRemote
|
||||
|
||||
|
||||
def Remote(repo, **kwargs):
|
||||
|
||||
+37
-42
@@ -1,63 +1,39 @@
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime, timedelta
|
||||
from urllib.parse import urlparse
|
||||
import posixpath
|
||||
import threading
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from funcy import cached_property, wrap_prop
|
||||
|
||||
from dvc.path_info import CloudURLInfo
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.remote.base import RemoteBASE
|
||||
from dvc.remote.base import BaseRemote
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RemoteAZURE(RemoteBASE):
|
||||
class AzureRemote(BaseRemote):
|
||||
scheme = Schemes.AZURE
|
||||
path_cls = CloudURLInfo
|
||||
REGEX = (
|
||||
r"azure://((?P<path>[^=;]*)?|("
|
||||
# backward compatibility
|
||||
r"(ContainerName=(?P<container_name>[^;]+);?)?"
|
||||
r"(?P<connection_string>.+)?)?)$"
|
||||
)
|
||||
REQUIRES = {"azure-storage-blob": "azure.storage.blob"}
|
||||
PARAM_CHECKSUM = "etag"
|
||||
COPY_POLL_SECONDS = 5
|
||||
LIST_OBJECT_PAGE_SIZE = 5000
|
||||
|
||||
def __init__(self, repo, config):
|
||||
super().__init__(repo, config)
|
||||
|
||||
url = config.get("url", "azure://")
|
||||
self.path_info = self.path_cls(url)
|
||||
|
||||
match = re.match(self.REGEX, url) # backward compatibility
|
||||
path = match.group("path")
|
||||
bucket = (
|
||||
urlparse(url if path else "").netloc
|
||||
or match.group("container_name") # backward compatibility
|
||||
or os.getenv("AZURE_STORAGE_CONTAINER_NAME")
|
||||
)
|
||||
if not self.path_info.bucket:
|
||||
container = os.getenv("AZURE_STORAGE_CONTAINER_NAME")
|
||||
self.path_info = self.path_cls(f"azure://{container}")
|
||||
|
||||
self.connection_string = (
|
||||
config.get("connection_string")
|
||||
or match.group("connection_string") # backward compatibility
|
||||
or os.getenv("AZURE_STORAGE_CONNECTION_STRING")
|
||||
)
|
||||
|
||||
if not bucket:
|
||||
raise ValueError("azure storage container name missing")
|
||||
|
||||
if not self.connection_string:
|
||||
raise ValueError("azure storage connection string missing")
|
||||
|
||||
self.path_info = (
|
||||
self.path_cls(url)
|
||||
if path
|
||||
else self.path_cls.from_parts(scheme=self.scheme, netloc=bucket)
|
||||
self.connection_string = config.get("connection_string") or os.getenv(
|
||||
"AZURE_STORAGE_CONNECTION_STRING"
|
||||
)
|
||||
|
||||
@wrap_prop(threading.Lock())
|
||||
@@ -66,12 +42,12 @@ class RemoteAZURE(RemoteBASE):
|
||||
from azure.storage.blob import BlockBlobService
|
||||
from azure.common import AzureMissingResourceHttpError
|
||||
|
||||
logger.debug("URL {}".format(self.path_info))
|
||||
logger.debug("Connection string {}".format(self.connection_string))
|
||||
logger.debug(f"URL {self.path_info}")
|
||||
logger.debug(f"Connection string {self.connection_string}")
|
||||
blob_service = BlockBlobService(
|
||||
connection_string=self.connection_string
|
||||
)
|
||||
logger.debug("Container name {}".format(self.path_info.bucket))
|
||||
logger.debug(f"Container name {self.path_info.bucket}")
|
||||
try: # verify that container exists
|
||||
blob_service.list_blobs(
|
||||
self.path_info.bucket, delimiter="/", num_results=1
|
||||
@@ -80,14 +56,23 @@ class RemoteAZURE(RemoteBASE):
|
||||
blob_service.create_container(self.path_info.bucket)
|
||||
return blob_service
|
||||
|
||||
def get_etag(self, path_info):
|
||||
etag = self.blob_service.get_blob_properties(
|
||||
path_info.bucket, path_info.path
|
||||
).properties.etag
|
||||
return etag.strip('"')
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
return self.get_etag(path_info)
|
||||
|
||||
def remove(self, path_info):
|
||||
if path_info.scheme != self.scheme:
|
||||
raise NotImplementedError
|
||||
|
||||
logger.debug("Removing {}".format(path_info))
|
||||
logger.debug(f"Removing {path_info}")
|
||||
self.blob_service.delete_blob(path_info.bucket, path_info.path)
|
||||
|
||||
def _list_paths(self, bucket, prefix):
|
||||
def _list_paths(self, bucket, prefix, progress_callback=None):
|
||||
blob_service = self.blob_service
|
||||
next_marker = None
|
||||
while True:
|
||||
@@ -96,6 +81,8 @@ class RemoteAZURE(RemoteBASE):
|
||||
)
|
||||
|
||||
for blob in blobs:
|
||||
if progress_callback:
|
||||
progress_callback()
|
||||
yield blob.name
|
||||
|
||||
if not blobs.next_marker:
|
||||
@@ -103,8 +90,16 @@ class RemoteAZURE(RemoteBASE):
|
||||
|
||||
next_marker = blobs.next_marker
|
||||
|
||||
def list_cache_paths(self):
|
||||
return self._list_paths(self.path_info.bucket, self.path_info.path)
|
||||
def list_cache_paths(self, prefix=None, progress_callback=None):
|
||||
if prefix:
|
||||
prefix = posixpath.join(
|
||||
self.path_info.path, prefix[:2], prefix[2:]
|
||||
)
|
||||
else:
|
||||
prefix = self.path_info.path
|
||||
return self._list_paths(
|
||||
self.path_info.bucket, prefix, progress_callback
|
||||
)
|
||||
|
||||
def _upload(
|
||||
self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
|
||||
|
||||
+460
-150
@@ -1,32 +1,35 @@
|
||||
import errno
|
||||
import hashlib
|
||||
import itertools
|
||||
import json
|
||||
import logging
|
||||
import tempfile
|
||||
from urllib.parse import urlparse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from copy import copy
|
||||
from functools import partial
|
||||
from functools import partial, wraps
|
||||
from multiprocessing import cpu_count
|
||||
from operator import itemgetter
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from shortuuid import uuid
|
||||
|
||||
import dvc.prompt as prompt
|
||||
from dvc.exceptions import (
|
||||
CheckoutError,
|
||||
DvcException,
|
||||
ConfirmRemoveError,
|
||||
DvcException,
|
||||
DvcIgnoreInCollectedDirError,
|
||||
RemoteCacheRequiredError,
|
||||
)
|
||||
from dvc.ignore import DvcIgnore
|
||||
from dvc.path_info import PathInfo, URLInfo
|
||||
from dvc.path_info import PathInfo, URLInfo, WindowsPathInfo
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.remote.index import RemoteIndex, RemoteIndexNoop
|
||||
from dvc.remote.slow_link_detection import slow_link_guard
|
||||
from dvc.scm.tree import is_working_tree
|
||||
from dvc.state import StateNoop
|
||||
from dvc.utils import tmp_fname
|
||||
from dvc.utils.fs import move, makedirs
|
||||
from dvc.utils.fs import makedirs, move
|
||||
from dvc.utils.http import open_url
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -55,7 +58,7 @@ class RemoteCmdError(DvcException):
|
||||
|
||||
class RemoteActionNotImplemented(DvcException):
|
||||
def __init__(self, action, scheme):
|
||||
m = "{} is not supported for {} remotes".format(action, scheme)
|
||||
m = f"{action} is not supported for {scheme} remotes"
|
||||
super().__init__(m)
|
||||
|
||||
|
||||
@@ -66,22 +69,40 @@ class RemoteMissingDepsError(DvcException):
|
||||
class DirCacheError(DvcException):
|
||||
def __init__(self, checksum):
|
||||
super().__init__(
|
||||
"Failed to load dir cache for hash value: '{}'.".format(checksum)
|
||||
f"Failed to load dir cache for hash value: '{checksum}'."
|
||||
)
|
||||
|
||||
|
||||
class RemoteBASE(object):
|
||||
def index_locked(f):
|
||||
@wraps(f)
|
||||
def wrapper(remote_obj, *args, **kwargs):
|
||||
remote = kwargs.get("remote", remote_obj)
|
||||
with remote.index:
|
||||
return f(remote_obj, *args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
class BaseRemote:
|
||||
scheme = "base"
|
||||
path_cls = URLInfo
|
||||
REQUIRES = {}
|
||||
JOBS = 4 * cpu_count()
|
||||
INDEX_CLS = RemoteIndex
|
||||
|
||||
PARAM_RELPATH = "relpath"
|
||||
CHECKSUM_DIR_SUFFIX = ".dir"
|
||||
CHECKSUM_JOBS = max(1, min(4, cpu_count() // 2))
|
||||
DEFAULT_CACHE_TYPES = ["copy"]
|
||||
DEFAULT_NO_TRAVERSE = True
|
||||
DEFAULT_VERIFY = False
|
||||
LIST_OBJECT_PAGE_SIZE = 1000
|
||||
TRAVERSE_WEIGHT_MULTIPLIER = 5
|
||||
TRAVERSE_PREFIX_LEN = 3
|
||||
TRAVERSE_THRESHOLD_SIZE = 500000
|
||||
CAN_TRAVERSE = True
|
||||
|
||||
CACHE_MODE = None
|
||||
SHARED_MODE_MAP = {None: (None, None), "group": (None, None)}
|
||||
|
||||
state = StateNoop()
|
||||
|
||||
@@ -90,34 +111,48 @@ class RemoteBASE(object):
|
||||
|
||||
self._check_requires(config)
|
||||
|
||||
shared = config.get("shared")
|
||||
self._file_mode, self._dir_mode = self.SHARED_MODE_MAP[shared]
|
||||
|
||||
self.checksum_jobs = (
|
||||
config.get("checksum_jobs")
|
||||
or (self.repo and self.repo.config["core"].get("checksum_jobs"))
|
||||
or self.CHECKSUM_JOBS
|
||||
)
|
||||
self.protected = False
|
||||
self.no_traverse = config.get("no_traverse", self.DEFAULT_NO_TRAVERSE)
|
||||
self.verify = config.get("verify", self.DEFAULT_VERIFY)
|
||||
self._dir_info = {}
|
||||
|
||||
self.cache_types = config.get("type") or copy(self.DEFAULT_CACHE_TYPES)
|
||||
self.cache_type_confirmed = False
|
||||
|
||||
def _check_requires(self, config):
|
||||
url = config.get("url")
|
||||
if url:
|
||||
index_name = hashlib.sha256(url.encode("utf-8")).hexdigest()
|
||||
self.index = self.INDEX_CLS(
|
||||
self.repo, index_name, dir_suffix=self.CHECKSUM_DIR_SUFFIX
|
||||
)
|
||||
else:
|
||||
self.index = RemoteIndexNoop()
|
||||
|
||||
@classmethod
|
||||
def get_missing_deps(cls):
|
||||
import importlib
|
||||
|
||||
missing = []
|
||||
|
||||
for package, module in self.REQUIRES.items():
|
||||
for package, module in cls.REQUIRES.items():
|
||||
try:
|
||||
importlib.import_module(module)
|
||||
except ImportError:
|
||||
missing.append(package)
|
||||
|
||||
return missing
|
||||
|
||||
def _check_requires(self, config):
|
||||
missing = self.get_missing_deps()
|
||||
if not missing:
|
||||
return
|
||||
|
||||
url = config.get("url", "{}://".format(self.scheme))
|
||||
url = config.get("url", f"{self.scheme}://")
|
||||
msg = (
|
||||
"URL '{}' is supported but requires these missing "
|
||||
"dependencies: {}. If you have installed dvc using pip, "
|
||||
@@ -164,34 +199,46 @@ class RemoteBASE(object):
|
||||
|
||||
def _calculate_checksums(self, file_infos):
|
||||
file_infos = list(file_infos)
|
||||
with ThreadPoolExecutor(max_workers=self.checksum_jobs) as executor:
|
||||
tasks = executor.map(self.get_file_checksum, file_infos)
|
||||
with Tqdm(
|
||||
tasks,
|
||||
total=len(file_infos),
|
||||
unit="md5",
|
||||
desc="Computing file/dir hashes (only done once)",
|
||||
) as tasks:
|
||||
with Tqdm(
|
||||
total=len(file_infos),
|
||||
unit="md5",
|
||||
desc="Computing file/dir hashes (only done once)",
|
||||
) as pbar:
|
||||
worker = pbar.wrap_fn(self.get_file_checksum)
|
||||
with ThreadPoolExecutor(
|
||||
max_workers=self.checksum_jobs
|
||||
) as executor:
|
||||
tasks = executor.map(worker, file_infos)
|
||||
checksums = dict(zip(file_infos, tasks))
|
||||
return checksums
|
||||
|
||||
def _collect_dir(self, path_info):
|
||||
def _collect_dir(self, path_info, tree=None, save_tree=False, **kwargs):
|
||||
file_infos = set()
|
||||
|
||||
for fname in self.walk_files(path_info):
|
||||
if tree:
|
||||
walk_files = tree.walk_files
|
||||
else:
|
||||
walk_files = self.walk_files
|
||||
|
||||
for fname in walk_files(path_info, **kwargs):
|
||||
if DvcIgnore.DVCIGNORE_FILE == fname.name:
|
||||
raise DvcIgnoreInCollectedDirError(fname.parent)
|
||||
|
||||
file_infos.add(fname)
|
||||
|
||||
checksums = {fi: self.state.get(fi) for fi in file_infos}
|
||||
not_in_state = {
|
||||
fi for fi, checksum in checksums.items() if checksum is None
|
||||
}
|
||||
if tree:
|
||||
checksums = {fi: tree.get_file_checksum(fi) for fi in file_infos}
|
||||
if save_tree:
|
||||
for fi, checksum in checksums.items():
|
||||
self._save_file(fi, checksum, tree=tree, **kwargs)
|
||||
else:
|
||||
checksums = {fi: self.state.get(fi) for fi in file_infos}
|
||||
not_in_state = {
|
||||
fi for fi, checksum in checksums.items() if checksum is None
|
||||
}
|
||||
|
||||
new_checksums = self._calculate_checksums(not_in_state)
|
||||
|
||||
checksums.update(new_checksums)
|
||||
new_checksums = self._calculate_checksums(not_in_state)
|
||||
checksums.update(new_checksums)
|
||||
|
||||
result = [
|
||||
{
|
||||
@@ -212,18 +259,25 @@ class RemoteBASE(object):
|
||||
# Sorting the list by path to ensure reproducibility
|
||||
return sorted(result, key=itemgetter(self.PARAM_RELPATH))
|
||||
|
||||
def get_dir_checksum(self, path_info):
|
||||
def get_dir_checksum(self, path_info, tree=None):
|
||||
if not self.cache:
|
||||
raise RemoteCacheRequiredError(path_info)
|
||||
|
||||
dir_info = self._collect_dir(path_info)
|
||||
dir_info = self._collect_dir(path_info, tree=None)
|
||||
if tree:
|
||||
# don't save state entry for path_info if it is a tree path
|
||||
path_info = None
|
||||
return self._save_dir_info(dir_info, path_info)
|
||||
|
||||
def _save_dir_info(self, dir_info, path_info=None):
|
||||
checksum, tmp_info = self._get_dir_info_checksum(dir_info)
|
||||
new_info = self.cache.checksum_to_path_info(checksum)
|
||||
if self.cache.changed_cache_file(checksum):
|
||||
self.cache.makedirs(new_info.parent)
|
||||
self.cache.move(tmp_info, new_info)
|
||||
self.cache.move(tmp_info, new_info, mode=self.CACHE_MODE)
|
||||
|
||||
self.state.save(path_info, checksum)
|
||||
if path_info:
|
||||
self.state.save(path_info, checksum)
|
||||
self.state.save(new_info, checksum)
|
||||
|
||||
return checksum
|
||||
@@ -271,19 +325,24 @@ class RemoteBASE(object):
|
||||
)
|
||||
return []
|
||||
|
||||
for info in d:
|
||||
# NOTE: here is a BUG, see comment to .as_posix() below
|
||||
relative_path = PathInfo.from_posix(info[self.PARAM_RELPATH])
|
||||
info[self.PARAM_RELPATH] = relative_path.fspath
|
||||
if self.path_cls == WindowsPathInfo:
|
||||
# only need to convert it for Windows
|
||||
for info in d:
|
||||
# NOTE: here is a BUG, see comment to .as_posix() below
|
||||
info[self.PARAM_RELPATH] = info[self.PARAM_RELPATH].replace(
|
||||
"/", self.path_cls.sep
|
||||
)
|
||||
|
||||
return d
|
||||
|
||||
@classmethod
|
||||
def is_dir_checksum(cls, checksum):
|
||||
if not checksum:
|
||||
return False
|
||||
return checksum.endswith(cls.CHECKSUM_DIR_SUFFIX)
|
||||
|
||||
def get_checksum(self, path_info):
|
||||
assert path_info.scheme == self.scheme
|
||||
assert isinstance(path_info, str) or path_info.scheme == self.scheme
|
||||
|
||||
if not self.exists(path_info):
|
||||
return None
|
||||
@@ -379,10 +438,10 @@ class RemoteBASE(object):
|
||||
if self.cache_type_confirmed:
|
||||
return
|
||||
|
||||
is_link = getattr(self, "is_{}".format(link_type), None)
|
||||
is_link = getattr(self, f"is_{link_type}", None)
|
||||
if is_link and not is_link(path_info):
|
||||
self.remove(path_info)
|
||||
raise DvcException("failed to verify {}".format(link_type))
|
||||
raise DvcException(f"failed to verify {link_type}")
|
||||
|
||||
self.cache_type_confirmed = True
|
||||
|
||||
@@ -405,46 +464,53 @@ class RemoteBASE(object):
|
||||
|
||||
def _do_link(self, from_info, to_info, link_method):
|
||||
if self.exists(to_info):
|
||||
raise DvcException("Link '{}' already exists!".format(to_info))
|
||||
raise DvcException(f"Link '{to_info}' already exists!")
|
||||
|
||||
link_method(from_info, to_info)
|
||||
|
||||
if self.protected:
|
||||
self.protect(to_info)
|
||||
|
||||
logger.debug(
|
||||
"Created %s'%s': %s -> %s",
|
||||
"protected " if self.protected else "",
|
||||
self.cache_types[0],
|
||||
from_info,
|
||||
to_info,
|
||||
"Created '%s': %s -> %s", self.cache_types[0], from_info, to_info,
|
||||
)
|
||||
|
||||
def _save_file(self, path_info, checksum, save_link=True):
|
||||
def _save_file(
|
||||
self, path_info, checksum, save_link=True, tree=None, **kwargs
|
||||
):
|
||||
assert checksum
|
||||
|
||||
cache_info = self.checksum_to_path_info(checksum)
|
||||
if self.changed_cache(checksum):
|
||||
self.move(path_info, cache_info)
|
||||
self.link(cache_info, path_info)
|
||||
elif self.iscopy(path_info) and self._cache_is_copy(path_info):
|
||||
# Default relink procedure involves unneeded copy
|
||||
if self.protected:
|
||||
self.protect(path_info)
|
||||
else:
|
||||
self.unprotect(path_info)
|
||||
if tree:
|
||||
if self.changed_cache(checksum):
|
||||
with tree.open(path_info, mode="rb") as fobj:
|
||||
# if tree has fetch enabled, DVC out will be fetched on
|
||||
# open and we do not need to read/copy any data
|
||||
if not (
|
||||
tree.isdvc(path_info, strict=False) and tree.fetch
|
||||
):
|
||||
self.copy_fobj(fobj, cache_info)
|
||||
callback = kwargs.get("download_callback")
|
||||
if callback:
|
||||
callback(1)
|
||||
else:
|
||||
self.remove(path_info)
|
||||
self.link(cache_info, path_info)
|
||||
if self.changed_cache(checksum):
|
||||
self.move(path_info, cache_info, mode=self.CACHE_MODE)
|
||||
self.link(cache_info, path_info)
|
||||
elif self.iscopy(path_info) and self._cache_is_copy(path_info):
|
||||
# Default relink procedure involves unneeded copy
|
||||
self.unprotect(path_info)
|
||||
else:
|
||||
self.remove(path_info)
|
||||
self.link(cache_info, path_info)
|
||||
|
||||
if save_link:
|
||||
self.state.save_link(path_info)
|
||||
if save_link:
|
||||
self.state.save_link(path_info)
|
||||
|
||||
# we need to update path and cache, since in case of reflink,
|
||||
# or copy cache type moving original file results in updates on
|
||||
# next executed command, which causes md5 recalculation
|
||||
self.state.save(path_info, checksum)
|
||||
if not tree or is_working_tree(tree):
|
||||
self.state.save(path_info, checksum)
|
||||
self.state.save(cache_info, checksum)
|
||||
return {self.PARAM_CHECKSUM: checksum}
|
||||
|
||||
def _cache_is_copy(self, path_info):
|
||||
"""Checks whether cache uses copies."""
|
||||
@@ -468,20 +534,32 @@ class RemoteBASE(object):
|
||||
self.cache_type_confirmed = True
|
||||
return self.cache_types[0] == "copy"
|
||||
|
||||
def _save_dir(self, path_info, checksum):
|
||||
def _save_dir(
|
||||
self, path_info, checksum, save_link=True, tree=None, **kwargs
|
||||
):
|
||||
if tree:
|
||||
dir_info = self._collect_dir(
|
||||
path_info, tree=tree, save_tree=True, **kwargs
|
||||
)
|
||||
checksum = self._save_dir_info(dir_info)
|
||||
else:
|
||||
dir_info = self.get_dir_cache(checksum)
|
||||
|
||||
for entry in Tqdm(
|
||||
dir_info, desc="Saving " + path_info.name, unit="file"
|
||||
):
|
||||
entry_info = path_info / entry[self.PARAM_RELPATH]
|
||||
entry_checksum = entry[self.PARAM_CHECKSUM]
|
||||
self._save_file(entry_info, entry_checksum, save_link=False)
|
||||
|
||||
if save_link:
|
||||
self.state.save_link(path_info)
|
||||
|
||||
cache_info = self.checksum_to_path_info(checksum)
|
||||
dir_info = self.get_dir_cache(checksum)
|
||||
|
||||
for entry in Tqdm(
|
||||
dir_info, desc="Saving " + path_info.name, unit="file"
|
||||
):
|
||||
entry_info = path_info / entry[self.PARAM_RELPATH]
|
||||
entry_checksum = entry[self.PARAM_CHECKSUM]
|
||||
self._save_file(entry_info, entry_checksum, save_link=False)
|
||||
|
||||
self.state.save_link(path_info)
|
||||
self.state.save(cache_info, checksum)
|
||||
self.state.save(path_info, checksum)
|
||||
if not tree or is_working_tree(tree):
|
||||
self.state.save(path_info, checksum)
|
||||
return {self.PARAM_CHECKSUM: checksum}
|
||||
|
||||
def is_empty(self, path_info):
|
||||
return False
|
||||
@@ -510,23 +588,42 @@ class RemoteBASE(object):
|
||||
def protect(path_info):
|
||||
pass
|
||||
|
||||
def save(self, path_info, checksum_info):
|
||||
def save(
|
||||
self, path_info, checksum_info, save_link=True, tree=None, **kwargs
|
||||
):
|
||||
if path_info.scheme != self.scheme:
|
||||
raise RemoteActionNotImplemented(
|
||||
"save {} -> {}".format(path_info.scheme, self.scheme),
|
||||
self.scheme,
|
||||
f"save {path_info.scheme} -> {self.scheme}", self.scheme,
|
||||
)
|
||||
|
||||
checksum = checksum_info[self.PARAM_CHECKSUM]
|
||||
self._save(path_info, checksum)
|
||||
if tree:
|
||||
if tree.isdir(path_info):
|
||||
# save checksum will be computed during tree walk
|
||||
checksum = None
|
||||
else:
|
||||
checksum = tree.get_file_checksum(path_info)
|
||||
else:
|
||||
checksum = checksum_info[self.PARAM_CHECKSUM]
|
||||
return self._save(path_info, checksum, save_link, tree, **kwargs)
|
||||
|
||||
def _save(self, path_info, checksum):
|
||||
to_info = self.checksum_to_path_info(checksum)
|
||||
logger.debug("Saving '%s' to '%s'.", path_info, to_info)
|
||||
if self.isdir(path_info):
|
||||
self._save_dir(path_info, checksum)
|
||||
return
|
||||
self._save_file(path_info, checksum)
|
||||
def _save(self, path_info, checksum, save_link=True, tree=None, **kwargs):
|
||||
if tree:
|
||||
logger.debug("Saving tree path '%s' to cache.", path_info)
|
||||
else:
|
||||
to_info = self.checksum_to_path_info(checksum)
|
||||
logger.debug("Saving '%s' to '%s'.", path_info, to_info)
|
||||
|
||||
if tree:
|
||||
isdir = tree.isdir
|
||||
save_link = False
|
||||
else:
|
||||
isdir = self.isdir
|
||||
|
||||
if isdir(path_info):
|
||||
return self._save_dir(
|
||||
path_info, checksum, save_link, tree, **kwargs
|
||||
)
|
||||
return self._save_file(path_info, checksum, save_link, tree, **kwargs)
|
||||
|
||||
def _handle_transfer_exception(
|
||||
self, from_info, to_info, exception, operation
|
||||
@@ -605,22 +702,23 @@ class RemoteBASE(object):
|
||||
to_info / info.relative_to(from_info) for info in from_infos
|
||||
)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=self.JOBS) as executor:
|
||||
download_files = partial(
|
||||
self._download_file,
|
||||
name=name,
|
||||
no_progress_bar=True,
|
||||
file_mode=file_mode,
|
||||
dir_mode=dir_mode,
|
||||
with Tqdm(
|
||||
total=len(from_infos),
|
||||
desc="Downloading directory",
|
||||
unit="Files",
|
||||
disable=no_progress_bar,
|
||||
) as pbar:
|
||||
download_files = pbar.wrap_fn(
|
||||
partial(
|
||||
self._download_file,
|
||||
name=name,
|
||||
no_progress_bar=True,
|
||||
file_mode=file_mode,
|
||||
dir_mode=dir_mode,
|
||||
)
|
||||
)
|
||||
futures = executor.map(download_files, from_infos, to_infos)
|
||||
with Tqdm(
|
||||
futures,
|
||||
total=len(from_infos),
|
||||
desc="Downloading directory",
|
||||
unit="Files",
|
||||
disable=no_progress_bar,
|
||||
) as futures:
|
||||
with ThreadPoolExecutor(max_workers=self.JOBS) as executor:
|
||||
futures = executor.map(download_files, from_infos, to_infos)
|
||||
return sum(futures)
|
||||
|
||||
def _download_file(
|
||||
@@ -656,13 +754,17 @@ class RemoteBASE(object):
|
||||
def remove(self, path_info):
|
||||
raise RemoteActionNotImplemented("remove", self.scheme)
|
||||
|
||||
def move(self, from_info, to_info):
|
||||
def move(self, from_info, to_info, mode=None):
|
||||
assert mode is None
|
||||
self.copy(from_info, to_info)
|
||||
self.remove(from_info)
|
||||
|
||||
def copy(self, from_info, to_info):
|
||||
raise RemoteActionNotImplemented("copy", self.scheme)
|
||||
|
||||
def copy_fobj(self, fobj, to_info):
|
||||
raise RemoteActionNotImplemented("copy_fobj", self.scheme)
|
||||
|
||||
def symlink(self, from_info, to_info):
|
||||
raise RemoteActionNotImplemented("symlink", self.scheme)
|
||||
|
||||
@@ -679,45 +781,83 @@ class RemoteBASE(object):
|
||||
parts = self.path_cls(path).parts[-2:]
|
||||
|
||||
if not (len(parts) == 2 and parts[0] and len(parts[0]) == 2):
|
||||
raise ValueError("Bad cache file path")
|
||||
raise ValueError(f"Bad cache file path '{path}'")
|
||||
|
||||
return "".join(parts)
|
||||
|
||||
def checksum_to_path_info(self, checksum):
|
||||
return self.path_info / checksum[0:2] / checksum[2:]
|
||||
|
||||
def list_cache_paths(self):
|
||||
# Return path as a string instead of PathInfo for remotes which support
|
||||
# string paths (see local)
|
||||
checksum_to_path = checksum_to_path_info
|
||||
|
||||
def list_cache_paths(self, prefix=None, progress_callback=None):
|
||||
raise NotImplementedError
|
||||
|
||||
def all(self):
|
||||
# NOTE: The list might be way too big(e.g. 100M entries, md5 for each
|
||||
# is 32 bytes, so ~3200Mb list) and we don't really need all of it at
|
||||
# the same time, so it makes sense to use a generator to gradually
|
||||
# iterate over it, without keeping all of it in memory.
|
||||
for path in self.list_cache_paths():
|
||||
def cache_checksums(self, prefix=None, progress_callback=None):
|
||||
"""Iterate over remote cache checksums.
|
||||
|
||||
If `prefix` is specified, only checksums which begin with `prefix`
|
||||
will be returned.
|
||||
"""
|
||||
for path in self.list_cache_paths(prefix, progress_callback):
|
||||
try:
|
||||
yield self.path_to_checksum(path)
|
||||
except ValueError:
|
||||
# We ignore all the non-cache looking files
|
||||
pass
|
||||
logger.debug(
|
||||
"'%s' doesn't look like a cache file, skipping", path
|
||||
)
|
||||
|
||||
def gc(self, named_cache):
|
||||
used = self.extract_used_local_checksums(named_cache)
|
||||
def all(self, jobs=None, name=None):
|
||||
"""Iterate over all checksums in the remote cache.
|
||||
|
||||
Checksums will be fetched in parallel threads according to prefix
|
||||
(except for small remotes) and a progress bar will be displayed.
|
||||
"""
|
||||
logger.debug(
|
||||
"Fetching all checksums from '{}'".format(
|
||||
name if name else "remote cache"
|
||||
)
|
||||
)
|
||||
|
||||
if not self.CAN_TRAVERSE:
|
||||
return self.cache_checksums()
|
||||
|
||||
remote_size, remote_checksums = self._estimate_cache_size(name=name)
|
||||
return self._cache_checksums_traverse(
|
||||
remote_size, remote_checksums, jobs, name
|
||||
)
|
||||
|
||||
@index_locked
|
||||
def gc(self, named_cache, jobs=None):
|
||||
used = set(named_cache.scheme_keys("local"))
|
||||
|
||||
if self.scheme != "":
|
||||
used.update(named_cache[self.scheme])
|
||||
used.update(named_cache.scheme_keys(self.scheme))
|
||||
|
||||
removed = False
|
||||
for checksum in self.all():
|
||||
# checksums must be sorted to ensure we always remove .dir files first
|
||||
for checksum in sorted(
|
||||
self.all(jobs, str(self.path_info)),
|
||||
key=self.is_dir_checksum,
|
||||
reverse=True,
|
||||
):
|
||||
if checksum in used:
|
||||
continue
|
||||
path_info = self.checksum_to_path_info(checksum)
|
||||
if self.is_dir_checksum(checksum):
|
||||
# backward compatibility
|
||||
self._remove_unpacked_dir(checksum)
|
||||
self.remove(path_info)
|
||||
removed = True
|
||||
if removed:
|
||||
self.index.clear()
|
||||
return removed
|
||||
|
||||
def is_protected(self, path_info):
|
||||
return False
|
||||
|
||||
def changed_cache_file(self, checksum):
|
||||
"""Compare the given checksum with the (corresponding) actual one.
|
||||
|
||||
@@ -730,7 +870,14 @@ class RemoteBASE(object):
|
||||
|
||||
- Remove the file from cache if it doesn't match the actual checksum
|
||||
"""
|
||||
cache_info = self.checksum_to_path_info(checksum)
|
||||
# Prefer string path over PathInfo when possible due to performance
|
||||
cache_info = self.checksum_to_path(checksum)
|
||||
if self.is_protected(cache_info):
|
||||
logger.debug(
|
||||
"Assuming '%s' is unchanged since it is read-only", cache_info
|
||||
)
|
||||
return False
|
||||
|
||||
actual = self.get_checksum(cache_info)
|
||||
|
||||
logger.debug(
|
||||
@@ -744,6 +891,9 @@ class RemoteBASE(object):
|
||||
return True
|
||||
|
||||
if actual.split(".")[0] == checksum.split(".")[0]:
|
||||
# making cache file read-only so we don't need to check it
|
||||
# next time
|
||||
self.protect(cache_info)
|
||||
return False
|
||||
|
||||
if self.exists(cache_info):
|
||||
@@ -756,11 +906,6 @@ class RemoteBASE(object):
|
||||
if self.changed_cache_file(checksum):
|
||||
return True
|
||||
|
||||
if not (path_info and filter_info) and not self._changed_unpacked_dir(
|
||||
checksum
|
||||
):
|
||||
return False
|
||||
|
||||
for entry in self.get_dir_cache(checksum):
|
||||
entry_checksum = entry[self.PARAM_CHECKSUM]
|
||||
|
||||
@@ -772,9 +917,6 @@ class RemoteBASE(object):
|
||||
if self.changed_cache_file(entry_checksum):
|
||||
return True
|
||||
|
||||
if not (path_info and filter_info):
|
||||
self._update_unpacked_dir(checksum)
|
||||
|
||||
return False
|
||||
|
||||
def changed_cache(self, checksum, path_info=None, filter_info=None):
|
||||
@@ -789,11 +931,13 @@ class RemoteBASE(object):
|
||||
|
||||
There are two ways of performing this check:
|
||||
|
||||
- Traverse: Get a list of all the files in the remote
|
||||
- Traverse method: Get a list of all the files in the remote
|
||||
(traversing the cache directory) and compare it with
|
||||
the given checksums.
|
||||
the given checksums. Cache entries will be retrieved in parallel
|
||||
threads according to prefix (i.e. entries starting with, "00...",
|
||||
"01...", and so on) and a progress bar will be displayed.
|
||||
|
||||
- No traverse: For each given checksum, run the `exists`
|
||||
- Exists method: For each given checksum, run the `exists`
|
||||
method and filter the checksums that aren't on the remote.
|
||||
This is done in parallel threads.
|
||||
It also shows a progress bar when performing the check.
|
||||
@@ -804,12 +948,192 @@ class RemoteBASE(object):
|
||||
check if particular file exists much quicker, use their own
|
||||
implementation of cache_exists (see ssh, local).
|
||||
|
||||
Which method to use will be automatically determined after estimating
|
||||
the size of the remote cache, and comparing the estimated size with
|
||||
len(checksums). To estimate the size of the remote cache, we fetch
|
||||
a small subset of cache entries (i.e. entries starting with "00...").
|
||||
Based on the number of entries in that subset, the size of the full
|
||||
cache can be estimated, since the cache is evenly distributed according
|
||||
to checksum.
|
||||
|
||||
Returns:
|
||||
A list with checksums that were found in the remote
|
||||
"""
|
||||
if not self.no_traverse:
|
||||
return list(set(checksums) & set(self.all()))
|
||||
# Remotes which do not use traverse prefix should override
|
||||
# cache_exists() (see ssh, local)
|
||||
assert self.TRAVERSE_PREFIX_LEN >= 2
|
||||
|
||||
checksums = set(checksums)
|
||||
indexed_checksums = set(self.index.intersection(checksums))
|
||||
checksums -= indexed_checksums
|
||||
logger.debug(
|
||||
"Matched '{}' indexed checksums".format(len(indexed_checksums))
|
||||
)
|
||||
if not checksums:
|
||||
return indexed_checksums
|
||||
|
||||
if len(checksums) == 1 or not self.CAN_TRAVERSE:
|
||||
remote_checksums = self._cache_object_exists(checksums, jobs, name)
|
||||
return list(indexed_checksums) + remote_checksums
|
||||
|
||||
# Max remote size allowed for us to use traverse method
|
||||
remote_size, remote_checksums = self._estimate_cache_size(
|
||||
checksums, name
|
||||
)
|
||||
|
||||
traverse_pages = remote_size / self.LIST_OBJECT_PAGE_SIZE
|
||||
# For sufficiently large remotes, traverse must be weighted to account
|
||||
# for performance overhead from large lists/sets.
|
||||
# From testing with S3, for remotes with 1M+ files, object_exists is
|
||||
# faster until len(checksums) is at least 10k~100k
|
||||
if remote_size > self.TRAVERSE_THRESHOLD_SIZE:
|
||||
traverse_weight = traverse_pages * self.TRAVERSE_WEIGHT_MULTIPLIER
|
||||
else:
|
||||
traverse_weight = traverse_pages
|
||||
if len(checksums) < traverse_weight:
|
||||
logger.debug(
|
||||
"Large remote ('{}' checksums < '{}' traverse weight), "
|
||||
"using object_exists for remaining checksums".format(
|
||||
len(checksums), traverse_weight
|
||||
)
|
||||
)
|
||||
return (
|
||||
list(indexed_checksums)
|
||||
+ list(checksums & remote_checksums)
|
||||
+ self._cache_object_exists(
|
||||
checksums - remote_checksums, jobs, name
|
||||
)
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
"Querying '{}' checksums via traverse".format(len(checksums))
|
||||
)
|
||||
remote_checksums = set(
|
||||
self._cache_checksums_traverse(
|
||||
remote_size, remote_checksums, jobs, name
|
||||
)
|
||||
)
|
||||
return list(indexed_checksums) + list(
|
||||
checksums & set(remote_checksums)
|
||||
)
|
||||
|
||||
def _checksums_with_limit(
|
||||
self, limit, prefix=None, progress_callback=None
|
||||
):
|
||||
count = 0
|
||||
for checksum in self.cache_checksums(prefix, progress_callback):
|
||||
yield checksum
|
||||
count += 1
|
||||
if count > limit:
|
||||
logger.debug(
|
||||
"`cache_checksums()` returned max '{}' checksums, "
|
||||
"skipping remaining results".format(limit)
|
||||
)
|
||||
return
|
||||
|
||||
def _max_estimation_size(self, checksums):
|
||||
# Max remote size allowed for us to use traverse method
|
||||
return max(
|
||||
self.TRAVERSE_THRESHOLD_SIZE,
|
||||
len(checksums)
|
||||
/ self.TRAVERSE_WEIGHT_MULTIPLIER
|
||||
* self.LIST_OBJECT_PAGE_SIZE,
|
||||
)
|
||||
|
||||
def _estimate_cache_size(self, checksums=None, name=None):
|
||||
"""Estimate remote cache size based on number of entries beginning with
|
||||
"00..." prefix.
|
||||
"""
|
||||
prefix = "0" * self.TRAVERSE_PREFIX_LEN
|
||||
total_prefixes = pow(16, self.TRAVERSE_PREFIX_LEN)
|
||||
if checksums:
|
||||
max_checksums = self._max_estimation_size(checksums)
|
||||
else:
|
||||
max_checksums = None
|
||||
|
||||
with Tqdm(
|
||||
desc="Estimating size of "
|
||||
+ (f"cache in '{name}'" if name else "remote cache"),
|
||||
unit="file",
|
||||
) as pbar:
|
||||
|
||||
def update(n=1):
|
||||
pbar.update(n * total_prefixes)
|
||||
|
||||
if max_checksums:
|
||||
checksums = self._checksums_with_limit(
|
||||
max_checksums / total_prefixes, prefix, update
|
||||
)
|
||||
else:
|
||||
checksums = self.cache_checksums(prefix, update)
|
||||
|
||||
remote_checksums = set(checksums)
|
||||
if remote_checksums:
|
||||
remote_size = total_prefixes * len(remote_checksums)
|
||||
else:
|
||||
remote_size = total_prefixes
|
||||
logger.debug(f"Estimated remote size: {remote_size} files")
|
||||
return remote_size, remote_checksums
|
||||
|
||||
def _cache_checksums_traverse(
|
||||
self, remote_size, remote_checksums, jobs=None, name=None
|
||||
):
|
||||
"""Iterate over all checksums in the remote cache.
|
||||
Checksums are fetched in parallel according to prefix, except in
|
||||
cases where the remote size is very small.
|
||||
|
||||
All checksums from the remote (including any from the size
|
||||
estimation step passed via the `remote_checksums` argument) will be
|
||||
returned.
|
||||
|
||||
NOTE: For large remotes the list of checksums will be very
|
||||
big(e.g. 100M entries, md5 for each is 32 bytes, so ~3200Mb list)
|
||||
and we don't really need all of it at the same time, so it makes
|
||||
sense to use a generator to gradually iterate over it, without
|
||||
keeping all of it in memory.
|
||||
"""
|
||||
num_pages = remote_size / self.LIST_OBJECT_PAGE_SIZE
|
||||
if num_pages < 256 / self.JOBS:
|
||||
# Fetching prefixes in parallel requires at least 255 more
|
||||
# requests, for small enough remotes it will be faster to fetch
|
||||
# entire cache without splitting it into prefixes.
|
||||
#
|
||||
# NOTE: this ends up re-fetching checksums that were already
|
||||
# fetched during remote size estimation
|
||||
traverse_prefixes = [None]
|
||||
initial = 0
|
||||
else:
|
||||
yield from remote_checksums
|
||||
initial = len(remote_checksums)
|
||||
traverse_prefixes = [f"{i:02x}" for i in range(1, 256)]
|
||||
if self.TRAVERSE_PREFIX_LEN > 2:
|
||||
traverse_prefixes += [
|
||||
"{0:0{1}x}".format(i, self.TRAVERSE_PREFIX_LEN)
|
||||
for i in range(1, pow(16, self.TRAVERSE_PREFIX_LEN - 2))
|
||||
]
|
||||
with Tqdm(
|
||||
desc="Querying "
|
||||
+ (f"cache in '{name}'" if name else "remote cache"),
|
||||
total=remote_size,
|
||||
initial=initial,
|
||||
unit="file",
|
||||
) as pbar:
|
||||
|
||||
def list_with_update(prefix):
|
||||
return list(
|
||||
self.cache_checksums(
|
||||
prefix=prefix, progress_callback=pbar.update
|
||||
)
|
||||
)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=jobs or self.JOBS) as executor:
|
||||
in_remote = executor.map(list_with_update, traverse_prefixes,)
|
||||
yield from itertools.chain.from_iterable(in_remote)
|
||||
|
||||
def _cache_object_exists(self, checksums, jobs=None, name=None):
|
||||
logger.debug(
|
||||
"Querying {} checksums via object_exists".format(len(checksums))
|
||||
)
|
||||
with Tqdm(
|
||||
desc="Querying "
|
||||
+ ("cache in " + name if name else "remote cache"),
|
||||
@@ -819,7 +1143,7 @@ class RemoteBASE(object):
|
||||
|
||||
def exists_with_progress(path_info):
|
||||
ret = self.exists(path_info)
|
||||
pbar.update_desc(str(path_info))
|
||||
pbar.update_msg(str(path_info))
|
||||
return ret
|
||||
|
||||
with ThreadPoolExecutor(max_workers=jobs or self.JOBS) as executor:
|
||||
@@ -1031,19 +1355,5 @@ class RemoteBASE(object):
|
||||
def unprotect(path_info):
|
||||
pass
|
||||
|
||||
def _get_unpacked_dir_names(self, checksums):
|
||||
return set()
|
||||
|
||||
def extract_used_local_checksums(self, named_cache):
|
||||
used = set(named_cache["local"])
|
||||
unpacked = self._get_unpacked_dir_names(used)
|
||||
return used | unpacked
|
||||
|
||||
def _changed_unpacked_dir(self, checksum):
|
||||
return True
|
||||
|
||||
def _update_unpacked_dir(self, checksum):
|
||||
pass
|
||||
|
||||
def _remove_unpacked_dir(self, checksum):
|
||||
pass
|
||||
|
||||
+333
-202
@@ -1,54 +1,72 @@
|
||||
from collections import defaultdict
|
||||
import logging
|
||||
import os
|
||||
import posixpath
|
||||
import logging
|
||||
import re
|
||||
import threading
|
||||
from collections import defaultdict
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from funcy import retry, wrap_with, wrap_prop, cached_property
|
||||
from funcy import cached_property, retry, wrap_prop, wrap_with
|
||||
from funcy.py3 import cat
|
||||
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.path_info import CloudURLInfo
|
||||
from dvc.remote.base import RemoteBASE
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.utils import tmp_fname, format_link
|
||||
from dvc.path_info import CloudURLInfo
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.remote.base import BaseRemote
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.utils import format_link, tmp_fname
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"
|
||||
|
||||
|
||||
class GDrivePathNotFound(DvcException):
|
||||
def __init__(self, path_info):
|
||||
super().__init__("Google Drive path '{}' not found.".format(path_info))
|
||||
def __init__(self, path_info, hint):
|
||||
hint = "" if hint is None else f" {hint}"
|
||||
super().__init__(f"GDrive path '{path_info}' not found.{hint}")
|
||||
|
||||
|
||||
class GDriveAccessTokenRefreshError(DvcException):
|
||||
def __init__(self):
|
||||
super().__init__("Google Drive access token refreshment is failed.")
|
||||
class GDriveAuthError(DvcException):
|
||||
def __init__(self, cred_location):
|
||||
|
||||
if cred_location:
|
||||
message = (
|
||||
"GDrive remote auth failed with credentials in '{}'.\n"
|
||||
"Backup first, remove of fix them, and run DVC again.\n"
|
||||
"It should do auth again and refresh the credentials.\n\n"
|
||||
"Details:".format(cred_location)
|
||||
)
|
||||
else:
|
||||
message = "Failed to authenticate GDrive remote"
|
||||
|
||||
super().__init__(message)
|
||||
|
||||
|
||||
class GDriveMissedCredentialKeyError(DvcException):
|
||||
def __init__(self, path):
|
||||
super().__init__(
|
||||
"Google Drive user credentials file '{}' "
|
||||
"misses value for key.".format(path)
|
||||
)
|
||||
|
||||
|
||||
def gdrive_retry(func):
|
||||
retry_re = re.compile(r"HttpError (403|500|502|503|504)")
|
||||
|
||||
def _gdrive_retry(func):
|
||||
def should_retry(exc):
|
||||
from pydrive2.files import ApiRequestError
|
||||
|
||||
return isinstance(exc, ApiRequestError) and retry_re.search(str(exc))
|
||||
if not isinstance(exc, ApiRequestError):
|
||||
return False
|
||||
|
||||
# 15 tries, start at 0.5s, multiply by golden ratio, cap at 20s
|
||||
error_code = exc.error.get("code", 0)
|
||||
result = False
|
||||
if 500 <= error_code < 600:
|
||||
result = True
|
||||
|
||||
if error_code == 403:
|
||||
result = exc.GetField("reason") in [
|
||||
"userRateLimitExceeded",
|
||||
"rateLimitExceeded",
|
||||
]
|
||||
if result:
|
||||
logger.debug(f"Retrying GDrive API call, error: {exc}.")
|
||||
|
||||
return result
|
||||
|
||||
# 16 tries, start at 0.5s, multiply by golden ratio, cap at 20s
|
||||
return retry(
|
||||
15,
|
||||
16,
|
||||
timeout=lambda a: min(0.5 * 1.618 ** a, 20),
|
||||
filter_errors=should_retry,
|
||||
)(func)
|
||||
@@ -69,30 +87,35 @@ class GDriveURLInfo(CloudURLInfo):
|
||||
self._spath = re.sub("/{2,}", "/", self._spath.rstrip("/"))
|
||||
|
||||
|
||||
class RemoteGDrive(RemoteBASE):
|
||||
class GDriveRemote(BaseRemote):
|
||||
scheme = Schemes.GDRIVE
|
||||
path_cls = GDriveURLInfo
|
||||
REQUIRES = {"pydrive2": "pydrive2"}
|
||||
DEFAULT_NO_TRAVERSE = False
|
||||
DEFAULT_VERIFY = True
|
||||
# Always prefer traverse for GDrive since API usage quotas are a concern.
|
||||
TRAVERSE_WEIGHT_MULTIPLIER = 1
|
||||
TRAVERSE_PREFIX_LEN = 2
|
||||
|
||||
GDRIVE_CREDENTIALS_DATA = "GDRIVE_CREDENTIALS_DATA"
|
||||
DEFAULT_USER_CREDENTIALS_FILE = "gdrive-user-credentials.json"
|
||||
|
||||
DEFAULT_GDRIVE_CLIENT_ID = "710796635688-iivsgbgsb6uv1fap6635dhvuei09o66c.apps.googleusercontent.com" # noqa: E501
|
||||
DEFAULT_GDRIVE_CLIENT_SECRET = "a1Fz59uTpVNeG_VGuSKDLJXv"
|
||||
|
||||
def __init__(self, repo, config):
|
||||
super().__init__(repo, config)
|
||||
self.path_info = self.path_cls(config["url"])
|
||||
|
||||
if not self.path_info.bucket:
|
||||
raise DvcException(
|
||||
"Empty Google Drive URL '{}'. Learn more at "
|
||||
"{}.".format(
|
||||
"Empty GDrive URL '{}'. Learn more at {}".format(
|
||||
config["url"],
|
||||
format_link("https://man.dvc.org/remote/add"),
|
||||
)
|
||||
)
|
||||
|
||||
self._bucket = self.path_info.bucket
|
||||
self._trash_only = config.get("gdrive_trash_only")
|
||||
self._use_service_account = config.get("gdrive_use_service_account")
|
||||
self._service_account_email = config.get(
|
||||
"gdrive_service_account_email"
|
||||
@@ -108,7 +131,7 @@ class RemoteGDrive(RemoteBASE):
|
||||
self._validate_config()
|
||||
self._gdrive_user_credentials_path = (
|
||||
tmp_fname(os.path.join(self.repo.tmp_dir, ""))
|
||||
if os.getenv(RemoteGDrive.GDRIVE_CREDENTIALS_DATA)
|
||||
if os.getenv(GDriveRemote.GDRIVE_CREDENTIALS_DATA)
|
||||
else config.get(
|
||||
"gdrive_user_credentials_file",
|
||||
os.path.join(
|
||||
@@ -117,8 +140,6 @@ class RemoteGDrive(RemoteBASE):
|
||||
)
|
||||
)
|
||||
|
||||
self._list_params = None
|
||||
|
||||
def _validate_config(self):
|
||||
# Validate Service Account configuration
|
||||
if self._use_service_account and (
|
||||
@@ -126,125 +147,208 @@ class RemoteGDrive(RemoteBASE):
|
||||
or not self._service_account_p12_file_path
|
||||
):
|
||||
raise DvcException(
|
||||
"To use service account please specify {}, {} and "
|
||||
"{} in DVC config. Learn more at "
|
||||
"{}.".format(
|
||||
"gdrive_service_account_email",
|
||||
"gdrive_service_account_p12_file_path",
|
||||
"gdrive_service_account_user_email (optional)",
|
||||
format_link("https://man.dvc.org/remote/modify"),
|
||||
"To use service account, set `gdrive_service_account_email`,\n"
|
||||
"`gdrive_service_account_p12_file_path`, and optionally "
|
||||
"`gdrive_service_account_user_email`\nin DVC config. "
|
||||
"Learn more at {}".format(
|
||||
format_link("https://man.dvc.org/remote/modify")
|
||||
)
|
||||
)
|
||||
|
||||
# Validate OAuth 2.0 Client ID configuration
|
||||
if not self._use_service_account and (
|
||||
not self._client_id or not self._client_secret
|
||||
):
|
||||
raise DvcException(
|
||||
"Please specify Google Drive's client id and "
|
||||
"secret in DVC config. Learn more at "
|
||||
"{}.".format(format_link("https://man.dvc.org/remote/modify"))
|
||||
)
|
||||
if not self._use_service_account:
|
||||
if bool(self._client_id) != bool(self._client_secret):
|
||||
raise DvcException(
|
||||
"Please specify GDrive's client ID and secret in "
|
||||
"DVC config or omit both to use the defaults.\n"
|
||||
"Learn more at {}".format(
|
||||
format_link("https://man.dvc.org/remote/modify")
|
||||
)
|
||||
)
|
||||
|
||||
@cached_property
|
||||
def credentials_location(self):
|
||||
"""
|
||||
Helper to determine where will GDrive remote read credentials from.
|
||||
Useful for tests, exception messages, etc. Returns either env variable
|
||||
name if it's set or actual path to the credentials file.
|
||||
"""
|
||||
if os.getenv(GDriveRemote.GDRIVE_CREDENTIALS_DATA):
|
||||
return GDriveRemote.GDRIVE_CREDENTIALS_DATA
|
||||
if os.path.exists(self._gdrive_user_credentials_path):
|
||||
return self._gdrive_user_credentials_path
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _validate_credentials(auth, settings):
|
||||
"""
|
||||
Detects discrepancy in DVC config and cached credentials file.
|
||||
Usually happens when a second remote is added and it is using
|
||||
the same credentials default file. Or when someones decides to change
|
||||
DVC config client id or secret but forgets to remove the cached
|
||||
credentials file.
|
||||
"""
|
||||
if not os.getenv(GDriveRemote.GDRIVE_CREDENTIALS_DATA):
|
||||
if (
|
||||
settings["client_config"]["client_id"]
|
||||
!= auth.credentials.client_id
|
||||
or settings["client_config"]["client_secret"]
|
||||
!= auth.credentials.client_secret
|
||||
):
|
||||
logger.warning(
|
||||
"Client ID and secret configured do not match the "
|
||||
"actual ones used\nto access the remote. Do you "
|
||||
"use multiple GDrive remotes and forgot to\nset "
|
||||
"`gdrive_user_credentials_file` for one or more of them? "
|
||||
"Learn more at\n{}.\n".format(
|
||||
format_link("https://man.dvc.org/remote/modify")
|
||||
)
|
||||
)
|
||||
|
||||
@wrap_prop(threading.RLock())
|
||||
@cached_property
|
||||
def drive(self):
|
||||
from pydrive2.auth import RefreshError
|
||||
def _drive(self):
|
||||
from pydrive2.auth import GoogleAuth
|
||||
from pydrive2.drive import GoogleDrive
|
||||
|
||||
if os.getenv(RemoteGDrive.GDRIVE_CREDENTIALS_DATA):
|
||||
with open(
|
||||
self._gdrive_user_credentials_path, "w"
|
||||
) as credentials_file:
|
||||
credentials_file.write(
|
||||
os.getenv(RemoteGDrive.GDRIVE_CREDENTIALS_DATA)
|
||||
if os.getenv(GDriveRemote.GDRIVE_CREDENTIALS_DATA):
|
||||
with open(self._gdrive_user_credentials_path, "w") as cred_file:
|
||||
cred_file.write(
|
||||
os.getenv(GDriveRemote.GDRIVE_CREDENTIALS_DATA)
|
||||
)
|
||||
|
||||
GoogleAuth.DEFAULT_SETTINGS["client_config_backend"] = "settings"
|
||||
auth_settings = {
|
||||
"client_config_backend": "settings",
|
||||
"save_credentials": True,
|
||||
"save_credentials_backend": "file",
|
||||
"save_credentials_file": self._gdrive_user_credentials_path,
|
||||
"get_refresh_token": True,
|
||||
"oauth_scope": [
|
||||
"https://www.googleapis.com/auth/drive",
|
||||
"https://www.googleapis.com/auth/drive.appdata",
|
||||
],
|
||||
}
|
||||
|
||||
if self._use_service_account:
|
||||
GoogleAuth.DEFAULT_SETTINGS["service_config"] = {
|
||||
auth_settings["service_config"] = {
|
||||
"client_service_email": self._service_account_email,
|
||||
"client_user_email": self._service_account_user_email,
|
||||
"client_pkcs12_file_path": self._service_account_p12_file_path,
|
||||
}
|
||||
else:
|
||||
GoogleAuth.DEFAULT_SETTINGS["client_config"] = {
|
||||
"client_id": self._client_id,
|
||||
"client_secret": self._client_secret,
|
||||
auth_settings["client_config"] = {
|
||||
"client_id": self._client_id or self.DEFAULT_GDRIVE_CLIENT_ID,
|
||||
"client_secret": self._client_secret
|
||||
or self.DEFAULT_GDRIVE_CLIENT_SECRET,
|
||||
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
|
||||
"token_uri": "https://oauth2.googleapis.com/token",
|
||||
"revoke_uri": "https://oauth2.googleapis.com/revoke",
|
||||
"redirect_uri": "",
|
||||
}
|
||||
GoogleAuth.DEFAULT_SETTINGS["save_credentials"] = True
|
||||
GoogleAuth.DEFAULT_SETTINGS["save_credentials_backend"] = "file"
|
||||
GoogleAuth.DEFAULT_SETTINGS[
|
||||
"save_credentials_file"
|
||||
] = self._gdrive_user_credentials_path
|
||||
GoogleAuth.DEFAULT_SETTINGS["get_refresh_token"] = True
|
||||
GoogleAuth.DEFAULT_SETTINGS["oauth_scope"] = [
|
||||
"https://www.googleapis.com/auth/drive",
|
||||
"https://www.googleapis.com/auth/drive.appdata",
|
||||
]
|
||||
|
||||
# Pass non existent settings path to force DEFAULT_SETTINGS loading
|
||||
GoogleAuth.DEFAULT_SETTINGS.update(auth_settings)
|
||||
|
||||
# Pass non existent settings path to force DEFAULT_SETTINGS loadings
|
||||
gauth = GoogleAuth(settings_file="")
|
||||
|
||||
try:
|
||||
logger.debug(
|
||||
"GDrive remote auth with config '{}'.".format(
|
||||
GoogleAuth.DEFAULT_SETTINGS
|
||||
)
|
||||
)
|
||||
if self._use_service_account:
|
||||
gauth.ServiceAuth()
|
||||
else:
|
||||
gauth.CommandLineAuth()
|
||||
except RefreshError as exc:
|
||||
raise GDriveAccessTokenRefreshError from exc
|
||||
except KeyError as exc:
|
||||
raise GDriveMissedCredentialKeyError(
|
||||
self._gdrive_user_credentials_path
|
||||
) from exc
|
||||
# Handle pydrive2.auth.AuthenticationError and other auth failures
|
||||
GDriveRemote._validate_credentials(gauth, auth_settings)
|
||||
|
||||
# Handle AuthenticationError, RefreshError and other auth failures
|
||||
# It's hard to come up with a narrow exception, since PyDrive throws
|
||||
# a lot of different errors - broken credentials file, refresh token
|
||||
# expired, flow failed, etc.
|
||||
except Exception as exc:
|
||||
raise DvcException("Google Drive authentication failed") from exc
|
||||
raise GDriveAuthError(self.credentials_location) from exc
|
||||
finally:
|
||||
if os.getenv(RemoteGDrive.GDRIVE_CREDENTIALS_DATA):
|
||||
if os.getenv(GDriveRemote.GDRIVE_CREDENTIALS_DATA):
|
||||
os.remove(self._gdrive_user_credentials_path)
|
||||
|
||||
return GoogleDrive(gauth)
|
||||
|
||||
@wrap_prop(threading.RLock())
|
||||
@cached_property
|
||||
def cache(self):
|
||||
cache = {"dirs": defaultdict(list), "ids": {}}
|
||||
def _ids_cache(self):
|
||||
cache = {
|
||||
"dirs": defaultdict(list),
|
||||
"ids": {},
|
||||
"root_id": self._get_item_id(
|
||||
self.path_info,
|
||||
use_cache=False,
|
||||
hint="Confirm the directory exists and you can access it.",
|
||||
),
|
||||
}
|
||||
|
||||
cache["root_id"] = self._get_remote_id(self.path_info)
|
||||
cache["dirs"][self.path_info.path] = [cache["root_id"]]
|
||||
self._cache_path(self.path_info.path, cache["root_id"], cache)
|
||||
self._cache_path_id(self.path_info.path, cache["root_id"], cache)
|
||||
|
||||
for item in self.gdrive_list_item(
|
||||
for item in self._gdrive_list(
|
||||
"'{}' in parents and trashed=false".format(cache["root_id"])
|
||||
):
|
||||
remote_path = (self.path_info / item["title"]).path
|
||||
self._cache_path(remote_path, item["id"], cache)
|
||||
item_path = (self.path_info / item["title"]).path
|
||||
self._cache_path_id(item_path, item["id"], cache)
|
||||
|
||||
return cache
|
||||
|
||||
def _cache_path(self, remote_path, remote_id, cache=None):
|
||||
cache = cache or self.cache
|
||||
cache["dirs"][remote_path].append(remote_id)
|
||||
cache["ids"][remote_id] = remote_path
|
||||
def _cache_path_id(self, path, item_id, cache=None):
|
||||
cache = cache or self._ids_cache
|
||||
cache["dirs"][path].append(item_id)
|
||||
cache["ids"][item_id] = path
|
||||
|
||||
@cached_property
|
||||
def list_params(self):
|
||||
def _list_params(self):
|
||||
params = {"corpora": "default"}
|
||||
if self._bucket != "root" and self._bucket != "appDataFolder":
|
||||
drive_id = self._get_remote_drive_id(self._bucket)
|
||||
drive_id = self._gdrive_shared_drive_id(self._bucket)
|
||||
if drive_id:
|
||||
logger.debug(
|
||||
"GDrive remote '{}' is using shared drive id '{}'.".format(
|
||||
self.path_info, drive_id
|
||||
)
|
||||
)
|
||||
params["driveId"] = drive_id
|
||||
params["corpora"] = "drive"
|
||||
return params
|
||||
|
||||
@gdrive_retry
|
||||
def gdrive_upload_file(
|
||||
@_gdrive_retry
|
||||
def _gdrive_shared_drive_id(self, item_id):
|
||||
from pydrive2.files import ApiRequestError
|
||||
|
||||
param = {"id": item_id}
|
||||
# it does not create a file on the remote
|
||||
item = self._drive.CreateFile(param)
|
||||
# ID of the shared drive the item resides in.
|
||||
# Only populated for items in shared drives.
|
||||
try:
|
||||
item.FetchMetadata("driveId")
|
||||
except ApiRequestError as exc:
|
||||
error_code = exc.error.get("code", 0)
|
||||
if error_code == 404:
|
||||
raise DvcException(
|
||||
"'{}' for '{}':\n\n"
|
||||
"1. Confirm the directory exists and you can access it.\n"
|
||||
"2. Make sure that credentials in '{}'\n"
|
||||
" are correct for this remote e.g. "
|
||||
"use the `gdrive_user_credentials_file` config\n"
|
||||
" option if you use multiple GDrive remotes with "
|
||||
"different email accounts.\n\nDetails".format(
|
||||
item_id, self.path_info, self._credentials_location
|
||||
)
|
||||
) from exc
|
||||
raise
|
||||
|
||||
return item.get("driveId", None)
|
||||
|
||||
@_gdrive_retry
|
||||
def _gdrive_upload_file(
|
||||
self,
|
||||
parent_id,
|
||||
title,
|
||||
@@ -252,7 +356,7 @@ class RemoteGDrive(RemoteBASE):
|
||||
from_file="",
|
||||
progress_name="",
|
||||
):
|
||||
item = self.drive.CreateFile(
|
||||
item = self._drive.CreateFile(
|
||||
{"title": title, "parents": [{"id": parent_id}]}
|
||||
)
|
||||
|
||||
@@ -272,161 +376,188 @@ class RemoteGDrive(RemoteBASE):
|
||||
item.Upload()
|
||||
return item
|
||||
|
||||
@gdrive_retry
|
||||
def gdrive_download_file(
|
||||
self, file_id, to_file, progress_name, no_progress_bar
|
||||
@_gdrive_retry
|
||||
def _gdrive_download_file(
|
||||
self, item_id, to_file, progress_desc, no_progress_bar
|
||||
):
|
||||
param = {"id": file_id}
|
||||
param = {"id": item_id}
|
||||
# it does not create a file on the remote
|
||||
gdrive_file = self.drive.CreateFile(param)
|
||||
bar_format = (
|
||||
"Downloading {desc:{ncols_desc}.{ncols_desc}}... "
|
||||
+ Tqdm.format_sizeof(int(gdrive_file["fileSize"]), "B", 1024)
|
||||
)
|
||||
gdrive_file = self._drive.CreateFile(param)
|
||||
|
||||
with Tqdm(
|
||||
bar_format=bar_format, desc=progress_name, disable=no_progress_bar
|
||||
):
|
||||
gdrive_file.GetContentFile(to_file)
|
||||
desc=progress_desc,
|
||||
disable=no_progress_bar,
|
||||
bytes=True,
|
||||
# explicit `bar_format` as `total` will be set by `update_to`
|
||||
bar_format=Tqdm.BAR_FMT_DEFAULT,
|
||||
) as pbar:
|
||||
gdrive_file.GetContentFile(to_file, callback=pbar.update_to)
|
||||
|
||||
def gdrive_list_item(self, query):
|
||||
@_gdrive_retry
|
||||
def _gdrive_delete_file(self, item_id):
|
||||
from pydrive2.files import ApiRequestError
|
||||
|
||||
param = {"id": item_id}
|
||||
# it does not create a file on the remote
|
||||
item = self._drive.CreateFile(param)
|
||||
|
||||
try:
|
||||
item.Trash() if self._trash_only else item.Delete()
|
||||
except ApiRequestError as exc:
|
||||
http_error_code = exc.error.get("code", 0)
|
||||
if (
|
||||
http_error_code == 403
|
||||
and self._list_params["corpora"] == "drive"
|
||||
and exc.GetField("location") == "file.permissions"
|
||||
):
|
||||
raise DvcException(
|
||||
"Insufficient permissions to {}. You should have {} "
|
||||
"access level for the used shared drive. More details "
|
||||
"at {}.".format(
|
||||
"move the file into Trash"
|
||||
if self._trash_only
|
||||
else "permanently delete the file",
|
||||
"Manager or Content Manager"
|
||||
if self._trash_only
|
||||
else "Manager",
|
||||
"https://support.google.com/a/answer/7337554",
|
||||
)
|
||||
) from exc
|
||||
raise
|
||||
|
||||
def _gdrive_list(self, query):
|
||||
param = {"q": query, "maxResults": 1000}
|
||||
param.update(self.list_params)
|
||||
param.update(self._list_params)
|
||||
file_list = self._drive.ListFile(param)
|
||||
|
||||
file_list = self.drive.ListFile(param)
|
||||
# Isolate and decorate fetching of remote drive items in pages.
|
||||
get_list = _gdrive_retry(lambda: next(file_list, None))
|
||||
|
||||
# Isolate and decorate fetching of remote drive items in pages
|
||||
get_list = gdrive_retry(lambda: next(file_list, None))
|
||||
|
||||
# Fetch pages until None is received, lazily flatten the thing
|
||||
# Fetch pages until None is received, lazily flatten the thing.
|
||||
return cat(iter(get_list, None))
|
||||
|
||||
@wrap_with(threading.RLock())
|
||||
def gdrive_create_dir(self, parent_id, title, remote_path):
|
||||
cached = self.cache["dirs"].get(remote_path)
|
||||
if cached:
|
||||
return cached[0]
|
||||
|
||||
item = self._create_remote_dir(parent_id, title)
|
||||
|
||||
if parent_id == self.cache["root_id"]:
|
||||
self._cache_path(remote_path, item["id"])
|
||||
|
||||
return item["id"]
|
||||
|
||||
@gdrive_retry
|
||||
def _create_remote_dir(self, parent_id, title):
|
||||
@_gdrive_retry
|
||||
def _gdrive_create_dir(self, parent_id, title):
|
||||
parent = {"id": parent_id}
|
||||
item = self.drive.CreateFile(
|
||||
item = self._drive.CreateFile(
|
||||
{"title": title, "parents": [parent], "mimeType": FOLDER_MIME_TYPE}
|
||||
)
|
||||
item.Upload()
|
||||
return item
|
||||
|
||||
@gdrive_retry
|
||||
def _delete_remote_file(self, remote_id):
|
||||
param = {"id": remote_id}
|
||||
# it does not create a file on the remote
|
||||
item = self.drive.CreateFile(param)
|
||||
item.Delete()
|
||||
@wrap_with(threading.RLock())
|
||||
def _create_dir(self, parent_id, title, remote_path):
|
||||
cached = self._ids_cache["dirs"].get(remote_path)
|
||||
if cached:
|
||||
return cached[0]
|
||||
|
||||
@gdrive_retry
|
||||
def _get_remote_item(self, name, parents_ids):
|
||||
if not parents_ids:
|
||||
item = self._gdrive_create_dir(parent_id, title)
|
||||
|
||||
if parent_id == self._ids_cache["root_id"]:
|
||||
self._cache_path_id(remote_path, item["id"])
|
||||
|
||||
return item["id"]
|
||||
|
||||
def _get_remote_item_ids(self, parent_ids, title):
|
||||
if not parent_ids:
|
||||
return None
|
||||
query = "({})".format(
|
||||
query = "trashed=false and ({})".format(
|
||||
" or ".join(
|
||||
"'{}' in parents".format(parent_id)
|
||||
for parent_id in parents_ids
|
||||
f"'{parent_id}' in parents" for parent_id in parent_ids
|
||||
)
|
||||
)
|
||||
query += " and title='{}'".format(title.replace("'", "\\'"))
|
||||
|
||||
query += " and trashed=false and title='{}'".format(name)
|
||||
# GDrive list API is case insensitive, we need to compare
|
||||
# all results and pick the ones with the right title
|
||||
return [
|
||||
item["id"]
|
||||
for item in self._gdrive_list(query)
|
||||
if item["title"] == title
|
||||
]
|
||||
|
||||
# Remote might contain items with duplicated path (titles).
|
||||
# We thus limit number of items.
|
||||
param = {"q": query, "maxResults": 1}
|
||||
param.update(self.list_params)
|
||||
|
||||
# Limit found remote items count to 1 in response
|
||||
item_list = self.drive.ListFile(param).GetList()
|
||||
return next(iter(item_list), None)
|
||||
|
||||
@gdrive_retry
|
||||
def _get_remote_drive_id(self, remote_id):
|
||||
param = {"id": remote_id}
|
||||
# it does not create a file on the remote
|
||||
item = self.drive.CreateFile(param)
|
||||
item.FetchMetadata("driveId")
|
||||
return item.get("driveId", None)
|
||||
|
||||
def _get_cached_remote_ids(self, path):
|
||||
def _get_cached_item_ids(self, path, use_cache):
|
||||
if not path:
|
||||
return [self._bucket]
|
||||
if "cache" in self.__dict__:
|
||||
return self.cache["dirs"].get(path, [])
|
||||
if use_cache:
|
||||
return self._ids_cache["dirs"].get(path, [])
|
||||
return []
|
||||
|
||||
def _path_to_remote_ids(self, path, create):
|
||||
remote_ids = self._get_cached_remote_ids(path)
|
||||
if remote_ids:
|
||||
return remote_ids
|
||||
def _path_to_item_ids(self, path, create, use_cache):
|
||||
item_ids = self._get_cached_item_ids(path, use_cache)
|
||||
if item_ids:
|
||||
return item_ids
|
||||
|
||||
parent_path, part = posixpath.split(path)
|
||||
parent_ids = self._path_to_remote_ids(parent_path, create)
|
||||
item = self._get_remote_item(part, parent_ids)
|
||||
parent_path, title = posixpath.split(path)
|
||||
parent_ids = self._path_to_item_ids(parent_path, create, use_cache)
|
||||
item_ids = self._get_remote_item_ids(parent_ids, title)
|
||||
if item_ids:
|
||||
return item_ids
|
||||
|
||||
if not item:
|
||||
return (
|
||||
[self.gdrive_create_dir(parent_ids[0], part, path)]
|
||||
if create
|
||||
else []
|
||||
)
|
||||
return (
|
||||
[self._create_dir(min(parent_ids), title, path)] if create else []
|
||||
)
|
||||
|
||||
return [item["id"]]
|
||||
|
||||
def _get_remote_id(self, path_info, create=False):
|
||||
def _get_item_id(self, path_info, create=False, use_cache=True, hint=None):
|
||||
assert path_info.bucket == self._bucket
|
||||
|
||||
remote_ids = self._path_to_remote_ids(path_info.path, create)
|
||||
if not remote_ids:
|
||||
raise GDrivePathNotFound(path_info)
|
||||
item_ids = self._path_to_item_ids(path_info.path, create, use_cache)
|
||||
if item_ids:
|
||||
return min(item_ids)
|
||||
|
||||
return remote_ids[0]
|
||||
assert not create
|
||||
raise GDrivePathNotFound(path_info, hint)
|
||||
|
||||
def exists(self, path_info):
|
||||
try:
|
||||
self._get_remote_id(path_info)
|
||||
self._get_item_id(path_info)
|
||||
except GDrivePathNotFound:
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
|
||||
def _upload(self, from_file, to_info, name, no_progress_bar):
|
||||
def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
|
||||
dirname = to_info.parent
|
||||
assert dirname
|
||||
parent_id = self._get_remote_id(dirname, True)
|
||||
parent_id = self._get_item_id(dirname, True)
|
||||
|
||||
self.gdrive_upload_file(
|
||||
self._gdrive_upload_file(
|
||||
parent_id, to_info.name, no_progress_bar, from_file, name
|
||||
)
|
||||
|
||||
def _download(self, from_info, to_file, name, no_progress_bar):
|
||||
file_id = self._get_remote_id(from_info)
|
||||
self.gdrive_download_file(file_id, to_file, name, no_progress_bar)
|
||||
def _download(self, from_info, to_file, name=None, no_progress_bar=False):
|
||||
item_id = self._get_item_id(from_info)
|
||||
self._gdrive_download_file(item_id, to_file, name, no_progress_bar)
|
||||
|
||||
def list_cache_paths(self):
|
||||
if not self.cache["ids"]:
|
||||
def list_cache_paths(self, prefix=None, progress_callback=None):
|
||||
if not self._ids_cache["ids"]:
|
||||
return
|
||||
|
||||
if prefix:
|
||||
dir_ids = self._ids_cache["dirs"].get(prefix[:2])
|
||||
if not dir_ids:
|
||||
return
|
||||
else:
|
||||
dir_ids = self._ids_cache["ids"]
|
||||
parents_query = " or ".join(
|
||||
"'{}' in parents".format(dir_id) for dir_id in self.cache["ids"]
|
||||
f"'{dir_id}' in parents" for dir_id in dir_ids
|
||||
)
|
||||
query = "({}) and trashed=false".format(parents_query)
|
||||
query = f"({parents_query}) and trashed=false"
|
||||
|
||||
for item in self.gdrive_list_item(query):
|
||||
for item in self._gdrive_list(query):
|
||||
if progress_callback:
|
||||
progress_callback()
|
||||
parent_id = item["parents"][0]["id"]
|
||||
yield posixpath.join(self.cache["ids"][parent_id], item["title"])
|
||||
yield posixpath.join(
|
||||
self._ids_cache["ids"][parent_id], item["title"]
|
||||
)
|
||||
|
||||
def remove(self, path_info):
|
||||
remote_id = self._get_remote_id(path_info)
|
||||
self._delete_remote_file(remote_id)
|
||||
item_id = self._get_item_id(path_info)
|
||||
self._gdrive_delete_file(item_id)
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
raise NotImplementedError
|
||||
|
||||
def walk_files(self, path_info):
|
||||
raise NotImplementedError
|
||||
|
||||
+25
-12
@@ -1,16 +1,16 @@
|
||||
import logging
|
||||
import os.path
|
||||
import posixpath
|
||||
import threading
|
||||
from datetime import timedelta
|
||||
from functools import wraps
|
||||
import io
|
||||
import os.path
|
||||
import threading
|
||||
|
||||
from funcy import cached_property, wrap_prop
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.path_info import CloudURLInfo
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.remote.base import RemoteBASE
|
||||
from dvc.remote.base import BaseRemote
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -54,7 +54,7 @@ def _upload_to_bucket(
|
||||
no_progress_bar=False,
|
||||
):
|
||||
blob = bucket.blob(to_info.path, chunk_size=chunk_size)
|
||||
with io.open(from_file, mode="rb") as fobj:
|
||||
with open(from_file, mode="rb") as fobj:
|
||||
with Tqdm.wrapattr(
|
||||
fobj,
|
||||
"read",
|
||||
@@ -65,7 +65,7 @@ def _upload_to_bucket(
|
||||
blob.upload_from_file(wrapped)
|
||||
|
||||
|
||||
class RemoteGS(RemoteBASE):
|
||||
class GSRemote(BaseRemote):
|
||||
scheme = Schemes.GS
|
||||
path_cls = CloudURLInfo
|
||||
REQUIRES = {"google-cloud-storage": "google.cloud.storage"}
|
||||
@@ -109,7 +109,7 @@ class RemoteGS(RemoteBASE):
|
||||
from_bucket = self.gs.bucket(from_info.bucket)
|
||||
blob = from_bucket.get_blob(from_info.path)
|
||||
if not blob:
|
||||
msg = "'{}' doesn't exist in the cloud".format(from_info.path)
|
||||
msg = f"'{from_info.path}' doesn't exist in the cloud"
|
||||
raise DvcException(msg)
|
||||
|
||||
to_bucket = self.gs.bucket(to_info.bucket)
|
||||
@@ -119,21 +119,31 @@ class RemoteGS(RemoteBASE):
|
||||
if path_info.scheme != "gs":
|
||||
raise NotImplementedError
|
||||
|
||||
logger.debug("Removing gs://{}".format(path_info))
|
||||
logger.debug(f"Removing gs://{path_info}")
|
||||
blob = self.gs.bucket(path_info.bucket).get_blob(path_info.path)
|
||||
if not blob:
|
||||
return
|
||||
|
||||
blob.delete()
|
||||
|
||||
def _list_paths(self, path_info, max_items=None):
|
||||
def _list_paths(
|
||||
self, path_info, max_items=None, prefix=None, progress_callback=None
|
||||
):
|
||||
if prefix:
|
||||
prefix = posixpath.join(path_info.path, prefix[:2], prefix[2:])
|
||||
else:
|
||||
prefix = path_info.path
|
||||
for blob in self.gs.bucket(path_info.bucket).list_blobs(
|
||||
prefix=path_info.path, max_results=max_items
|
||||
):
|
||||
if progress_callback:
|
||||
progress_callback()
|
||||
yield blob.name
|
||||
|
||||
def list_cache_paths(self):
|
||||
return self._list_paths(self.path_info)
|
||||
def list_cache_paths(self, prefix=None, progress_callback=None):
|
||||
return self._list_paths(
|
||||
self.path_info, prefix=prefix, progress_callback=progress_callback
|
||||
)
|
||||
|
||||
def walk_files(self, path_info):
|
||||
for fname in self._list_paths(path_info / ""):
|
||||
@@ -143,6 +153,9 @@ class RemoteGS(RemoteBASE):
|
||||
yield path_info.replace(fname)
|
||||
|
||||
def makedirs(self, path_info):
|
||||
if not path_info.path:
|
||||
return
|
||||
|
||||
self.gs.bucket(path_info.bucket).blob(
|
||||
(path_info / "").path
|
||||
).upload_from_string("")
|
||||
@@ -179,7 +192,7 @@ class RemoteGS(RemoteBASE):
|
||||
def _download(self, from_info, to_file, name=None, no_progress_bar=False):
|
||||
bucket = self.gs.bucket(from_info.bucket)
|
||||
blob = bucket.get_blob(from_info.path)
|
||||
with io.open(to_file, mode="wb") as fobj:
|
||||
with open(to_file, mode="wb") as fobj:
|
||||
with Tqdm.wrapattr(
|
||||
fobj,
|
||||
"write",
|
||||
|
||||
+31
-14
@@ -8,19 +8,21 @@ from collections import deque
|
||||
from contextlib import closing, contextmanager
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from .base import RemoteBASE, RemoteCmdError
|
||||
from .pool import get_connection
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.utils import fix_env, tmp_fname
|
||||
|
||||
from .base import BaseRemote, RemoteCmdError
|
||||
from .pool import get_connection
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RemoteHDFS(RemoteBASE):
|
||||
class HDFSRemote(BaseRemote):
|
||||
scheme = Schemes.HDFS
|
||||
REGEX = r"^hdfs://((?P<user>.*)@)?.*$"
|
||||
PARAM_CHECKSUM = "checksum"
|
||||
REQUIRES = {"pyarrow": "pyarrow"}
|
||||
TRAVERSE_PREFIX_LEN = 2
|
||||
|
||||
def __init__(self, repo, config):
|
||||
super().__init__(repo, config)
|
||||
@@ -53,7 +55,7 @@ class RemoteHDFS(RemoteBASE):
|
||||
def hadoop_fs(self, cmd, user=None):
|
||||
cmd = "hadoop fs -" + cmd
|
||||
if user:
|
||||
cmd = "HADOOP_USER_NAME={} ".format(user) + cmd
|
||||
cmd = f"HADOOP_USER_NAME={user} " + cmd
|
||||
|
||||
# NOTE: close_fds doesn't work with redirected stdin/stdout/stderr.
|
||||
# See https://github.com/iterative/dvc/issues/1197.
|
||||
@@ -85,7 +87,7 @@ class RemoteHDFS(RemoteBASE):
|
||||
# NOTE: pyarrow doesn't support checksum, so we need to use hadoop
|
||||
regex = r".*\t.*\t(?P<checksum>.*)"
|
||||
stdout = self.hadoop_fs(
|
||||
"checksum {}".format(path_info.path), user=path_info.user
|
||||
f"checksum {path_info.path}", user=path_info.user
|
||||
)
|
||||
return self._group(regex, stdout, "checksum")
|
||||
|
||||
@@ -110,7 +112,7 @@ class RemoteHDFS(RemoteBASE):
|
||||
raise NotImplementedError
|
||||
|
||||
if self.exists(path_info):
|
||||
logger.debug("Removing {}".format(path_info.path))
|
||||
logger.debug(f"Removing {path_info.path}")
|
||||
with self.hdfs(path_info) as hdfs:
|
||||
hdfs.rm(path_info.path)
|
||||
|
||||
@@ -145,23 +147,38 @@ class RemoteHDFS(RemoteBASE):
|
||||
yield fd
|
||||
else:
|
||||
yield io.TextIOWrapper(fd, encoding=encoding)
|
||||
except IOError as e:
|
||||
except OSError as e:
|
||||
# Empty .errno and not specific enough error class in pyarrow,
|
||||
# see https://issues.apache.org/jira/browse/ARROW-6248
|
||||
if "file does not exist" in str(e):
|
||||
raise FileNotFoundError(*e.args)
|
||||
raise
|
||||
|
||||
def list_cache_paths(self):
|
||||
def list_cache_paths(self, prefix=None, progress_callback=None):
|
||||
if not self.exists(self.path_info):
|
||||
return
|
||||
|
||||
dirs = deque([self.path_info.path])
|
||||
if prefix:
|
||||
root = posixpath.join(self.path_info.path, prefix[:2])
|
||||
else:
|
||||
root = self.path_info.path
|
||||
dirs = deque([root])
|
||||
|
||||
with self.hdfs(self.path_info) as hdfs:
|
||||
if prefix and not hdfs.exists(root):
|
||||
return
|
||||
while dirs:
|
||||
for entry in hdfs.ls(dirs.pop(), detail=True):
|
||||
if entry["kind"] == "directory":
|
||||
dirs.append(urlparse(entry["name"]).path)
|
||||
elif entry["kind"] == "file":
|
||||
yield urlparse(entry["name"]).path
|
||||
try:
|
||||
entries = hdfs.ls(dirs.pop(), detail=True)
|
||||
for entry in entries:
|
||||
if entry["kind"] == "directory":
|
||||
dirs.append(urlparse(entry["name"]).path)
|
||||
elif entry["kind"] == "file":
|
||||
if progress_callback:
|
||||
progress_callback()
|
||||
yield urlparse(entry["name"]).path
|
||||
except OSError as e:
|
||||
# When searching for a specific prefix pyarrow raises an
|
||||
# exception if the specified cache dir does not exist
|
||||
if not prefix:
|
||||
raise e
|
||||
|
||||
+34
-35
@@ -5,10 +5,10 @@ import threading
|
||||
from funcy import cached_property, memoize, wrap_prop, wrap_with
|
||||
|
||||
import dvc.prompt as prompt
|
||||
from dvc.config import ConfigError
|
||||
from dvc.exceptions import DvcException, HTTPError
|
||||
from dvc.path_info import HTTPURLInfo
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.remote.base import RemoteBASE
|
||||
from dvc.remote.base import BaseRemote
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -23,13 +23,15 @@ def ask_password(host, user):
|
||||
)
|
||||
|
||||
|
||||
class RemoteHTTP(RemoteBASE):
|
||||
class HTTPRemote(BaseRemote):
|
||||
scheme = Schemes.HTTP
|
||||
path_cls = HTTPURLInfo
|
||||
SESSION_RETRIES = 5
|
||||
SESSION_BACKOFF_FACTOR = 0.1
|
||||
REQUEST_TIMEOUT = 10
|
||||
CHUNK_SIZE = 2 ** 16
|
||||
PARAM_CHECKSUM = "etag"
|
||||
CAN_TRAVERSE = False
|
||||
|
||||
def __init__(self, repo, config):
|
||||
super().__init__(repo, config)
|
||||
@@ -43,12 +45,6 @@ class RemoteHTTP(RemoteBASE):
|
||||
else:
|
||||
self.path_info = None
|
||||
|
||||
if not self.no_traverse:
|
||||
raise ConfigError(
|
||||
"HTTP doesn't support traversing the remote to list existing "
|
||||
"files. Use: `dvc remote modify <name> no_traverse true`"
|
||||
)
|
||||
|
||||
self.auth = config.get("auth", None)
|
||||
self.custom_auth_header = config.get("custom_auth_header", None)
|
||||
self.password = config.get("password", None)
|
||||
@@ -59,39 +55,42 @@ class RemoteHTTP(RemoteBASE):
|
||||
response = self._request("GET", from_info.url, stream=True)
|
||||
if response.status_code != 200:
|
||||
raise HTTPError(response.status_code, response.reason)
|
||||
with Tqdm(
|
||||
total=None if no_progress_bar else self._content_length(response),
|
||||
leave=False,
|
||||
bytes=True,
|
||||
desc=from_info.url if name is None else name,
|
||||
disable=no_progress_bar,
|
||||
) as pbar:
|
||||
with open(to_file, "wb") as fd:
|
||||
with open(to_file, "wb") as fd:
|
||||
with Tqdm.wrapattr(
|
||||
fd,
|
||||
"write",
|
||||
total=None
|
||||
if no_progress_bar
|
||||
else self._content_length(response),
|
||||
leave=False,
|
||||
desc=from_info.url if name is None else name,
|
||||
disable=no_progress_bar,
|
||||
) as fd_wrapped:
|
||||
for chunk in response.iter_content(chunk_size=self.CHUNK_SIZE):
|
||||
fd.write(chunk)
|
||||
pbar.update(len(chunk))
|
||||
fd_wrapped.write(chunk)
|
||||
|
||||
def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
|
||||
with Tqdm(
|
||||
total=None if no_progress_bar else os.path.getsize(from_file),
|
||||
leave=False,
|
||||
bytes=True,
|
||||
desc=to_info.url if name is None else name,
|
||||
disable=no_progress_bar,
|
||||
) as pbar:
|
||||
|
||||
def chunks():
|
||||
with open(from_file, "rb") as fd:
|
||||
def chunks():
|
||||
with open(from_file, "rb") as fd:
|
||||
with Tqdm.wrapattr(
|
||||
fd,
|
||||
"read",
|
||||
total=None
|
||||
if no_progress_bar
|
||||
else os.path.getsize(from_file),
|
||||
leave=False,
|
||||
desc=to_info.url if name is None else name,
|
||||
disable=no_progress_bar,
|
||||
) as fd_wrapped:
|
||||
while True:
|
||||
chunk = fd.read(self.CHUNK_SIZE)
|
||||
chunk = fd_wrapped.read(self.CHUNK_SIZE)
|
||||
if not chunk:
|
||||
break
|
||||
pbar.update(len(chunk))
|
||||
yield chunk
|
||||
|
||||
response = self._request("POST", to_info.url, data=chunks())
|
||||
if response.status_code not in (200, 201):
|
||||
raise HTTPError(response.status_code, response.reason)
|
||||
response = self._request("POST", to_info.url, data=chunks())
|
||||
if response.status_code not in (200, 201):
|
||||
raise HTTPError(response.status_code, response.reason)
|
||||
|
||||
def exists(self, path_info):
|
||||
return bool(self._request("HEAD", path_info.url))
|
||||
@@ -186,7 +185,7 @@ class RemoteHTTP(RemoteBASE):
|
||||
return res
|
||||
|
||||
except requests.exceptions.RequestException:
|
||||
raise DvcException("could not perform a {} request".format(method))
|
||||
raise DvcException(f"could not perform a {method} request")
|
||||
|
||||
def gc(self):
|
||||
raise NotImplementedError
|
||||
|
||||
+3
-2
@@ -1,6 +1,7 @@
|
||||
from .http import RemoteHTTP
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
from .http import HTTPRemote
|
||||
|
||||
class RemoteHTTPS(RemoteHTTP):
|
||||
|
||||
class HTTPSRemote(HTTPRemote):
|
||||
scheme = Schemes.HTTPS
|
||||
|
||||
@@ -0,0 +1,212 @@
|
||||
import logging
|
||||
import os
|
||||
import sqlite3
|
||||
import threading
|
||||
|
||||
from funcy import lchunks
|
||||
|
||||
from dvc.state import _connect_sqlite
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RemoteIndexNoop:
|
||||
"""No-op class for remotes which are not indexed (i.e. local)."""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
pass
|
||||
|
||||
def __enter__(self):
|
||||
pass
|
||||
|
||||
def __exit__(self, typ, value, tbck):
|
||||
pass
|
||||
|
||||
def __iter__(self):
|
||||
return iter([])
|
||||
|
||||
def __contains__(self, checksum):
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def checksums():
|
||||
return []
|
||||
|
||||
@staticmethod
|
||||
def dir_checksums():
|
||||
return []
|
||||
|
||||
def load(self):
|
||||
pass
|
||||
|
||||
def dump(self):
|
||||
pass
|
||||
|
||||
def clear(self):
|
||||
pass
|
||||
|
||||
def update(self, *args):
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def intersection(*args):
|
||||
return []
|
||||
|
||||
|
||||
class RemoteIndex:
|
||||
"""Class for indexing remote checksums in a sqlite3 database.
|
||||
|
||||
Args:
|
||||
repo: repo for this remote index.
|
||||
name: name for this index. Index db will be loaded from and saved to
|
||||
``.dvc/tmp/index/{name}.idx``.
|
||||
dir_suffix: suffix used for naming directory checksums
|
||||
"""
|
||||
|
||||
INDEX_SUFFIX = ".idx"
|
||||
VERSION = 1
|
||||
INDEX_TABLE = "remote_index"
|
||||
INDEX_TABLE_LAYOUT = "checksum TEXT PRIMARY KEY, " "dir INTEGER NOT NULL"
|
||||
|
||||
def __init__(self, repo, name, dir_suffix=".dir"):
|
||||
self.path = os.path.join(repo.index_dir, f"{name}{self.INDEX_SUFFIX}")
|
||||
|
||||
self.dir_suffix = dir_suffix
|
||||
self.database = None
|
||||
self.cursor = None
|
||||
self.modified = False
|
||||
self.lock = threading.Lock()
|
||||
|
||||
def __iter__(self):
|
||||
cmd = f"SELECT checksum FROM {self.INDEX_TABLE}"
|
||||
for (checksum,) in self._execute(cmd):
|
||||
yield checksum
|
||||
|
||||
def __enter__(self):
|
||||
self.lock.acquire()
|
||||
self.load()
|
||||
|
||||
def __exit__(self, typ, value, tbck):
|
||||
self.dump()
|
||||
self.lock.release()
|
||||
|
||||
def __contains__(self, checksum):
|
||||
cmd = "SELECT checksum FROM {} WHERE checksum = (?)".format(
|
||||
self.INDEX_TABLE
|
||||
)
|
||||
self._execute(cmd, (checksum,))
|
||||
return self.cursor.fetchone() is not None
|
||||
|
||||
def checksums(self):
|
||||
"""Iterate over checksums stored in the index."""
|
||||
return iter(self)
|
||||
|
||||
def dir_checksums(self):
|
||||
"""Iterate over .dir checksums stored in the index."""
|
||||
cmd = f"SELECT checksum FROM {self.INDEX_TABLE} WHERE dir = 1"
|
||||
for (checksum,) in self._execute(cmd):
|
||||
yield checksum
|
||||
|
||||
def is_dir_checksum(self, checksum):
|
||||
return checksum.endswith(self.dir_suffix)
|
||||
|
||||
def _execute(self, cmd, parameters=()):
|
||||
return self.cursor.execute(cmd, parameters)
|
||||
|
||||
def _executemany(self, cmd, seq_of_parameters):
|
||||
return self.cursor.executemany(cmd, seq_of_parameters)
|
||||
|
||||
def _prepare_db(self, empty=False):
|
||||
if not empty:
|
||||
cmd = "PRAGMA user_version;"
|
||||
self._execute(cmd)
|
||||
ret = self.cursor.fetchall()
|
||||
assert len(ret) == 1
|
||||
assert len(ret[0]) == 1
|
||||
assert isinstance(ret[0][0], int)
|
||||
version = ret[0][0]
|
||||
|
||||
if version != self.VERSION:
|
||||
logger.error(
|
||||
"Index file version '{}' will be reformatted "
|
||||
"to the current version '{}'.".format(
|
||||
version, self.VERSION,
|
||||
)
|
||||
)
|
||||
cmd = "DROP TABLE IF EXISTS {};"
|
||||
self._execute(cmd.format(self.INDEX_TABLE))
|
||||
|
||||
cmd = "CREATE TABLE IF NOT EXISTS {} ({})"
|
||||
self._execute(cmd.format(self.INDEX_TABLE, self.INDEX_TABLE_LAYOUT))
|
||||
|
||||
cmd = "PRAGMA user_version = {};"
|
||||
self._execute(cmd.format(self.VERSION))
|
||||
|
||||
def load(self):
|
||||
"""(Re)load this index database."""
|
||||
retries = 1
|
||||
while True:
|
||||
assert self.database is None
|
||||
assert self.cursor is None
|
||||
|
||||
empty = not os.path.isfile(self.path)
|
||||
self.database = _connect_sqlite(self.path, {"nolock": 1})
|
||||
self.cursor = self.database.cursor()
|
||||
|
||||
try:
|
||||
self._prepare_db(empty=empty)
|
||||
return
|
||||
except sqlite3.DatabaseError:
|
||||
self.cursor.close()
|
||||
self.database.close()
|
||||
self.database = None
|
||||
self.cursor = None
|
||||
if retries > 0:
|
||||
os.unlink(self.path)
|
||||
retries -= 1
|
||||
else:
|
||||
raise
|
||||
|
||||
def dump(self):
|
||||
"""Save this index database."""
|
||||
assert self.database is not None
|
||||
|
||||
self.database.commit()
|
||||
self.cursor.close()
|
||||
self.database.close()
|
||||
self.database = None
|
||||
self.cursor = None
|
||||
|
||||
def clear(self):
|
||||
"""Clear this index (to force re-indexing later).
|
||||
|
||||
Changes to the index will not committed until dump() is called.
|
||||
"""
|
||||
cmd = f"DELETE FROM {self.INDEX_TABLE}"
|
||||
self._execute(cmd)
|
||||
|
||||
def update(self, dir_checksums, file_checksums):
|
||||
"""Update this index, adding the specified checksums.
|
||||
|
||||
Changes to the index will not committed until dump() is called.
|
||||
"""
|
||||
cmd = "INSERT OR IGNORE INTO {} (checksum, dir) VALUES (?, ?)".format(
|
||||
self.INDEX_TABLE
|
||||
)
|
||||
self._executemany(
|
||||
cmd, ((checksum, True) for checksum in dir_checksums)
|
||||
)
|
||||
self._executemany(
|
||||
cmd, ((checksum, False) for checksum in file_checksums)
|
||||
)
|
||||
|
||||
def intersection(self, checksums):
|
||||
"""Iterate over values from `checksums` which exist in the index."""
|
||||
# sqlite has a compile time limit of 999, see:
|
||||
# https://www.sqlite.org/c3ref/c_limit_attached.html#sqlitelimitvariablenumber
|
||||
for chunk in lchunks(999, checksums):
|
||||
cmd = "SELECT checksum FROM {} WHERE checksum IN ({})".format(
|
||||
self.INDEX_TABLE, ",".join("?" for checksum in chunk)
|
||||
)
|
||||
for (checksum,) in self._execute(cmd, chunk):
|
||||
yield checksum
|
||||
+336
-143
@@ -2,49 +2,57 @@ import errno
|
||||
import logging
|
||||
import os
|
||||
import stat
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from functools import partial
|
||||
|
||||
from funcy import cached_property, concat
|
||||
from shortuuid import uuid
|
||||
|
||||
from dvc.compat import fspath_py35
|
||||
from dvc.exceptions import DvcException, DownloadError, UploadError
|
||||
from dvc.exceptions import DownloadError, DvcException, UploadError
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.remote.base import RemoteBASE, STATUS_MAP
|
||||
from dvc.remote.base import STATUS_DELETED, STATUS_MISSING, STATUS_NEW
|
||||
from dvc.remote.base import (
|
||||
STATUS_DELETED,
|
||||
STATUS_MAP,
|
||||
STATUS_MISSING,
|
||||
STATUS_NEW,
|
||||
BaseRemote,
|
||||
index_locked,
|
||||
)
|
||||
from dvc.remote.index import RemoteIndexNoop
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.scm.tree import is_working_tree
|
||||
from dvc.scm.tree import WorkingTree, is_working_tree
|
||||
from dvc.system import System
|
||||
from dvc.utils import file_md5, relpath, tmp_fname
|
||||
from dvc.utils.fs import copyfile, move, makedirs, remove, walk_files
|
||||
from dvc.utils.fs import (
|
||||
copy_fobj_to_file,
|
||||
copyfile,
|
||||
makedirs,
|
||||
move,
|
||||
remove,
|
||||
walk_files,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RemoteLOCAL(RemoteBASE):
|
||||
class LocalRemote(BaseRemote):
|
||||
scheme = Schemes.LOCAL
|
||||
path_cls = PathInfo
|
||||
PARAM_CHECKSUM = "md5"
|
||||
PARAM_PATH = "path"
|
||||
TRAVERSE_PREFIX_LEN = 2
|
||||
INDEX_CLS = RemoteIndexNoop
|
||||
|
||||
UNPACKED_DIR_SUFFIX = ".unpacked"
|
||||
|
||||
DEFAULT_CACHE_TYPES = ["reflink", "copy"]
|
||||
|
||||
CACHE_MODE = 0o444
|
||||
SHARED_MODE_MAP = {None: (0o644, 0o755), "group": (0o664, 0o775)}
|
||||
|
||||
def __init__(self, repo, config):
|
||||
super().__init__(repo, config)
|
||||
self.protected = config.get("protected", False)
|
||||
|
||||
shared = config.get("shared")
|
||||
self._file_mode, self._dir_mode = self.SHARED_MODE_MAP[shared]
|
||||
|
||||
if self.protected:
|
||||
# cache files are set to be read-only for everyone
|
||||
self._file_mode = stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH
|
||||
|
||||
self.cache_dir = config.get("url")
|
||||
self._dir_info = {}
|
||||
|
||||
@@ -60,13 +68,51 @@ class RemoteLOCAL(RemoteBASE):
|
||||
def cache_dir(self, value):
|
||||
self.path_info = PathInfo(value) if value else None
|
||||
|
||||
@cached_property
|
||||
def _work_tree(self):
|
||||
if self.repo:
|
||||
return WorkingTree(self.repo.root_dir)
|
||||
return None
|
||||
|
||||
@property
|
||||
def work_tree(self):
|
||||
# When using repo.brancher, repo.tree may change to/from WorkingTree to
|
||||
# GitTree arbitarily. When repo.tree is GitTree, local cache needs to
|
||||
# use its own WorkingTree instance.
|
||||
if self.repo and not is_working_tree(self.repo.tree):
|
||||
return self._work_tree
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def supported(cls, config):
|
||||
return True
|
||||
|
||||
def list_cache_paths(self):
|
||||
@cached_property
|
||||
def cache_path(self):
|
||||
return os.path.abspath(self.cache_dir)
|
||||
|
||||
def checksum_to_path(self, checksum):
|
||||
# NOTE: `self.cache_path` is already normalized so we can simply use
|
||||
# `os.sep` instead of `os.path.join`. This results in this helper
|
||||
# being ~5.5 times faster.
|
||||
return (
|
||||
f"{self.cache_path}{os.sep}{checksum[0:2]}{os.sep}{checksum[2:]}"
|
||||
)
|
||||
|
||||
def list_cache_paths(self, prefix=None, progress_callback=None):
|
||||
assert self.path_info is not None
|
||||
return walk_files(self.path_info)
|
||||
if prefix:
|
||||
path_info = self.path_info / prefix[:2]
|
||||
if not self.exists(path_info):
|
||||
return
|
||||
else:
|
||||
path_info = self.path_info
|
||||
if progress_callback:
|
||||
for path in walk_files(path_info):
|
||||
progress_callback()
|
||||
yield path
|
||||
else:
|
||||
yield from walk_files(path_info)
|
||||
|
||||
def get(self, md5):
|
||||
if not md5:
|
||||
@@ -74,10 +120,13 @@ class RemoteLOCAL(RemoteBASE):
|
||||
|
||||
return self.checksum_to_path_info(md5).url
|
||||
|
||||
@staticmethod
|
||||
def exists(path_info):
|
||||
assert path_info.scheme == "local"
|
||||
return os.path.lexists(fspath_py35(path_info))
|
||||
def exists(self, path_info):
|
||||
assert isinstance(path_info, str) or path_info.scheme == "local"
|
||||
if not self.repo:
|
||||
return os.path.exists(path_info)
|
||||
if self.work_tree and self.work_tree.exists(path_info):
|
||||
return True
|
||||
return self.repo.tree.exists(path_info)
|
||||
|
||||
def makedirs(self, path_info):
|
||||
makedirs(path_info, exist_ok=True, mode=self._dir_mode)
|
||||
@@ -109,13 +158,19 @@ class RemoteLOCAL(RemoteBASE):
|
||||
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def isfile(path_info):
|
||||
return os.path.isfile(fspath_py35(path_info))
|
||||
def isfile(self, path_info):
|
||||
if not self.repo:
|
||||
return os.path.isfile(path_info)
|
||||
if self.work_tree and self.work_tree.isfile(path_info):
|
||||
return True
|
||||
return self.repo.tree.isfile(path_info)
|
||||
|
||||
@staticmethod
|
||||
def isdir(path_info):
|
||||
return os.path.isdir(fspath_py35(path_info))
|
||||
def isdir(self, path_info):
|
||||
if not self.repo:
|
||||
return os.path.isdir(path_info)
|
||||
if self.work_tree and self.work_tree.isdir(path_info):
|
||||
return True
|
||||
return self.repo.tree.isdir(path_info)
|
||||
|
||||
def iscopy(self, path_info):
|
||||
return not (
|
||||
@@ -124,34 +179,41 @@ class RemoteLOCAL(RemoteBASE):
|
||||
|
||||
@staticmethod
|
||||
def getsize(path_info):
|
||||
return os.path.getsize(fspath_py35(path_info))
|
||||
return os.path.getsize(path_info)
|
||||
|
||||
def walk_files(self, path_info):
|
||||
assert is_working_tree(self.repo.tree)
|
||||
|
||||
for fname in self.repo.tree.walk_files(path_info):
|
||||
if self.work_tree:
|
||||
tree = self.work_tree
|
||||
else:
|
||||
tree = self.repo.tree
|
||||
for fname in tree.walk_files(path_info):
|
||||
yield PathInfo(fname)
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
return file_md5(path_info)[0]
|
||||
|
||||
def remove(self, path_info):
|
||||
if path_info.scheme != "local":
|
||||
raise NotImplementedError
|
||||
if isinstance(path_info, PathInfo):
|
||||
if path_info.scheme != "local":
|
||||
raise NotImplementedError
|
||||
path = path_info.fspath
|
||||
else:
|
||||
path = path_info
|
||||
|
||||
if self.exists(path_info):
|
||||
remove(path_info.fspath)
|
||||
if self.exists(path):
|
||||
remove(path)
|
||||
|
||||
def move(self, from_info, to_info):
|
||||
def move(self, from_info, to_info, mode=None):
|
||||
if from_info.scheme != "local" or to_info.scheme != "local":
|
||||
raise NotImplementedError
|
||||
|
||||
self.makedirs(to_info.parent)
|
||||
|
||||
if self.isfile(from_info):
|
||||
mode = self._file_mode
|
||||
else:
|
||||
mode = self._dir_mode
|
||||
if mode is None:
|
||||
if self.isfile(from_info):
|
||||
mode = self._file_mode
|
||||
else:
|
||||
mode = self._dir_mode
|
||||
|
||||
move(from_info, to_info, mode=mode)
|
||||
|
||||
@@ -159,7 +221,19 @@ class RemoteLOCAL(RemoteBASE):
|
||||
tmp_info = to_info.parent / tmp_fname(to_info.name)
|
||||
try:
|
||||
System.copy(from_info, tmp_info)
|
||||
os.rename(fspath_py35(tmp_info), fspath_py35(to_info))
|
||||
os.chmod(tmp_info, self._file_mode)
|
||||
os.rename(tmp_info, to_info)
|
||||
except Exception:
|
||||
self.remove(tmp_info)
|
||||
raise
|
||||
|
||||
def copy_fobj(self, fobj, to_info):
|
||||
self.makedirs(to_info.parent)
|
||||
tmp_info = to_info.parent / tmp_fname(to_info.name)
|
||||
try:
|
||||
copy_fobj_to_file(fobj, tmp_info)
|
||||
os.chmod(tmp_info, self._file_mode)
|
||||
os.rename(tmp_info, to_info)
|
||||
except Exception:
|
||||
self.remove(tmp_info)
|
||||
raise
|
||||
@@ -202,9 +276,13 @@ class RemoteLOCAL(RemoteBASE):
|
||||
def is_hardlink(path_info):
|
||||
return System.is_hardlink(path_info)
|
||||
|
||||
@staticmethod
|
||||
def reflink(from_info, to_info):
|
||||
System.reflink(from_info, to_info)
|
||||
def reflink(self, from_info, to_info):
|
||||
tmp_info = to_info.parent / tmp_fname(to_info.name)
|
||||
System.reflink(from_info, tmp_info)
|
||||
# NOTE: reflink has its own separate inode, so you can set permissions
|
||||
# that are different from the source.
|
||||
os.chmod(tmp_info, self._file_mode)
|
||||
os.rename(tmp_info, to_info)
|
||||
|
||||
def cache_exists(self, checksums, jobs=None, name=None):
|
||||
return [
|
||||
@@ -227,7 +305,9 @@ class RemoteLOCAL(RemoteBASE):
|
||||
copyfile(
|
||||
from_file, tmp_file, name=name, no_progress_bar=no_progress_bar
|
||||
)
|
||||
os.rename(tmp_file, fspath_py35(to_info))
|
||||
|
||||
self.protect(tmp_file)
|
||||
os.rename(tmp_file, to_info)
|
||||
|
||||
def _download(
|
||||
self, from_info, to_file, name=None, no_progress_bar=False, **_kwargs
|
||||
@@ -238,8 +318,9 @@ class RemoteLOCAL(RemoteBASE):
|
||||
|
||||
@staticmethod
|
||||
def open(path_info, mode="r", encoding=None):
|
||||
return open(fspath_py35(path_info), mode=mode, encoding=encoding)
|
||||
return open(path_info, mode=mode, encoding=encoding)
|
||||
|
||||
@index_locked
|
||||
def status(
|
||||
self,
|
||||
named_cache,
|
||||
@@ -248,37 +329,128 @@ class RemoteLOCAL(RemoteBASE):
|
||||
show_checksums=False,
|
||||
download=False,
|
||||
):
|
||||
logger.debug(
|
||||
"Preparing to collect status from {}".format(remote.path_info)
|
||||
# Return flattened dict containing all status info
|
||||
dir_status, file_status, _ = self._status(
|
||||
named_cache,
|
||||
remote,
|
||||
jobs=jobs,
|
||||
show_checksums=show_checksums,
|
||||
download=download,
|
||||
)
|
||||
md5s = list(named_cache[self.scheme])
|
||||
return dict(dir_status, **file_status)
|
||||
|
||||
def _status(
|
||||
self,
|
||||
named_cache,
|
||||
remote,
|
||||
jobs=None,
|
||||
show_checksums=False,
|
||||
download=False,
|
||||
):
|
||||
"""Return a tuple of (dir_status_info, file_status_info, dir_contents).
|
||||
|
||||
dir_status_info contains status for .dir files, file_status_info
|
||||
contains status for all other files, and dir_contents is a dict of
|
||||
{dir_checksum: set(file_checksum, ...)} which can be used to map
|
||||
a .dir file to its file contents.
|
||||
"""
|
||||
logger.debug(f"Preparing to collect status from {remote.path_info}")
|
||||
md5s = set(named_cache.scheme_keys(self.scheme))
|
||||
|
||||
logger.debug("Collecting information from local cache...")
|
||||
local_exists = self.cache_exists(md5s, jobs=jobs, name=self.cache_dir)
|
||||
local_exists = frozenset(
|
||||
self.cache_exists(md5s, jobs=jobs, name=self.cache_dir)
|
||||
)
|
||||
|
||||
# This is a performance optimization. We can safely assume that,
|
||||
# if the resources that we want to fetch are already cached,
|
||||
# there's no need to check the remote storage for the existence of
|
||||
# those files.
|
||||
if download and sorted(local_exists) == sorted(md5s):
|
||||
if download and local_exists == md5s:
|
||||
remote_exists = local_exists
|
||||
else:
|
||||
logger.debug("Collecting information from remote cache...")
|
||||
remote_exists = list(
|
||||
remote.cache_exists(
|
||||
md5s, jobs=jobs, name=str(remote.path_info)
|
||||
remote_exists = set()
|
||||
dir_md5s = set(named_cache.dir_keys(self.scheme))
|
||||
if dir_md5s:
|
||||
remote_exists.update(
|
||||
self._indexed_dir_checksums(named_cache, remote, dir_md5s)
|
||||
)
|
||||
md5s.difference_update(remote_exists)
|
||||
if md5s:
|
||||
remote_exists.update(
|
||||
remote.cache_exists(
|
||||
md5s, jobs=jobs, name=str(remote.path_info)
|
||||
)
|
||||
)
|
||||
return self._make_status(
|
||||
named_cache, show_checksums, local_exists, remote_exists
|
||||
)
|
||||
|
||||
def _make_status(
|
||||
self, named_cache, show_checksums, local_exists, remote_exists
|
||||
):
|
||||
def make_names(checksum, names):
|
||||
return {"name": checksum if show_checksums else " ".join(names)}
|
||||
|
||||
dir_status = {}
|
||||
file_status = {}
|
||||
dir_contents = {}
|
||||
for checksum, item in named_cache[self.scheme].items():
|
||||
if item.children:
|
||||
dir_status[checksum] = make_names(checksum, item.names)
|
||||
dir_contents[checksum] = set()
|
||||
for child_checksum, child in item.children.items():
|
||||
file_status[child_checksum] = make_names(
|
||||
child_checksum, child.names
|
||||
)
|
||||
dir_contents[checksum].add(child_checksum)
|
||||
else:
|
||||
file_status[checksum] = make_names(checksum, item.names)
|
||||
|
||||
self._fill_statuses(dir_status, local_exists, remote_exists)
|
||||
self._fill_statuses(file_status, local_exists, remote_exists)
|
||||
|
||||
self._log_missing_caches(dict(dir_status, **file_status))
|
||||
|
||||
return dir_status, file_status, dir_contents
|
||||
|
||||
def _indexed_dir_checksums(self, named_cache, remote, dir_md5s):
|
||||
# Validate our index by verifying all indexed .dir checksums
|
||||
# still exist on the remote
|
||||
indexed_dirs = set(remote.index.dir_checksums())
|
||||
indexed_dir_exists = set()
|
||||
if indexed_dirs:
|
||||
indexed_dir_exists.update(
|
||||
remote._cache_object_exists(indexed_dirs)
|
||||
)
|
||||
missing_dirs = indexed_dirs.difference(indexed_dir_exists)
|
||||
if missing_dirs:
|
||||
logger.debug(
|
||||
"Remote cache missing indexed .dir checksums '{}', "
|
||||
"clearing remote index".format(", ".join(missing_dirs))
|
||||
)
|
||||
remote.index.clear()
|
||||
|
||||
ret = {
|
||||
checksum: {"name": checksum if show_checksums else " ".join(names)}
|
||||
for checksum, names in named_cache[self.scheme].items()
|
||||
}
|
||||
self._fill_statuses(ret, local_exists, remote_exists)
|
||||
# Check if non-indexed (new) dir checksums exist on remote
|
||||
dir_exists = dir_md5s.intersection(indexed_dir_exists)
|
||||
dir_exists.update(remote._cache_object_exists(dir_md5s - dir_exists))
|
||||
|
||||
self._log_missing_caches(ret)
|
||||
|
||||
return ret
|
||||
# If .dir checksum exists on the remote, assume directory contents
|
||||
# still exists on the remote
|
||||
for dir_checksum in dir_exists:
|
||||
file_checksums = list(
|
||||
named_cache.child_keys(self.scheme, dir_checksum)
|
||||
)
|
||||
if dir_checksum not in remote.index:
|
||||
logger.debug(
|
||||
"Indexing new .dir '{}' with '{}' nested files".format(
|
||||
dir_checksum, len(file_checksums)
|
||||
)
|
||||
)
|
||||
remote.index.update([dir_checksum], file_checksums)
|
||||
yield dir_checksum
|
||||
yield from file_checksums
|
||||
|
||||
@staticmethod
|
||||
def _fill_statuses(checksum_info_dir, local_exists, remote_exists):
|
||||
@@ -294,6 +466,7 @@ class RemoteLOCAL(RemoteBASE):
|
||||
cache = []
|
||||
path_infos = []
|
||||
names = []
|
||||
checksums = []
|
||||
for md5, info in Tqdm(
|
||||
status_info.items(), desc="Analysing status", unit="file"
|
||||
):
|
||||
@@ -301,6 +474,7 @@ class RemoteLOCAL(RemoteBASE):
|
||||
cache.append(self.checksum_to_path_info(md5))
|
||||
path_infos.append(remote.checksum_to_path_info(md5))
|
||||
names.append(info["name"])
|
||||
checksums.append(md5)
|
||||
|
||||
if download:
|
||||
to_infos = cache
|
||||
@@ -309,7 +483,7 @@ class RemoteLOCAL(RemoteBASE):
|
||||
to_infos = path_infos
|
||||
from_infos = cache
|
||||
|
||||
return from_infos, to_infos, names
|
||||
return from_infos, to_infos, names, checksums
|
||||
|
||||
def _process(
|
||||
self,
|
||||
@@ -333,14 +507,16 @@ class RemoteLOCAL(RemoteBASE):
|
||||
file_mode=self._file_mode,
|
||||
)
|
||||
status = STATUS_DELETED
|
||||
desc = "Downloading"
|
||||
else:
|
||||
func = remote.upload
|
||||
status = STATUS_NEW
|
||||
desc = "Uploading"
|
||||
|
||||
if jobs is None:
|
||||
jobs = remote.JOBS
|
||||
|
||||
status_info = self.status(
|
||||
dir_status, file_status, dir_contents = self._status(
|
||||
named_cache,
|
||||
remote,
|
||||
jobs=jobs,
|
||||
@@ -348,24 +524,87 @@ class RemoteLOCAL(RemoteBASE):
|
||||
download=download,
|
||||
)
|
||||
|
||||
plans = self._get_plans(download, remote, status_info, status)
|
||||
dir_plans = self._get_plans(download, remote, dir_status, status)
|
||||
file_plans = self._get_plans(download, remote, file_status, status)
|
||||
|
||||
if len(plans[0]) == 0:
|
||||
total = len(dir_plans[0]) + len(file_plans[0])
|
||||
if total == 0:
|
||||
return 0
|
||||
|
||||
if jobs > 1:
|
||||
with Tqdm(total=total, unit="file", desc=desc) as pbar:
|
||||
func = pbar.wrap_fn(func)
|
||||
with ThreadPoolExecutor(max_workers=jobs) as executor:
|
||||
fails = sum(executor.map(func, *plans))
|
||||
else:
|
||||
fails = sum(map(func, *plans))
|
||||
if download:
|
||||
fails = sum(executor.map(func, *dir_plans))
|
||||
fails += sum(executor.map(func, *file_plans))
|
||||
else:
|
||||
# for uploads, push files first, and any .dir files last
|
||||
|
||||
file_futures = {}
|
||||
for from_info, to_info, name, checksum in zip(*file_plans):
|
||||
file_futures[checksum] = executor.submit(
|
||||
func, from_info, to_info, name
|
||||
)
|
||||
dir_futures = {}
|
||||
for from_info, to_info, name, dir_checksum in zip(
|
||||
*dir_plans
|
||||
):
|
||||
wait_futures = {
|
||||
future
|
||||
for file_checksum, future in file_futures.items()
|
||||
if file_checksum in dir_contents[dir_checksum]
|
||||
}
|
||||
dir_futures[dir_checksum] = executor.submit(
|
||||
self._dir_upload,
|
||||
func,
|
||||
wait_futures,
|
||||
from_info,
|
||||
to_info,
|
||||
name,
|
||||
)
|
||||
fails = sum(
|
||||
future.result()
|
||||
for future in concat(
|
||||
file_futures.values(), dir_futures.values()
|
||||
)
|
||||
)
|
||||
|
||||
if fails:
|
||||
if download:
|
||||
remote.index.clear()
|
||||
raise DownloadError(fails)
|
||||
raise UploadError(fails)
|
||||
|
||||
return len(plans[0])
|
||||
if not download:
|
||||
# index successfully pushed dirs
|
||||
for dir_checksum, future in dir_futures.items():
|
||||
if future.result() == 0:
|
||||
file_checksums = dir_contents[dir_checksum]
|
||||
logger.debug(
|
||||
"Indexing pushed dir '{}' with "
|
||||
"'{}' nested files".format(
|
||||
dir_checksum, len(file_checksums)
|
||||
)
|
||||
)
|
||||
remote.index.update([dir_checksum], file_checksums)
|
||||
|
||||
return len(dir_plans[0]) + len(file_plans[0])
|
||||
|
||||
@staticmethod
|
||||
def _dir_upload(func, futures, from_info, to_info, name):
|
||||
for future in as_completed(futures):
|
||||
if future.result():
|
||||
# do not upload this .dir file if any file in this
|
||||
# directory failed to upload
|
||||
logger.debug(
|
||||
"failed to upload full contents of '{}', "
|
||||
"aborting .dir file upload".format(name)
|
||||
)
|
||||
logger.error(f"failed to upload '{from_info}' to '{to_info}'")
|
||||
return 1
|
||||
return func(from_info, to_info, name)
|
||||
|
||||
@index_locked
|
||||
def push(self, named_cache, remote, jobs=None, show_checksums=False):
|
||||
return self._process(
|
||||
named_cache,
|
||||
@@ -375,6 +614,7 @@ class RemoteLOCAL(RemoteBASE):
|
||||
download=False,
|
||||
)
|
||||
|
||||
@index_locked
|
||||
def pull(self, named_cache, remote, jobs=None, show_checksums=False):
|
||||
return self._process(
|
||||
named_cache,
|
||||
@@ -402,10 +642,9 @@ class RemoteLOCAL(RemoteBASE):
|
||||
)
|
||||
logger.warning(msg)
|
||||
|
||||
@staticmethod
|
||||
def _unprotect_file(path):
|
||||
def _unprotect_file(self, path):
|
||||
if System.is_symlink(path) or System.is_hardlink(path):
|
||||
logger.debug("Unprotecting '{}'".format(path))
|
||||
logger.debug(f"Unprotecting '{path}'")
|
||||
tmp = os.path.join(os.path.dirname(path), "." + uuid())
|
||||
|
||||
# The operations order is important here - if some application
|
||||
@@ -423,99 +662,53 @@ class RemoteLOCAL(RemoteBASE):
|
||||
"a symlink or a hardlink.".format(path)
|
||||
)
|
||||
|
||||
os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
|
||||
os.chmod(path, self._file_mode)
|
||||
|
||||
def _unprotect_dir(self, path):
|
||||
assert is_working_tree(self.repo.tree)
|
||||
|
||||
for fname in self.repo.tree.walk_files(path):
|
||||
RemoteLOCAL._unprotect_file(fname)
|
||||
self._unprotect_file(fname)
|
||||
|
||||
def unprotect(self, path_info):
|
||||
path = path_info.fspath
|
||||
if not os.path.exists(path):
|
||||
raise DvcException(
|
||||
"can't unprotect non-existing data '{}'".format(path)
|
||||
)
|
||||
raise DvcException(f"can't unprotect non-existing data '{path}'")
|
||||
|
||||
if os.path.isdir(path):
|
||||
self._unprotect_dir(path)
|
||||
else:
|
||||
RemoteLOCAL._unprotect_file(path)
|
||||
self._unprotect_file(path)
|
||||
|
||||
@staticmethod
|
||||
def protect(path_info):
|
||||
path = fspath_py35(path_info)
|
||||
mode = stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH
|
||||
def protect(self, path_info):
|
||||
path = os.fspath(path_info)
|
||||
mode = self.CACHE_MODE
|
||||
|
||||
try:
|
||||
os.chmod(path, mode)
|
||||
except OSError as exc:
|
||||
# In share cache scenario, we might not own the cache file, so we
|
||||
# There is nothing we need to do in case of a read-only file system
|
||||
if exc.errno == errno.EROFS:
|
||||
return
|
||||
|
||||
# In shared cache scenario, we might not own the cache file, so we
|
||||
# need to check if cache file is already protected.
|
||||
if exc.errno not in [errno.EPERM, errno.EACCES]:
|
||||
raise
|
||||
|
||||
actual = os.stat(path).st_mode
|
||||
if actual & mode != mode:
|
||||
actual = stat.S_IMODE(os.stat(path).st_mode)
|
||||
if actual != mode:
|
||||
raise
|
||||
|
||||
def _get_unpacked_dir_path_info(self, checksum):
|
||||
info = self.checksum_to_path_info(checksum)
|
||||
return info.with_name(info.name + self.UNPACKED_DIR_SUFFIX)
|
||||
|
||||
def _remove_unpacked_dir(self, checksum):
|
||||
path_info = self._get_unpacked_dir_path_info(checksum)
|
||||
info = self.checksum_to_path_info(checksum)
|
||||
path_info = info.with_name(info.name + self.UNPACKED_DIR_SUFFIX)
|
||||
self.remove(path_info)
|
||||
|
||||
def _path_info_changed(self, path_info):
|
||||
if self.exists(path_info) and self.state.get(path_info):
|
||||
return False
|
||||
return True
|
||||
|
||||
def _update_unpacked_dir(self, checksum):
|
||||
unpacked_dir_info = self._get_unpacked_dir_path_info(checksum)
|
||||
|
||||
if not self._path_info_changed(unpacked_dir_info):
|
||||
return
|
||||
|
||||
self.remove(unpacked_dir_info)
|
||||
|
||||
def is_protected(self, path_info):
|
||||
try:
|
||||
dir_info = self.get_dir_cache(checksum)
|
||||
self._create_unpacked_dir(checksum, dir_info, unpacked_dir_info)
|
||||
except DvcException:
|
||||
logger.warning("Could not create '{}'".format(unpacked_dir_info))
|
||||
mode = os.stat(path_info).st_mode
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
|
||||
self.remove(unpacked_dir_info)
|
||||
|
||||
def _create_unpacked_dir(self, checksum, dir_info, unpacked_dir_info):
|
||||
self.makedirs(unpacked_dir_info)
|
||||
|
||||
for entry in Tqdm(dir_info, desc="Creating unpacked dir", unit="file"):
|
||||
entry_cache_info = self.checksum_to_path_info(
|
||||
entry[self.PARAM_CHECKSUM]
|
||||
)
|
||||
relative_path = entry[self.PARAM_RELPATH]
|
||||
# In shared cache mode some cache files might not be owned by the
|
||||
# user, so we need to use symlinks because, unless
|
||||
# /proc/sys/fs/protected_hardlinks is disabled, the user is not
|
||||
# allowed to create hardlinks to files that he doesn't own.
|
||||
link_types = ["hardlink", "symlink"]
|
||||
self._link(
|
||||
entry_cache_info, unpacked_dir_info / relative_path, link_types
|
||||
)
|
||||
|
||||
self.state.save(unpacked_dir_info, checksum)
|
||||
|
||||
def _changed_unpacked_dir(self, checksum):
|
||||
status_unpacked_dir_info = self._get_unpacked_dir_path_info(checksum)
|
||||
|
||||
return not self.state.get(status_unpacked_dir_info)
|
||||
|
||||
def _get_unpacked_dir_names(self, checksums):
|
||||
unpacked = set()
|
||||
for c in checksums:
|
||||
if self.is_dir_checksum(c):
|
||||
unpacked.add(c + self.UNPACKED_DIR_SUFFIX)
|
||||
return unpacked
|
||||
return stat.S_IMODE(mode) == self.CACHE_MODE
|
||||
|
||||
+19
-10
@@ -1,19 +1,19 @@
|
||||
import logging
|
||||
import os
|
||||
import posixpath
|
||||
import threading
|
||||
|
||||
from funcy import cached_property, wrap_prop
|
||||
|
||||
from dvc.path_info import CloudURLInfo
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.remote.base import RemoteBASE
|
||||
from dvc.remote.base import BaseRemote
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RemoteOSS(RemoteBASE):
|
||||
class OSSRemote(BaseRemote):
|
||||
"""
|
||||
oss2 document:
|
||||
https://www.alibabacloud.com/help/doc-detail/32026.htm
|
||||
@@ -37,6 +37,7 @@ class RemoteOSS(RemoteBASE):
|
||||
REQUIRES = {"oss2": "oss2"}
|
||||
PARAM_CHECKSUM = "etag"
|
||||
COPY_POLL_SECONDS = 5
|
||||
LIST_OBJECT_PAGE_SIZE = 100
|
||||
|
||||
def __init__(self, repo, config):
|
||||
super().__init__(repo, config)
|
||||
@@ -63,9 +64,9 @@ class RemoteOSS(RemoteBASE):
|
||||
def oss_service(self):
|
||||
import oss2
|
||||
|
||||
logger.debug("URL: {}".format(self.path_info))
|
||||
logger.debug("key id: {}".format(self.key_id))
|
||||
logger.debug("key secret: {}".format(self.key_secret))
|
||||
logger.debug(f"URL: {self.path_info}")
|
||||
logger.debug(f"key id: {self.key_id}")
|
||||
logger.debug(f"key secret: {self.key_secret}")
|
||||
|
||||
auth = oss2.Auth(self.key_id, self.key_secret)
|
||||
bucket = oss2.Bucket(auth, self.endpoint, self.path_info.bucket)
|
||||
@@ -86,17 +87,25 @@ class RemoteOSS(RemoteBASE):
|
||||
if path_info.scheme != self.scheme:
|
||||
raise NotImplementedError
|
||||
|
||||
logger.debug("Removing oss://{}".format(path_info))
|
||||
logger.debug(f"Removing oss://{path_info}")
|
||||
self.oss_service.delete_object(path_info.path)
|
||||
|
||||
def _list_paths(self, prefix):
|
||||
def _list_paths(self, prefix, progress_callback=None):
|
||||
import oss2
|
||||
|
||||
for blob in oss2.ObjectIterator(self.oss_service, prefix=prefix):
|
||||
if progress_callback:
|
||||
progress_callback()
|
||||
yield blob.key
|
||||
|
||||
def list_cache_paths(self):
|
||||
return self._list_paths(self.path_info.path)
|
||||
def list_cache_paths(self, prefix=None, progress_callback=None):
|
||||
if prefix:
|
||||
prefix = posixpath.join(
|
||||
self.path_info.path, prefix[:2], prefix[2:]
|
||||
)
|
||||
else:
|
||||
prefix = self.path_info.path
|
||||
return self._list_paths(prefix, progress_callback)
|
||||
|
||||
def _upload(
|
||||
self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
|
||||
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
import threading
|
||||
from collections import deque
|
||||
from contextlib import contextmanager
|
||||
import threading
|
||||
|
||||
from funcy import memoize, wrap_with
|
||||
|
||||
@@ -30,7 +30,7 @@ def close_pools():
|
||||
get_pool.memory.clear()
|
||||
|
||||
|
||||
class Pool(object):
|
||||
class Pool:
|
||||
def __init__(self, conn_func, *conn_args, **conn_kwargs):
|
||||
self._conn_func = conn_func
|
||||
self._conn_args = conn_args
|
||||
|
||||
+37
-17
@@ -1,23 +1,21 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import logging
|
||||
import os
|
||||
import posixpath
|
||||
import threading
|
||||
|
||||
from funcy import cached_property, wrap_prop
|
||||
|
||||
from dvc.config import ConfigError
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.exceptions import ETagMismatchError
|
||||
from dvc.exceptions import DvcException, ETagMismatchError
|
||||
from dvc.path_info import CloudURLInfo
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.remote.base import RemoteBASE
|
||||
from dvc.remote.base import BaseRemote
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RemoteS3(RemoteBASE):
|
||||
class S3Remote(BaseRemote):
|
||||
scheme = Schemes.S3
|
||||
path_cls = CloudURLInfo
|
||||
REQUIRES = {"boto3": "boto3"}
|
||||
@@ -46,6 +44,10 @@ class RemoteS3(RemoteBASE):
|
||||
if self.sse:
|
||||
self.extra_args["ServerSideEncryption"] = self.sse
|
||||
|
||||
self.sse_kms_key_id = config.get("sse_kms_key_id")
|
||||
if self.sse_kms_key_id:
|
||||
self.extra_args["SSEKMSKeyId"] = self.sse_kms_key_id
|
||||
|
||||
self.acl = config.get("acl")
|
||||
if self.acl:
|
||||
self.extra_args["ACL"] = self.acl
|
||||
@@ -84,9 +86,7 @@ class RemoteS3(RemoteBASE):
|
||||
try:
|
||||
obj = s3.head_object(Bucket=bucket, Key=path, *args, **kwargs)
|
||||
except Exception as exc:
|
||||
raise DvcException(
|
||||
"s3://{}/{} does not exist".format(bucket, path)
|
||||
) from exc
|
||||
raise DvcException(f"s3://{bucket}/{path} does not exist") from exc
|
||||
return obj
|
||||
|
||||
@classmethod
|
||||
@@ -109,7 +109,7 @@ class RemoteS3(RemoteBASE):
|
||||
if lastbyte > size:
|
||||
lastbyte = size - 1
|
||||
|
||||
srange = "bytes={}-{}".format(byte_position, lastbyte)
|
||||
srange = f"bytes={byte_position}-{lastbyte}"
|
||||
|
||||
part = s3.upload_part_copy(
|
||||
Bucket=to_info.bucket,
|
||||
@@ -187,27 +187,44 @@ class RemoteS3(RemoteBASE):
|
||||
if path_info.scheme != "s3":
|
||||
raise NotImplementedError
|
||||
|
||||
logger.debug("Removing {}".format(path_info))
|
||||
logger.debug(f"Removing {path_info}")
|
||||
self.s3.delete_object(Bucket=path_info.bucket, Key=path_info.path)
|
||||
|
||||
def _list_objects(self, path_info, max_items=None):
|
||||
def _list_objects(
|
||||
self, path_info, max_items=None, prefix=None, progress_callback=None
|
||||
):
|
||||
""" Read config for list object api, paginate through list objects."""
|
||||
kwargs = {
|
||||
"Bucket": path_info.bucket,
|
||||
"Prefix": path_info.path,
|
||||
"PaginationConfig": {"MaxItems": max_items},
|
||||
}
|
||||
if prefix:
|
||||
kwargs["Prefix"] = posixpath.join(path_info.path, prefix[:2])
|
||||
paginator = self.s3.get_paginator(self.list_objects_api)
|
||||
for page in paginator.paginate(**kwargs):
|
||||
yield from page.get("Contents", ())
|
||||
contents = page.get("Contents", ())
|
||||
if progress_callback:
|
||||
for item in contents:
|
||||
progress_callback()
|
||||
yield item
|
||||
else:
|
||||
yield from contents
|
||||
|
||||
def _list_paths(self, path_info, max_items=None):
|
||||
def _list_paths(
|
||||
self, path_info, max_items=None, prefix=None, progress_callback=None
|
||||
):
|
||||
return (
|
||||
item["Key"] for item in self._list_objects(path_info, max_items)
|
||||
item["Key"]
|
||||
for item in self._list_objects(
|
||||
path_info, max_items, prefix, progress_callback
|
||||
)
|
||||
)
|
||||
|
||||
def list_cache_paths(self):
|
||||
return self._list_paths(self.path_info)
|
||||
def list_cache_paths(self, prefix=None, progress_callback=None):
|
||||
return self._list_paths(
|
||||
self.path_info, prefix=prefix, progress_callback=progress_callback
|
||||
)
|
||||
|
||||
def isfile(self, path_info):
|
||||
from botocore.exceptions import ClientError
|
||||
@@ -238,6 +255,9 @@ class RemoteS3(RemoteBASE):
|
||||
#
|
||||
# We are not creating directory objects for every parent prefix,
|
||||
# as it is not required.
|
||||
if not path_info.path:
|
||||
return
|
||||
|
||||
dir_path = path_info / ""
|
||||
self.s3.put_object(Bucket=path_info.bucket, Key=dir_path.path, Body="")
|
||||
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
import logging
|
||||
import sys
|
||||
import time
|
||||
|
||||
from functools import wraps
|
||||
import colorama
|
||||
|
||||
import colorama
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
this = sys.modules[__name__]
|
||||
|
||||
+32
-10
@@ -4,16 +4,17 @@ import io
|
||||
import itertools
|
||||
import logging
|
||||
import os
|
||||
import posixpath
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from contextlib import closing, contextmanager
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from funcy import memoize, wrap_with, silent, first
|
||||
from funcy import first, memoize, silent, wrap_with
|
||||
|
||||
import dvc.prompt as prompt
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.remote.base import RemoteBASE
|
||||
from dvc.remote.base import BaseRemote
|
||||
from dvc.remote.pool import get_connection
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.utils import to_chunks
|
||||
@@ -32,7 +33,7 @@ def ask_password(host, user, port):
|
||||
)
|
||||
|
||||
|
||||
class RemoteSSH(RemoteBASE):
|
||||
class SSHRemote(BaseRemote):
|
||||
scheme = Schemes.SSH
|
||||
REQUIRES = {"paramiko": "paramiko"}
|
||||
|
||||
@@ -44,6 +45,7 @@ class RemoteSSH(RemoteBASE):
|
||||
# paramiko stuff, so we would ideally have it double of server processors.
|
||||
# We use conservative setting of 4 instead to not exhaust max sessions.
|
||||
CHECKSUM_JOBS = 4
|
||||
TRAVERSE_PREFIX_LEN = 2
|
||||
|
||||
DEFAULT_CACHE_TYPES = ["copy"]
|
||||
|
||||
@@ -85,6 +87,13 @@ class RemoteSSH(RemoteBASE):
|
||||
self.password = config.get("password", None)
|
||||
self.ask_password = config.get("ask_password", False)
|
||||
self.gss_auth = config.get("gss_auth", False)
|
||||
proxy_command = user_ssh_config.get("proxycommand", False)
|
||||
if proxy_command:
|
||||
import paramiko
|
||||
|
||||
self.sock = paramiko.ProxyCommand(proxy_command)
|
||||
else:
|
||||
self.sock = None
|
||||
|
||||
@staticmethod
|
||||
def ssh_config_filename():
|
||||
@@ -94,7 +103,7 @@ class RemoteSSH(RemoteBASE):
|
||||
def _load_user_ssh_config(hostname):
|
||||
import paramiko
|
||||
|
||||
user_config_file = RemoteSSH.ssh_config_filename()
|
||||
user_config_file = SSHRemote.ssh_config_filename()
|
||||
user_ssh_config = {}
|
||||
if hostname and os.path.exists(user_config_file):
|
||||
ssh_config = paramiko.SSHConfig()
|
||||
@@ -136,6 +145,7 @@ class RemoteSSH(RemoteBASE):
|
||||
timeout=self.timeout,
|
||||
password=self.password,
|
||||
gss_auth=self.gss_auth,
|
||||
sock=self.sock,
|
||||
)
|
||||
|
||||
def exists(self, path_info):
|
||||
@@ -209,7 +219,8 @@ class RemoteSSH(RemoteBASE):
|
||||
with self.ssh(path_info) as ssh:
|
||||
ssh.remove(path_info.path)
|
||||
|
||||
def move(self, from_info, to_info):
|
||||
def move(self, from_info, to_info, mode=None):
|
||||
assert mode is None
|
||||
if from_info.scheme != self.scheme or to_info.scheme != self.scheme:
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -248,10 +259,21 @@ class RemoteSSH(RemoteBASE):
|
||||
else:
|
||||
yield io.TextIOWrapper(fd, encoding=encoding)
|
||||
|
||||
def list_cache_paths(self):
|
||||
def list_cache_paths(self, prefix=None, progress_callback=None):
|
||||
if prefix:
|
||||
root = posixpath.join(self.path_info.path, prefix[:2])
|
||||
else:
|
||||
root = self.path_info.path
|
||||
with self.ssh(self.path_info) as ssh:
|
||||
if prefix and not ssh.exists(root):
|
||||
return
|
||||
# If we simply return an iterator then with above closes instantly
|
||||
yield from ssh.walk_files(self.path_info.path)
|
||||
if progress_callback:
|
||||
for path in ssh.walk_files(root):
|
||||
progress_callback()
|
||||
yield path
|
||||
else:
|
||||
yield from ssh.walk_files(root)
|
||||
|
||||
def walk_files(self, path_info):
|
||||
with self.ssh(path_info) as ssh:
|
||||
@@ -270,7 +292,7 @@ class RemoteSSH(RemoteBASE):
|
||||
try:
|
||||
channel.stat(path)
|
||||
ret.append(True)
|
||||
except IOError as exc:
|
||||
except OSError as exc:
|
||||
if exc.errno != errno.ENOENT:
|
||||
raise
|
||||
ret.append(False)
|
||||
@@ -296,7 +318,7 @@ class RemoteSSH(RemoteBASE):
|
||||
faster than current approach (relying on exists(path_info)) applied in
|
||||
remote/base.
|
||||
"""
|
||||
if not self.no_traverse:
|
||||
if not self.CAN_TRAVERSE:
|
||||
return list(set(checksums) & set(self.all()))
|
||||
|
||||
# possibly prompt for credentials before "Querying" progress output
|
||||
@@ -310,7 +332,7 @@ class RemoteSSH(RemoteBASE):
|
||||
) as pbar:
|
||||
|
||||
def exists_with_progress(chunks):
|
||||
return self.batch_exists(chunks, callback=pbar.update_desc)
|
||||
return self.batch_exists(chunks, callback=pbar.update_msg)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=jobs or self.JOBS) as executor:
|
||||
path_infos = [self.checksum_to_path_info(x) for x in checksums]
|
||||
|
||||
@@ -7,16 +7,16 @@ from contextlib import suppress
|
||||
|
||||
from funcy import cached_property
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.remote.base import RemoteCmdError
|
||||
from dvc.utils import tmp_fname
|
||||
|
||||
try:
|
||||
import paramiko
|
||||
except ImportError:
|
||||
paramiko = None
|
||||
|
||||
from dvc.utils import tmp_fname
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.remote.base import RemoteCmdError
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -25,9 +25,9 @@ def sizeof_fmt(num, suffix="B"):
|
||||
""" Convert number of bytes to human-readable string """
|
||||
for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]:
|
||||
if abs(num) < 1024.0:
|
||||
return "%3.1f%s%s" % (num, unit, suffix)
|
||||
return f"{num:3.1f}{unit}{suffix}"
|
||||
num /= 1024.0
|
||||
return "%.1f%s%s" % (num, "Y", suffix)
|
||||
return "{:.1f}{}{}".format(num, "Y", suffix)
|
||||
|
||||
|
||||
class SSHConnection:
|
||||
@@ -41,6 +41,14 @@ class SSHConnection:
|
||||
self.timeout = kwargs.get("timeout", 1800)
|
||||
|
||||
self._ssh = paramiko.SSHClient()
|
||||
|
||||
# Explicitly disable paramiko logger. Due to how paramiko dynamically
|
||||
# loads loggers, it is not disabled by DVC disable_other_loggers().
|
||||
# See https://github.com/iterative/dvc/issues/3482
|
||||
self._ssh.set_log_channel("dvc.paramiko")
|
||||
logging.getLogger("dvc.paramiko").disabled = True
|
||||
logging.getLogger("dvc.paramiko.sftp").disabled = True
|
||||
|
||||
self._ssh.load_system_host_keys()
|
||||
self._ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
|
||||
@@ -92,7 +100,7 @@ class SSHConnection:
|
||||
|
||||
if stat.S_ISREG(st_mode) or stat.S_ISLNK(st_mode):
|
||||
raise DvcException(
|
||||
"a file with the same name '{}' already exists".format(path)
|
||||
f"a file with the same name '{path}' already exists"
|
||||
)
|
||||
|
||||
head, tail = posixpath.split(path)
|
||||
@@ -103,12 +111,12 @@ class SSHConnection:
|
||||
if tail:
|
||||
try:
|
||||
self.sftp.mkdir(path)
|
||||
except IOError as exc:
|
||||
except OSError as exc:
|
||||
# Since paramiko errors are very vague we need to recheck
|
||||
# whether it's because path already exists or something else
|
||||
if exc.errno == errno.EACCES or not self.exists(path):
|
||||
raise DvcException(
|
||||
"unable to create remote directory '{}'".format(path)
|
||||
f"unable to create remote directory '{path}'"
|
||||
) from exc
|
||||
|
||||
def walk(self, directory, topdown=True):
|
||||
@@ -118,7 +126,7 @@ class SSHConnection:
|
||||
# [1] https://github.com/python/cpython/blob/master/Lib/os.py
|
||||
try:
|
||||
dir_entries = self.sftp.listdir_attr(directory)
|
||||
except IOError as exc:
|
||||
except OSError as exc:
|
||||
raise DvcException(
|
||||
"couldn't get the '{}' remote directory files list".format(
|
||||
directory
|
||||
@@ -284,14 +292,14 @@ class SSHConnection:
|
||||
md5 = self.execute("md5 " + path).split()[-1]
|
||||
else:
|
||||
raise DvcException(
|
||||
"'{}' is not supported as a SSH remote".format(self.uname)
|
||||
f"'{self.uname}' is not supported as a SSH remote"
|
||||
)
|
||||
|
||||
assert len(md5) == 32
|
||||
return md5
|
||||
|
||||
def copy(self, src, dest):
|
||||
self.execute("cp {} {}".format(src, dest))
|
||||
self.execute(f"cp {src} {dest}")
|
||||
|
||||
def open_max_sftp_channels(self):
|
||||
# If there are more than 1 it means we've already opened max amount
|
||||
@@ -313,14 +321,12 @@ class SSHConnection:
|
||||
|
||||
def reflink(self, src, dest):
|
||||
if self.uname == "Linux":
|
||||
return self.execute("cp --reflink {} {}".format(src, dest))
|
||||
return self.execute(f"cp --reflink {src} {dest}")
|
||||
|
||||
if self.uname == "Darwin":
|
||||
return self.execute("cp -c {} {}".format(src, dest))
|
||||
return self.execute(f"cp -c {src} {dest}")
|
||||
|
||||
raise DvcException(
|
||||
"'{}' is not supported as a SSH remote".format(self.uname)
|
||||
)
|
||||
raise DvcException(f"'{self.uname}' is not supported as a SSH remote")
|
||||
|
||||
def hardlink(self, src, dest):
|
||||
self.execute("ln {} {}".format(src, dest))
|
||||
self.execute(f"ln {src} {dest}")
|
||||
|
||||
+212
-124
@@ -1,24 +1,30 @@
|
||||
import logging
|
||||
import os
|
||||
from contextlib import contextmanager
|
||||
from functools import wraps
|
||||
|
||||
from dvc.ignore import CleanTree
|
||||
from dvc.compat import fspath_py35
|
||||
|
||||
from funcy import cached_property, cat, first
|
||||
|
||||
from dvc.config import Config
|
||||
from dvc.dvcfile import PIPELINE_FILE, Dvcfile, is_valid_filename
|
||||
from dvc.exceptions import FileMissingError
|
||||
from dvc.exceptions import IsADirectoryError as DvcIsADirectoryError
|
||||
from dvc.exceptions import (
|
||||
FileMissingError,
|
||||
IsADirectoryError,
|
||||
NoOutputOrStageError,
|
||||
NotDvcRepoError,
|
||||
OutputNotFoundError,
|
||||
)
|
||||
from dvc.ignore import CleanTree
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.remote.base import RemoteActionNotImplemented
|
||||
from dvc.repo.tree import RepoTree
|
||||
from dvc.utils.fs import path_isin
|
||||
|
||||
from ..stage.exceptions import StageFileDoesNotExistError, StageNotFound
|
||||
from ..utils import parse_target
|
||||
from .graph import check_acyclic, get_pipeline, get_pipelines
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def locked(f):
|
||||
@wraps(f)
|
||||
@@ -33,7 +39,7 @@ def locked(f):
|
||||
return wrapper
|
||||
|
||||
|
||||
class Repo(object):
|
||||
class Repo:
|
||||
DVC_DIR = ".dvc"
|
||||
|
||||
from dvc.repo.destroy import destroy
|
||||
@@ -60,49 +66,64 @@ class Repo(object):
|
||||
from dvc.repo.get_url import get_url
|
||||
from dvc.repo.update import update
|
||||
|
||||
def __init__(self, root_dir=None):
|
||||
from dvc.state import State
|
||||
def __init__(self, root_dir=None, scm=None, rev=None):
|
||||
from dvc.state import State, StateNoop
|
||||
from dvc.lock import make_lock
|
||||
from dvc.scm import SCM
|
||||
from dvc.cache import Cache
|
||||
from dvc.data_cloud import DataCloud
|
||||
from dvc.repo.metrics import Metrics
|
||||
from dvc.repo.plots import Plots
|
||||
from dvc.repo.params import Params
|
||||
from dvc.scm.tree import WorkingTree
|
||||
from dvc.repo.tag import Tag
|
||||
from dvc.utils.fs import makedirs
|
||||
from dvc.stage.cache import StageCache
|
||||
|
||||
root_dir = self.find_root(root_dir)
|
||||
if scm:
|
||||
# use GitTree instead of WorkingTree as default repo tree instance
|
||||
tree = scm.get_tree(rev)
|
||||
self.root_dir = self.find_root(root_dir, tree)
|
||||
self.scm = scm
|
||||
self.tree = tree
|
||||
self.state = StateNoop()
|
||||
else:
|
||||
root_dir = self.find_root(root_dir)
|
||||
self.root_dir = os.path.abspath(os.path.realpath(root_dir))
|
||||
|
||||
self.root_dir = os.path.abspath(os.path.realpath(root_dir))
|
||||
self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR)
|
||||
|
||||
self.config = Config(self.dvc_dir)
|
||||
|
||||
no_scm = self.config["core"].get("no_scm", False)
|
||||
self.scm = SCM(self.root_dir, no_scm=no_scm)
|
||||
|
||||
self.tree = WorkingTree(self.root_dir)
|
||||
if not scm:
|
||||
no_scm = self.config["core"].get("no_scm", False)
|
||||
self.scm = SCM(self.root_dir, no_scm=no_scm)
|
||||
self.tree = WorkingTree(self.root_dir)
|
||||
|
||||
self.tmp_dir = os.path.join(self.dvc_dir, "tmp")
|
||||
makedirs(self.tmp_dir, exist_ok=True)
|
||||
self.index_dir = os.path.join(self.tmp_dir, "index")
|
||||
makedirs(self.index_dir, exist_ok=True)
|
||||
|
||||
hardlink_lock = self.config["core"].get("hardlink_lock", False)
|
||||
self.lock = make_lock(
|
||||
os.path.join(self.dvc_dir, "lock"),
|
||||
tmp_dir=os.path.join(self.dvc_dir, "tmp"),
|
||||
os.path.join(self.tmp_dir, "lock"),
|
||||
tmp_dir=self.tmp_dir,
|
||||
hardlink_lock=hardlink_lock,
|
||||
friendly=True,
|
||||
)
|
||||
|
||||
# NOTE: storing state and link_state in the repository itself to avoid
|
||||
# any possible state corruption in 'shared cache dir' scenario.
|
||||
self.state = State(self)
|
||||
if not scm:
|
||||
# NOTE: storing state and link_state in the repository itself to
|
||||
# avoid any possible state corruption in 'shared cache dir'
|
||||
# scenario.
|
||||
self.state = State(self)
|
||||
|
||||
self.cache = Cache(self)
|
||||
self.cloud = DataCloud(self)
|
||||
|
||||
self.stage_cache = StageCache(self)
|
||||
|
||||
self.metrics = Metrics(self)
|
||||
self.tag = Tag(self)
|
||||
self.plots = Plots(self)
|
||||
self.params = Params(self)
|
||||
|
||||
self._ignore()
|
||||
|
||||
@@ -118,14 +139,19 @@ class Repo(object):
|
||||
self._reset()
|
||||
|
||||
def __repr__(self):
|
||||
return "{}: '{}'".format(self.__class__.__name__, self.root_dir)
|
||||
return f"{self.__class__.__name__}: '{self.root_dir}'"
|
||||
|
||||
@classmethod
|
||||
def find_root(cls, root=None):
|
||||
def find_root(cls, root=None, tree=None):
|
||||
root_dir = os.path.realpath(root or os.curdir)
|
||||
|
||||
if tree:
|
||||
if tree.isdir(os.path.join(root_dir, cls.DVC_DIR)):
|
||||
return root_dir
|
||||
raise NotDvcRepoError(f"'{root}' does not contain DVC directory")
|
||||
|
||||
if not os.path.isdir(root_dir):
|
||||
raise NotDvcRepoError("directory '{}' does not exist".format(root))
|
||||
raise NotDvcRepoError(f"directory '{root}' does not exist")
|
||||
|
||||
while True:
|
||||
dvc_dir = os.path.join(root_dir, cls.DVC_DIR)
|
||||
@@ -157,64 +183,142 @@ class Repo(object):
|
||||
return self.cache.local.unprotect(PathInfo(target))
|
||||
|
||||
def _ignore(self):
|
||||
from dvc.updater import Updater
|
||||
|
||||
updater = Updater(self.dvc_dir)
|
||||
|
||||
flist = (
|
||||
[self.config.files["local"], updater.updater_file]
|
||||
+ [self.lock.lockfile, updater.lock.lockfile, self.tmp_dir]
|
||||
+ self.state.files
|
||||
)
|
||||
flist = [self.config.files["local"], self.tmp_dir]
|
||||
|
||||
if path_isin(self.cache.local.cache_dir, self.root_dir):
|
||||
flist += [self.cache.local.cache_dir]
|
||||
|
||||
self.scm.ignore_list(flist)
|
||||
|
||||
def get_stage(self, path=None, name=None):
|
||||
if not path:
|
||||
path = PIPELINE_FILE
|
||||
logger.debug("Assuming '%s' to be a stage inside '%s'", name, path)
|
||||
|
||||
dvcfile = Dvcfile(self, path)
|
||||
return dvcfile.stages[name]
|
||||
|
||||
def get_stages(self, path=None, name=None):
|
||||
if not path:
|
||||
path = PIPELINE_FILE
|
||||
logger.debug("Assuming '%s' to be a stage inside '%s'", name, path)
|
||||
|
||||
if name:
|
||||
return [self.get_stage(path, name)]
|
||||
|
||||
dvcfile = Dvcfile(self, path)
|
||||
return list(dvcfile.stages.values())
|
||||
|
||||
def check_modified_graph(self, new_stages):
|
||||
"""Generate graph including the new stage to check for errors"""
|
||||
self._collect_graph(self.stages + new_stages)
|
||||
# Building graph might be costly for the ones with many DVC-files,
|
||||
# so we provide this undocumented hack to skip it. See [1] for
|
||||
# more details. The hack can be used as:
|
||||
#
|
||||
# repo = Repo(...)
|
||||
# repo._skip_graph_checks = True
|
||||
# repo.add(...)
|
||||
#
|
||||
# A user should care about not duplicating outs and not adding cycles,
|
||||
# otherwise DVC might have an undefined behaviour.
|
||||
#
|
||||
# [1] https://github.com/iterative/dvc/issues/2671
|
||||
if not getattr(self, "_skip_graph_checks", False):
|
||||
self._collect_graph(self.stages + new_stages)
|
||||
|
||||
def collect(self, target, with_deps=False, recursive=False, graph=None):
|
||||
def _collect_inside(self, path, graph):
|
||||
import networkx as nx
|
||||
from dvc.stage import Stage
|
||||
|
||||
stages = nx.dfs_postorder_nodes(graph)
|
||||
return [stage for stage in stages if path_isin(stage.path, path)]
|
||||
|
||||
def collect(
|
||||
self, target=None, with_deps=False, recursive=False, graph=None
|
||||
):
|
||||
if not target:
|
||||
return list(graph) if graph else self.stages
|
||||
|
||||
target = os.path.abspath(target)
|
||||
|
||||
if recursive and os.path.isdir(target):
|
||||
stages = nx.dfs_postorder_nodes(graph or self.graph)
|
||||
return [stage for stage in stages if path_isin(stage.path, target)]
|
||||
return self._collect_inside(
|
||||
os.path.abspath(target), graph or self.graph
|
||||
)
|
||||
|
||||
stage = Stage.load(self, target)
|
||||
|
||||
# Optimization: do not collect the graph for a specific target
|
||||
path, name = parse_target(target)
|
||||
stages = self.get_stages(path, name)
|
||||
if not with_deps:
|
||||
return [stage]
|
||||
return stages
|
||||
|
||||
res = set()
|
||||
for stage in stages:
|
||||
res.update(self._collect_pipeline(stage, graph=graph))
|
||||
return res
|
||||
|
||||
def _collect_pipeline(self, stage, graph=None):
|
||||
import networkx as nx
|
||||
|
||||
pipeline = get_pipeline(get_pipelines(graph or self.graph), stage)
|
||||
return list(nx.dfs_postorder_nodes(pipeline, stage))
|
||||
return nx.dfs_postorder_nodes(pipeline, stage)
|
||||
|
||||
def collect_granular(self, target, *args, **kwargs):
|
||||
from dvc.stage import Stage
|
||||
def _collect_from_default_dvcfile(self, target):
|
||||
dvcfile = Dvcfile(self, PIPELINE_FILE)
|
||||
if dvcfile.exists():
|
||||
return dvcfile.stages.get(target)
|
||||
|
||||
def collect_granular(
|
||||
self, target=None, with_deps=False, recursive=False, graph=None
|
||||
):
|
||||
"""
|
||||
Priority is in the order of following in case of ambiguity:
|
||||
- .dvc file or .yaml file
|
||||
- dir if recursive and directory exists
|
||||
- stage_name
|
||||
- output file
|
||||
"""
|
||||
if not target:
|
||||
return [(stage, None) for stage in self.stages]
|
||||
|
||||
# Optimization: do not collect the graph for a specific .dvc target
|
||||
if Stage.is_valid_filename(target) and not kwargs.get("with_deps"):
|
||||
return [(Stage.load(self, target), None)]
|
||||
file, name = parse_target(target)
|
||||
stages = []
|
||||
|
||||
try:
|
||||
(out,) = self.find_outs_by_path(target, strict=False)
|
||||
filter_info = PathInfo(os.path.abspath(target))
|
||||
return [(out.stage, filter_info)]
|
||||
except OutputNotFoundError:
|
||||
stages = self.collect(target, *args, **kwargs)
|
||||
return [(stage, None) for stage in stages]
|
||||
# Optimization: do not collect the graph for a specific target
|
||||
if not file:
|
||||
# parsing is ambiguous when it does not have a colon
|
||||
# or if it's not a dvcfile, as it can be a stage name
|
||||
# in `dvc.yaml` or, an output in a stage.
|
||||
logger.debug(
|
||||
"Checking if stage '%s' is in '%s'", target, PIPELINE_FILE
|
||||
)
|
||||
if not (recursive and os.path.isdir(target)):
|
||||
stage = self._collect_from_default_dvcfile(target)
|
||||
if stage:
|
||||
stages = (
|
||||
self._collect_pipeline(stage) if with_deps else [stage]
|
||||
)
|
||||
elif not with_deps and is_valid_filename(file):
|
||||
stages = self.get_stages(file, name)
|
||||
|
||||
if not stages:
|
||||
if not (recursive and os.path.isdir(target)):
|
||||
try:
|
||||
(out,) = self.find_outs_by_path(target, strict=False)
|
||||
filter_info = PathInfo(os.path.abspath(target))
|
||||
return [(out.stage, filter_info)]
|
||||
except OutputNotFoundError:
|
||||
pass
|
||||
|
||||
try:
|
||||
stages = self.collect(target, with_deps, recursive, graph)
|
||||
except StageFileDoesNotExistError as exc:
|
||||
# collect() might try to use `target` as a stage name
|
||||
# and throw error that dvc.yaml does not exist, whereas it
|
||||
# should say that both stage name and file does not exist.
|
||||
if file and is_valid_filename(file):
|
||||
raise
|
||||
raise NoOutputOrStageError(target, exc.file) from exc
|
||||
except StageNotFound as exc:
|
||||
raise NoOutputOrStageError(target, exc.file) from exc
|
||||
|
||||
return [(stage, None) for stage in stages]
|
||||
|
||||
def used_cache(
|
||||
self,
|
||||
@@ -227,6 +331,7 @@ class Repo(object):
|
||||
force=False,
|
||||
jobs=None,
|
||||
recursive=False,
|
||||
used_run_cache=None,
|
||||
):
|
||||
"""Get the stages related to the given target and collect
|
||||
the `info` of its outputs.
|
||||
@@ -238,8 +343,9 @@ class Repo(object):
|
||||
`all_branches`/`all_tags`/`all_commits` to expand the scope.
|
||||
|
||||
Returns:
|
||||
A dictionary with Schemes (representing output's location) as keys,
|
||||
and a list with the outputs' `dumpd` as values.
|
||||
A dictionary with Schemes (representing output's location) mapped
|
||||
to items containing the output's `dumpd` names and the output's
|
||||
children (if the given output is a directory).
|
||||
"""
|
||||
from dvc.cache import NamedCache
|
||||
|
||||
@@ -259,7 +365,7 @@ class Repo(object):
|
||||
for target in targets
|
||||
)
|
||||
|
||||
suffix = "({})".format(branch) if branch else ""
|
||||
suffix = f"({branch})" if branch else ""
|
||||
for stage, filter_info in pairs:
|
||||
used_cache = stage.get_used_cache(
|
||||
remote=remote,
|
||||
@@ -269,9 +375,15 @@ class Repo(object):
|
||||
)
|
||||
cache.update(used_cache, suffix=suffix)
|
||||
|
||||
if used_run_cache:
|
||||
used_cache = self.stage_cache.get_used_cache(
|
||||
used_run_cache, remote=remote, force=force, jobs=jobs,
|
||||
)
|
||||
cache.update(used_cache)
|
||||
|
||||
return cache
|
||||
|
||||
def _collect_graph(self, stages=None):
|
||||
def _collect_graph(self, stages):
|
||||
"""Generate a graph by using the given stages on the given directory
|
||||
|
||||
The nodes of the graph are the stage's path relative to the root.
|
||||
@@ -320,10 +432,9 @@ class Repo(object):
|
||||
|
||||
G = nx.DiGraph()
|
||||
stages = stages or self.stages
|
||||
stages = [stage for stage in stages if stage]
|
||||
outs = Trie() # Use trie to efficiently find overlapping outs and deps
|
||||
|
||||
for stage in stages:
|
||||
for stage in filter(bool, stages):
|
||||
for out in stage.outs:
|
||||
out_key = out.path_info.parts
|
||||
|
||||
@@ -346,9 +457,9 @@ class Repo(object):
|
||||
"rerun command with non overlapping outs paths."
|
||||
).format(
|
||||
str(parent),
|
||||
parent.stage.relpath,
|
||||
parent.stage.addressing,
|
||||
str(overlapping),
|
||||
overlapping.stage.relpath,
|
||||
overlapping.stage.addressing,
|
||||
)
|
||||
raise OverlappingOutputPathsError(parent, overlapping, msg)
|
||||
|
||||
@@ -372,14 +483,13 @@ class Repo(object):
|
||||
overlapping.extend(outs.values(prefix=dep_key))
|
||||
|
||||
G.add_edges_from((stage, out.stage) for out in overlapping)
|
||||
|
||||
check_acyclic(G)
|
||||
|
||||
return G
|
||||
|
||||
@cached_property
|
||||
def graph(self):
|
||||
return self._collect_graph()
|
||||
return self._collect_graph(self.stages)
|
||||
|
||||
@cached_property
|
||||
def pipelines(self):
|
||||
@@ -396,25 +506,30 @@ class Repo(object):
|
||||
NOTE: For large repos, this could be an expensive
|
||||
operation. Consider using some memoization.
|
||||
"""
|
||||
from dvc.stage import Stage
|
||||
return self._collect_stages()
|
||||
|
||||
@cached_property
|
||||
def plot_templates(self):
|
||||
from .plots.template import PlotTemplates
|
||||
|
||||
return PlotTemplates(self.dvc_dir)
|
||||
|
||||
def _collect_stages(self):
|
||||
stages = []
|
||||
outs = set()
|
||||
|
||||
for root, dirs, files in self.tree.walk(self.root_dir):
|
||||
for fname in files:
|
||||
path = os.path.join(root, fname)
|
||||
if not Stage.is_valid_filename(path):
|
||||
continue
|
||||
stage = Stage.load(self, path)
|
||||
stages.append(stage)
|
||||
|
||||
for out in stage.outs:
|
||||
if out.scheme == "local":
|
||||
outs.add(out.fspath)
|
||||
|
||||
for file_name in filter(is_valid_filename, files):
|
||||
path = os.path.join(root, file_name)
|
||||
stages.extend(self.get_stages(path))
|
||||
outs.update(
|
||||
out.fspath
|
||||
for stage in stages
|
||||
for out in (
|
||||
out for out in stage.outs if out.scheme == "local"
|
||||
)
|
||||
)
|
||||
dirs[:] = [d for d in dirs if os.path.join(root, d) not in outs]
|
||||
|
||||
return stages
|
||||
|
||||
def find_outs_by_path(self, path, outs=None, recursive=False, strict=True):
|
||||
@@ -452,49 +567,22 @@ class Repo(object):
|
||||
@contextmanager
|
||||
def open_by_relpath(self, path, remote=None, mode="r", encoding=None):
|
||||
"""Opens a specified resource as a file descriptor"""
|
||||
cause = None
|
||||
|
||||
tree = RepoTree(self, stream=True)
|
||||
path = os.path.join(self.root_dir, path)
|
||||
try:
|
||||
out = self.find_out_by_relpath(path)
|
||||
except OutputNotFoundError as exc:
|
||||
out = None
|
||||
cause = exc
|
||||
|
||||
if out and out.use_cache:
|
||||
try:
|
||||
with self._open_cached(out, remote, mode, encoding) as fd:
|
||||
yield fd
|
||||
return
|
||||
except FileNotFoundError as exc:
|
||||
raise FileMissingError(path) from exc
|
||||
|
||||
abs_path = os.path.join(self.root_dir, path)
|
||||
if os.path.exists(abs_path):
|
||||
with open(abs_path, mode=mode, encoding=encoding) as fd:
|
||||
yield fd
|
||||
return
|
||||
|
||||
raise FileMissingError(path) from cause
|
||||
|
||||
def _open_cached(self, out, remote=None, mode="r", encoding=None):
|
||||
if out.isdir():
|
||||
raise IsADirectoryError("Can't open a dir")
|
||||
|
||||
cache_file = self.cache.local.checksum_to_path_info(out.checksum)
|
||||
cache_file = fspath_py35(cache_file)
|
||||
|
||||
if os.path.exists(cache_file):
|
||||
return open(cache_file, mode=mode, encoding=encoding)
|
||||
|
||||
try:
|
||||
remote_obj = self.cloud.get_remote(remote)
|
||||
remote_info = remote_obj.checksum_to_path_info(out.checksum)
|
||||
return remote_obj.open(remote_info, mode=mode, encoding=encoding)
|
||||
except RemoteActionNotImplemented:
|
||||
with self.state:
|
||||
cache_info = out.get_used_cache(remote=remote)
|
||||
self.cloud.pull(cache_info, remote=remote)
|
||||
|
||||
return open(cache_file, mode=mode, encoding=encoding)
|
||||
with tree.open(
|
||||
os.path.join(self.root_dir, path),
|
||||
mode=mode,
|
||||
encoding=encoding,
|
||||
remote=remote,
|
||||
) as fobj:
|
||||
yield fobj
|
||||
except FileNotFoundError as exc:
|
||||
raise FileMissingError(path) from exc
|
||||
except IsADirectoryError as exc:
|
||||
raise DvcIsADirectoryError(f"'{path}' is a directory") from exc
|
||||
|
||||
def close(self):
|
||||
self.scm.close()
|
||||
|
||||
+21
-11
@@ -3,16 +3,18 @@ import os
|
||||
|
||||
import colorama
|
||||
|
||||
from . import locked
|
||||
from dvc.dvcfile import Dvcfile, is_dvc_file
|
||||
|
||||
from ..exceptions import (
|
||||
RecursiveAddingWhileUsingFilename,
|
||||
OutputDuplicationError,
|
||||
OverlappingOutputPathsError,
|
||||
RecursiveAddingWhileUsingFilename,
|
||||
)
|
||||
from ..output.base import OutputDoesNotExistError
|
||||
from ..progress import Tqdm
|
||||
from ..repo.scm_context import scm_context
|
||||
from ..stage import Stage
|
||||
from ..utils import LARGE_DIR_SIZE
|
||||
from ..utils import LARGE_DIR_SIZE, resolve_paths
|
||||
from . import locked
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -63,11 +65,15 @@ def add(repo, targets, recursive=False, no_commit=False, fname=None):
|
||||
).format(
|
||||
out=exc.overlapping_out.path_info,
|
||||
parent=exc.parent.path_info,
|
||||
parent_stage=exc.parent.stage.relpath,
|
||||
parent_stage=exc.parent.stage.addressing,
|
||||
)
|
||||
raise OverlappingOutputPathsError(
|
||||
exc.parent, exc.overlapping_out, msg
|
||||
)
|
||||
except OutputDuplicationError as exc:
|
||||
raise OutputDuplicationError(
|
||||
exc.output, set(exc.stages) - set(stages)
|
||||
)
|
||||
|
||||
with Tqdm(
|
||||
total=len(stages),
|
||||
@@ -85,7 +91,7 @@ def add(repo, targets, recursive=False, no_commit=False, fname=None):
|
||||
if not no_commit:
|
||||
stage.commit()
|
||||
|
||||
stage.dump()
|
||||
Dvcfile(repo, stage.path).dump(stage)
|
||||
pbar_stages.update()
|
||||
|
||||
stages_list += stages
|
||||
@@ -107,7 +113,7 @@ def _find_all_targets(repo, target, recursive):
|
||||
unit="file",
|
||||
)
|
||||
if not repo.is_dvc_internal(fname)
|
||||
if not Stage.is_stage_file(fname)
|
||||
if not is_dvc_file(fname)
|
||||
if not repo.scm.belongs_to_scm(fname)
|
||||
if not repo.scm.is_tracked(fname)
|
||||
]
|
||||
@@ -115,6 +121,8 @@ def _find_all_targets(repo, target, recursive):
|
||||
|
||||
|
||||
def _create_stages(repo, targets, fname, pbar=None):
|
||||
from dvc.stage import Stage, create_stage
|
||||
|
||||
stages = []
|
||||
|
||||
for out in Tqdm(
|
||||
@@ -123,9 +131,11 @@ def _create_stages(repo, targets, fname, pbar=None):
|
||||
disable=len(targets) < LARGE_DIR_SIZE,
|
||||
unit="file",
|
||||
):
|
||||
stage = Stage.create(
|
||||
repo, outs=[out], accompany_outs=True, fname=fname
|
||||
)
|
||||
path, wdir, out = resolve_paths(repo, out)
|
||||
stage = create_stage(Stage, repo, fname or path, wdir=wdir, outs=[out])
|
||||
if stage:
|
||||
Dvcfile(repo, stage.path).remove_with_prompt(force=True)
|
||||
|
||||
repo._reset()
|
||||
|
||||
if not stage:
|
||||
@@ -135,6 +145,6 @@ def _create_stages(repo, targets, fname, pbar=None):
|
||||
|
||||
stages.append(stage)
|
||||
if pbar is not None:
|
||||
pbar.update_desc(out)
|
||||
pbar.update_msg(out)
|
||||
|
||||
return stages
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user