Compare commits

..
16 Commits
Author SHA1 Message Date
Ruslan Kuprieiev 2e01ce1974 dvc: bump to 2.0.0a1 2021-02-23 20:48:42 +02:00
Batuhan Taskaya d587d41d6c utils: proper file interface for HashedStreamReader (#5508) 2021-02-23 02:47:36 +02:00
Batuhan Taskaya 27c2401f59 azure: transfer bigger chunks of data (#5504) 2021-02-22 14:08:26 +02:00
Batuhan Taskaya 174621696d azure: migrate AzureFileSystem to fsspec (#5463)
* azure: migrate AzureFileSystem to fsspec

* client => fs

* principal auth

* better login error handling
2021-02-22 12:05:15 +02:00
Batuhan Taskaya e1a193f00b update: forbid updating repo-imports with --to-remote (#5475) 2021-02-22 11:55:00 +02:00
Saugat PachhaiandDave Berenbaum cccead3441 stage:list: update output and help text (#5499)
* stage:list: update output and help text

* Replaced 'Produces' with 'Outputs'
* Replaced 'Stage depends on' with just 'Depends on'
* Changes help text of --names-only and --fail

* Update dvc/command/stage.py

Co-authored-by: Dave Berenbaum <[email protected]>

* s/Generates/Reports

Co-authored-by: Dave Berenbaum <[email protected]>
2021-02-20 13:21:39 +05:45
Batuhan Taskaya 8102ede5cc add, import-url: always resolve the output in the local path for --to-remote (#5473)
* add, import-url: always resolve the output in the local path for --to-remote

* windows

* use the proper fixture
2021-02-19 15:22:59 +02:00
Saugat Pachhai 14944dd9e0 Minor refactoring stage:list command (#5498)
Moving text truncation to utils and adding tests for it
2021-02-19 18:27:37 +05:45
Paweł Redzyński 63f32936b2 live integration: turn on tests (#5487) 2021-02-19 09:25:28 +01:00
Peter Rowlands (변기호) 7d13d24a1f pygit/dulwich: implement create branch (#5497)
* pygit: implement create branch

* dulwich: implement create branch
2021-02-19 17:18:15 +09:00
Peter Rowlands (변기호) 9cb8187cb3 pygit: implement resolve_commit (#5496)
* git: return backend-agnostic object for resolve_commit

* use int timestamps instead of datetime

* make branch_revs Git method instead of backend

* pygit: implement resolve_commit
2021-02-19 17:00:22 +09:00
Peter Rowlands (변기호) 98590779eb pygit: implement get_refs_containing (#5495)
* git: make is_sha a Git class method instead of backend function

* pygit: implement get_refs_containing

* use merge base instead of parents
2021-02-19 16:42:04 +09:00
Batuhan Taskaya 9ae8bcdf52 gdrive: support service auth with json (#5485)
* gdrive: support service auth with json

* retry logic
2021-02-18 17:10:35 +02:00
Paweł Redzyński 14958ff85f plots: fix bug preventing usage from being displayed (#5492) 2021-02-18 14:24:47 +01:00
Peter Rowlands (변기호) 9909ee85a1 git: implement dulwich add --update (#5491)
* dulwich: implement add with --update

* add git backend test for add()

* fix list comprehension in dulwich status()

* allow adding single string path instead of list

* handle windows paths in add(update=...)
2021-02-18 17:56:56 +09:00
Saugat Pachhai 45477a2a81 dvc.lock: raise error if the file is gitignored (#5484) 2021-02-18 03:08:00 +02:00
39 changed files with 654 additions and 325 deletions
+9 -1
View File
@@ -27,7 +27,11 @@ jobs:
with:
python-version: 3.8
- name: Install requirements
run: pip install ".[all,tests]" Pygments collective.checkdocs pre-commit git+https://github.com/intake/gdrivefs.git
run: pip install ".[all,tests]" Pygments collective.checkdocs pre-commit
- name: install temporary dependencies
run: |
pip install git+https://github.com/isidentical/gdrivefs@service-account
pip install git+https://github.com/isidentical/pydata-google-auth@service-account
- name: Check README
run: python setup.py checkdocs
- uses: pre-commit/[email protected]
@@ -69,6 +73,10 @@ jobs:
run: |
pip install wheel
pip install -e ".[all,tests]"
- name: install temporary dependencies
run: |
pip install git+https://github.com/isidentical/gdrivefs@service-account
pip install git+https://github.com/isidentical/pydata-google-auth@service-account
- name: setup git
run: |
git config --global user.email "[email protected]"
+1 -1
View File
@@ -293,7 +293,7 @@ ignored-classes=optparse.Values,thread._local,_thread._local,Dvcfile
# (useful for modules/projects where namespaces are manipulated during runtime
# and thus existing member attributes cannot be deduced by static analysis. It
# supports qualified module names, as well as Unix pattern matching.
ignored-modules=
ignored-modules=azure
# Show a hint with possible names when a member name was not found. The aspect
# of finding the hint is based on edit distance.
+1 -1
View File
@@ -138,7 +138,7 @@ def add_parser(subparsers, parent_parser):
help=(
"Specific plots file(s) to visualize "
"(even if not found as `plots` in `dvc.yaml`). "
"Shows all tracked plots by default.",
"Shows all tracked plots by default."
),
metavar="<paths>",
).complete = completion.FILE
+8 -10
View File
@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Dict, Iterable, List
from dvc.command import completion
from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
from dvc.utils.cli_parse import parse_params
from dvc.utils.humanize import truncate_text
if TYPE_CHECKING:
from dvc.output.base import BaseOutput
@@ -14,7 +15,6 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
MAX_TEXT_LENGTH = 80
ELLIPSIS = "…"
def generate_description(stage: "Stage") -> str:
@@ -22,10 +22,10 @@ def generate_description(stage: "Stage") -> str:
return ", ".join(out.def_path for out in outs)
if not stage.deps and not stage.outs:
return "No outputs or dependencies."
return "No outputs or dependencies"
if not stage.outs and stage.deps:
return "Stage depends on " + part_desc(stage.deps)
return "Depends on " + part_desc(stage.deps)
def is_plot_or_metric(out: "BaseOutput"):
return bool(out.plot) or bool(out.metric)
@@ -34,11 +34,11 @@ def generate_description(stage: "Stage") -> str:
outs = list(filterfalse(is_plot_or_metric, stage.outs))
if outs:
desc.append("Produces " + part_desc(outs))
desc.append("Outputs " + part_desc(outs))
plots_and_metrics = list(filter(is_plot_or_metric, stage.outs))
if plots_and_metrics:
desc.append("Generates " + part_desc(plots_and_metrics))
desc.append("Reports " + part_desc(plots_and_metrics))
return "; ".join(desc)
@@ -47,9 +47,7 @@ def prepare_description(
stage: "Stage", max_length: int = MAX_TEXT_LENGTH
) -> str:
desc = stage.short_description() or generate_description(stage)
if len(desc) > max_length:
return desc[: max_length - 1] + ELLIPSIS
return desc
return truncate_text(desc, max_length)
def prepare_stages_data(
@@ -348,7 +346,7 @@ def add_parser(subparsers, parent_parser):
"--fail",
action="store_true",
default=False,
help="Fail immediately if there's an error.",
help="Fail immediately, do not suppress any syntax errors.",
)
stage_list_parser.add_argument(
"-R",
@@ -361,6 +359,6 @@ def add_parser(subparsers, parent_parser):
"--names-only",
action="store_true",
default=False,
help="List only the name of the stages.",
help="List only stage names.",
)
stage_list_parser.set_defaults(func=CmdStageList)
+1 -1
View File
@@ -236,7 +236,7 @@ class Config(dict):
str: {
"url": func,
"gdrive_user_credentials_file": func,
"gdrive_service_account_p12_file_path": func,
"gdrive_service_account_json_file_path": func,
"credentialpath": func,
"keyfile": func,
"cert_path": func,
+10 -3
View File
@@ -179,7 +179,15 @@ SCHEMA = {
"webhdfs_alias": str,
**REMOTE_COMMON,
},
"azure": {"connection_string": str, **REMOTE_COMMON},
"azure": {
"connection_string": str,
"sas_token": str,
"account_name": str,
"tenant_id": str,
"client_id": str,
"client_secret": str,
**REMOTE_COMMON,
},
"oss": {
"oss_key_id": str,
"oss_key_secret": str,
@@ -191,9 +199,8 @@ SCHEMA = {
"gdrive_client_id": str,
"gdrive_client_secret": str,
"gdrive_user_credentials_file": str,
"gdrive_service_account_email": str,
"gdrive_service_account_user_email": str,
"gdrive_service_account_p12_file_path": str,
"gdrive_service_account_json_file_path": str,
Optional("gdrive_trash_only", default=False): Bool,
**REMOTE_COMMON,
},
+24 -11
View File
@@ -20,7 +20,6 @@ from dvc.utils import relpath
from dvc.utils.collections import apply_diff
from dvc.utils.serialize import (
dump_yaml,
load_yaml,
modify_yaml,
parse_yaml,
parse_yaml_for_update,
@@ -115,6 +114,14 @@ class FileMixin:
self.repo.fs, LocalFileSystem
) and self.repo.scm.is_ignored(self.path)
def _verify_filename(self):
if self.verify:
check_dvc_filename(self.path)
def _check_gitignored(self):
if self._is_git_ignored():
raise FileIsGitIgnored(self.path)
def _load(self):
# it raises the proper exceptions by priority:
# 1. when the file doesn't exists
@@ -125,12 +132,11 @@ class FileMixin:
is_ignored = self.repo.fs.exists(self.path, use_dvcignore=False)
raise StageFileDoesNotExistError(self.path, dvc_ignored=is_ignored)
if self.verify:
check_dvc_filename(self.path)
self._verify_filename()
if not self.repo.fs.isfile(self.path):
raise StageFileIsNotDvcFileError(self.path)
if self._is_git_ignored():
raise FileIsGitIgnored(self.path)
self._check_gitignored()
with self.repo.fs.open(self.path, encoding="utf-8") as fd:
stage_text = fd.read()
d = parse_yaml(stage_text, self.path)
@@ -330,18 +336,25 @@ class Lockfile(FileMixin):
except MultipleInvalid as exc:
raise StageFileFormatError(f"'{fname}' format error: {exc}")
def load(self):
if not self.exists():
return {}
def _verify_filename(self):
pass # lockfile path is hardcoded, so no need to verify here
data = load_yaml(self.path, fs=self.repo.fs)
def _load(self):
try:
data = self.validate(data, fname=self.relpath)
return super()._load()
except StageFileDoesNotExistError:
# we still need to account for git-ignored dvc.lock file
# even though it may not exist or have been .dvcignored
self._check_gitignored()
return {}, ""
except StageFileFormatError as exc:
raise LockfileCorruptedError(
f"Lockfile '{self.relpath}' is corrupted."
) from exc
return data
def load(self):
d, _ = self._load()
return d
@property
def latest_version_info(self):
+134 -143
View File
@@ -1,58 +1,94 @@
import logging
import os
import shutil
import threading
from datetime import datetime, timedelta
from funcy import cached_property, wrap_prop
from dvc.exceptions import DvcException
from dvc.path_info import CloudURLInfo
from dvc.progress import Tqdm
from dvc.scheme import Schemes
from dvc.utils import format_link
from .base import BaseFileSystem
logger = logging.getLogger(__name__)
_DEFAULT_CREDS_STEPS = (
"https://azuresdkdocs.blob.core.windows.net/$web/python/"
"azure-identity/1.4.0/azure.identity.html#azure.identity"
".DefaultAzureCredential"
)
class AzureAuthError(DvcException):
pass
class AzureFileSystem(BaseFileSystem):
scheme = Schemes.AZURE
PATH_CLS = CloudURLInfo
REQUIRES = {
"azure-storage-blob": "azure.storage.blob",
"knack": "knack",
}
PARAM_CHECKSUM = "etag"
DETAIL_FIELDS = frozenset(("etag", "size"))
COPY_POLL_SECONDS = 5
LIST_OBJECT_PAGE_SIZE = 5000
REQUIRES = {
"adlfs": "adlfs",
"knack": "knack",
"azure-identity": "azure.identity",
}
def __init__(self, repo, config):
super().__init__(repo, config)
url = config.get("url", "azure://")
url = config.get("url")
self.path_info = self.PATH_CLS(url)
self.bucket = self.path_info.bucket
if not self.bucket:
container = self._az_config.get("storage", "container_name", None)
self.path_info = self.PATH_CLS(f"azure://{container}")
self.bucket = self.path_info.bucket
url = f"azure://{container}"
self._conn_str = config.get(
"connection_string"
) or self._az_config.get("storage", "connection_string", None)
self.path_info = self.PATH_CLS(url)
self.bucket = self.path_info.bucket
self._account_url = None
if not self._conn_str:
name = self._az_config.get("storage", "account", None)
self._account_url = f"https://{name}.blob.core.windows.net"
self.login_method, self.login_info = self._prepare_credentials(config)
self._credential = config.get("sas_token") or self._az_config.get(
"storage", "sas_token", None
def _prepare_credentials(self, config):
from azure.identity.aio import DefaultAzureCredential
login_info = {}
login_info["connection_string"] = config.get(
"connection_string",
self._az_config.get("storage", "connection_string", None),
)
if not self._credential:
self._credential = self._az_config.get("storage", "key", None)
login_info["account_name"] = config.get(
"account_name", self._az_config.get("storage", "account", None)
)
login_info["account_key"] = config.get(
"account_key", self._az_config.get("storage", "key", None)
)
login_info["sas_token"] = config.get(
"sas_token", self._az_config.get("storage", "sas_token", None)
)
login_info["tenant_id"] = config.get("tenant_id")
login_info["client_id"] = config.get("client_id")
login_info["client_secret"] = config.get("client_secret")
if not any(login_info.values()):
login_info["credentials"] = DefaultAzureCredential()
for login_method, required_keys in [ # noqa
("connection string", ["connection_string"]),
("account key", ["account_name", "account_key"]),
("SAS token", ["account_name", "sas_token"]),
("anonymous login", ["account_name"]),
(f"default credentials ({_DEFAULT_CREDS_STEPS})", ["credentials"]),
]:
if all(login_info.get(key) is not None for key in required_keys):
break
else:
login_method = None
return login_method, login_info
@cached_property
def _az_config(self):
@@ -68,158 +104,113 @@ class AzureFileSystem(BaseFileSystem):
@wrap_prop(threading.Lock())
@cached_property
def blob_service(self):
# pylint: disable=no-name-in-module
from azure.core.exceptions import (
HttpResponseError,
ResourceNotFoundError,
)
from azure.storage.blob import BlobServiceClient
def fs(self):
from adlfs import AzureBlobFileSystem
from azure.core.exceptions import AzureError
if self._conn_str:
logger.debug(f"Using connection string '{self._conn_str}'")
blob_service = BlobServiceClient.from_connection_string(
self._conn_str, credential=self._credential
)
else:
logger.debug(f"Using account url '{self._account_url}'")
blob_service = BlobServiceClient(
self._account_url, credential=self._credential
)
try:
file_system = AzureBlobFileSystem(**self.login_info)
if self.bucket not in [
container.rstrip("/") for container in file_system.ls("/")
]:
file_system.mkdir(self.bucket)
except (ValueError, AzureError) as e:
raise AzureAuthError(
f"Authentication to Azure Blob Storage via {self.login_method}"
" failed.\nLearn more about configuration settings at"
f" {format_link('https://man.dvc.org/remote/modify')}"
) from e
logger.debug(f"Container name {self.bucket}")
container_client = blob_service.get_container_client(self.bucket)
return file_system
try: # verify that container exists
container_client.get_container_properties()
except ResourceNotFoundError:
container_client.create_container()
except HttpResponseError as exc:
# client may not have account-level privileges
if exc.status_code != 403:
raise
def _with_bucket(self, path):
if isinstance(path, self.PATH_CLS):
return f"{path.bucket}/{path.path}"
return path
return blob_service
def open(
self, path_info, mode="r", **kwargs
): # pylint: disable=arguments-differ
return self.fs.open(self._with_bucket(path_info), mode=mode)
def get_etag(self, path_info):
blob_client = self.blob_service.get_blob_client(
path_info.bucket, path_info.path
)
etag = blob_client.get_blob_properties().etag
return etag.strip('"')
def exists(self, path_info, use_dvcignore=False):
return self.fs.exists(self._with_bucket(path_info))
def _generate_download_url(self, path_info, expires=3600):
from azure.storage.blob import ( # pylint:disable=no-name-in-module
BlobSasPermissions,
generate_blob_sas,
)
def _strip_bucket(self, entry):
_, entry = entry.split("/", 1)
return entry
expires_at = datetime.utcnow() + timedelta(seconds=expires)
blob_client = self.blob_service.get_blob_client(
path_info.bucket, path_info.path
)
sas_token = generate_blob_sas(
blob_client.account_name,
blob_client.container_name,
blob_client.blob_name,
account_key=blob_client.credential.account_key,
permission=BlobSasPermissions(read=True),
expiry=expires_at,
)
return blob_client.url + "?" + sas_token
def exists(self, path_info, use_dvcignore=True):
paths = self._list_paths(path_info.bucket, path_info.path)
return any(path_info.path == path for path in paths)
def _list_paths(self, bucket, prefix):
container_client = self.blob_service.get_container_client(bucket)
for blob in container_client.list_blobs(name_starts_with=prefix):
yield blob.name
def walk_files(self, path_info, **kwargs):
if not kwargs.pop("prefix", False):
path_info = path_info / ""
for fname in self._list_paths(
path_info.bucket, path_info.path, **kwargs
):
if fname.endswith("/"):
continue
yield path_info.replace(path=fname)
def _strip_buckets(self, entries, detail, prefix=None):
for entry in entries:
if detail:
entry = entry.copy()
entry["name"] = self._strip_bucket(entry["name"])
else:
entry = self._strip_bucket(
f"{prefix}/{entry}" if prefix else entry
)
yield entry
def ls(
self, path_info, detail=False, recursive=False
): # pylint: disable=arguments-differ
assert recursive
path = self._with_bucket(path_info)
if recursive:
for root, _, files in self.fs.walk(path, detail=detail):
if detail:
files = files.values()
yield from self._strip_buckets(files, detail, prefix=root)
return None
container_client = self.blob_service.get_container_client(
path_info.bucket
)
for blob in container_client.list_blobs(
name_starts_with=path_info.path
):
if detail:
yield {
"type": "file",
"name": blob.name,
"size": blob.size,
"etag": blob.etag.strip('"'),
}
else:
yield blob.name
yield from self._strip_buckets(self.ls(path, detail=detail), detail)
def walk_files(self, path_info, **kwargs):
for file in self.ls(path_info, recursive=True):
yield path_info.replace(path=file)
def remove(self, path_info):
if path_info.scheme != self.scheme:
raise NotImplementedError
logger.debug(f"Removing {path_info}")
self.blob_service.get_blob_client(
path_info.bucket, path_info.path
).delete_blob()
self.fs.delete(self._with_bucket(path_info))
def info(self, path_info):
blob_client = self.blob_service.get_blob_client(
path_info.bucket, path_info.path
)
properties = blob_client.get_blob_properties()
return {
"type": "file",
"size": properties.size,
"etag": properties.etag.strip('"'),
}
info = self.fs.info(self._with_bucket(path_info)).copy()
info["name"] = self._strip_bucket(info["name"])
return info
def _upload_fobj(self, fobj, to_info):
blob_client = self.blob_service.get_blob_client(
to_info.bucket, to_info.path
)
blob_client.upload_blob(fobj, overwrite=True)
from adlfs import AzureBlobFile
with self.open(to_info, "wb") as fdest:
shutil.copyfileobj(
fobj, fdest, length=AzureBlobFile.DEFAULT_BLOCK_SIZE
)
def _upload(
self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
self, from_file, to_info, name=None, no_progress_bar=False, **kwargs
):
total = os.path.getsize(from_file)
with open(from_file, "rb") as fobj:
self.upload_fobj(
fobj,
to_info,
self._with_bucket(to_info),
desc=name,
total=total,
no_progress_bar=no_progress_bar,
)
self.fs.invalidate_cache(self._with_bucket(to_info.parent))
def _download(
self, from_info, to_file, name=None, no_progress_bar=False, **_kwargs
self, from_info, to_file, name=None, no_progress_bar=False, **pbar_args
):
blob_client = self.blob_service.get_blob_client(
from_info.bucket, from_info.path
)
total = blob_client.get_blob_properties().size
stream = blob_client.download_blob()
with open(to_file, "wb") as fobj:
total = self.getsize(self._with_bucket(from_info))
with self.open(from_info, "rb") as fobj:
with Tqdm.wrapattr(
fobj, "write", desc=name, total=total, disable=no_progress_bar
fobj,
"read",
desc=name,
disable=no_progress_bar,
bytes=True,
total=total,
**pbar_args,
) as wrapped:
stream.readinto(wrapped)
with open(to_file, "wb") as fdest:
shutil.copyfileobj(wrapped, fdest)
+11 -14
View File
@@ -117,14 +117,11 @@ class GDriveFileSystem(BaseFileSystem):
self._path = self.path_info.path
self._trash_only = config.get("gdrive_trash_only")
self._use_service_account = config.get("gdrive_use_service_account")
self._service_account_email = config.get(
"gdrive_service_account_email"
)
self._service_account_user_email = config.get(
"gdrive_service_account_user_email"
)
self._service_account_p12_file_path = config.get(
"gdrive_service_account_p12_file_path"
self._service_account_json_file_path = config.get(
"gdrive_service_account_json_file_path"
)
self._client_id = config.get("gdrive_client_id")
self._client_secret = config.get("gdrive_client_secret")
@@ -142,14 +139,14 @@ class GDriveFileSystem(BaseFileSystem):
def _validate_config(self):
# Validate Service Account configuration
if self._use_service_account and (
not self._service_account_email
or not self._service_account_p12_file_path
if (
self._use_service_account
and not self._service_account_json_file_path
):
raise DvcException(
"To use service account, set `gdrive_service_account_email`,\n"
"`gdrive_service_account_p12_file_path`, and optionally "
"`gdrive_service_account_user_email`\nin DVC config. "
"To use service account, set "
"`gdrive_service_account_json_file_path`, and optionally"
"`gdrive_service_account_user_email` in DVC config\n"
"Learn more at {}".format(
format_link("https://man.dvc.org/remote/modify")
)
@@ -220,7 +217,6 @@ class GDriveFileSystem(BaseFileSystem):
auth_settings = {
"client_config_backend": "settings",
"save_credentials": True,
"save_credentials_backend": "file",
"save_credentials_file": self._gdrive_user_credentials_path,
"get_refresh_token": True,
"oauth_scope": [
@@ -230,12 +226,13 @@ class GDriveFileSystem(BaseFileSystem):
}
if self._use_service_account:
auth_settings["save_credentials_backend"] = "json"
auth_settings["service_config"] = {
"client_service_email": self._service_account_email,
"client_user_email": self._service_account_user_email,
"client_pkcs12_file_path": self._service_account_p12_file_path,
"client_json_file_path": self._service_account_json_file_path,
}
else:
auth_settings["save_credentials_backend"] = "file"
auth_settings["client_config"] = {
"client_id": self._client_id or self.DEFAULT_GDRIVE_CLIENT_ID,
"client_secret": self._client_secret
+12 -2
View File
@@ -52,6 +52,8 @@ def add( # noqa: C901
invalid_opt = "--no-commit option"
elif recursive:
invalid_opt = "--recursive option"
elif kwargs.get("external"):
invalid_opt = "--external option"
else:
message = "{option} can't be used without --to-remote"
if kwargs.get("remote"):
@@ -88,7 +90,12 @@ def add( # noqa: C901
)
stages = _create_stages(
repo, sub_targets, fname, pbar=pbar, **kwargs,
repo,
sub_targets,
fname,
pbar=pbar,
transfer=to_remote or to_cache,
**kwargs,
)
try:
@@ -230,6 +237,7 @@ def _create_stages(
external=False,
glob=False,
desc=None,
transfer=False,
**kwargs,
):
from dvc.dvcfile import Dvcfile
@@ -246,7 +254,9 @@ def _create_stages(
):
if kwargs.get("out"):
out = resolve_output(out, kwargs["out"])
path, wdir, out = resolve_paths(repo, out)
path, wdir, out = resolve_paths(
repo, out, always_local=transfer and not kwargs.get("out")
)
stage = create_stage(
Stage,
repo,
+1 -1
View File
@@ -761,7 +761,7 @@ class Experiments:
# if we can't tell from branch name, fall back to parent commit
exp_commit = self.scm.resolve_commit(exp_rev)
if exp_commit:
exp_baseline = first(exp_commit.parents).hexsha
exp_baseline = first(exp_commit.parents)
if exp_baseline == baseline_sha:
return exp_baseline
raise BaselineMismatchError(exp_baseline, baseline_sha)
+2 -2
View File
@@ -19,7 +19,7 @@ def _collect_experiment_commit(repo, rev, stash=False, sha_only=True):
res["timestamp"] = None
else:
commit = repo.scm.resolve_commit(rev)
res["timestamp"] = datetime.fromtimestamp(commit.committed_date)
res["timestamp"] = datetime.fromtimestamp(commit.commit_time)
configs = _collect_configs(repo, rev=rev)
params = _read_params(repo, configs, rev)
@@ -121,7 +121,7 @@ def show(
for ref in repo.scm.iter_refs(base=str(ref_info))
]
for exp_ref, _ in sorted(
commits, key=lambda x: x[1].committed_date, reverse=True,
commits, key=lambda x: x[1].commit_time, reverse=True,
):
ref_info = ExpRefInfo.from_ref(exp_ref)
assert ref_info.baseline_sha == rev
+5 -4
View File
@@ -1,4 +1,4 @@
from typing import TYPE_CHECKING, Generator, Iterable
from typing import TYPE_CHECKING, Generator, Iterable, Set
from .base import EXEC_NAMESPACE, EXPS_NAMESPACE, EXPS_STASH, ExpRefInfo
@@ -75,13 +75,14 @@ def remote_exp_refs_by_baseline(
def exp_commits(
scm: "Git", ref_infos: Iterable["ExpRefInfo"] = None
) -> Generator[str, None, None]:
) -> Iterable[str]:
"""Iterate over all experiment commits."""
shas = set()
shas: Set["str"] = set()
refs = ref_infos if ref_infos else exp_refs(scm)
for ref_info in refs:
shas.update(scm.branch_revs(str(ref_info), ref_info.baseline_sha))
shas.add(ref_info.baseline_sha)
if ref_info.baseline_sha:
shas.add(ref_info.baseline_sha)
yield from shas
+3 -1
View File
@@ -27,7 +27,9 @@ def imp_url(
from dvc.stage import Stage, create_stage, restore_meta
out = resolve_output(url, out)
path, wdir, out = resolve_paths(self, out)
path, wdir, out = resolve_paths(
self, out, always_local=to_remote and not out
)
if to_remote and no_exec:
raise InvalidArgumentError(
+19 -7
View File
@@ -2,6 +2,7 @@
import logging
import os
import re
import shlex
from collections.abc import Mapping
from contextlib import contextmanager
@@ -75,6 +76,7 @@ class Git(Base):
GITIGNORE = ".gitignore"
GIT_DIR = ".git"
LOCAL_BRANCH_PREFIX = "refs/heads/"
RE_HEXSHA = re.compile(r"^[0-9A-Fa-f]{4,40}$")
def __init__(
self, *args, backends: Optional[Iterable[str]] = None, **kwargs
@@ -119,12 +121,7 @@ class Git(Base):
@classmethod
def is_sha(cls, rev):
for _, backend in GitBackends.DEFAULT.items():
try:
return backend.is_sha(rev)
except NotImplementedError:
pass
raise NoGitBackendError("is_sha")
return rev and cls.RE_HEXSHA.search(rev)
@staticmethod
def _get_git_dir(root_dir):
@@ -368,7 +365,6 @@ class Git(Base):
get_rev = partialmethod(_backend_func, "get_rev")
_resolve_rev = partialmethod(_backend_func, "resolve_rev")
resolve_commit = partialmethod(_backend_func, "resolve_commit")
branch_revs = partialmethod(_backend_func, "branch_revs")
set_ref = partialmethod(_backend_func, "set_ref")
get_ref = partialmethod(_backend_func, "get_ref")
@@ -405,6 +401,22 @@ class Git(Base):
raise RevError(f"ambiguous Git revision '{rev}'")
raise
def branch_revs(
self, branch: str, end_rev: Optional[str] = None
) -> Iterable[str]:
"""Iterate over revisions in a given branch (from newest to oldest).
If end_rev is set, iterator will stop when the specified revision is
reached.
"""
commit = self.resolve_commit(branch)
while commit is not None:
yield commit.hexsha
parent = first(commit.parents)
if parent is None or parent == end_rev:
return
commit = self.resolve_commit(parent)
@contextmanager
def detach_head(self, rev: Optional[str] = None):
"""Context manager for performing detached HEAD SCM operations.
+14 -14
View File
@@ -1,11 +1,22 @@
import os
from abc import ABC, abstractmethod
from typing import Callable, Iterable, Mapping, Optional, Tuple
from typing import (
TYPE_CHECKING,
Callable,
Iterable,
Mapping,
Optional,
Tuple,
Union,
)
from dvc.scm.base import SCMError
from ..objects import GitObject
if TYPE_CHECKING:
from ..objects import GitCommit
class NoGitBackendError(SCMError):
def __init__(self, func):
@@ -41,18 +52,13 @@ class BaseGitBackend(ABC):
):
pass
@staticmethod
@abstractmethod
def is_sha(rev: str) -> bool:
pass
@property
@abstractmethod
def dir(self) -> str:
pass
@abstractmethod
def add(self, paths: Iterable[str], update=False):
def add(self, paths: Union[str, Iterable[str]], update=False):
pass
@abstractmethod
@@ -122,13 +128,7 @@ class BaseGitBackend(ABC):
pass
@abstractmethod
def resolve_commit(self, rev: str) -> str:
pass
@abstractmethod
def branch_revs(
self, branch: str, end_rev: Optional[str] = None
) -> Iterable[str]:
def resolve_commit(self, rev: str) -> "GitCommit":
pass
@abstractmethod
+52 -21
View File
@@ -4,7 +4,17 @@ import logging
import os
import stat
from io import BytesIO, StringIO
from typing import Callable, Dict, Iterable, Mapping, Optional, Tuple
from typing import (
TYPE_CHECKING,
Callable,
Dict,
Iterable,
List,
Mapping,
Optional,
Tuple,
Union,
)
from funcy import cached_property
@@ -16,6 +26,9 @@ from dvc.utils import relpath
from ..objects import GitObject
from .base import BaseGitBackend
if TYPE_CHECKING:
from ..objects import GitCommit
logger = logging.getLogger(__name__)
@@ -109,24 +122,23 @@ class DulwichBackend(BaseGitBackend): # pylint:disable=abstract-method
):
raise NotImplementedError
@staticmethod
def is_sha(rev: str) -> bool:
raise NotImplementedError
@property
def dir(self) -> str:
return self.repo.commondir()
def add(self, paths: Iterable[str], update=False):
def add(self, paths: Union[str, Iterable[str]], update=False):
from dvc.utils.fs import walk_files
if update:
raise NotImplementedError
assert paths or update
if isinstance(paths, str):
paths = [paths]
files = []
if update and not paths:
self.repo.stage(list(self.repo.open_index()))
return
files: List[bytes] = []
for path in paths:
if not os.path.isabs(path) and self._submodules:
# NOTE: If path is inside a submodule, Dulwich expects the
@@ -142,13 +154,30 @@ class DulwichBackend(BaseGitBackend): # pylint:disable=abstract-method
)
break
if os.path.isdir(path):
files.extend(walk_files(path))
files.extend(
os.fsencode(relpath(fpath, self.root_dir))
for fpath in walk_files(path)
)
else:
files.append(path)
files.append(os.fsencode(relpath(path, self.root_dir)))
for fpath in files:
# NOTE: this doesn't check gitignore, same as GitPythonBackend.add
self.repo.stage(relpath(fpath, self.root_dir))
# NOTE: this doesn't check gitignore, same as GitPythonBackend.add
if update:
index = self.repo.open_index()
if os.name == "nt":
# NOTE: we need git/unix separator to compare against index
# paths but repo.stage() expects to be called with OS paths
self.repo.stage(
[
fname
for fname in files
if fname.replace(b"\\", b"/") in index
]
)
else:
self.repo.stage([fname for fname in files if fname in index])
else:
self.repo.stage(files)
def commit(self, msg: str, no_verify: bool = False):
from dulwich.errors import CommitError
@@ -171,7 +200,12 @@ class DulwichBackend(BaseGitBackend): # pylint:disable=abstract-method
raise NotImplementedError
def branch(self, branch: str):
raise NotImplementedError
from dulwich.porcelain import Error, branch_create
try:
branch_create(self.root_dir, branch)
except Error as exc:
raise SCMError(f"Failed to create branch '{branch}'") from exc
def tag(self, tag: str):
raise NotImplementedError
@@ -215,10 +249,7 @@ class DulwichBackend(BaseGitBackend): # pylint:disable=abstract-method
def resolve_rev(self, rev: str) -> str:
raise NotImplementedError
def resolve_commit(self, rev: str) -> str:
raise NotImplementedError
def branch_revs(self, branch: str, end_rev: Optional[str] = None):
def resolve_commit(self, rev: str) -> "GitCommit":
raise NotImplementedError
def _get_stash(self, ref: str):
@@ -561,9 +592,9 @@ class DulwichBackend(BaseGitBackend): # pylint:disable=abstract-method
)
return (
{
status: [os.fsdecode(name)]
status: [os.fsdecode(name) for name in paths]
for status, paths in staged.items()
for name in paths
if paths
},
[os.fsdecode(name) for name in unstaged],
[os.fsdecode(name) for name in untracked],
+15 -22
View File
@@ -3,15 +3,15 @@ import locale
import logging
import os
from functools import partial
from typing import Callable, Iterable, List, Mapping, Optional, Tuple
from typing import Callable, Iterable, List, Mapping, Optional, Tuple, Union
from funcy import first, ignore
from funcy import ignore
from dvc.progress import Tqdm
from dvc.scm.base import CloneError, MergeConflictError, RevError, SCMError
from dvc.utils import fix_env, is_binary, relpath
from ..objects import GitObject
from ..objects import GitCommit, GitObject
from .base import BaseGitBackend
logger = logging.getLogger(__name__)
@@ -188,11 +188,13 @@ class GitPythonBackend(BaseGitBackend): # pylint:disable=abstract-method
def dir(self) -> str:
return self.repo.git_dir
def add(self, paths: Iterable[str], update=False):
def add(self, paths: Union[str, Iterable[str]], update=False):
# NOTE: GitPython is not currently able to handle index version >= 3.
# See https://github.com/iterative/dvc/issues/610 for more details.
try:
if update:
if isinstance(paths, str):
paths = [paths]
self.git.add(*paths, update=True)
else:
self.repo.index.add(paths)
@@ -313,7 +315,7 @@ class GitPythonBackend(BaseGitBackend): # pylint:disable=abstract-method
raise RevError(f"unknown Git revision '{rev}'")
def resolve_commit(self, rev):
def resolve_commit(self, rev: str) -> "GitCommit":
"""Return Commit object for the specified revision."""
from git.exc import BadName, GitCommandError
from git.objects.tag import TagObject
@@ -321,25 +323,16 @@ class GitPythonBackend(BaseGitBackend): # pylint:disable=abstract-method
try:
commit = self.repo.rev_parse(rev)
except (BadName, GitCommandError):
commit = None
raise SCMError(f"Invalid commit '{rev}'")
if isinstance(commit, TagObject):
commit = commit.object
return commit
def branch_revs(
self, branch: str, end_rev: Optional[str] = None
) -> Iterable[str]:
"""Iterate over revisions in a given branch (from newest to oldest).
If end_rev is set, iterator will stop when the specified revision is
reached.
"""
commit = self.resolve_commit(branch)
while commit is not None:
yield commit.hexsha
commit = first(commit.parents)
if commit and commit.hexsha == end_rev:
return
return GitCommit(
commit.hexsha,
commit.committed_date,
commit.committer_tz_offset,
commit.message,
[str(parent) for parent in commit.parents],
)
def set_ref(
self,
+58 -16
View File
@@ -2,12 +2,12 @@ import locale
import logging
import os
from io import BytesIO, StringIO
from typing import Callable, Iterable, List, Mapping, Optional, Tuple
from typing import Callable, Iterable, List, Mapping, Optional, Tuple, Union
from dvc.scm.base import MergeConflictError, RevError, SCMError
from dvc.utils import relpath
from ..objects import GitObject
from ..objects import GitCommit, GitObject
from .base import BaseGitBackend
logger = logging.getLogger(__name__)
@@ -76,15 +76,11 @@ class Pygit2Backend(BaseGitBackend): # pylint:disable=abstract-method
):
raise NotImplementedError
@staticmethod
def is_sha(rev: str) -> bool:
raise NotImplementedError
@property
def dir(self) -> str:
raise NotImplementedError
def add(self, paths: Iterable[str], update=False):
def add(self, paths: Union[str, Iterable[str]], update=False):
raise NotImplementedError
def commit(self, msg: str, no_verify: bool = False):
@@ -120,7 +116,13 @@ class Pygit2Backend(BaseGitBackend): # pylint:disable=abstract-method
raise NotImplementedError
def branch(self, branch: str):
raise NotImplementedError
from pygit2 import GitError
try:
commit = self.repo[self.repo.head.target]
self.repo.create_branch(branch, commit)
except GitError as exc:
raise SCMError(f"Failed to create branch '{branch}'") from exc
def tag(self, tag: str):
raise NotImplementedError
@@ -173,11 +175,20 @@ class Pygit2Backend(BaseGitBackend): # pylint:disable=abstract-method
return shas.pop() # type: ignore
raise RevError(f"unknown Git revision '{rev}'")
def resolve_commit(self, rev: str) -> str:
raise NotImplementedError
def resolve_commit(self, rev: str) -> "GitCommit":
from pygit2 import GitError
def branch_revs(self, branch: str, end_rev: Optional[str] = None):
raise NotImplementedError
try:
commit, _ref = self.repo.resolve_refish(rev)
except (KeyError, GitError):
raise SCMError(f"Invalid commit '{rev}'")
return GitCommit(
str(commit.id),
commit.commit_time,
commit.commit_time_offset,
commit.message,
[str(parent) for parent in commit.parent_ids],
)
def _get_stash(self, ref: str):
raise NotImplementedError
@@ -225,12 +236,43 @@ class Pygit2Backend(BaseGitBackend): # pylint:disable=abstract-method
ref.delete()
def iter_refs(self, base: Optional[str] = None):
for ref in self.repo.references:
if ref.startswith(base):
yield ref
if base:
for ref in self.repo.references:
if ref.startswith(base):
yield ref
else:
yield from self.repo.references
def get_refs_containing(self, rev: str, pattern: Optional[str] = None):
raise NotImplementedError
import fnmatch
from pygit2 import GitError
def _contains(repo, ref, search_commit):
commit, _ref = self.repo.resolve_refish(ref)
base = repo.merge_base(search_commit.id, commit.id)
return base == search_commit.id
try:
search_commit, _ref = self.repo.resolve_refish(rev)
except (KeyError, GitError):
raise SCMError(f"Invalid rev '{rev}'")
if not pattern:
yield from (
ref
for ref in self.iter_refs()
if _contains(self.repo, ref, search_commit)
)
return
literal = pattern.rstrip("/").split("/")
for ref in self.iter_refs():
if (
ref.split("/")[: len(literal)] == literal
or fnmatch.fnmatch(ref, pattern)
) and _contains(self.repo, ref, search_commit):
yield ref
def push_refspec(
self,
+11 -1
View File
@@ -1,7 +1,8 @@
import os
import stat
from abc import ABC, abstractmethod
from typing import Iterable, Optional
from dataclasses import dataclass
from typing import Iterable, List, Optional
from pygtrie import Trie
@@ -118,3 +119,12 @@ class GitTrie:
def stat(self, key: tuple) -> os.stat_result:
obj = self.trie[key]
return os.stat_result((obj.mode, 0, 0, 0, 0, 0, 0, 0, 0, 0))
@dataclass
class GitCommit:
hexsha: str
commit_time: int
commit_time_offset: int
message: str
parents: List[str]
+7
View File
@@ -1,9 +1,16 @@
import logging
from dvc.exceptions import InvalidArgumentError
logger = logging.getLogger(__name__)
def _update_import_on_remote(stage, remote, jobs):
if stage.is_repo_import:
raise InvalidArgumentError(
"Can't update a repo import with --to-remote"
)
url = stage.deps[0].path_info.url
stage.outs[0].hash_info = stage.repo.cloud.transfer(
url, jobs=jobs, remote=remote, command="update"
+4 -1
View File
@@ -357,7 +357,7 @@ def resolve_output(inp, out):
return ret
def resolve_paths(repo, out):
def resolve_paths(repo, out, always_local=False):
from urllib.parse import urlparse
from ..dvcfile import DVC_FILE_SUFFIX
@@ -393,6 +393,9 @@ def resolve_paths(repo, out):
wdir = dirname
out = base
if always_local:
out = base
path = os.path.join(wdir, base + DVC_FILE_SUFFIX)
return (path, wdir, out)
+12
View File
@@ -25,3 +25,15 @@ def get_summary(stats):
"{} file{} {}".format(num, "s" if num > 1 else "", state)
for state, num in status
)
ELLIPSIS = "…"
def truncate_text(
text: str, max_length: int, with_ellipsis: bool = True
) -> str:
if with_ellipsis and len(text) > max_length:
return text[: max_length - 1] + ELLIPSIS
return text[:max_length]
+18 -3
View File
@@ -1,6 +1,8 @@
import hashlib
import io
from funcy import cached_property
from dvc.hash_info import HashInfo
from dvc.istextfile import istextblock
from dvc.utils import dos2unix
@@ -61,17 +63,30 @@ class IterStream(io.RawIOBase):
return self.leftover[:n]
class HashedStreamReader:
class HashedStreamReader(io.IOBase):
PARAM_CHECKSUM = "md5"
def __init__(self, fobj):
self.fobj = fobj
self.md5 = hashlib.md5()
self.is_text_file = None
self.reader = fobj.read1 if hasattr(fobj, "read1") else fobj.read
super().__init__()
def readable(self):
return True
def tell(self):
return self.fobj.tell()
@cached_property
def _reader(self):
if hasattr(self.fobj, "read1"):
return self.fobj.read1
return self.fobj.read
def read(self, n=-1):
chunk = self.reader(n)
chunk = self._reader(n)
if self.is_text_file is None:
self.is_text_file = istextblock(chunk)
+1 -1
View File
@@ -6,7 +6,7 @@
import os
import subprocess
_BASE_VERSION = "2.0.0a0"
_BASE_VERSION = "2.0.0a1"
def _generate_version(base_version):
+3 -2
View File
@@ -88,15 +88,16 @@ install_requires = [
"pyparsing==2.4.7",
"typing_extensions>=3.7.4",
"fsspec>=0.8.5",
"dvclive>=0.0.1a0",
]
# Extra dependencies for remote integrations
gs = ["google-cloud-storage==1.19.0"]
gdrive = ["pydrive2>=1.6.3", "six >= 1.13.0"]
gdrive = ["pydrive2>=1.7.3", "six >= 1.13.0"]
s3 = ["boto3>=1.9.201"]
azure = ["azure-storage-blob>=12.0", "knack"]
azure = ["adlfs>=0.6.3", "azure.identity>=1.4.0", "knack"]
# https://github.com/Legrandin/pycryptodome/issues/465
oss = ["oss2==2.6.1", "pycryptodome<3.9.9"]
ssh = ["paramiko[invoke]>=2.7.0"]
+26 -3
View File
@@ -27,7 +27,10 @@ from dvc.objects.db import ODBManager
from dvc.output.base import OutputAlreadyTrackedError, OutputIsStageFileError
from dvc.repo import Repo as DvcRepo
from dvc.stage import Stage
from dvc.stage.exceptions import StagePathNotFoundError
from dvc.stage.exceptions import (
StageExternalOutputsError,
StagePathNotFoundError,
)
from dvc.system import System
from dvc.utils import LARGE_DIR_SIZE, file_md5, relpath
from dvc.utils.fs import path_isin
@@ -324,8 +327,6 @@ def test_add_filtered_files_in_dir(
indirect=["workspace"],
)
def test_add_external_file(tmp_dir, dvc, workspace, hash_name, hash_value):
from dvc.stage.exceptions import StageExternalOutputsError
workspace.gen("file", "file")
with pytest.raises(StageExternalOutputsError):
@@ -1013,12 +1014,34 @@ def test_add_to_remote(tmp_dir, dvc, local_cloud, local_remote):
assert local_remote.hash_to_path_info(hash_info.value).read_text() == "foo"
def test_add_to_remote_absolute(tmp_dir, make_tmp_dir, dvc, local_remote):
tmp_abs_dir = make_tmp_dir("abs")
tmp_foo = tmp_abs_dir / "foo"
tmp_foo.write_text("foo")
dvc.add(str(tmp_foo), to_remote=True)
tmp_foo.unlink()
foo = tmp_dir / "foo"
assert foo.with_suffix(".dvc").exists()
assert not os.path.exists(tmp_foo)
dvc.pull("foo")
assert not os.path.exists(tmp_foo)
assert foo.read_text() == "foo"
with pytest.raises(StageExternalOutputsError):
tmp_bar = tmp_abs_dir / "bar"
dvc.add(str(tmp_foo), out=str(tmp_bar), to_remote=True)
@pytest.mark.parametrize(
"invalid_opt, kwargs",
[
("multiple targets", {"targets": ["foo", "bar", "baz"]}),
("--no-commit", {"targets": ["foo"], "no_commit": True}),
("--recursive", {"targets": ["foo"], "recursive": True},),
("--external", {"targets": ["foo"], "external": True}),
],
)
def test_add_to_remote_invalid_combinations(dvc, invalid_opt, kwargs):
+1 -1
View File
@@ -218,7 +218,7 @@ def test_config_loads_without_error_for_non_dvc_repo(tmp_dir):
("keyfile", "ssh://[email protected]:1234/path/to/dir"),
("cert_path", "webdavs://example.com/files/USERNAME/"),
("key_path", "webdavs://example.com/files/USERNAME/"),
("gdrive_service_account_p12_file_path", "gdrive://root/test"),
("gdrive_service_account_json_file_path", "gdrive://root/test"),
("gdrive_user_credentials_file", "gdrive://root/test"),
],
)
+15
View File
@@ -340,6 +340,21 @@ def test_import_url_to_remote_directory(tmp_dir, dvc, workspace, local_remote):
)
def test_import_url_to_remote_absolute(
tmp_dir, make_tmp_dir, dvc, local_remote
):
tmp_abs_dir = make_tmp_dir("abs")
tmp_foo = tmp_abs_dir / "foo"
tmp_foo.write_text("foo")
stage = dvc.imp_url(str(tmp_foo), to_remote=True)
foo = tmp_dir / "foo"
assert stage.deps[0].fspath == str(tmp_foo)
assert stage.outs[0].fspath == os.fspath(foo)
assert foo.with_suffix(".dvc").exists()
def test_import_url_to_remote_invalid_combinations(dvc):
with pytest.raises(InvalidArgumentError, match="--no-exec"):
dvc.imp_url("s3://bucket/foo", no_exec=True, to_remote=True)
+5 -10
View File
@@ -1,3 +1,4 @@
import os
from copy import deepcopy
from textwrap import dedent
@@ -53,9 +54,6 @@ LIVE_CHECKPOINT_SCRIPT = dedent(
@pytest.fixture
def live_stage(tmp_dir, scm, dvc):
pytest.skip("dvclive does not exist yet")
def make(summary=True, html=True, live=None, live_no_cache=None):
assert bool(live) != bool(live_no_cache)
tmp_dir.gen("train.py", LIVE_SCRITP)
@@ -130,8 +128,8 @@ def test_live_provides_metrics(tmp_dir, dvc, live_stage):
assert (tmp_dir / "logs").is_dir()
plots = dvc.plots.show()
assert "logs/accuracy.tsv" in plots
assert "logs/loss.tsv" in plots
assert os.path.join("logs", "accuracy.tsv") in plots
assert os.path.join("logs", "loss.tsv") in plots
def test_live_provides_no_metrics(tmp_dir, dvc, live_stage):
@@ -143,8 +141,8 @@ def test_live_provides_no_metrics(tmp_dir, dvc, live_stage):
assert (tmp_dir / "logs").is_dir()
plots = dvc.plots.show()
assert "logs/accuracy.tsv" in plots
assert "logs/loss.tsv" in plots
assert os.path.join("logs", "accuracy.tsv") in plots
assert os.path.join("logs", "loss.tsv") in plots
@pytest.mark.parametrize("typ", ("live", "live_no_cache"))
@@ -169,9 +167,6 @@ def test_live_html(tmp_dir, dvc, live_stage, html):
@pytest.fixture
def live_checkpoint_stage(tmp_dir, scm, dvc):
pytest.skip("dvclive does not exist yet")
def make(live=None, live_no_cache=None):
assert bool(live) != bool(live_no_cache)
+9
View File
@@ -3,6 +3,7 @@ import os
import pytest
from dvc.dvcfile import Dvcfile
from dvc.exceptions import InvalidArgumentError
from tests.unit.fs.test_repo import make_subrepo
@@ -317,6 +318,14 @@ def test_update_from_subrepos(tmp_dir, dvc, erepo_dir, is_dvc):
}
def test_update_import_to_remote(tmp_dir, dvc, erepo_dir, local_remote):
erepo_dir.scm_gen({"foo": "foo"}, commit="add foo")
stage = dvc.imp(os.fspath(erepo_dir), "foo")
erepo_dir.scm_gen({"foo": "bar"}, commit="update foo")
with pytest.raises(InvalidArgumentError):
dvc.update(stage.path, to_remote=True)
@pytest.mark.parametrize(
"workspace",
[
+55 -23
View File
@@ -1,10 +1,11 @@
# pylint:disable=abstract-method
import json
import os
import uuid
from functools import partial, partialmethod
from functools import partialmethod
import pytest
from funcy import cached_property
from funcy import cached_property, retry
from dvc.fs.gdrive import GDriveFileSystem
from dvc.path_info import CloudURLInfo
@@ -14,10 +15,36 @@ from .base import Base
TEST_GDRIVE_REPO_BUCKET = "root"
CREDENTIALS_DIR = os.path.join(os.path.expanduser("~"), ".config", "pydata")
CREDENTIALS_FILE = os.path.join(
CREDENTIALS_DIR, "pydata_google_credentials.json"
)
def _gdrive_retry(func):
def should_retry(exc):
from googleapiclient.errors import HttpError
if not isinstance(exc, HttpError):
return False
if 500 <= exc.resp.status < 600:
return True
if exc.resp.status == 403:
try:
reason = json.loads(exc.content)["error"]["errors"][0][
"reason"
]
except (ValueError, LookupError):
return False
return reason in [
"userRateLimitExceeded",
"rateLimitExceeded",
]
# 16 tries, start at 0.5s, multiply by golden ratio, cap at 20s
return retry(
16,
timeout=lambda a: min(0.5 * 1.618 ** a, 20),
filter_errors=should_retry,
)(func)
class GDrive(Base, CloudURLInfo):
@@ -27,11 +54,23 @@ class GDrive(Base, CloudURLInfo):
@cached_property
def config(self):
tmp_path = tmp_fname()
with open(tmp_path, "w") as stream:
raw_credentials = os.getenv(
GDriveFileSystem.GDRIVE_CREDENTIALS_DATA
)
try:
credentials = json.loads(raw_credentials)
except ValueError:
credentials = {}
use_service_account = credentials.get("type") == "service_account"
stream.write(raw_credentials)
return {
"url": self.url,
"gdrive_service_account_email": "test",
"gdrive_service_account_p12_file_path": "test.p12",
"gdrive_use_service_account": True,
"gdrive_service_account_json_file_path": tmp_path,
"gdrive_use_service_account": use_service_account,
}
@staticmethod
@@ -45,45 +84,38 @@ class GDrive(Base, CloudURLInfo):
@cached_property
def client(self):
import pydata_google_auth
try:
from gdrivefs import GoogleDriveFileSystem
except ImportError:
pytest.skip("gdrivefs is not installed")
tmp_path = tmp_fname()
with open(tmp_path, "w") as stream:
stream.write(os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA))
GoogleDriveFileSystem._connect_cache = partial(
pydata_google_auth.load_user_credentials, tmp_path
return GoogleDriveFileSystem(
token="cache",
tokens_file=self.config["gdrive_service_account_json_file_path"],
service_account=self.config["gdrive_use_service_account"],
)
return GoogleDriveFileSystem(token="cache")
@_gdrive_retry
def mkdir(self, mode=0o777, parents=False, exist_ok=False):
if not self.client.exists(self.path):
self.client.mkdir(self.path)
@_gdrive_retry
def write_bytes(self, contents):
with self.client.open(self.path, mode="wb") as stream:
stream.write(contents)
@_gdrive_retry
def _read(self, mode):
with self.client.open(self.path, mode=mode) as stream:
return stream.read()
def cleanup(self):
self.client.delete(self.path, recursive=True)
read_text = partialmethod(_read, mode="r")
read_bytes = partialmethod(_read, mode="rb")
@pytest.fixture
def gdrive(test_config, make_tmp_dir):
pytest.skip("temporarily disabled")
test_config.requires("gdrive")
if not GDrive.should_test():
pytest.skip("no gdrive")
BIN
View File
Binary file not shown.
Binary file not shown.
-2
View File
@@ -17,7 +17,6 @@ def test_init_env_var(monkeypatch, dvc):
config = {"url": "azure://"}
fs = AzureFileSystem(dvc, config)
assert fs.path_info == "azure://" + container_name
assert fs._conn_str == connection_string
def test_init(dvc):
@@ -26,7 +25,6 @@ def test_init(dvc):
config = {"url": url, "connection_string": connection_string}
fs = AzureFileSystem(dvc, config)
assert fs.path_info == url
assert fs._conn_str == connection_string
def test_info(tmp_dir, azure):
+27
View File
@@ -513,3 +513,30 @@ def test_remind_to_track(scm, caplog):
scm.files_to_track = ["fname with spaces.txt", "тест", "foo"]
scm.remind_to_track()
assert "git add 'fname with spaces.txt' 'тест' foo" in caplog.text
def test_add(tmp_dir, scm, git):
if git.test_backend == "pygit2":
pytest.skip()
tmp_dir.gen({"foo": "foo", "bar": "bar", "dir": {"baz": "baz"}})
git.add(["foo", "dir"])
staged, unstaged, untracked = scm.status()
assert set(staged["add"]) == {"foo", "dir/baz"}
assert len(unstaged) == 0
assert len(untracked) == 1
scm.commit("commit")
tmp_dir.gen({"foo": "bar", "dir": {"baz": "bar"}})
git.add([], update=True)
staged, unstaged, _ = scm.status()
assert set(staged["modify"]) == {"foo", "dir/baz"}
assert len(unstaged) == 0
assert len(untracked) == 1
scm.reset()
git.add(["dir"], update=True)
staged, unstaged, _ = scm.status()
assert set(staged["modify"]) == {"dir/baz"}
assert len(unstaged) == 1
assert len(untracked) == 1
+28 -1
View File
@@ -1,6 +1,6 @@
import pytest
from dvc.dvcfile import Lockfile, LockfileCorruptedError
from dvc.dvcfile import FileIsGitIgnored, Lockfile, LockfileCorruptedError
from dvc.stage import PipelineStage
from dvc.utils.serialize import dump_yaml
@@ -83,3 +83,30 @@ def test_load_when_lockfile_is_corrupted(tmp_dir, dvc, corrupt_data):
with pytest.raises(LockfileCorruptedError) as exc_info:
lockfile.load()
assert "Dvcfile.lock" in str(exc_info.value)
@pytest.mark.parametrize("dvcignored", [True, False])
@pytest.mark.parametrize("file_exists", [True, False])
def test_try_loading_lockfile_that_is_gitignored(
tmp_dir, dvc, scm, dvcignored, file_exists
):
# it should raise error if the file is git-ignored, even if:
# 1. The file does not exist at all.
# 2. Or, is dvc-ignored.
files = [".gitignore"]
if dvcignored:
files.append(".dvcignore")
for file in files:
with (tmp_dir / file).open(mode="a+") as fd:
fd.write("dvc.lock")
if file_exists:
(tmp_dir / "dvc.lock").write_text("")
scm._reset()
with pytest.raises(FileIsGitIgnored) as exc_info:
Lockfile(dvc, "dvc.lock").load()
assert str(exc_info.value) == "'dvc.lock' is git-ignored."
+35 -1
View File
@@ -1,6 +1,8 @@
from collections import OrderedDict
from dvc.utils.humanize import get_summary
import pytest
from dvc.utils.humanize import get_summary, truncate_text
def test_get_summary():
@@ -38,3 +40,35 @@ def test_get_summary():
assert get_summary([]) == ""
assert get_summary([("x", 0), ("y", [])]) == ""
assert get_summary([("x", 1), ("y", [])]) == "1 file x"
def test_truncate_text():
text = "lorem ipsum"
length = 5
truncated = truncate_text(text, length)
# length should not cross the max length
assert len(truncated) == length
assert truncated[:-1] == text[: length - 1]
# last character should be ellipsis
assert truncated[-1] == "…"
truncated = truncate_text(text, length, with_ellipsis=False)
# length should not cross the max length
assert len(truncated) == length
assert truncated == text[:length]
@pytest.mark.parametrize("with_ellipsis", [True, False])
def test_truncate_text_smaller_than_max_length(with_ellipsis):
text = "lorem ipsum"
# exact match as length
truncated = truncate_text(text, len(text), with_ellipsis=with_ellipsis)
assert len(truncated) == len(text)
assert truncated == text
# max_length > len(text)
truncated = truncate_text(text, len(text) + 1, with_ellipsis=with_ellipsis)
assert len(truncated) == len(text)
assert truncated == text
+17 -1
View File
@@ -9,7 +9,15 @@ def test_hashed_stream_reader(tmp_dir):
foo = tmp_dir / "foo"
with open(foo, "rb") as fobj:
stream_reader = HashedStreamReader(fobj)
assert stream_reader.read(3) == b"foo"
assert stream_reader.readable()
assert not stream_reader.seekable()
assert stream_reader.read(2) == b"fo"
assert stream_reader.tell() == 2
assert stream_reader.read(1) == b"o"
assert stream_reader.tell() == 3
hex_digest = file_md5(foo, LocalFileSystem(None, {}))
assert stream_reader.is_text_file
@@ -20,13 +28,21 @@ def test_hashed_stream_reader_as_chunks(tmp_dir):
tmp_dir.gen({"foo": b"foo \x00" * 16})
foo = tmp_dir / "foo"
actual_size = len(foo.read_bytes())
with open(foo, "rb") as fobj:
stream_reader = HashedStreamReader(fobj)
total_read = 0
while True:
chunk = stream_reader.read(16)
total_read += len(chunk)
assert stream_reader.tell() == total_read
if not chunk:
break
assert stream_reader.tell() == actual_size == total_read
hex_digest = file_md5(foo, LocalFileSystem(None, {}))
assert not stream_reader.is_text_file
assert hex_digest == stream_reader.hash_info.value