mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-18 23:10:14 +01:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2e01ce1974 | ||
|
|
d587d41d6c | ||
|
|
27c2401f59 | ||
|
|
174621696d | ||
|
|
e1a193f00b | ||
|
|
cccead3441 | ||
|
|
8102ede5cc | ||
|
|
14944dd9e0 | ||
|
|
63f32936b2 | ||
|
|
7d13d24a1f | ||
|
|
9cb8187cb3 | ||
|
|
98590779eb | ||
|
|
9ae8bcdf52 | ||
|
|
14958ff85f | ||
|
|
9909ee85a1 | ||
|
|
45477a2a81 |
@@ -27,7 +27,11 @@ jobs:
|
||||
with:
|
||||
python-version: 3.8
|
||||
- name: Install requirements
|
||||
run: pip install ".[all,tests]" Pygments collective.checkdocs pre-commit git+https://github.com/intake/gdrivefs.git
|
||||
run: pip install ".[all,tests]" Pygments collective.checkdocs pre-commit
|
||||
- name: install temporary dependencies
|
||||
run: |
|
||||
pip install git+https://github.com/isidentical/gdrivefs@service-account
|
||||
pip install git+https://github.com/isidentical/pydata-google-auth@service-account
|
||||
- name: Check README
|
||||
run: python setup.py checkdocs
|
||||
- uses: pre-commit/[email protected]
|
||||
@@ -69,6 +73,10 @@ jobs:
|
||||
run: |
|
||||
pip install wheel
|
||||
pip install -e ".[all,tests]"
|
||||
- name: install temporary dependencies
|
||||
run: |
|
||||
pip install git+https://github.com/isidentical/gdrivefs@service-account
|
||||
pip install git+https://github.com/isidentical/pydata-google-auth@service-account
|
||||
- name: setup git
|
||||
run: |
|
||||
git config --global user.email "[email protected]"
|
||||
|
||||
@@ -293,7 +293,7 @@ ignored-classes=optparse.Values,thread._local,_thread._local,Dvcfile
|
||||
# (useful for modules/projects where namespaces are manipulated during runtime
|
||||
# and thus existing member attributes cannot be deduced by static analysis. It
|
||||
# supports qualified module names, as well as Unix pattern matching.
|
||||
ignored-modules=
|
||||
ignored-modules=azure
|
||||
|
||||
# Show a hint with possible names when a member name was not found. The aspect
|
||||
# of finding the hint is based on edit distance.
|
||||
|
||||
@@ -138,7 +138,7 @@ def add_parser(subparsers, parent_parser):
|
||||
help=(
|
||||
"Specific plots file(s) to visualize "
|
||||
"(even if not found as `plots` in `dvc.yaml`). "
|
||||
"Shows all tracked plots by default.",
|
||||
"Shows all tracked plots by default."
|
||||
),
|
||||
metavar="<paths>",
|
||||
).complete = completion.FILE
|
||||
|
||||
+8
-10
@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Dict, Iterable, List
|
||||
from dvc.command import completion
|
||||
from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
|
||||
from dvc.utils.cli_parse import parse_params
|
||||
from dvc.utils.humanize import truncate_text
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from dvc.output.base import BaseOutput
|
||||
@@ -14,7 +15,6 @@ if TYPE_CHECKING:
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
MAX_TEXT_LENGTH = 80
|
||||
ELLIPSIS = "…"
|
||||
|
||||
|
||||
def generate_description(stage: "Stage") -> str:
|
||||
@@ -22,10 +22,10 @@ def generate_description(stage: "Stage") -> str:
|
||||
return ", ".join(out.def_path for out in outs)
|
||||
|
||||
if not stage.deps and not stage.outs:
|
||||
return "No outputs or dependencies."
|
||||
return "No outputs or dependencies"
|
||||
|
||||
if not stage.outs and stage.deps:
|
||||
return "Stage depends on " + part_desc(stage.deps)
|
||||
return "Depends on " + part_desc(stage.deps)
|
||||
|
||||
def is_plot_or_metric(out: "BaseOutput"):
|
||||
return bool(out.plot) or bool(out.metric)
|
||||
@@ -34,11 +34,11 @@ def generate_description(stage: "Stage") -> str:
|
||||
|
||||
outs = list(filterfalse(is_plot_or_metric, stage.outs))
|
||||
if outs:
|
||||
desc.append("Produces " + part_desc(outs))
|
||||
desc.append("Outputs " + part_desc(outs))
|
||||
|
||||
plots_and_metrics = list(filter(is_plot_or_metric, stage.outs))
|
||||
if plots_and_metrics:
|
||||
desc.append("Generates " + part_desc(plots_and_metrics))
|
||||
desc.append("Reports " + part_desc(plots_and_metrics))
|
||||
|
||||
return "; ".join(desc)
|
||||
|
||||
@@ -47,9 +47,7 @@ def prepare_description(
|
||||
stage: "Stage", max_length: int = MAX_TEXT_LENGTH
|
||||
) -> str:
|
||||
desc = stage.short_description() or generate_description(stage)
|
||||
if len(desc) > max_length:
|
||||
return desc[: max_length - 1] + ELLIPSIS
|
||||
return desc
|
||||
return truncate_text(desc, max_length)
|
||||
|
||||
|
||||
def prepare_stages_data(
|
||||
@@ -348,7 +346,7 @@ def add_parser(subparsers, parent_parser):
|
||||
"--fail",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Fail immediately if there's an error.",
|
||||
help="Fail immediately, do not suppress any syntax errors.",
|
||||
)
|
||||
stage_list_parser.add_argument(
|
||||
"-R",
|
||||
@@ -361,6 +359,6 @@ def add_parser(subparsers, parent_parser):
|
||||
"--names-only",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="List only the name of the stages.",
|
||||
help="List only stage names.",
|
||||
)
|
||||
stage_list_parser.set_defaults(func=CmdStageList)
|
||||
|
||||
+1
-1
@@ -236,7 +236,7 @@ class Config(dict):
|
||||
str: {
|
||||
"url": func,
|
||||
"gdrive_user_credentials_file": func,
|
||||
"gdrive_service_account_p12_file_path": func,
|
||||
"gdrive_service_account_json_file_path": func,
|
||||
"credentialpath": func,
|
||||
"keyfile": func,
|
||||
"cert_path": func,
|
||||
|
||||
+10
-3
@@ -179,7 +179,15 @@ SCHEMA = {
|
||||
"webhdfs_alias": str,
|
||||
**REMOTE_COMMON,
|
||||
},
|
||||
"azure": {"connection_string": str, **REMOTE_COMMON},
|
||||
"azure": {
|
||||
"connection_string": str,
|
||||
"sas_token": str,
|
||||
"account_name": str,
|
||||
"tenant_id": str,
|
||||
"client_id": str,
|
||||
"client_secret": str,
|
||||
**REMOTE_COMMON,
|
||||
},
|
||||
"oss": {
|
||||
"oss_key_id": str,
|
||||
"oss_key_secret": str,
|
||||
@@ -191,9 +199,8 @@ SCHEMA = {
|
||||
"gdrive_client_id": str,
|
||||
"gdrive_client_secret": str,
|
||||
"gdrive_user_credentials_file": str,
|
||||
"gdrive_service_account_email": str,
|
||||
"gdrive_service_account_user_email": str,
|
||||
"gdrive_service_account_p12_file_path": str,
|
||||
"gdrive_service_account_json_file_path": str,
|
||||
Optional("gdrive_trash_only", default=False): Bool,
|
||||
**REMOTE_COMMON,
|
||||
},
|
||||
|
||||
+24
-11
@@ -20,7 +20,6 @@ from dvc.utils import relpath
|
||||
from dvc.utils.collections import apply_diff
|
||||
from dvc.utils.serialize import (
|
||||
dump_yaml,
|
||||
load_yaml,
|
||||
modify_yaml,
|
||||
parse_yaml,
|
||||
parse_yaml_for_update,
|
||||
@@ -115,6 +114,14 @@ class FileMixin:
|
||||
self.repo.fs, LocalFileSystem
|
||||
) and self.repo.scm.is_ignored(self.path)
|
||||
|
||||
def _verify_filename(self):
|
||||
if self.verify:
|
||||
check_dvc_filename(self.path)
|
||||
|
||||
def _check_gitignored(self):
|
||||
if self._is_git_ignored():
|
||||
raise FileIsGitIgnored(self.path)
|
||||
|
||||
def _load(self):
|
||||
# it raises the proper exceptions by priority:
|
||||
# 1. when the file doesn't exists
|
||||
@@ -125,12 +132,11 @@ class FileMixin:
|
||||
is_ignored = self.repo.fs.exists(self.path, use_dvcignore=False)
|
||||
raise StageFileDoesNotExistError(self.path, dvc_ignored=is_ignored)
|
||||
|
||||
if self.verify:
|
||||
check_dvc_filename(self.path)
|
||||
self._verify_filename()
|
||||
if not self.repo.fs.isfile(self.path):
|
||||
raise StageFileIsNotDvcFileError(self.path)
|
||||
if self._is_git_ignored():
|
||||
raise FileIsGitIgnored(self.path)
|
||||
|
||||
self._check_gitignored()
|
||||
with self.repo.fs.open(self.path, encoding="utf-8") as fd:
|
||||
stage_text = fd.read()
|
||||
d = parse_yaml(stage_text, self.path)
|
||||
@@ -330,18 +336,25 @@ class Lockfile(FileMixin):
|
||||
except MultipleInvalid as exc:
|
||||
raise StageFileFormatError(f"'{fname}' format error: {exc}")
|
||||
|
||||
def load(self):
|
||||
if not self.exists():
|
||||
return {}
|
||||
def _verify_filename(self):
|
||||
pass # lockfile path is hardcoded, so no need to verify here
|
||||
|
||||
data = load_yaml(self.path, fs=self.repo.fs)
|
||||
def _load(self):
|
||||
try:
|
||||
data = self.validate(data, fname=self.relpath)
|
||||
return super()._load()
|
||||
except StageFileDoesNotExistError:
|
||||
# we still need to account for git-ignored dvc.lock file
|
||||
# even though it may not exist or have been .dvcignored
|
||||
self._check_gitignored()
|
||||
return {}, ""
|
||||
except StageFileFormatError as exc:
|
||||
raise LockfileCorruptedError(
|
||||
f"Lockfile '{self.relpath}' is corrupted."
|
||||
) from exc
|
||||
return data
|
||||
|
||||
def load(self):
|
||||
d, _ = self._load()
|
||||
return d
|
||||
|
||||
@property
|
||||
def latest_version_info(self):
|
||||
|
||||
+134
-143
@@ -1,58 +1,94 @@
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import threading
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from funcy import cached_property, wrap_prop
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.path_info import CloudURLInfo
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.utils import format_link
|
||||
|
||||
from .base import BaseFileSystem
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_DEFAULT_CREDS_STEPS = (
|
||||
"https://azuresdkdocs.blob.core.windows.net/$web/python/"
|
||||
"azure-identity/1.4.0/azure.identity.html#azure.identity"
|
||||
".DefaultAzureCredential"
|
||||
)
|
||||
|
||||
|
||||
class AzureAuthError(DvcException):
|
||||
pass
|
||||
|
||||
|
||||
class AzureFileSystem(BaseFileSystem):
|
||||
scheme = Schemes.AZURE
|
||||
PATH_CLS = CloudURLInfo
|
||||
REQUIRES = {
|
||||
"azure-storage-blob": "azure.storage.blob",
|
||||
"knack": "knack",
|
||||
}
|
||||
PARAM_CHECKSUM = "etag"
|
||||
DETAIL_FIELDS = frozenset(("etag", "size"))
|
||||
|
||||
COPY_POLL_SECONDS = 5
|
||||
LIST_OBJECT_PAGE_SIZE = 5000
|
||||
REQUIRES = {
|
||||
"adlfs": "adlfs",
|
||||
"knack": "knack",
|
||||
"azure-identity": "azure.identity",
|
||||
}
|
||||
|
||||
def __init__(self, repo, config):
|
||||
super().__init__(repo, config)
|
||||
|
||||
url = config.get("url", "azure://")
|
||||
url = config.get("url")
|
||||
self.path_info = self.PATH_CLS(url)
|
||||
self.bucket = self.path_info.bucket
|
||||
|
||||
if not self.bucket:
|
||||
container = self._az_config.get("storage", "container_name", None)
|
||||
self.path_info = self.PATH_CLS(f"azure://{container}")
|
||||
self.bucket = self.path_info.bucket
|
||||
url = f"azure://{container}"
|
||||
|
||||
self._conn_str = config.get(
|
||||
"connection_string"
|
||||
) or self._az_config.get("storage", "connection_string", None)
|
||||
self.path_info = self.PATH_CLS(url)
|
||||
self.bucket = self.path_info.bucket
|
||||
|
||||
self._account_url = None
|
||||
if not self._conn_str:
|
||||
name = self._az_config.get("storage", "account", None)
|
||||
self._account_url = f"https://{name}.blob.core.windows.net"
|
||||
self.login_method, self.login_info = self._prepare_credentials(config)
|
||||
|
||||
self._credential = config.get("sas_token") or self._az_config.get(
|
||||
"storage", "sas_token", None
|
||||
def _prepare_credentials(self, config):
|
||||
from azure.identity.aio import DefaultAzureCredential
|
||||
|
||||
login_info = {}
|
||||
login_info["connection_string"] = config.get(
|
||||
"connection_string",
|
||||
self._az_config.get("storage", "connection_string", None),
|
||||
)
|
||||
if not self._credential:
|
||||
self._credential = self._az_config.get("storage", "key", None)
|
||||
login_info["account_name"] = config.get(
|
||||
"account_name", self._az_config.get("storage", "account", None)
|
||||
)
|
||||
login_info["account_key"] = config.get(
|
||||
"account_key", self._az_config.get("storage", "key", None)
|
||||
)
|
||||
login_info["sas_token"] = config.get(
|
||||
"sas_token", self._az_config.get("storage", "sas_token", None)
|
||||
)
|
||||
login_info["tenant_id"] = config.get("tenant_id")
|
||||
login_info["client_id"] = config.get("client_id")
|
||||
login_info["client_secret"] = config.get("client_secret")
|
||||
|
||||
if not any(login_info.values()):
|
||||
login_info["credentials"] = DefaultAzureCredential()
|
||||
|
||||
for login_method, required_keys in [ # noqa
|
||||
("connection string", ["connection_string"]),
|
||||
("account key", ["account_name", "account_key"]),
|
||||
("SAS token", ["account_name", "sas_token"]),
|
||||
("anonymous login", ["account_name"]),
|
||||
(f"default credentials ({_DEFAULT_CREDS_STEPS})", ["credentials"]),
|
||||
]:
|
||||
if all(login_info.get(key) is not None for key in required_keys):
|
||||
break
|
||||
else:
|
||||
login_method = None
|
||||
|
||||
return login_method, login_info
|
||||
|
||||
@cached_property
|
||||
def _az_config(self):
|
||||
@@ -68,158 +104,113 @@ class AzureFileSystem(BaseFileSystem):
|
||||
|
||||
@wrap_prop(threading.Lock())
|
||||
@cached_property
|
||||
def blob_service(self):
|
||||
# pylint: disable=no-name-in-module
|
||||
from azure.core.exceptions import (
|
||||
HttpResponseError,
|
||||
ResourceNotFoundError,
|
||||
)
|
||||
from azure.storage.blob import BlobServiceClient
|
||||
def fs(self):
|
||||
from adlfs import AzureBlobFileSystem
|
||||
from azure.core.exceptions import AzureError
|
||||
|
||||
if self._conn_str:
|
||||
logger.debug(f"Using connection string '{self._conn_str}'")
|
||||
blob_service = BlobServiceClient.from_connection_string(
|
||||
self._conn_str, credential=self._credential
|
||||
)
|
||||
else:
|
||||
logger.debug(f"Using account url '{self._account_url}'")
|
||||
blob_service = BlobServiceClient(
|
||||
self._account_url, credential=self._credential
|
||||
)
|
||||
try:
|
||||
file_system = AzureBlobFileSystem(**self.login_info)
|
||||
if self.bucket not in [
|
||||
container.rstrip("/") for container in file_system.ls("/")
|
||||
]:
|
||||
file_system.mkdir(self.bucket)
|
||||
except (ValueError, AzureError) as e:
|
||||
raise AzureAuthError(
|
||||
f"Authentication to Azure Blob Storage via {self.login_method}"
|
||||
" failed.\nLearn more about configuration settings at"
|
||||
f" {format_link('https://man.dvc.org/remote/modify')}"
|
||||
) from e
|
||||
|
||||
logger.debug(f"Container name {self.bucket}")
|
||||
container_client = blob_service.get_container_client(self.bucket)
|
||||
return file_system
|
||||
|
||||
try: # verify that container exists
|
||||
container_client.get_container_properties()
|
||||
except ResourceNotFoundError:
|
||||
container_client.create_container()
|
||||
except HttpResponseError as exc:
|
||||
# client may not have account-level privileges
|
||||
if exc.status_code != 403:
|
||||
raise
|
||||
def _with_bucket(self, path):
|
||||
if isinstance(path, self.PATH_CLS):
|
||||
return f"{path.bucket}/{path.path}"
|
||||
return path
|
||||
|
||||
return blob_service
|
||||
def open(
|
||||
self, path_info, mode="r", **kwargs
|
||||
): # pylint: disable=arguments-differ
|
||||
return self.fs.open(self._with_bucket(path_info), mode=mode)
|
||||
|
||||
def get_etag(self, path_info):
|
||||
blob_client = self.blob_service.get_blob_client(
|
||||
path_info.bucket, path_info.path
|
||||
)
|
||||
etag = blob_client.get_blob_properties().etag
|
||||
return etag.strip('"')
|
||||
def exists(self, path_info, use_dvcignore=False):
|
||||
return self.fs.exists(self._with_bucket(path_info))
|
||||
|
||||
def _generate_download_url(self, path_info, expires=3600):
|
||||
from azure.storage.blob import ( # pylint:disable=no-name-in-module
|
||||
BlobSasPermissions,
|
||||
generate_blob_sas,
|
||||
)
|
||||
def _strip_bucket(self, entry):
|
||||
_, entry = entry.split("/", 1)
|
||||
return entry
|
||||
|
||||
expires_at = datetime.utcnow() + timedelta(seconds=expires)
|
||||
|
||||
blob_client = self.blob_service.get_blob_client(
|
||||
path_info.bucket, path_info.path
|
||||
)
|
||||
|
||||
sas_token = generate_blob_sas(
|
||||
blob_client.account_name,
|
||||
blob_client.container_name,
|
||||
blob_client.blob_name,
|
||||
account_key=blob_client.credential.account_key,
|
||||
permission=BlobSasPermissions(read=True),
|
||||
expiry=expires_at,
|
||||
)
|
||||
return blob_client.url + "?" + sas_token
|
||||
|
||||
def exists(self, path_info, use_dvcignore=True):
|
||||
paths = self._list_paths(path_info.bucket, path_info.path)
|
||||
return any(path_info.path == path for path in paths)
|
||||
|
||||
def _list_paths(self, bucket, prefix):
|
||||
container_client = self.blob_service.get_container_client(bucket)
|
||||
for blob in container_client.list_blobs(name_starts_with=prefix):
|
||||
yield blob.name
|
||||
|
||||
def walk_files(self, path_info, **kwargs):
|
||||
if not kwargs.pop("prefix", False):
|
||||
path_info = path_info / ""
|
||||
for fname in self._list_paths(
|
||||
path_info.bucket, path_info.path, **kwargs
|
||||
):
|
||||
if fname.endswith("/"):
|
||||
continue
|
||||
|
||||
yield path_info.replace(path=fname)
|
||||
def _strip_buckets(self, entries, detail, prefix=None):
|
||||
for entry in entries:
|
||||
if detail:
|
||||
entry = entry.copy()
|
||||
entry["name"] = self._strip_bucket(entry["name"])
|
||||
else:
|
||||
entry = self._strip_bucket(
|
||||
f"{prefix}/{entry}" if prefix else entry
|
||||
)
|
||||
yield entry
|
||||
|
||||
def ls(
|
||||
self, path_info, detail=False, recursive=False
|
||||
): # pylint: disable=arguments-differ
|
||||
assert recursive
|
||||
path = self._with_bucket(path_info)
|
||||
if recursive:
|
||||
for root, _, files in self.fs.walk(path, detail=detail):
|
||||
if detail:
|
||||
files = files.values()
|
||||
yield from self._strip_buckets(files, detail, prefix=root)
|
||||
return None
|
||||
|
||||
container_client = self.blob_service.get_container_client(
|
||||
path_info.bucket
|
||||
)
|
||||
for blob in container_client.list_blobs(
|
||||
name_starts_with=path_info.path
|
||||
):
|
||||
if detail:
|
||||
yield {
|
||||
"type": "file",
|
||||
"name": blob.name,
|
||||
"size": blob.size,
|
||||
"etag": blob.etag.strip('"'),
|
||||
}
|
||||
else:
|
||||
yield blob.name
|
||||
yield from self._strip_buckets(self.ls(path, detail=detail), detail)
|
||||
|
||||
def walk_files(self, path_info, **kwargs):
|
||||
for file in self.ls(path_info, recursive=True):
|
||||
yield path_info.replace(path=file)
|
||||
|
||||
def remove(self, path_info):
|
||||
if path_info.scheme != self.scheme:
|
||||
raise NotImplementedError
|
||||
|
||||
logger.debug(f"Removing {path_info}")
|
||||
self.blob_service.get_blob_client(
|
||||
path_info.bucket, path_info.path
|
||||
).delete_blob()
|
||||
self.fs.delete(self._with_bucket(path_info))
|
||||
|
||||
def info(self, path_info):
|
||||
blob_client = self.blob_service.get_blob_client(
|
||||
path_info.bucket, path_info.path
|
||||
)
|
||||
properties = blob_client.get_blob_properties()
|
||||
return {
|
||||
"type": "file",
|
||||
"size": properties.size,
|
||||
"etag": properties.etag.strip('"'),
|
||||
}
|
||||
info = self.fs.info(self._with_bucket(path_info)).copy()
|
||||
info["name"] = self._strip_bucket(info["name"])
|
||||
return info
|
||||
|
||||
def _upload_fobj(self, fobj, to_info):
|
||||
blob_client = self.blob_service.get_blob_client(
|
||||
to_info.bucket, to_info.path
|
||||
)
|
||||
blob_client.upload_blob(fobj, overwrite=True)
|
||||
from adlfs import AzureBlobFile
|
||||
|
||||
with self.open(to_info, "wb") as fdest:
|
||||
shutil.copyfileobj(
|
||||
fobj, fdest, length=AzureBlobFile.DEFAULT_BLOCK_SIZE
|
||||
)
|
||||
|
||||
def _upload(
|
||||
self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
|
||||
self, from_file, to_info, name=None, no_progress_bar=False, **kwargs
|
||||
):
|
||||
total = os.path.getsize(from_file)
|
||||
with open(from_file, "rb") as fobj:
|
||||
self.upload_fobj(
|
||||
fobj,
|
||||
to_info,
|
||||
self._with_bucket(to_info),
|
||||
desc=name,
|
||||
total=total,
|
||||
no_progress_bar=no_progress_bar,
|
||||
)
|
||||
self.fs.invalidate_cache(self._with_bucket(to_info.parent))
|
||||
|
||||
def _download(
|
||||
self, from_info, to_file, name=None, no_progress_bar=False, **_kwargs
|
||||
self, from_info, to_file, name=None, no_progress_bar=False, **pbar_args
|
||||
):
|
||||
blob_client = self.blob_service.get_blob_client(
|
||||
from_info.bucket, from_info.path
|
||||
)
|
||||
total = blob_client.get_blob_properties().size
|
||||
stream = blob_client.download_blob()
|
||||
with open(to_file, "wb") as fobj:
|
||||
total = self.getsize(self._with_bucket(from_info))
|
||||
with self.open(from_info, "rb") as fobj:
|
||||
with Tqdm.wrapattr(
|
||||
fobj, "write", desc=name, total=total, disable=no_progress_bar
|
||||
fobj,
|
||||
"read",
|
||||
desc=name,
|
||||
disable=no_progress_bar,
|
||||
bytes=True,
|
||||
total=total,
|
||||
**pbar_args,
|
||||
) as wrapped:
|
||||
stream.readinto(wrapped)
|
||||
with open(to_file, "wb") as fdest:
|
||||
shutil.copyfileobj(wrapped, fdest)
|
||||
|
||||
+11
-14
@@ -117,14 +117,11 @@ class GDriveFileSystem(BaseFileSystem):
|
||||
self._path = self.path_info.path
|
||||
self._trash_only = config.get("gdrive_trash_only")
|
||||
self._use_service_account = config.get("gdrive_use_service_account")
|
||||
self._service_account_email = config.get(
|
||||
"gdrive_service_account_email"
|
||||
)
|
||||
self._service_account_user_email = config.get(
|
||||
"gdrive_service_account_user_email"
|
||||
)
|
||||
self._service_account_p12_file_path = config.get(
|
||||
"gdrive_service_account_p12_file_path"
|
||||
self._service_account_json_file_path = config.get(
|
||||
"gdrive_service_account_json_file_path"
|
||||
)
|
||||
self._client_id = config.get("gdrive_client_id")
|
||||
self._client_secret = config.get("gdrive_client_secret")
|
||||
@@ -142,14 +139,14 @@ class GDriveFileSystem(BaseFileSystem):
|
||||
|
||||
def _validate_config(self):
|
||||
# Validate Service Account configuration
|
||||
if self._use_service_account and (
|
||||
not self._service_account_email
|
||||
or not self._service_account_p12_file_path
|
||||
if (
|
||||
self._use_service_account
|
||||
and not self._service_account_json_file_path
|
||||
):
|
||||
raise DvcException(
|
||||
"To use service account, set `gdrive_service_account_email`,\n"
|
||||
"`gdrive_service_account_p12_file_path`, and optionally "
|
||||
"`gdrive_service_account_user_email`\nin DVC config. "
|
||||
"To use service account, set "
|
||||
"`gdrive_service_account_json_file_path`, and optionally"
|
||||
"`gdrive_service_account_user_email` in DVC config\n"
|
||||
"Learn more at {}".format(
|
||||
format_link("https://man.dvc.org/remote/modify")
|
||||
)
|
||||
@@ -220,7 +217,6 @@ class GDriveFileSystem(BaseFileSystem):
|
||||
auth_settings = {
|
||||
"client_config_backend": "settings",
|
||||
"save_credentials": True,
|
||||
"save_credentials_backend": "file",
|
||||
"save_credentials_file": self._gdrive_user_credentials_path,
|
||||
"get_refresh_token": True,
|
||||
"oauth_scope": [
|
||||
@@ -230,12 +226,13 @@ class GDriveFileSystem(BaseFileSystem):
|
||||
}
|
||||
|
||||
if self._use_service_account:
|
||||
auth_settings["save_credentials_backend"] = "json"
|
||||
auth_settings["service_config"] = {
|
||||
"client_service_email": self._service_account_email,
|
||||
"client_user_email": self._service_account_user_email,
|
||||
"client_pkcs12_file_path": self._service_account_p12_file_path,
|
||||
"client_json_file_path": self._service_account_json_file_path,
|
||||
}
|
||||
else:
|
||||
auth_settings["save_credentials_backend"] = "file"
|
||||
auth_settings["client_config"] = {
|
||||
"client_id": self._client_id or self.DEFAULT_GDRIVE_CLIENT_ID,
|
||||
"client_secret": self._client_secret
|
||||
|
||||
+12
-2
@@ -52,6 +52,8 @@ def add( # noqa: C901
|
||||
invalid_opt = "--no-commit option"
|
||||
elif recursive:
|
||||
invalid_opt = "--recursive option"
|
||||
elif kwargs.get("external"):
|
||||
invalid_opt = "--external option"
|
||||
else:
|
||||
message = "{option} can't be used without --to-remote"
|
||||
if kwargs.get("remote"):
|
||||
@@ -88,7 +90,12 @@ def add( # noqa: C901
|
||||
)
|
||||
|
||||
stages = _create_stages(
|
||||
repo, sub_targets, fname, pbar=pbar, **kwargs,
|
||||
repo,
|
||||
sub_targets,
|
||||
fname,
|
||||
pbar=pbar,
|
||||
transfer=to_remote or to_cache,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
try:
|
||||
@@ -230,6 +237,7 @@ def _create_stages(
|
||||
external=False,
|
||||
glob=False,
|
||||
desc=None,
|
||||
transfer=False,
|
||||
**kwargs,
|
||||
):
|
||||
from dvc.dvcfile import Dvcfile
|
||||
@@ -246,7 +254,9 @@ def _create_stages(
|
||||
):
|
||||
if kwargs.get("out"):
|
||||
out = resolve_output(out, kwargs["out"])
|
||||
path, wdir, out = resolve_paths(repo, out)
|
||||
path, wdir, out = resolve_paths(
|
||||
repo, out, always_local=transfer and not kwargs.get("out")
|
||||
)
|
||||
stage = create_stage(
|
||||
Stage,
|
||||
repo,
|
||||
|
||||
@@ -761,7 +761,7 @@ class Experiments:
|
||||
# if we can't tell from branch name, fall back to parent commit
|
||||
exp_commit = self.scm.resolve_commit(exp_rev)
|
||||
if exp_commit:
|
||||
exp_baseline = first(exp_commit.parents).hexsha
|
||||
exp_baseline = first(exp_commit.parents)
|
||||
if exp_baseline == baseline_sha:
|
||||
return exp_baseline
|
||||
raise BaselineMismatchError(exp_baseline, baseline_sha)
|
||||
|
||||
@@ -19,7 +19,7 @@ def _collect_experiment_commit(repo, rev, stash=False, sha_only=True):
|
||||
res["timestamp"] = None
|
||||
else:
|
||||
commit = repo.scm.resolve_commit(rev)
|
||||
res["timestamp"] = datetime.fromtimestamp(commit.committed_date)
|
||||
res["timestamp"] = datetime.fromtimestamp(commit.commit_time)
|
||||
|
||||
configs = _collect_configs(repo, rev=rev)
|
||||
params = _read_params(repo, configs, rev)
|
||||
@@ -121,7 +121,7 @@ def show(
|
||||
for ref in repo.scm.iter_refs(base=str(ref_info))
|
||||
]
|
||||
for exp_ref, _ in sorted(
|
||||
commits, key=lambda x: x[1].committed_date, reverse=True,
|
||||
commits, key=lambda x: x[1].commit_time, reverse=True,
|
||||
):
|
||||
ref_info = ExpRefInfo.from_ref(exp_ref)
|
||||
assert ref_info.baseline_sha == rev
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from typing import TYPE_CHECKING, Generator, Iterable
|
||||
from typing import TYPE_CHECKING, Generator, Iterable, Set
|
||||
|
||||
from .base import EXEC_NAMESPACE, EXPS_NAMESPACE, EXPS_STASH, ExpRefInfo
|
||||
|
||||
@@ -75,13 +75,14 @@ def remote_exp_refs_by_baseline(
|
||||
|
||||
def exp_commits(
|
||||
scm: "Git", ref_infos: Iterable["ExpRefInfo"] = None
|
||||
) -> Generator[str, None, None]:
|
||||
) -> Iterable[str]:
|
||||
"""Iterate over all experiment commits."""
|
||||
shas = set()
|
||||
shas: Set["str"] = set()
|
||||
refs = ref_infos if ref_infos else exp_refs(scm)
|
||||
for ref_info in refs:
|
||||
shas.update(scm.branch_revs(str(ref_info), ref_info.baseline_sha))
|
||||
shas.add(ref_info.baseline_sha)
|
||||
if ref_info.baseline_sha:
|
||||
shas.add(ref_info.baseline_sha)
|
||||
yield from shas
|
||||
|
||||
|
||||
|
||||
+3
-1
@@ -27,7 +27,9 @@ def imp_url(
|
||||
from dvc.stage import Stage, create_stage, restore_meta
|
||||
|
||||
out = resolve_output(url, out)
|
||||
path, wdir, out = resolve_paths(self, out)
|
||||
path, wdir, out = resolve_paths(
|
||||
self, out, always_local=to_remote and not out
|
||||
)
|
||||
|
||||
if to_remote and no_exec:
|
||||
raise InvalidArgumentError(
|
||||
|
||||
+19
-7
@@ -2,6 +2,7 @@
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
from collections.abc import Mapping
|
||||
from contextlib import contextmanager
|
||||
@@ -75,6 +76,7 @@ class Git(Base):
|
||||
GITIGNORE = ".gitignore"
|
||||
GIT_DIR = ".git"
|
||||
LOCAL_BRANCH_PREFIX = "refs/heads/"
|
||||
RE_HEXSHA = re.compile(r"^[0-9A-Fa-f]{4,40}$")
|
||||
|
||||
def __init__(
|
||||
self, *args, backends: Optional[Iterable[str]] = None, **kwargs
|
||||
@@ -119,12 +121,7 @@ class Git(Base):
|
||||
|
||||
@classmethod
|
||||
def is_sha(cls, rev):
|
||||
for _, backend in GitBackends.DEFAULT.items():
|
||||
try:
|
||||
return backend.is_sha(rev)
|
||||
except NotImplementedError:
|
||||
pass
|
||||
raise NoGitBackendError("is_sha")
|
||||
return rev and cls.RE_HEXSHA.search(rev)
|
||||
|
||||
@staticmethod
|
||||
def _get_git_dir(root_dir):
|
||||
@@ -368,7 +365,6 @@ class Git(Base):
|
||||
get_rev = partialmethod(_backend_func, "get_rev")
|
||||
_resolve_rev = partialmethod(_backend_func, "resolve_rev")
|
||||
resolve_commit = partialmethod(_backend_func, "resolve_commit")
|
||||
branch_revs = partialmethod(_backend_func, "branch_revs")
|
||||
|
||||
set_ref = partialmethod(_backend_func, "set_ref")
|
||||
get_ref = partialmethod(_backend_func, "get_ref")
|
||||
@@ -405,6 +401,22 @@ class Git(Base):
|
||||
raise RevError(f"ambiguous Git revision '{rev}'")
|
||||
raise
|
||||
|
||||
def branch_revs(
|
||||
self, branch: str, end_rev: Optional[str] = None
|
||||
) -> Iterable[str]:
|
||||
"""Iterate over revisions in a given branch (from newest to oldest).
|
||||
|
||||
If end_rev is set, iterator will stop when the specified revision is
|
||||
reached.
|
||||
"""
|
||||
commit = self.resolve_commit(branch)
|
||||
while commit is not None:
|
||||
yield commit.hexsha
|
||||
parent = first(commit.parents)
|
||||
if parent is None or parent == end_rev:
|
||||
return
|
||||
commit = self.resolve_commit(parent)
|
||||
|
||||
@contextmanager
|
||||
def detach_head(self, rev: Optional[str] = None):
|
||||
"""Context manager for performing detached HEAD SCM operations.
|
||||
|
||||
+14
-14
@@ -1,11 +1,22 @@
|
||||
import os
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Callable, Iterable, Mapping, Optional, Tuple
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Callable,
|
||||
Iterable,
|
||||
Mapping,
|
||||
Optional,
|
||||
Tuple,
|
||||
Union,
|
||||
)
|
||||
|
||||
from dvc.scm.base import SCMError
|
||||
|
||||
from ..objects import GitObject
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..objects import GitCommit
|
||||
|
||||
|
||||
class NoGitBackendError(SCMError):
|
||||
def __init__(self, func):
|
||||
@@ -41,18 +52,13 @@ class BaseGitBackend(ABC):
|
||||
):
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
@abstractmethod
|
||||
def is_sha(rev: str) -> bool:
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def dir(self) -> str:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def add(self, paths: Iterable[str], update=False):
|
||||
def add(self, paths: Union[str, Iterable[str]], update=False):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
@@ -122,13 +128,7 @@ class BaseGitBackend(ABC):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def resolve_commit(self, rev: str) -> str:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def branch_revs(
|
||||
self, branch: str, end_rev: Optional[str] = None
|
||||
) -> Iterable[str]:
|
||||
def resolve_commit(self, rev: str) -> "GitCommit":
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
|
||||
@@ -4,7 +4,17 @@ import logging
|
||||
import os
|
||||
import stat
|
||||
from io import BytesIO, StringIO
|
||||
from typing import Callable, Dict, Iterable, Mapping, Optional, Tuple
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Callable,
|
||||
Dict,
|
||||
Iterable,
|
||||
List,
|
||||
Mapping,
|
||||
Optional,
|
||||
Tuple,
|
||||
Union,
|
||||
)
|
||||
|
||||
from funcy import cached_property
|
||||
|
||||
@@ -16,6 +26,9 @@ from dvc.utils import relpath
|
||||
from ..objects import GitObject
|
||||
from .base import BaseGitBackend
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..objects import GitCommit
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -109,24 +122,23 @@ class DulwichBackend(BaseGitBackend): # pylint:disable=abstract-method
|
||||
):
|
||||
raise NotImplementedError
|
||||
|
||||
@staticmethod
|
||||
def is_sha(rev: str) -> bool:
|
||||
raise NotImplementedError
|
||||
|
||||
@property
|
||||
def dir(self) -> str:
|
||||
return self.repo.commondir()
|
||||
|
||||
def add(self, paths: Iterable[str], update=False):
|
||||
def add(self, paths: Union[str, Iterable[str]], update=False):
|
||||
from dvc.utils.fs import walk_files
|
||||
|
||||
if update:
|
||||
raise NotImplementedError
|
||||
assert paths or update
|
||||
|
||||
if isinstance(paths, str):
|
||||
paths = [paths]
|
||||
|
||||
files = []
|
||||
if update and not paths:
|
||||
self.repo.stage(list(self.repo.open_index()))
|
||||
return
|
||||
|
||||
files: List[bytes] = []
|
||||
for path in paths:
|
||||
if not os.path.isabs(path) and self._submodules:
|
||||
# NOTE: If path is inside a submodule, Dulwich expects the
|
||||
@@ -142,13 +154,30 @@ class DulwichBackend(BaseGitBackend): # pylint:disable=abstract-method
|
||||
)
|
||||
break
|
||||
if os.path.isdir(path):
|
||||
files.extend(walk_files(path))
|
||||
files.extend(
|
||||
os.fsencode(relpath(fpath, self.root_dir))
|
||||
for fpath in walk_files(path)
|
||||
)
|
||||
else:
|
||||
files.append(path)
|
||||
files.append(os.fsencode(relpath(path, self.root_dir)))
|
||||
|
||||
for fpath in files:
|
||||
# NOTE: this doesn't check gitignore, same as GitPythonBackend.add
|
||||
self.repo.stage(relpath(fpath, self.root_dir))
|
||||
# NOTE: this doesn't check gitignore, same as GitPythonBackend.add
|
||||
if update:
|
||||
index = self.repo.open_index()
|
||||
if os.name == "nt":
|
||||
# NOTE: we need git/unix separator to compare against index
|
||||
# paths but repo.stage() expects to be called with OS paths
|
||||
self.repo.stage(
|
||||
[
|
||||
fname
|
||||
for fname in files
|
||||
if fname.replace(b"\\", b"/") in index
|
||||
]
|
||||
)
|
||||
else:
|
||||
self.repo.stage([fname for fname in files if fname in index])
|
||||
else:
|
||||
self.repo.stage(files)
|
||||
|
||||
def commit(self, msg: str, no_verify: bool = False):
|
||||
from dulwich.errors import CommitError
|
||||
@@ -171,7 +200,12 @@ class DulwichBackend(BaseGitBackend): # pylint:disable=abstract-method
|
||||
raise NotImplementedError
|
||||
|
||||
def branch(self, branch: str):
|
||||
raise NotImplementedError
|
||||
from dulwich.porcelain import Error, branch_create
|
||||
|
||||
try:
|
||||
branch_create(self.root_dir, branch)
|
||||
except Error as exc:
|
||||
raise SCMError(f"Failed to create branch '{branch}'") from exc
|
||||
|
||||
def tag(self, tag: str):
|
||||
raise NotImplementedError
|
||||
@@ -215,10 +249,7 @@ class DulwichBackend(BaseGitBackend): # pylint:disable=abstract-method
|
||||
def resolve_rev(self, rev: str) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
def resolve_commit(self, rev: str) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
def branch_revs(self, branch: str, end_rev: Optional[str] = None):
|
||||
def resolve_commit(self, rev: str) -> "GitCommit":
|
||||
raise NotImplementedError
|
||||
|
||||
def _get_stash(self, ref: str):
|
||||
@@ -561,9 +592,9 @@ class DulwichBackend(BaseGitBackend): # pylint:disable=abstract-method
|
||||
)
|
||||
return (
|
||||
{
|
||||
status: [os.fsdecode(name)]
|
||||
status: [os.fsdecode(name) for name in paths]
|
||||
for status, paths in staged.items()
|
||||
for name in paths
|
||||
if paths
|
||||
},
|
||||
[os.fsdecode(name) for name in unstaged],
|
||||
[os.fsdecode(name) for name in untracked],
|
||||
|
||||
@@ -3,15 +3,15 @@ import locale
|
||||
import logging
|
||||
import os
|
||||
from functools import partial
|
||||
from typing import Callable, Iterable, List, Mapping, Optional, Tuple
|
||||
from typing import Callable, Iterable, List, Mapping, Optional, Tuple, Union
|
||||
|
||||
from funcy import first, ignore
|
||||
from funcy import ignore
|
||||
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.scm.base import CloneError, MergeConflictError, RevError, SCMError
|
||||
from dvc.utils import fix_env, is_binary, relpath
|
||||
|
||||
from ..objects import GitObject
|
||||
from ..objects import GitCommit, GitObject
|
||||
from .base import BaseGitBackend
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -188,11 +188,13 @@ class GitPythonBackend(BaseGitBackend): # pylint:disable=abstract-method
|
||||
def dir(self) -> str:
|
||||
return self.repo.git_dir
|
||||
|
||||
def add(self, paths: Iterable[str], update=False):
|
||||
def add(self, paths: Union[str, Iterable[str]], update=False):
|
||||
# NOTE: GitPython is not currently able to handle index version >= 3.
|
||||
# See https://github.com/iterative/dvc/issues/610 for more details.
|
||||
try:
|
||||
if update:
|
||||
if isinstance(paths, str):
|
||||
paths = [paths]
|
||||
self.git.add(*paths, update=True)
|
||||
else:
|
||||
self.repo.index.add(paths)
|
||||
@@ -313,7 +315,7 @@ class GitPythonBackend(BaseGitBackend): # pylint:disable=abstract-method
|
||||
|
||||
raise RevError(f"unknown Git revision '{rev}'")
|
||||
|
||||
def resolve_commit(self, rev):
|
||||
def resolve_commit(self, rev: str) -> "GitCommit":
|
||||
"""Return Commit object for the specified revision."""
|
||||
from git.exc import BadName, GitCommandError
|
||||
from git.objects.tag import TagObject
|
||||
@@ -321,25 +323,16 @@ class GitPythonBackend(BaseGitBackend): # pylint:disable=abstract-method
|
||||
try:
|
||||
commit = self.repo.rev_parse(rev)
|
||||
except (BadName, GitCommandError):
|
||||
commit = None
|
||||
raise SCMError(f"Invalid commit '{rev}'")
|
||||
if isinstance(commit, TagObject):
|
||||
commit = commit.object
|
||||
return commit
|
||||
|
||||
def branch_revs(
|
||||
self, branch: str, end_rev: Optional[str] = None
|
||||
) -> Iterable[str]:
|
||||
"""Iterate over revisions in a given branch (from newest to oldest).
|
||||
|
||||
If end_rev is set, iterator will stop when the specified revision is
|
||||
reached.
|
||||
"""
|
||||
commit = self.resolve_commit(branch)
|
||||
while commit is not None:
|
||||
yield commit.hexsha
|
||||
commit = first(commit.parents)
|
||||
if commit and commit.hexsha == end_rev:
|
||||
return
|
||||
return GitCommit(
|
||||
commit.hexsha,
|
||||
commit.committed_date,
|
||||
commit.committer_tz_offset,
|
||||
commit.message,
|
||||
[str(parent) for parent in commit.parents],
|
||||
)
|
||||
|
||||
def set_ref(
|
||||
self,
|
||||
|
||||
@@ -2,12 +2,12 @@ import locale
|
||||
import logging
|
||||
import os
|
||||
from io import BytesIO, StringIO
|
||||
from typing import Callable, Iterable, List, Mapping, Optional, Tuple
|
||||
from typing import Callable, Iterable, List, Mapping, Optional, Tuple, Union
|
||||
|
||||
from dvc.scm.base import MergeConflictError, RevError, SCMError
|
||||
from dvc.utils import relpath
|
||||
|
||||
from ..objects import GitObject
|
||||
from ..objects import GitCommit, GitObject
|
||||
from .base import BaseGitBackend
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -76,15 +76,11 @@ class Pygit2Backend(BaseGitBackend): # pylint:disable=abstract-method
|
||||
):
|
||||
raise NotImplementedError
|
||||
|
||||
@staticmethod
|
||||
def is_sha(rev: str) -> bool:
|
||||
raise NotImplementedError
|
||||
|
||||
@property
|
||||
def dir(self) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
def add(self, paths: Iterable[str], update=False):
|
||||
def add(self, paths: Union[str, Iterable[str]], update=False):
|
||||
raise NotImplementedError
|
||||
|
||||
def commit(self, msg: str, no_verify: bool = False):
|
||||
@@ -120,7 +116,13 @@ class Pygit2Backend(BaseGitBackend): # pylint:disable=abstract-method
|
||||
raise NotImplementedError
|
||||
|
||||
def branch(self, branch: str):
|
||||
raise NotImplementedError
|
||||
from pygit2 import GitError
|
||||
|
||||
try:
|
||||
commit = self.repo[self.repo.head.target]
|
||||
self.repo.create_branch(branch, commit)
|
||||
except GitError as exc:
|
||||
raise SCMError(f"Failed to create branch '{branch}'") from exc
|
||||
|
||||
def tag(self, tag: str):
|
||||
raise NotImplementedError
|
||||
@@ -173,11 +175,20 @@ class Pygit2Backend(BaseGitBackend): # pylint:disable=abstract-method
|
||||
return shas.pop() # type: ignore
|
||||
raise RevError(f"unknown Git revision '{rev}'")
|
||||
|
||||
def resolve_commit(self, rev: str) -> str:
|
||||
raise NotImplementedError
|
||||
def resolve_commit(self, rev: str) -> "GitCommit":
|
||||
from pygit2 import GitError
|
||||
|
||||
def branch_revs(self, branch: str, end_rev: Optional[str] = None):
|
||||
raise NotImplementedError
|
||||
try:
|
||||
commit, _ref = self.repo.resolve_refish(rev)
|
||||
except (KeyError, GitError):
|
||||
raise SCMError(f"Invalid commit '{rev}'")
|
||||
return GitCommit(
|
||||
str(commit.id),
|
||||
commit.commit_time,
|
||||
commit.commit_time_offset,
|
||||
commit.message,
|
||||
[str(parent) for parent in commit.parent_ids],
|
||||
)
|
||||
|
||||
def _get_stash(self, ref: str):
|
||||
raise NotImplementedError
|
||||
@@ -225,12 +236,43 @@ class Pygit2Backend(BaseGitBackend): # pylint:disable=abstract-method
|
||||
ref.delete()
|
||||
|
||||
def iter_refs(self, base: Optional[str] = None):
|
||||
for ref in self.repo.references:
|
||||
if ref.startswith(base):
|
||||
yield ref
|
||||
if base:
|
||||
for ref in self.repo.references:
|
||||
if ref.startswith(base):
|
||||
yield ref
|
||||
else:
|
||||
yield from self.repo.references
|
||||
|
||||
def get_refs_containing(self, rev: str, pattern: Optional[str] = None):
|
||||
raise NotImplementedError
|
||||
import fnmatch
|
||||
|
||||
from pygit2 import GitError
|
||||
|
||||
def _contains(repo, ref, search_commit):
|
||||
commit, _ref = self.repo.resolve_refish(ref)
|
||||
base = repo.merge_base(search_commit.id, commit.id)
|
||||
return base == search_commit.id
|
||||
|
||||
try:
|
||||
search_commit, _ref = self.repo.resolve_refish(rev)
|
||||
except (KeyError, GitError):
|
||||
raise SCMError(f"Invalid rev '{rev}'")
|
||||
|
||||
if not pattern:
|
||||
yield from (
|
||||
ref
|
||||
for ref in self.iter_refs()
|
||||
if _contains(self.repo, ref, search_commit)
|
||||
)
|
||||
return
|
||||
|
||||
literal = pattern.rstrip("/").split("/")
|
||||
for ref in self.iter_refs():
|
||||
if (
|
||||
ref.split("/")[: len(literal)] == literal
|
||||
or fnmatch.fnmatch(ref, pattern)
|
||||
) and _contains(self.repo, ref, search_commit):
|
||||
yield ref
|
||||
|
||||
def push_refspec(
|
||||
self,
|
||||
|
||||
+11
-1
@@ -1,7 +1,8 @@
|
||||
import os
|
||||
import stat
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Iterable, Optional
|
||||
from dataclasses import dataclass
|
||||
from typing import Iterable, List, Optional
|
||||
|
||||
from pygtrie import Trie
|
||||
|
||||
@@ -118,3 +119,12 @@ class GitTrie:
|
||||
def stat(self, key: tuple) -> os.stat_result:
|
||||
obj = self.trie[key]
|
||||
return os.stat_result((obj.mode, 0, 0, 0, 0, 0, 0, 0, 0, 0))
|
||||
|
||||
|
||||
@dataclass
|
||||
class GitCommit:
|
||||
hexsha: str
|
||||
commit_time: int
|
||||
commit_time_offset: int
|
||||
message: str
|
||||
parents: List[str]
|
||||
|
||||
@@ -1,9 +1,16 @@
|
||||
import logging
|
||||
|
||||
from dvc.exceptions import InvalidArgumentError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _update_import_on_remote(stage, remote, jobs):
|
||||
if stage.is_repo_import:
|
||||
raise InvalidArgumentError(
|
||||
"Can't update a repo import with --to-remote"
|
||||
)
|
||||
|
||||
url = stage.deps[0].path_info.url
|
||||
stage.outs[0].hash_info = stage.repo.cloud.transfer(
|
||||
url, jobs=jobs, remote=remote, command="update"
|
||||
|
||||
@@ -357,7 +357,7 @@ def resolve_output(inp, out):
|
||||
return ret
|
||||
|
||||
|
||||
def resolve_paths(repo, out):
|
||||
def resolve_paths(repo, out, always_local=False):
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from ..dvcfile import DVC_FILE_SUFFIX
|
||||
@@ -393,6 +393,9 @@ def resolve_paths(repo, out):
|
||||
wdir = dirname
|
||||
out = base
|
||||
|
||||
if always_local:
|
||||
out = base
|
||||
|
||||
path = os.path.join(wdir, base + DVC_FILE_SUFFIX)
|
||||
|
||||
return (path, wdir, out)
|
||||
|
||||
@@ -25,3 +25,15 @@ def get_summary(stats):
|
||||
"{} file{} {}".format(num, "s" if num > 1 else "", state)
|
||||
for state, num in status
|
||||
)
|
||||
|
||||
|
||||
ELLIPSIS = "…"
|
||||
|
||||
|
||||
def truncate_text(
|
||||
text: str, max_length: int, with_ellipsis: bool = True
|
||||
) -> str:
|
||||
if with_ellipsis and len(text) > max_length:
|
||||
return text[: max_length - 1] + ELLIPSIS
|
||||
|
||||
return text[:max_length]
|
||||
|
||||
+18
-3
@@ -1,6 +1,8 @@
|
||||
import hashlib
|
||||
import io
|
||||
|
||||
from funcy import cached_property
|
||||
|
||||
from dvc.hash_info import HashInfo
|
||||
from dvc.istextfile import istextblock
|
||||
from dvc.utils import dos2unix
|
||||
@@ -61,17 +63,30 @@ class IterStream(io.RawIOBase):
|
||||
return self.leftover[:n]
|
||||
|
||||
|
||||
class HashedStreamReader:
|
||||
class HashedStreamReader(io.IOBase):
|
||||
|
||||
PARAM_CHECKSUM = "md5"
|
||||
|
||||
def __init__(self, fobj):
|
||||
self.fobj = fobj
|
||||
self.md5 = hashlib.md5()
|
||||
self.is_text_file = None
|
||||
self.reader = fobj.read1 if hasattr(fobj, "read1") else fobj.read
|
||||
super().__init__()
|
||||
|
||||
def readable(self):
|
||||
return True
|
||||
|
||||
def tell(self):
|
||||
return self.fobj.tell()
|
||||
|
||||
@cached_property
|
||||
def _reader(self):
|
||||
if hasattr(self.fobj, "read1"):
|
||||
return self.fobj.read1
|
||||
return self.fobj.read
|
||||
|
||||
def read(self, n=-1):
|
||||
chunk = self.reader(n)
|
||||
chunk = self._reader(n)
|
||||
if self.is_text_file is None:
|
||||
self.is_text_file = istextblock(chunk)
|
||||
|
||||
|
||||
+1
-1
@@ -6,7 +6,7 @@
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
_BASE_VERSION = "2.0.0a0"
|
||||
_BASE_VERSION = "2.0.0a1"
|
||||
|
||||
|
||||
def _generate_version(base_version):
|
||||
|
||||
@@ -88,15 +88,16 @@ install_requires = [
|
||||
"pyparsing==2.4.7",
|
||||
"typing_extensions>=3.7.4",
|
||||
"fsspec>=0.8.5",
|
||||
"dvclive>=0.0.1a0",
|
||||
]
|
||||
|
||||
|
||||
# Extra dependencies for remote integrations
|
||||
|
||||
gs = ["google-cloud-storage==1.19.0"]
|
||||
gdrive = ["pydrive2>=1.6.3", "six >= 1.13.0"]
|
||||
gdrive = ["pydrive2>=1.7.3", "six >= 1.13.0"]
|
||||
s3 = ["boto3>=1.9.201"]
|
||||
azure = ["azure-storage-blob>=12.0", "knack"]
|
||||
azure = ["adlfs>=0.6.3", "azure.identity>=1.4.0", "knack"]
|
||||
# https://github.com/Legrandin/pycryptodome/issues/465
|
||||
oss = ["oss2==2.6.1", "pycryptodome<3.9.9"]
|
||||
ssh = ["paramiko[invoke]>=2.7.0"]
|
||||
|
||||
+26
-3
@@ -27,7 +27,10 @@ from dvc.objects.db import ODBManager
|
||||
from dvc.output.base import OutputAlreadyTrackedError, OutputIsStageFileError
|
||||
from dvc.repo import Repo as DvcRepo
|
||||
from dvc.stage import Stage
|
||||
from dvc.stage.exceptions import StagePathNotFoundError
|
||||
from dvc.stage.exceptions import (
|
||||
StageExternalOutputsError,
|
||||
StagePathNotFoundError,
|
||||
)
|
||||
from dvc.system import System
|
||||
from dvc.utils import LARGE_DIR_SIZE, file_md5, relpath
|
||||
from dvc.utils.fs import path_isin
|
||||
@@ -324,8 +327,6 @@ def test_add_filtered_files_in_dir(
|
||||
indirect=["workspace"],
|
||||
)
|
||||
def test_add_external_file(tmp_dir, dvc, workspace, hash_name, hash_value):
|
||||
from dvc.stage.exceptions import StageExternalOutputsError
|
||||
|
||||
workspace.gen("file", "file")
|
||||
|
||||
with pytest.raises(StageExternalOutputsError):
|
||||
@@ -1013,12 +1014,34 @@ def test_add_to_remote(tmp_dir, dvc, local_cloud, local_remote):
|
||||
assert local_remote.hash_to_path_info(hash_info.value).read_text() == "foo"
|
||||
|
||||
|
||||
def test_add_to_remote_absolute(tmp_dir, make_tmp_dir, dvc, local_remote):
|
||||
tmp_abs_dir = make_tmp_dir("abs")
|
||||
tmp_foo = tmp_abs_dir / "foo"
|
||||
tmp_foo.write_text("foo")
|
||||
|
||||
dvc.add(str(tmp_foo), to_remote=True)
|
||||
tmp_foo.unlink()
|
||||
|
||||
foo = tmp_dir / "foo"
|
||||
assert foo.with_suffix(".dvc").exists()
|
||||
assert not os.path.exists(tmp_foo)
|
||||
|
||||
dvc.pull("foo")
|
||||
assert not os.path.exists(tmp_foo)
|
||||
assert foo.read_text() == "foo"
|
||||
|
||||
with pytest.raises(StageExternalOutputsError):
|
||||
tmp_bar = tmp_abs_dir / "bar"
|
||||
dvc.add(str(tmp_foo), out=str(tmp_bar), to_remote=True)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"invalid_opt, kwargs",
|
||||
[
|
||||
("multiple targets", {"targets": ["foo", "bar", "baz"]}),
|
||||
("--no-commit", {"targets": ["foo"], "no_commit": True}),
|
||||
("--recursive", {"targets": ["foo"], "recursive": True},),
|
||||
("--external", {"targets": ["foo"], "external": True}),
|
||||
],
|
||||
)
|
||||
def test_add_to_remote_invalid_combinations(dvc, invalid_opt, kwargs):
|
||||
|
||||
@@ -218,7 +218,7 @@ def test_config_loads_without_error_for_non_dvc_repo(tmp_dir):
|
||||
("keyfile", "ssh://[email protected]:1234/path/to/dir"),
|
||||
("cert_path", "webdavs://example.com/files/USERNAME/"),
|
||||
("key_path", "webdavs://example.com/files/USERNAME/"),
|
||||
("gdrive_service_account_p12_file_path", "gdrive://root/test"),
|
||||
("gdrive_service_account_json_file_path", "gdrive://root/test"),
|
||||
("gdrive_user_credentials_file", "gdrive://root/test"),
|
||||
],
|
||||
)
|
||||
|
||||
@@ -340,6 +340,21 @@ def test_import_url_to_remote_directory(tmp_dir, dvc, workspace, local_remote):
|
||||
)
|
||||
|
||||
|
||||
def test_import_url_to_remote_absolute(
|
||||
tmp_dir, make_tmp_dir, dvc, local_remote
|
||||
):
|
||||
tmp_abs_dir = make_tmp_dir("abs")
|
||||
tmp_foo = tmp_abs_dir / "foo"
|
||||
tmp_foo.write_text("foo")
|
||||
|
||||
stage = dvc.imp_url(str(tmp_foo), to_remote=True)
|
||||
|
||||
foo = tmp_dir / "foo"
|
||||
assert stage.deps[0].fspath == str(tmp_foo)
|
||||
assert stage.outs[0].fspath == os.fspath(foo)
|
||||
assert foo.with_suffix(".dvc").exists()
|
||||
|
||||
|
||||
def test_import_url_to_remote_invalid_combinations(dvc):
|
||||
with pytest.raises(InvalidArgumentError, match="--no-exec"):
|
||||
dvc.imp_url("s3://bucket/foo", no_exec=True, to_remote=True)
|
||||
|
||||
+5
-10
@@ -1,3 +1,4 @@
|
||||
import os
|
||||
from copy import deepcopy
|
||||
from textwrap import dedent
|
||||
|
||||
@@ -53,9 +54,6 @@ LIVE_CHECKPOINT_SCRIPT = dedent(
|
||||
|
||||
@pytest.fixture
|
||||
def live_stage(tmp_dir, scm, dvc):
|
||||
|
||||
pytest.skip("dvclive does not exist yet")
|
||||
|
||||
def make(summary=True, html=True, live=None, live_no_cache=None):
|
||||
assert bool(live) != bool(live_no_cache)
|
||||
tmp_dir.gen("train.py", LIVE_SCRITP)
|
||||
@@ -130,8 +128,8 @@ def test_live_provides_metrics(tmp_dir, dvc, live_stage):
|
||||
|
||||
assert (tmp_dir / "logs").is_dir()
|
||||
plots = dvc.plots.show()
|
||||
assert "logs/accuracy.tsv" in plots
|
||||
assert "logs/loss.tsv" in plots
|
||||
assert os.path.join("logs", "accuracy.tsv") in plots
|
||||
assert os.path.join("logs", "loss.tsv") in plots
|
||||
|
||||
|
||||
def test_live_provides_no_metrics(tmp_dir, dvc, live_stage):
|
||||
@@ -143,8 +141,8 @@ def test_live_provides_no_metrics(tmp_dir, dvc, live_stage):
|
||||
|
||||
assert (tmp_dir / "logs").is_dir()
|
||||
plots = dvc.plots.show()
|
||||
assert "logs/accuracy.tsv" in plots
|
||||
assert "logs/loss.tsv" in plots
|
||||
assert os.path.join("logs", "accuracy.tsv") in plots
|
||||
assert os.path.join("logs", "loss.tsv") in plots
|
||||
|
||||
|
||||
@pytest.mark.parametrize("typ", ("live", "live_no_cache"))
|
||||
@@ -169,9 +167,6 @@ def test_live_html(tmp_dir, dvc, live_stage, html):
|
||||
|
||||
@pytest.fixture
|
||||
def live_checkpoint_stage(tmp_dir, scm, dvc):
|
||||
|
||||
pytest.skip("dvclive does not exist yet")
|
||||
|
||||
def make(live=None, live_no_cache=None):
|
||||
assert bool(live) != bool(live_no_cache)
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ import os
|
||||
import pytest
|
||||
|
||||
from dvc.dvcfile import Dvcfile
|
||||
from dvc.exceptions import InvalidArgumentError
|
||||
from tests.unit.fs.test_repo import make_subrepo
|
||||
|
||||
|
||||
@@ -317,6 +318,14 @@ def test_update_from_subrepos(tmp_dir, dvc, erepo_dir, is_dvc):
|
||||
}
|
||||
|
||||
|
||||
def test_update_import_to_remote(tmp_dir, dvc, erepo_dir, local_remote):
|
||||
erepo_dir.scm_gen({"foo": "foo"}, commit="add foo")
|
||||
stage = dvc.imp(os.fspath(erepo_dir), "foo")
|
||||
erepo_dir.scm_gen({"foo": "bar"}, commit="update foo")
|
||||
with pytest.raises(InvalidArgumentError):
|
||||
dvc.update(stage.path, to_remote=True)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"workspace",
|
||||
[
|
||||
|
||||
+55
-23
@@ -1,10 +1,11 @@
|
||||
# pylint:disable=abstract-method
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
from functools import partial, partialmethod
|
||||
from functools import partialmethod
|
||||
|
||||
import pytest
|
||||
from funcy import cached_property
|
||||
from funcy import cached_property, retry
|
||||
|
||||
from dvc.fs.gdrive import GDriveFileSystem
|
||||
from dvc.path_info import CloudURLInfo
|
||||
@@ -14,10 +15,36 @@ from .base import Base
|
||||
|
||||
TEST_GDRIVE_REPO_BUCKET = "root"
|
||||
|
||||
CREDENTIALS_DIR = os.path.join(os.path.expanduser("~"), ".config", "pydata")
|
||||
CREDENTIALS_FILE = os.path.join(
|
||||
CREDENTIALS_DIR, "pydata_google_credentials.json"
|
||||
)
|
||||
|
||||
def _gdrive_retry(func):
|
||||
def should_retry(exc):
|
||||
from googleapiclient.errors import HttpError
|
||||
|
||||
if not isinstance(exc, HttpError):
|
||||
return False
|
||||
|
||||
if 500 <= exc.resp.status < 600:
|
||||
return True
|
||||
|
||||
if exc.resp.status == 403:
|
||||
try:
|
||||
reason = json.loads(exc.content)["error"]["errors"][0][
|
||||
"reason"
|
||||
]
|
||||
except (ValueError, LookupError):
|
||||
return False
|
||||
|
||||
return reason in [
|
||||
"userRateLimitExceeded",
|
||||
"rateLimitExceeded",
|
||||
]
|
||||
|
||||
# 16 tries, start at 0.5s, multiply by golden ratio, cap at 20s
|
||||
return retry(
|
||||
16,
|
||||
timeout=lambda a: min(0.5 * 1.618 ** a, 20),
|
||||
filter_errors=should_retry,
|
||||
)(func)
|
||||
|
||||
|
||||
class GDrive(Base, CloudURLInfo):
|
||||
@@ -27,11 +54,23 @@ class GDrive(Base, CloudURLInfo):
|
||||
|
||||
@cached_property
|
||||
def config(self):
|
||||
tmp_path = tmp_fname()
|
||||
with open(tmp_path, "w") as stream:
|
||||
raw_credentials = os.getenv(
|
||||
GDriveFileSystem.GDRIVE_CREDENTIALS_DATA
|
||||
)
|
||||
try:
|
||||
credentials = json.loads(raw_credentials)
|
||||
except ValueError:
|
||||
credentials = {}
|
||||
|
||||
use_service_account = credentials.get("type") == "service_account"
|
||||
stream.write(raw_credentials)
|
||||
|
||||
return {
|
||||
"url": self.url,
|
||||
"gdrive_service_account_email": "test",
|
||||
"gdrive_service_account_p12_file_path": "test.p12",
|
||||
"gdrive_use_service_account": True,
|
||||
"gdrive_service_account_json_file_path": tmp_path,
|
||||
"gdrive_use_service_account": use_service_account,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
@@ -45,45 +84,38 @@ class GDrive(Base, CloudURLInfo):
|
||||
|
||||
@cached_property
|
||||
def client(self):
|
||||
import pydata_google_auth
|
||||
|
||||
try:
|
||||
from gdrivefs import GoogleDriveFileSystem
|
||||
except ImportError:
|
||||
pytest.skip("gdrivefs is not installed")
|
||||
|
||||
tmp_path = tmp_fname()
|
||||
with open(tmp_path, "w") as stream:
|
||||
stream.write(os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA))
|
||||
|
||||
GoogleDriveFileSystem._connect_cache = partial(
|
||||
pydata_google_auth.load_user_credentials, tmp_path
|
||||
return GoogleDriveFileSystem(
|
||||
token="cache",
|
||||
tokens_file=self.config["gdrive_service_account_json_file_path"],
|
||||
service_account=self.config["gdrive_use_service_account"],
|
||||
)
|
||||
return GoogleDriveFileSystem(token="cache")
|
||||
|
||||
@_gdrive_retry
|
||||
def mkdir(self, mode=0o777, parents=False, exist_ok=False):
|
||||
if not self.client.exists(self.path):
|
||||
self.client.mkdir(self.path)
|
||||
|
||||
@_gdrive_retry
|
||||
def write_bytes(self, contents):
|
||||
with self.client.open(self.path, mode="wb") as stream:
|
||||
stream.write(contents)
|
||||
|
||||
@_gdrive_retry
|
||||
def _read(self, mode):
|
||||
with self.client.open(self.path, mode=mode) as stream:
|
||||
return stream.read()
|
||||
|
||||
def cleanup(self):
|
||||
self.client.delete(self.path, recursive=True)
|
||||
|
||||
read_text = partialmethod(_read, mode="r")
|
||||
read_bytes = partialmethod(_read, mode="rb")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def gdrive(test_config, make_tmp_dir):
|
||||
pytest.skip("temporarily disabled")
|
||||
|
||||
test_config.requires("gdrive")
|
||||
if not GDrive.should_test():
|
||||
pytest.skip("no gdrive")
|
||||
|
||||
Binary file not shown.
Binary file not shown.
@@ -17,7 +17,6 @@ def test_init_env_var(monkeypatch, dvc):
|
||||
config = {"url": "azure://"}
|
||||
fs = AzureFileSystem(dvc, config)
|
||||
assert fs.path_info == "azure://" + container_name
|
||||
assert fs._conn_str == connection_string
|
||||
|
||||
|
||||
def test_init(dvc):
|
||||
@@ -26,7 +25,6 @@ def test_init(dvc):
|
||||
config = {"url": url, "connection_string": connection_string}
|
||||
fs = AzureFileSystem(dvc, config)
|
||||
assert fs.path_info == url
|
||||
assert fs._conn_str == connection_string
|
||||
|
||||
|
||||
def test_info(tmp_dir, azure):
|
||||
|
||||
@@ -513,3 +513,30 @@ def test_remind_to_track(scm, caplog):
|
||||
scm.files_to_track = ["fname with spaces.txt", "тест", "foo"]
|
||||
scm.remind_to_track()
|
||||
assert "git add 'fname with spaces.txt' 'тест' foo" in caplog.text
|
||||
|
||||
|
||||
def test_add(tmp_dir, scm, git):
|
||||
if git.test_backend == "pygit2":
|
||||
pytest.skip()
|
||||
|
||||
tmp_dir.gen({"foo": "foo", "bar": "bar", "dir": {"baz": "baz"}})
|
||||
git.add(["foo", "dir"])
|
||||
staged, unstaged, untracked = scm.status()
|
||||
assert set(staged["add"]) == {"foo", "dir/baz"}
|
||||
assert len(unstaged) == 0
|
||||
assert len(untracked) == 1
|
||||
|
||||
scm.commit("commit")
|
||||
tmp_dir.gen({"foo": "bar", "dir": {"baz": "bar"}})
|
||||
git.add([], update=True)
|
||||
staged, unstaged, _ = scm.status()
|
||||
assert set(staged["modify"]) == {"foo", "dir/baz"}
|
||||
assert len(unstaged) == 0
|
||||
assert len(untracked) == 1
|
||||
|
||||
scm.reset()
|
||||
git.add(["dir"], update=True)
|
||||
staged, unstaged, _ = scm.status()
|
||||
assert set(staged["modify"]) == {"dir/baz"}
|
||||
assert len(unstaged) == 1
|
||||
assert len(untracked) == 1
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import pytest
|
||||
|
||||
from dvc.dvcfile import Lockfile, LockfileCorruptedError
|
||||
from dvc.dvcfile import FileIsGitIgnored, Lockfile, LockfileCorruptedError
|
||||
from dvc.stage import PipelineStage
|
||||
from dvc.utils.serialize import dump_yaml
|
||||
|
||||
@@ -83,3 +83,30 @@ def test_load_when_lockfile_is_corrupted(tmp_dir, dvc, corrupt_data):
|
||||
with pytest.raises(LockfileCorruptedError) as exc_info:
|
||||
lockfile.load()
|
||||
assert "Dvcfile.lock" in str(exc_info.value)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dvcignored", [True, False])
|
||||
@pytest.mark.parametrize("file_exists", [True, False])
|
||||
def test_try_loading_lockfile_that_is_gitignored(
|
||||
tmp_dir, dvc, scm, dvcignored, file_exists
|
||||
):
|
||||
# it should raise error if the file is git-ignored, even if:
|
||||
# 1. The file does not exist at all.
|
||||
# 2. Or, is dvc-ignored.
|
||||
files = [".gitignore"]
|
||||
if dvcignored:
|
||||
files.append(".dvcignore")
|
||||
|
||||
for file in files:
|
||||
with (tmp_dir / file).open(mode="a+") as fd:
|
||||
fd.write("dvc.lock")
|
||||
|
||||
if file_exists:
|
||||
(tmp_dir / "dvc.lock").write_text("")
|
||||
|
||||
scm._reset()
|
||||
|
||||
with pytest.raises(FileIsGitIgnored) as exc_info:
|
||||
Lockfile(dvc, "dvc.lock").load()
|
||||
|
||||
assert str(exc_info.value) == "'dvc.lock' is git-ignored."
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
from collections import OrderedDict
|
||||
|
||||
from dvc.utils.humanize import get_summary
|
||||
import pytest
|
||||
|
||||
from dvc.utils.humanize import get_summary, truncate_text
|
||||
|
||||
|
||||
def test_get_summary():
|
||||
@@ -38,3 +40,35 @@ def test_get_summary():
|
||||
assert get_summary([]) == ""
|
||||
assert get_summary([("x", 0), ("y", [])]) == ""
|
||||
assert get_summary([("x", 1), ("y", [])]) == "1 file x"
|
||||
|
||||
|
||||
def test_truncate_text():
|
||||
text = "lorem ipsum"
|
||||
length = 5
|
||||
|
||||
truncated = truncate_text(text, length)
|
||||
# length should not cross the max length
|
||||
assert len(truncated) == length
|
||||
assert truncated[:-1] == text[: length - 1]
|
||||
# last character should be ellipsis
|
||||
assert truncated[-1] == "…"
|
||||
|
||||
truncated = truncate_text(text, length, with_ellipsis=False)
|
||||
# length should not cross the max length
|
||||
assert len(truncated) == length
|
||||
assert truncated == text[:length]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("with_ellipsis", [True, False])
|
||||
def test_truncate_text_smaller_than_max_length(with_ellipsis):
|
||||
text = "lorem ipsum"
|
||||
|
||||
# exact match as length
|
||||
truncated = truncate_text(text, len(text), with_ellipsis=with_ellipsis)
|
||||
assert len(truncated) == len(text)
|
||||
assert truncated == text
|
||||
|
||||
# max_length > len(text)
|
||||
truncated = truncate_text(text, len(text) + 1, with_ellipsis=with_ellipsis)
|
||||
assert len(truncated) == len(text)
|
||||
assert truncated == text
|
||||
|
||||
@@ -9,7 +9,15 @@ def test_hashed_stream_reader(tmp_dir):
|
||||
foo = tmp_dir / "foo"
|
||||
with open(foo, "rb") as fobj:
|
||||
stream_reader = HashedStreamReader(fobj)
|
||||
assert stream_reader.read(3) == b"foo"
|
||||
|
||||
assert stream_reader.readable()
|
||||
assert not stream_reader.seekable()
|
||||
|
||||
assert stream_reader.read(2) == b"fo"
|
||||
assert stream_reader.tell() == 2
|
||||
|
||||
assert stream_reader.read(1) == b"o"
|
||||
assert stream_reader.tell() == 3
|
||||
|
||||
hex_digest = file_md5(foo, LocalFileSystem(None, {}))
|
||||
assert stream_reader.is_text_file
|
||||
@@ -20,13 +28,21 @@ def test_hashed_stream_reader_as_chunks(tmp_dir):
|
||||
tmp_dir.gen({"foo": b"foo \x00" * 16})
|
||||
|
||||
foo = tmp_dir / "foo"
|
||||
|
||||
actual_size = len(foo.read_bytes())
|
||||
with open(foo, "rb") as fobj:
|
||||
stream_reader = HashedStreamReader(fobj)
|
||||
|
||||
total_read = 0
|
||||
while True:
|
||||
chunk = stream_reader.read(16)
|
||||
total_read += len(chunk)
|
||||
assert stream_reader.tell() == total_read
|
||||
if not chunk:
|
||||
break
|
||||
|
||||
assert stream_reader.tell() == actual_size == total_read
|
||||
|
||||
hex_digest = file_md5(foo, LocalFileSystem(None, {}))
|
||||
assert not stream_reader.is_text_file
|
||||
assert hex_digest == stream_reader.hash_info.value
|
||||
|
||||
Reference in New Issue
Block a user