mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 15:28:28 +01:00
gdrive: config option to move remote file to trash only (#3478)
This commit is contained in:
@@ -179,6 +179,7 @@ SCHEMA = {
|
||||
"gdrive_service_account_email": str,
|
||||
"gdrive_service_account_user_email": str,
|
||||
"gdrive_service_account_p12_file_path": str,
|
||||
Optional("gdrive_trash_only", default=False): Bool,
|
||||
**REMOTE_COMMON,
|
||||
},
|
||||
"http": {**HTTP_COMMON, **REMOTE_COMMON},
|
||||
|
||||
+45
-6
@@ -39,12 +39,14 @@ class GDriveMissedCredentialKeyError(DvcException):
|
||||
|
||||
|
||||
def gdrive_retry(func):
|
||||
retry_re = re.compile(r"HttpError (403|500|502|503|504)")
|
||||
|
||||
def should_retry(exc):
|
||||
from pydrive2.files import ApiRequestError
|
||||
|
||||
return isinstance(exc, ApiRequestError) and retry_re.search(str(exc))
|
||||
if not isinstance(exc, ApiRequestError):
|
||||
return False
|
||||
|
||||
retry_codes = [403, 500, 502, 503, 504]
|
||||
return exc.error.get("code", 0) in retry_codes
|
||||
|
||||
# 15 tries, start at 0.5s, multiply by golden ratio, cap at 20s
|
||||
return retry(
|
||||
@@ -54,6 +56,19 @@ def gdrive_retry(func):
|
||||
)(func)
|
||||
|
||||
|
||||
def _location(exc):
|
||||
from pydrive2.files import ApiRequestError
|
||||
|
||||
assert isinstance(exc, ApiRequestError)
|
||||
|
||||
# https://cloud.google.com/storage/docs/json_api/v1/status-codes#errorformat
|
||||
return (
|
||||
exc.error["errors"][0].get("location", "")
|
||||
if exc.error.get("errors", [])
|
||||
else ""
|
||||
)
|
||||
|
||||
|
||||
class GDriveURLInfo(CloudURLInfo):
|
||||
def __init__(self, url):
|
||||
super().__init__(url)
|
||||
@@ -95,6 +110,7 @@ class RemoteGDrive(RemoteBASE):
|
||||
)
|
||||
|
||||
self._bucket = self.path_info.bucket
|
||||
self._trash_only = config.get("gdrive_trash_only")
|
||||
self._use_service_account = config.get("gdrive_use_service_account")
|
||||
self._service_account_email = config.get(
|
||||
"gdrive_service_account_email"
|
||||
@@ -119,8 +135,6 @@ class RemoteGDrive(RemoteBASE):
|
||||
)
|
||||
)
|
||||
|
||||
self._list_params = None
|
||||
|
||||
def _validate_config(self):
|
||||
# Validate Service Account configuration
|
||||
if self._use_service_account and (
|
||||
@@ -326,10 +340,35 @@ class RemoteGDrive(RemoteBASE):
|
||||
|
||||
@gdrive_retry
|
||||
def _delete_remote_file(self, remote_id):
|
||||
from pydrive2.files import ApiRequestError
|
||||
|
||||
param = {"id": remote_id}
|
||||
# it does not create a file on the remote
|
||||
item = self.drive.CreateFile(param)
|
||||
item.Delete()
|
||||
|
||||
try:
|
||||
item.Trash() if self._trash_only else item.Delete()
|
||||
except ApiRequestError as exc:
|
||||
http_error_code = exc.error.get("code", 0)
|
||||
if (
|
||||
http_error_code == 403
|
||||
and self.list_params["corpora"] == "drive"
|
||||
and _location(exc) == "file.permissions"
|
||||
):
|
||||
raise DvcException(
|
||||
"Insufficient permissions to {}. You should have {} "
|
||||
"access level for the used shared drive. More details "
|
||||
"at {}.".format(
|
||||
"move the file into Trash"
|
||||
if self._trash_only
|
||||
else "permanently delete the file",
|
||||
"Manager or Content Manager"
|
||||
if self._trash_only
|
||||
else "Manager",
|
||||
"https://support.google.com/a/answer/7337554",
|
||||
)
|
||||
) from exc
|
||||
raise
|
||||
|
||||
@gdrive_retry
|
||||
def _get_remote_item(self, name, parents_ids):
|
||||
|
||||
Reference in New Issue
Block a user