gdrive: config option to move remote file to trash only (#3478)

This commit is contained in:
Max Hora
2020-03-25 20:39:39 +02:00
committed by GitHub
parent 9de8d1e0fa
commit 42e3b8aa25
3 changed files with 47 additions and 7 deletions
+1
View File
@@ -179,6 +179,7 @@ SCHEMA = {
"gdrive_service_account_email": str,
"gdrive_service_account_user_email": str,
"gdrive_service_account_p12_file_path": str,
Optional("gdrive_trash_only", default=False): Bool,
**REMOTE_COMMON,
},
"http": {**HTTP_COMMON, **REMOTE_COMMON},
+45 -6
View File
@@ -39,12 +39,14 @@ class GDriveMissedCredentialKeyError(DvcException):
def gdrive_retry(func):
retry_re = re.compile(r"HttpError (403|500|502|503|504)")
def should_retry(exc):
from pydrive2.files import ApiRequestError
return isinstance(exc, ApiRequestError) and retry_re.search(str(exc))
if not isinstance(exc, ApiRequestError):
return False
retry_codes = [403, 500, 502, 503, 504]
return exc.error.get("code", 0) in retry_codes
# 15 tries, start at 0.5s, multiply by golden ratio, cap at 20s
return retry(
@@ -54,6 +56,19 @@ def gdrive_retry(func):
)(func)
def _location(exc):
from pydrive2.files import ApiRequestError
assert isinstance(exc, ApiRequestError)
# https://cloud.google.com/storage/docs/json_api/v1/status-codes#errorformat
return (
exc.error["errors"][0].get("location", "")
if exc.error.get("errors", [])
else ""
)
class GDriveURLInfo(CloudURLInfo):
def __init__(self, url):
super().__init__(url)
@@ -95,6 +110,7 @@ class RemoteGDrive(RemoteBASE):
)
self._bucket = self.path_info.bucket
self._trash_only = config.get("gdrive_trash_only")
self._use_service_account = config.get("gdrive_use_service_account")
self._service_account_email = config.get(
"gdrive_service_account_email"
@@ -119,8 +135,6 @@ class RemoteGDrive(RemoteBASE):
)
)
self._list_params = None
def _validate_config(self):
# Validate Service Account configuration
if self._use_service_account and (
@@ -326,10 +340,35 @@ class RemoteGDrive(RemoteBASE):
@gdrive_retry
def _delete_remote_file(self, remote_id):
from pydrive2.files import ApiRequestError
param = {"id": remote_id}
# it does not create a file on the remote
item = self.drive.CreateFile(param)
item.Delete()
try:
item.Trash() if self._trash_only else item.Delete()
except ApiRequestError as exc:
http_error_code = exc.error.get("code", 0)
if (
http_error_code == 403
and self.list_params["corpora"] == "drive"
and _location(exc) == "file.permissions"
):
raise DvcException(
"Insufficient permissions to {}. You should have {} "
"access level for the used shared drive. More details "
"at {}.".format(
"move the file into Trash"
if self._trash_only
else "permanently delete the file",
"Manager or Content Manager"
if self._trash_only
else "Manager",
"https://support.google.com/a/answer/7337554",
)
) from exc
raise
@gdrive_retry
def _get_remote_item(self, name, parents_ids):
+1 -1
View File
@@ -88,7 +88,7 @@ install_requires = [
# Extra dependencies for remote integrations
gs = ["google-cloud-storage==1.19.0"]
gdrive = ["pydrive2>=1.4.6"]
gdrive = ["pydrive2>=1.4.8"]
s3 = ["boto3>=1.9.201"]
azure = ["azure-storage-blob==2.1.0"]
oss = ["oss2==2.6.1"]