diff --git a/dvc/config.py b/dvc/config.py index 50cd8feb3..6e1ccd7f5 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -179,6 +179,7 @@ SCHEMA = { "gdrive_service_account_email": str, "gdrive_service_account_user_email": str, "gdrive_service_account_p12_file_path": str, + Optional("gdrive_trash_only", default=False): Bool, **REMOTE_COMMON, }, "http": {**HTTP_COMMON, **REMOTE_COMMON}, diff --git a/dvc/remote/gdrive.py b/dvc/remote/gdrive.py index 484bd1a82..a695dd09b 100644 --- a/dvc/remote/gdrive.py +++ b/dvc/remote/gdrive.py @@ -39,12 +39,14 @@ class GDriveMissedCredentialKeyError(DvcException): def gdrive_retry(func): - retry_re = re.compile(r"HttpError (403|500|502|503|504)") - def should_retry(exc): from pydrive2.files import ApiRequestError - return isinstance(exc, ApiRequestError) and retry_re.search(str(exc)) + if not isinstance(exc, ApiRequestError): + return False + + retry_codes = [403, 500, 502, 503, 504] + return exc.error.get("code", 0) in retry_codes # 15 tries, start at 0.5s, multiply by golden ratio, cap at 20s return retry( @@ -54,6 +56,19 @@ def gdrive_retry(func): )(func) +def _location(exc): + from pydrive2.files import ApiRequestError + + assert isinstance(exc, ApiRequestError) + + # https://cloud.google.com/storage/docs/json_api/v1/status-codes#errorformat + return ( + exc.error["errors"][0].get("location", "") + if exc.error.get("errors", []) + else "" + ) + + class GDriveURLInfo(CloudURLInfo): def __init__(self, url): super().__init__(url) @@ -95,6 +110,7 @@ class RemoteGDrive(RemoteBASE): ) self._bucket = self.path_info.bucket + self._trash_only = config.get("gdrive_trash_only") self._use_service_account = config.get("gdrive_use_service_account") self._service_account_email = config.get( "gdrive_service_account_email" @@ -119,8 +135,6 @@ class RemoteGDrive(RemoteBASE): ) ) - self._list_params = None - def _validate_config(self): # Validate Service Account configuration if self._use_service_account and ( @@ -326,10 +340,35 @@ class RemoteGDrive(RemoteBASE): @gdrive_retry def _delete_remote_file(self, remote_id): + from pydrive2.files import ApiRequestError + param = {"id": remote_id} # it does not create a file on the remote item = self.drive.CreateFile(param) - item.Delete() + + try: + item.Trash() if self._trash_only else item.Delete() + except ApiRequestError as exc: + http_error_code = exc.error.get("code", 0) + if ( + http_error_code == 403 + and self.list_params["corpora"] == "drive" + and _location(exc) == "file.permissions" + ): + raise DvcException( + "Insufficient permissions to {}. You should have {} " + "access level for the used shared drive. More details " + "at {}.".format( + "move the file into Trash" + if self._trash_only + else "permanently delete the file", + "Manager or Content Manager" + if self._trash_only + else "Manager", + "https://support.google.com/a/answer/7337554", + ) + ) from exc + raise @gdrive_retry def _get_remote_item(self, name, parents_ids): diff --git a/setup.py b/setup.py index d9a29bbc3..573024882 100644 --- a/setup.py +++ b/setup.py @@ -88,7 +88,7 @@ install_requires = [ # Extra dependencies for remote integrations gs = ["google-cloud-storage==1.19.0"] -gdrive = ["pydrive2>=1.4.6"] +gdrive = ["pydrive2>=1.4.8"] s3 = ["boto3>=1.9.201"] azure = ["azure-storage-blob==2.1.0"] oss = ["oss2==2.6.1"]