make get-url consistent with ls-url

Also, move parse_external_url() to dvc.fs as it's now used both by
ls_url() and get_url().
This commit is contained in:
Ronan Lamy
2022-10-07 19:03:57 +03:00
committed by Ruslan Kuprieiev
parent 6a0d7f9c51
commit 10f60bed58
4 changed files with 21 additions and 23 deletions
+8
View File
@@ -56,6 +56,14 @@ def download(fs, fs_path, to, jobs=None):
fs.get(fs_path, to.fs_path, batch_size=jobs, callback=cb)
def parse_external_url(url, config=None):
remote_config = dict(config) if config else {}
remote_config["url"] = url
fs_cls, fs_config, fs_path = get_cloud_fs(None, **remote_config)
fs = fs_cls(**fs_config)
return fs, fs_path
def get_fs_config(config, **kwargs):
name = kwargs.get("name")
if name:
+10 -12
View File
@@ -1,19 +1,17 @@
import os
from dvc import output
from dvc.exceptions import URLMissingError
from dvc.fs import download, parse_external_url
from dvc.utils import resolve_output
def get_url(url, out=None, jobs=None):
from dvc import dependency, output
from dvc.utils import resolve_output
out = resolve_output(url, out)
if os.path.exists(url):
url = os.path.abspath(url)
out = os.path.abspath(out)
(dep,) = dependency.loads_from(None, [url])
(out,) = output.loads_from(None, [out], use_cache=False)
if not dep.exists:
raise dep.DoesNotExistError(dep)
dep.download(out, jobs=jobs)
src_fs, src_path = parse_external_url(url)
if not src_fs.exists(src_path):
raise URLMissingError(url)
download(src_fs, src_path, out, jobs=jobs)
+1 -9
View File
@@ -1,13 +1,5 @@
from dvc.exceptions import URLMissingError
from dvc.fs import get_cloud_fs
def parse_external_url(url, config=None):
remote_config = dict(config) if config else {}
remote_config["url"] = url
fs_cls, fs_config, fs_path = get_cloud_fs(None, **remote_config)
fs = fs_cls(**fs_config)
return fs, fs_path
from dvc.fs import parse_external_url
def ls_url(url, *, config=None, recursive=False):
+2 -2
View File
@@ -2,7 +2,7 @@ import os
import pytest
from dvc.dependency.base import DependencyDoesNotExistError
from dvc.exceptions import URLMissingError
from dvc.repo import Repo
@@ -36,5 +36,5 @@ def test_get_url_to_dir(tmp_dir, tmp_path_factory, dname):
def test_get_url_nonexistent(tmp_dir):
with pytest.raises(DependencyDoesNotExistError):
with pytest.raises(URLMissingError):
Repo.get_url("nonexistent")