mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-10 07:18:27 +01:00
make get-url consistent with ls-url
Also, move parse_external_url() to dvc.fs as it's now used both by ls_url() and get_url().
This commit is contained in:
committed by
Ruslan Kuprieiev
parent
6a0d7f9c51
commit
10f60bed58
@@ -56,6 +56,14 @@ def download(fs, fs_path, to, jobs=None):
|
||||
fs.get(fs_path, to.fs_path, batch_size=jobs, callback=cb)
|
||||
|
||||
|
||||
def parse_external_url(url, config=None):
|
||||
remote_config = dict(config) if config else {}
|
||||
remote_config["url"] = url
|
||||
fs_cls, fs_config, fs_path = get_cloud_fs(None, **remote_config)
|
||||
fs = fs_cls(**fs_config)
|
||||
return fs, fs_path
|
||||
|
||||
|
||||
def get_fs_config(config, **kwargs):
|
||||
name = kwargs.get("name")
|
||||
if name:
|
||||
|
||||
+10
-12
@@ -1,19 +1,17 @@
|
||||
import os
|
||||
|
||||
from dvc import output
|
||||
from dvc.exceptions import URLMissingError
|
||||
from dvc.fs import download, parse_external_url
|
||||
from dvc.utils import resolve_output
|
||||
|
||||
|
||||
def get_url(url, out=None, jobs=None):
|
||||
from dvc import dependency, output
|
||||
from dvc.utils import resolve_output
|
||||
|
||||
out = resolve_output(url, out)
|
||||
|
||||
if os.path.exists(url):
|
||||
url = os.path.abspath(url)
|
||||
|
||||
out = os.path.abspath(out)
|
||||
|
||||
(dep,) = dependency.loads_from(None, [url])
|
||||
(out,) = output.loads_from(None, [out], use_cache=False)
|
||||
if not dep.exists:
|
||||
raise dep.DoesNotExistError(dep)
|
||||
dep.download(out, jobs=jobs)
|
||||
|
||||
src_fs, src_path = parse_external_url(url)
|
||||
if not src_fs.exists(src_path):
|
||||
raise URLMissingError(url)
|
||||
download(src_fs, src_path, out, jobs=jobs)
|
||||
|
||||
+1
-9
@@ -1,13 +1,5 @@
|
||||
from dvc.exceptions import URLMissingError
|
||||
from dvc.fs import get_cloud_fs
|
||||
|
||||
|
||||
def parse_external_url(url, config=None):
|
||||
remote_config = dict(config) if config else {}
|
||||
remote_config["url"] = url
|
||||
fs_cls, fs_config, fs_path = get_cloud_fs(None, **remote_config)
|
||||
fs = fs_cls(**fs_config)
|
||||
return fs, fs_path
|
||||
from dvc.fs import parse_external_url
|
||||
|
||||
|
||||
def ls_url(url, *, config=None, recursive=False):
|
||||
|
||||
@@ -2,7 +2,7 @@ import os
|
||||
|
||||
import pytest
|
||||
|
||||
from dvc.dependency.base import DependencyDoesNotExistError
|
||||
from dvc.exceptions import URLMissingError
|
||||
from dvc.repo import Repo
|
||||
|
||||
|
||||
@@ -36,5 +36,5 @@ def test_get_url_to_dir(tmp_dir, tmp_path_factory, dname):
|
||||
|
||||
|
||||
def test_get_url_nonexistent(tmp_dir):
|
||||
with pytest.raises(DependencyDoesNotExistError):
|
||||
with pytest.raises(URLMissingError):
|
||||
Repo.get_url("nonexistent")
|
||||
|
||||
Reference in New Issue
Block a user