Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-11 23:10:10 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1463 from efiop/1451

Browse Source 
local: support remote:// syntax
This commit is contained in:
Ruslan Kuprieiev
2019-01-02 21:41:47 +02:00
committed by GitHub
parent 17f7f46bf4 8ca0d299b5
commit 0c30723cf3
11 changed files with 76 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/dependency/base.py
+19 -12
View File
@@ -20,26 +20,27 @@ class DependencyBase(object):
PARAM_PATH = 'path'
def __init__(self, stage, path):
def __init__(self, stage, path, info=None):
self.stage = stage
self.project = stage.project
self.path = path
self.url = path
self.info = info
def __repr__(self):
return "{class_name}: '{path}'".format(
return "{class_name}: '{url}'".format(
class_name=type(self).__name__,
path=(self.path or 'No path')
url=(self.url or 'No url')
)
def __str__(self):
return self.path
return self.url
@classmethod
def match(cls, url):
return re.match(cls.REGEX, url)
def group(self, name):
match = self.match(self.path)
match = self.match(self.url)
if not match:
return None
return match.group(name)
@@ -48,6 +49,14 @@ class DependencyBase(object):
def supported(cls, url):
return cls.match(url) is not None
@property
def scheme(self):
return self.path_info['scheme']
@property
def path(self):
return self.path_info['path']
@property
def sep(self):
return '/'
@@ -62,18 +71,16 @@ class DependencyBase(object):
def status(self):
if self.changed():
# FIXME better msgs
if self.path_info['scheme'] == 'local':
p = self.rel_path
else:
p = self.path
return {p: 'changed'}
return {str(self): 'changed'}
return {}
def save(self):
raise NotImplementedError
def dumpd(self):
return {self.PARAM_PATH: self.path}
ret = self.info.copy()
ret[self.PARAM_PATH] = self.url
return ret
def download(self, to_info):
self.remote.download([self.path_info], [to_info])
dvc/dependency/gs.py
+1 -2
View File
@@ -13,8 +13,7 @@ class DependencyGS(DependencyS3):
REGEX = RemoteGS.REGEX
def __init__(self, stage, path, info=None, remote=None):
super(DependencyGS, self).__init__(stage, path)
self.info = info
super(DependencyGS, self).__init__(stage, path, info=info)
self.remote = remote if remote else RemoteGS(stage.project, {})
bucket = remote.bucket if remote else urlparse(path).netloc
path = urlparse(path).path.lstrip('/')
dvc/dependency/hdfs.py
+1 -7
View File
@@ -13,8 +13,7 @@ class DependencyHDFS(DependencyBase):
REGEX = RemoteHDFS.REGEX
def __init__(self, stage, path, info=None, remote=None):
super(DependencyHDFS, self).__init__(stage, path)
self.info = info
super(DependencyHDFS, self).__init__(stage, path, info=info)
self.remote = remote if remote else RemoteHDFS(stage.project, {})
if remote:
path = posixpath.join(remote.url, urlparse(path).path.lstrip('/'))
@@ -31,8 +30,3 @@ class DependencyHDFS(DependencyBase):
def save(self):
self.info = self.remote.save_info(self.path_info)
def dumpd(self):
ret = self.info
ret[self.PARAM_PATH] = self.path
return ret
dvc/dependency/http.py
+1 -7
View File
@@ -11,9 +11,8 @@ class DependencyHTTP(DependencyBase):
REGEX = RemoteHTTP.REGEX
def __init__(self, stage, path, info=None, remote=None):
super(DependencyHTTP, self).__init__(stage, path)
super(DependencyHTTP, self).__init__(stage, path, info=info)
self.info = info
self.remote = remote or RemoteHTTP(stage.project, {})
if path.startswith('remote'):
@@ -27,11 +26,6 @@ class DependencyHTTP(DependencyBase):
def save(self):
self.info = self.remote.save_info(self.path_info)
def dumpd(self):
ret = self.info
ret[self.PARAM_PATH] = self.path
return ret
def changed(self):
if not self.exists:
return True
dvc/dependency/local.py
+13 -13
View File
@@ -19,33 +19,33 @@ class DependencyLOCAL(DependencyBase):
IsNotFileOrDirError = DependencyIsNotFileOrDirError
def __init__(self, stage, path, info=None, remote=None):
self.stage = stage
self.project = stage.project
self.info = info
super(DependencyLOCAL, self).__init__(stage, path, info)
if remote is not None:
self.remote = remote
else:
self.remote = RemoteLOCAL(stage.project, {})
if remote:
path = os.path.join(remote.prefix, urlparse(path).path.lstrip('/'))
p = os.path.join(remote.prefix,
urlparse(self.url).path.lstrip('/'))
else:
p = path
if not os.path.isabs(path):
path = self.remote.to_ospath(path)
path = os.path.join(stage.cwd, path)
self.path = os.path.abspath(os.path.normpath(path))
if not os.path.isabs(p):
p = self.remote.to_ospath(p)
p = os.path.join(stage.cwd, p)
p = os.path.abspath(os.path.normpath(p))
self.path_info = {'scheme': 'local',
'path': self.path}
'path': p}
def __str__(self):
return self.rel_path
@property
def is_local(self):
assert os.path.isabs(self.path)
assert os.path.isabs(self.project.root_dir)
return self.path.startswith(self.project.root_dir)
return (urlparse(self.url).scheme != 'remote'
and not os.path.isabs(self.url))
@property
def sep(self):
@@ -83,7 +83,7 @@ class DependencyLOCAL(DependencyBase):
path = self.remote.unixpath(os.path.relpath(self.path,
self.stage.cwd))
else:
path = self.path
path = self.url
info = self.info.copy()
info[self.PARAM_PATH] = path
dvc/dependency/s3.py
+1 -7
View File
@@ -13,8 +13,7 @@ class DependencyS3(DependencyBase):
REGEX = RemoteS3.REGEX
def __init__(self, stage, path, info=None, remote=None):
super(DependencyS3, self).__init__(stage, path)
self.info = info
super(DependencyS3, self).__init__(stage, path, info=info)
self.remote = remote or RemoteS3(stage.project, {})
bucket = remote.bucket if remote else urlparse(path).netloc
@@ -33,8 +32,3 @@ class DependencyS3(DependencyBase):
def save(self):
self.info = self.remote.save_info(self.path_info)
def dumpd(self):
ret = self.info
ret[self.PARAM_PATH] = self.path
return ret
dvc/dependency/ssh.py
+2 -8
View File
@@ -13,8 +13,7 @@ class DependencySSH(DependencyBase):
REGEX = RemoteSSH.REGEX
def __init__(self, stage, path, info=None, remote=None):
super(DependencySSH, self).__init__(stage, path)
self.info = info
super(DependencySSH, self).__init__(stage, path, info=info)
self.remote = remote if remote else RemoteSSH(stage.project, {})
host = remote.host if remote else self.group('host')
@@ -24,7 +23,7 @@ class DependencySSH(DependencyBase):
path = posixpath.join(remote.prefix,
urlparse(path).path.lstrip('/'))
else:
path = self.match(self.path).group('path')
path = self.match(self.url).group('path')
self.path_info = {'scheme': 'ssh',
'host': host,
@@ -40,8 +39,3 @@ class DependencySSH(DependencyBase):
def save(self):
self.info = self.remote.save_info(self.path_info)
def dumpd(self):
ret = self.info
ret[self.PARAM_PATH] = self.path
return ret
dvc/output/local.py
+1 -1
View File
@@ -127,7 +127,7 @@ class OutputLOCAL(DependencyLOCAL):
self.project.scm.ignore_remove(self.path)
self.remote.move(self.path_info, out.path_info)
self.path = out.path
self.url = out.url
self.path_info = out.path_info
self.save()
dvc/project.py
+2 -2
View File
@@ -653,7 +653,7 @@ class Project(object):
info['branch'] = branch
ret = [info]
if out.path_info['scheme'] != 'local':
if out.scheme != 'local':
return ret
md5 = info[out.remote.PARAM_CHECKSUM]
@@ -1098,7 +1098,7 @@ class Project(object):
msg = "unable to find file '{}' in the pipeline".format(path)
raise DvcException(msg)
if out.path_info['scheme'] != 'local':
if out.scheme != 'local':
msg = "output '{}' scheme '{}' is not supported for metrics"
raise DvcException(msg.format(out.path, out.path_info['scheme']))
dvc/stage.py
+3 -3
View File
@@ -224,7 +224,7 @@ class Stage(object):
def unprotect_outs(self):
for out in self.outs:
if out.path_info['scheme'] != 'local' or not out.exists:
if out.scheme != 'local' or not out.exists:
continue
self.project.unprotect(out.path)
@@ -292,7 +292,7 @@ class Stage(object):
return (cls.STAGE_FILE, cwd if cwd else os.getcwd())
out = outs[0]
if out.path_info['scheme'] == 'local':
if out.scheme == 'local':
path = os.path
else:
path = posixpath
@@ -583,7 +583,7 @@ class Stage(object):
def check_missing_outputs(self):
outs = [out for out in self.outs if not out.exists]
paths = [out.path if out.path_info['scheme'] != 'local' else
paths = [out.path if out.scheme != 'local' else
out.rel_path for out in outs]
if paths:
raise MissingDataSource(paths)
tests/test_add.py
+32
View File
@@ -1,4 +1,5 @@
import os
import yaml
import time
import shutil
import filecmp
@@ -8,6 +9,7 @@ from dvc.utils import file_md5
from dvc.stage import Stage
from dvc.exceptions import DvcException
from dvc.output.base import OutputAlreadyTrackedError
from dvc.project import Project
from tests.basic_env import TestDvc
@@ -149,6 +151,36 @@ class TestAddExternalLocalFile(TestDvc):
self.assertTrue(filecmp.cmp(fname, 'foo', shallow=False))
class TestAddLocalRemoteFile(TestDvc):
def test(self):
"""
Making sure that 'remote' syntax is handled properly for local outs.
"""
cwd = os.getcwd()
remote = 'myremote'
ret = main(['remote', 'add', remote, cwd])
self.assertEqual(ret, 0)
self.dvc = Project()
foo = 'remote://{}/{}'.format(remote, self.FOO)
ret = main(['add', foo])
self.assertEqual(ret, 0)
with open('foo.dvc', 'r') as fobj:
d = yaml.safe_load(fobj)
self.assertEqual(d['outs'][0]['path'], foo)
bar = os.path.join(cwd, self.BAR)
ret = main(['add', bar])
self.assertEqual(ret, 0)
with open('bar.dvc', 'r') as fobj:
d = yaml.safe_load(fobj)
self.assertEqual(d['outs'][0]['path'], bar)
class TestCmdAdd(TestDvc):
def test(self):
ret = main(['add',