Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-02 07:18:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

remote: use Remote() func to determine remote type

Browse Source
This commit is contained in:
Ruslan Kuprieiev
2018-06-04 22:28:14 +03:00
parent 42265339a8
commit 1ef21039c4
9 changed files with 60 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/cache.py
+17 -23
View File
@@ -1,10 +1,11 @@
from dvc.remote.local import RemoteLOCAL
from dvc.remote.s3 import RemoteS3
from dvc.remote.gs import RemoteGS
from dvc.remote.ssh import RemoteSSH
import os
from dvc.remote import Remote
class Cache(object):
CACHE_DIR = 'cache'
def __init__(self, project):
#FIXME
from dvc.config import Config
@@ -16,29 +17,22 @@ class Cache(object):
sect = project.config._config[Config.SECTION_REMOTE_FMT.format(local)]
else:
sect = {}
d = config.get(Config.SECTION_CACHE_DIR, None)
if d:
sect[Config.SECTION_REMOTE_URL] = d
cache_dir = os.path.abspath(os.path.realpath(os.path.join(project.dvc_dir, self.CACHE_DIR)))
sect[Config.SECTION_REMOTE_URL] = config.get(Config.SECTION_CACHE_DIR, cache_dir)
t = config.get(Config.SECTION_CACHE_TYPE, None)
if t:
sect[Config.SECTION_CACHE_TYPE] = t
self.local = RemoteLOCAL(project, sect)
self.local = Remote(project, sect)
self.s3 = None
s3 = config.get(Config.SECTION_CACHE_S3, None)
if s3:
sect = project.config._config[Config.SECTION_REMOTE_FMT.format(s3)]
self.s3 = RemoteS3(project, sect)
self.s3 = self._get_remote(project, config, Config.SECTION_CACHE_S3)
self.gs = self._get_remote(project, config, Config.SECTION_CACHE_GS)
self.ssh = self._get_remote(project, config, Config.SECTION_CACHE_SSH)
self.gs = None
gs = config.get(Config.SECTION_CACHE_GS, None)
if gs:
sect = project.config._config[Config.SECTION_REMOTE_FMT.format(gs)]
self.gs = RemoteGS(project, sect)
def _get_remote(self, project, config, name):
remote = config.get(name, None)
if not remote:
return None
# self.ssh = None
# ssh = config.get(Config.SECTION_CACHE_SSH, None)
# if ssh:
# sect = project.config._config[Config.SECTION_REMOTE_FMT.format(ssh)]
# self.ssh = RemoteSSH(project, sect)
sect = project.config._config[Config.SECTION_REMOTE_FMT.format(remote)]
return Remote(project, sect)
dvc/config.py
+1
View File
@@ -35,6 +35,7 @@ class Config(object):
SECTION_CACHE_LOCAL = 'local'
SECTION_CACHE_S3 = 's3'
SECTION_CACHE_GS = 'gs'
SECTION_CACHE_SSH = 'ssh'
SECTION_CACHE_SCHEMA = {
schema.Optional(SECTION_CACHE_LOCAL): str,
schema.Optional(SECTION_CACHE_S3): str,
dvc/output/__init__.py
+2 -2
View File
@@ -2,7 +2,7 @@ import schema
from dvc.exceptions import DvcException
import dvc.dependency as dependency
from dvc.dependency import SCHEMA as DEP_SCHEMA
from dvc.dependency.base import DependencyBase
from dvc.output.s3 import OutputS3
from dvc.output.gs import OutputGS
@@ -11,7 +11,7 @@ from dvc.output.local import OutputLOCAL
OUTS = [OutputS3, OutputGS, OutputLOCAL]
SCHEMA = dependency.SCHEMA
SCHEMA = DEP_SCHEMA
SCHEMA[schema.Optional(OutputLOCAL.PARAM_CACHE)] = bool
dvc/remote/__init__.py
+13
View File
@@ -0,0 +1,13 @@
from dvc.remote.local import RemoteLOCAL
from dvc.remote.s3 import RemoteS3
from dvc.remote.gs import RemoteGS
from dvc.config import Config
from dvc.exceptions import DvcException
def Remote(project, config):
for r in [RemoteLOCAL, RemoteS3, RemoteGS]:
if r.supported(config):
return r(project, config)
raise DvcException('Remote \'{}\' is not supported.'.format(config))
dvc/remote/base.py
+16
View File
@@ -1,7 +1,23 @@
import re
class RemoteBase(object):
REGEX = None
def __init__(self, project, config):
pass
@classmethod
def supported(cls, config):
#FIXME
from dvc.config import Config
url = config[Config.SECTION_REMOTE_URL]
return cls.match(url) != None
@classmethod
def match(cls, url):
return re.match(cls.REGEX, url)
def save_info(self, path_info):
pass
dvc/remote/gs.py
+1
View File
@@ -10,6 +10,7 @@ from dvc.remote.base import RemoteBase
class RemoteGS(RemoteBase):
REGEX = r'^gs://(?P<path>.*)$'
PARAM_ETAG = 'etag'
def __init__(self, project, config):
dvc/remote/local.py
+7 -14
View File
@@ -8,13 +8,12 @@ from dvc.remote.base import RemoteBase
from dvc.state import State
from dvc.logger import Logger
from dvc.utils import remove, move
from dvc.output.base import OutputAlreadyTrackedError
class RemoteLOCAL(RemoteBase):
REGEX = r'^(?P<path>(/+|.:\\+).*)$'
PARAM_MD5 = State.PARAM_MD5
CACHE_DIR = 'cache'
CACHE_TYPES = ['reflink', 'hardlink', 'symlink', 'copy']
CACHE_TYPE_MAP = {
'copy': shutil.copyfile,
@@ -28,23 +27,14 @@ class RemoteLOCAL(RemoteBase):
from dvc.config import Config
self.project = project
link_state = project.link_state
cache_dir = config.get(Config.SECTION_REMOTE_URL, self.CACHE_DIR)
cache_type = config.get(Config.SECTION_CACHE_TYPE, None)
self.cache_type = cache_type
cache_dir = cache_dir if cache_dir else self.CACHE_DIR
if os.path.isabs(cache_dir):
self.cache_dir = cache_dir
else:
self.cache_dir = os.path.abspath(os.path.realpath(os.path.join(project.dvc_dir, cache_dir)))
self.link_state = project.link_state
self.cache_dir = config[Config.SECTION_REMOTE_URL]
self.cache_type = config.get(Config.SECTION_CACHE_TYPE, None)
if not os.path.exists(self.cache_dir):
os.mkdir(self.cache_dir)
self.state = State(self.cache_dir)
self.link_state = link_state
def all(self):
clist = []
@@ -222,6 +212,9 @@ class RemoteLOCAL(RemoteBase):
return {self.PARAM_MD5: md5}
def save(self, path_info):
#FIXME
from dvc.output.base import OutputAlreadyTrackedError
if path_info['scheme'] != 'local':
raise NotImplementedError
dvc/remote/s3.py
+1
View File
@@ -10,6 +10,7 @@ from dvc.remote.base import RemoteBase
class RemoteS3(RemoteBase):
REGEX = r'^s3://(?P<path>.*)$'
PARAM_ETAG = 'etag'
def __init__(self, project, config):
dvc/remote/ssh.py
+2
View File
@@ -2,4 +2,6 @@ from dvc.remote.base import RemoteBase
class RemoteSSH(RemoteBase):
#NOTE: ~/ paths are temporarily forbidden
REGEX = r'^(?P<user>.*)@(?P<host>.*):(?P<path>/+.*)$'
pass