mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 15:28:28 +01:00
remote: support HDFS for push/pull/status from local repo
Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
@@ -8,6 +8,7 @@ from dvc.config import Config, ConfigError
|
||||
from dvc.remote.s3 import RemoteS3
|
||||
from dvc.remote.gs import RemoteGS
|
||||
from dvc.remote.ssh import RemoteSSH
|
||||
from dvc.remote.hdfs import RemoteHDFS
|
||||
from dvc.remote.local import RemoteLOCAL
|
||||
from dvc.remote.base import RemoteBase
|
||||
|
||||
@@ -19,6 +20,7 @@ class DataCloud(object):
|
||||
'aws' : RemoteS3,
|
||||
'gcp' : RemoteGS,
|
||||
'ssh' : RemoteSSH,
|
||||
'hdfs' : RemoteHDFS,
|
||||
'local' : RemoteLOCAL,
|
||||
}
|
||||
|
||||
|
||||
@@ -110,8 +110,31 @@ class RemoteHDFS(RemoteBase):
|
||||
|
||||
def md5s_to_path_infos(self, md5s):
|
||||
return [{'scheme': 'hdfs',
|
||||
'user': self.user,
|
||||
'url': posixpath.join(self.url, md5[0:2], md5[2:])} for md5 in md5s]
|
||||
|
||||
def exists(self, path_infos):
|
||||
try:
|
||||
stdout = self.hadoop_fs('ls -R {}'.format(self.url))
|
||||
except DvcException:
|
||||
return len(path_infos) * [False]
|
||||
|
||||
lines = stdout.split('\n')
|
||||
lurl = []
|
||||
for line in lines:
|
||||
if not line.startswith('-'):
|
||||
continue
|
||||
lurl.append(line.split()[-1])
|
||||
|
||||
ret = []
|
||||
for path_info in path_infos:
|
||||
exists = False
|
||||
if path_info['url'] in lurl:
|
||||
exists = True
|
||||
ret.append(exists)
|
||||
|
||||
return ret
|
||||
|
||||
def upload(self, paths, path_infos, names=None):
|
||||
assert isinstance(paths, list)
|
||||
assert isinstance(path_infos, list)
|
||||
|
||||
@@ -8,7 +8,7 @@ import tempfile
|
||||
|
||||
from dvc.main import main
|
||||
from dvc.config import Config, ConfigError
|
||||
from dvc.data_cloud import DataCloud, RemoteS3, RemoteGS, RemoteLOCAL, RemoteSSH
|
||||
from dvc.data_cloud import DataCloud, RemoteS3, RemoteGS, RemoteLOCAL, RemoteSSH, RemoteHDFS
|
||||
from dvc.remote.base import STATUS_OK, STATUS_NEW, STATUS_DELETED
|
||||
|
||||
from tests.basic_env import TestDvc
|
||||
@@ -63,6 +63,14 @@ def _should_test_ssh():
|
||||
return True
|
||||
|
||||
|
||||
def _should_test_hdfs():
|
||||
if os.name == 'nt':
|
||||
return False
|
||||
if os.getenv('HADOOP_CONTAINER_IP'):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def get_local_storagepath():
|
||||
return tempfile.mkdtemp()
|
||||
|
||||
@@ -75,6 +83,10 @@ def get_ssh_url():
|
||||
return 'ssh://{}@127.0.0.1:{}'.format(getpass.getuser(), get_local_storagepath())
|
||||
|
||||
|
||||
def get_hdfs_url():
|
||||
return 'hdfs://root@{}{}'.format(os.getenv('HADOOP_CONTAINER_IP'), get_local_storagepath())
|
||||
|
||||
|
||||
def get_aws_storagepath():
|
||||
return TEST_AWS_REPO_BUCKET + '/' + str(uuid.uuid4())
|
||||
|
||||
@@ -261,6 +273,20 @@ class TestRemoteSSH(TestDataCloudBase):
|
||||
super(TestRemoteSSH, self).test()
|
||||
|
||||
|
||||
class TestRemoteHDFS(TestDataCloudBase):
|
||||
def _should_test(self):
|
||||
return _should_test_hdfs()
|
||||
|
||||
def _get_url(self):
|
||||
return get_hdfs_url()
|
||||
|
||||
def _get_cloud_class(self):
|
||||
return RemoteHDFS
|
||||
|
||||
def test(self):
|
||||
super(TestRemoteHDFS, self).test()
|
||||
|
||||
|
||||
class TestDataCloudCLIBase(TestDvc):
|
||||
def main(self, args):
|
||||
ret = main(args)
|
||||
@@ -348,6 +374,18 @@ class TestRemoteSSHCLI(TestDataCloudCLIBase):
|
||||
self._test_cloud(TEST_REMOTE)
|
||||
|
||||
|
||||
class TestRemoteHDFSCLI(TestDataCloudCLIBase):
|
||||
def _should_test(self):
|
||||
return _should_test_hdfs()
|
||||
|
||||
def _test(self):
|
||||
url = get_hdfs_url()
|
||||
|
||||
self.main(['remote', 'add', TEST_REMOTE, url])
|
||||
|
||||
self._test_cloud(TEST_REMOTE)
|
||||
|
||||
|
||||
class TestCompatRemoteS3CLI(TestDataCloudCLIBase):
|
||||
def _should_test(self):
|
||||
return _should_test_aws()
|
||||
|
||||
+2
-6
@@ -22,7 +22,7 @@ from dvc.stage import Stage
|
||||
from tests.basic_env import TestDvc
|
||||
from tests.test_data_cloud import _should_test_aws, TEST_AWS_REPO_BUCKET
|
||||
from tests.test_data_cloud import _should_test_gcp, TEST_GCP_REPO_BUCKET
|
||||
from tests.test_data_cloud import _should_test_ssh
|
||||
from tests.test_data_cloud import _should_test_ssh, _should_test_hdfs
|
||||
|
||||
|
||||
class TestRepro(TestDvc):
|
||||
@@ -393,11 +393,7 @@ class TestReproExternalGS(TestReproExternalBase):
|
||||
|
||||
class TestReproExternalHDFS(TestReproExternalBase):
|
||||
def should_test(self):
|
||||
if os.name == 'nt':
|
||||
return False
|
||||
if os.getenv('HADOOP_CONTAINER_IP'):
|
||||
return True
|
||||
return False
|
||||
return _should_test_hdfs()
|
||||
|
||||
@property
|
||||
def scheme(self):
|
||||
|
||||
Reference in New Issue
Block a user