remote: support HDFS for push/pull/status from local repo

Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
Ruslan Kuprieiev
2018-06-30 22:54:49 +03:00
parent 5a3543b36f
commit 12edcbaa04
4 changed files with 66 additions and 7 deletions
+2
View File
@@ -8,6 +8,7 @@ from dvc.config import Config, ConfigError
from dvc.remote.s3 import RemoteS3
from dvc.remote.gs import RemoteGS
from dvc.remote.ssh import RemoteSSH
from dvc.remote.hdfs import RemoteHDFS
from dvc.remote.local import RemoteLOCAL
from dvc.remote.base import RemoteBase
@@ -19,6 +20,7 @@ class DataCloud(object):
'aws' : RemoteS3,
'gcp' : RemoteGS,
'ssh' : RemoteSSH,
'hdfs' : RemoteHDFS,
'local' : RemoteLOCAL,
}
+23
View File
@@ -110,8 +110,31 @@ class RemoteHDFS(RemoteBase):
def md5s_to_path_infos(self, md5s):
return [{'scheme': 'hdfs',
'user': self.user,
'url': posixpath.join(self.url, md5[0:2], md5[2:])} for md5 in md5s]
def exists(self, path_infos):
try:
stdout = self.hadoop_fs('ls -R {}'.format(self.url))
except DvcException:
return len(path_infos) * [False]
lines = stdout.split('\n')
lurl = []
for line in lines:
if not line.startswith('-'):
continue
lurl.append(line.split()[-1])
ret = []
for path_info in path_infos:
exists = False
if path_info['url'] in lurl:
exists = True
ret.append(exists)
return ret
def upload(self, paths, path_infos, names=None):
assert isinstance(paths, list)
assert isinstance(path_infos, list)
+39 -1
View File
@@ -8,7 +8,7 @@ import tempfile
from dvc.main import main
from dvc.config import Config, ConfigError
from dvc.data_cloud import DataCloud, RemoteS3, RemoteGS, RemoteLOCAL, RemoteSSH
from dvc.data_cloud import DataCloud, RemoteS3, RemoteGS, RemoteLOCAL, RemoteSSH, RemoteHDFS
from dvc.remote.base import STATUS_OK, STATUS_NEW, STATUS_DELETED
from tests.basic_env import TestDvc
@@ -63,6 +63,14 @@ def _should_test_ssh():
return True
def _should_test_hdfs():
if os.name == 'nt':
return False
if os.getenv('HADOOP_CONTAINER_IP'):
return True
return False
def get_local_storagepath():
return tempfile.mkdtemp()
@@ -75,6 +83,10 @@ def get_ssh_url():
return 'ssh://{}@127.0.0.1:{}'.format(getpass.getuser(), get_local_storagepath())
def get_hdfs_url():
return 'hdfs://root@{}{}'.format(os.getenv('HADOOP_CONTAINER_IP'), get_local_storagepath())
def get_aws_storagepath():
return TEST_AWS_REPO_BUCKET + '/' + str(uuid.uuid4())
@@ -261,6 +273,20 @@ class TestRemoteSSH(TestDataCloudBase):
super(TestRemoteSSH, self).test()
class TestRemoteHDFS(TestDataCloudBase):
def _should_test(self):
return _should_test_hdfs()
def _get_url(self):
return get_hdfs_url()
def _get_cloud_class(self):
return RemoteHDFS
def test(self):
super(TestRemoteHDFS, self).test()
class TestDataCloudCLIBase(TestDvc):
def main(self, args):
ret = main(args)
@@ -348,6 +374,18 @@ class TestRemoteSSHCLI(TestDataCloudCLIBase):
self._test_cloud(TEST_REMOTE)
class TestRemoteHDFSCLI(TestDataCloudCLIBase):
def _should_test(self):
return _should_test_hdfs()
def _test(self):
url = get_hdfs_url()
self.main(['remote', 'add', TEST_REMOTE, url])
self._test_cloud(TEST_REMOTE)
class TestCompatRemoteS3CLI(TestDataCloudCLIBase):
def _should_test(self):
return _should_test_aws()
+2 -6
View File
@@ -22,7 +22,7 @@ from dvc.stage import Stage
from tests.basic_env import TestDvc
from tests.test_data_cloud import _should_test_aws, TEST_AWS_REPO_BUCKET
from tests.test_data_cloud import _should_test_gcp, TEST_GCP_REPO_BUCKET
from tests.test_data_cloud import _should_test_ssh
from tests.test_data_cloud import _should_test_ssh, _should_test_hdfs
class TestRepro(TestDvc):
@@ -393,11 +393,7 @@ class TestReproExternalGS(TestReproExternalBase):
class TestReproExternalHDFS(TestReproExternalBase):
def should_test(self):
if os.name == 'nt':
return False
if os.getenv('HADOOP_CONTAINER_IP'):
return True
return False
return _should_test_hdfs()
@property
def scheme(self):