diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index 4f80bf2e4..e608da938 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -8,6 +8,7 @@ from dvc.config import Config, ConfigError from dvc.remote.s3 import RemoteS3 from dvc.remote.gs import RemoteGS from dvc.remote.ssh import RemoteSSH +from dvc.remote.hdfs import RemoteHDFS from dvc.remote.local import RemoteLOCAL from dvc.remote.base import RemoteBase @@ -19,6 +20,7 @@ class DataCloud(object): 'aws' : RemoteS3, 'gcp' : RemoteGS, 'ssh' : RemoteSSH, + 'hdfs' : RemoteHDFS, 'local' : RemoteLOCAL, } diff --git a/dvc/remote/hdfs.py b/dvc/remote/hdfs.py index ea19db0f1..773a8dbe6 100644 --- a/dvc/remote/hdfs.py +++ b/dvc/remote/hdfs.py @@ -110,8 +110,31 @@ class RemoteHDFS(RemoteBase): def md5s_to_path_infos(self, md5s): return [{'scheme': 'hdfs', + 'user': self.user, 'url': posixpath.join(self.url, md5[0:2], md5[2:])} for md5 in md5s] + def exists(self, path_infos): + try: + stdout = self.hadoop_fs('ls -R {}'.format(self.url)) + except DvcException: + return len(path_infos) * [False] + + lines = stdout.split('\n') + lurl = [] + for line in lines: + if not line.startswith('-'): + continue + lurl.append(line.split()[-1]) + + ret = [] + for path_info in path_infos: + exists = False + if path_info['url'] in lurl: + exists = True + ret.append(exists) + + return ret + def upload(self, paths, path_infos, names=None): assert isinstance(paths, list) assert isinstance(path_infos, list) diff --git a/tests/test_data_cloud.py b/tests/test_data_cloud.py index 00165c759..066f6c6a6 100644 --- a/tests/test_data_cloud.py +++ b/tests/test_data_cloud.py @@ -8,7 +8,7 @@ import tempfile from dvc.main import main from dvc.config import Config, ConfigError -from dvc.data_cloud import DataCloud, RemoteS3, RemoteGS, RemoteLOCAL, RemoteSSH +from dvc.data_cloud import DataCloud, RemoteS3, RemoteGS, RemoteLOCAL, RemoteSSH, RemoteHDFS from dvc.remote.base import STATUS_OK, STATUS_NEW, STATUS_DELETED from tests.basic_env import TestDvc @@ -63,6 +63,14 @@ def _should_test_ssh(): return True +def _should_test_hdfs(): + if os.name == 'nt': + return False + if os.getenv('HADOOP_CONTAINER_IP'): + return True + return False + + def get_local_storagepath(): return tempfile.mkdtemp() @@ -75,6 +83,10 @@ def get_ssh_url(): return 'ssh://{}@127.0.0.1:{}'.format(getpass.getuser(), get_local_storagepath()) +def get_hdfs_url(): + return 'hdfs://root@{}{}'.format(os.getenv('HADOOP_CONTAINER_IP'), get_local_storagepath()) + + def get_aws_storagepath(): return TEST_AWS_REPO_BUCKET + '/' + str(uuid.uuid4()) @@ -261,6 +273,20 @@ class TestRemoteSSH(TestDataCloudBase): super(TestRemoteSSH, self).test() +class TestRemoteHDFS(TestDataCloudBase): + def _should_test(self): + return _should_test_hdfs() + + def _get_url(self): + return get_hdfs_url() + + def _get_cloud_class(self): + return RemoteHDFS + + def test(self): + super(TestRemoteHDFS, self).test() + + class TestDataCloudCLIBase(TestDvc): def main(self, args): ret = main(args) @@ -348,6 +374,18 @@ class TestRemoteSSHCLI(TestDataCloudCLIBase): self._test_cloud(TEST_REMOTE) +class TestRemoteHDFSCLI(TestDataCloudCLIBase): + def _should_test(self): + return _should_test_hdfs() + + def _test(self): + url = get_hdfs_url() + + self.main(['remote', 'add', TEST_REMOTE, url]) + + self._test_cloud(TEST_REMOTE) + + class TestCompatRemoteS3CLI(TestDataCloudCLIBase): def _should_test(self): return _should_test_aws() diff --git a/tests/test_repro.py b/tests/test_repro.py index a6688a929..b178c1268 100644 --- a/tests/test_repro.py +++ b/tests/test_repro.py @@ -22,7 +22,7 @@ from dvc.stage import Stage from tests.basic_env import TestDvc from tests.test_data_cloud import _should_test_aws, TEST_AWS_REPO_BUCKET from tests.test_data_cloud import _should_test_gcp, TEST_GCP_REPO_BUCKET -from tests.test_data_cloud import _should_test_ssh +from tests.test_data_cloud import _should_test_ssh, _should_test_hdfs class TestRepro(TestDvc): @@ -393,11 +393,7 @@ class TestReproExternalGS(TestReproExternalBase): class TestReproExternalHDFS(TestReproExternalBase): def should_test(self): - if os.name == 'nt': - return False - if os.getenv('HADOOP_CONTAINER_IP'): - return True - return False + return _should_test_hdfs() @property def scheme(self):