Remove cmd refactoring and unit tests

2026-11-02 07:18:25 +01:00 · 2017-03-25 17:22:12 -07:00
parent 0a68f3d7f9
commit c011bafe80
8 changed files with 457 additions and 107 deletions
@@ -11,15 +11,22 @@ from dvc.utils import cached_property
 class CmdBase(object):
    CONFIG = 'dvc.conf'

-    def __init__(self, parse_config=True):
-        self._git = GitWrapper()
+    def __init__(self, parse_config=True, git_obj=None, config_obj=None):
+        if git_obj:
+            self._git = git_obj
+        else:
+            self._git = GitWrapper()
+
+        if config_obj:
+            self._config = config_obj
+        else:
+            self._config = None
+            if parse_config:
+                self._config = Config(os.path.realpath(os.path.join(self.git.git_dir, self.CONFIG)))
+
        self._args = None
        self._dvc_home = None

-        self._config = None
-        if parse_config:
-            self._config = Config(os.path.realpath(os.path.join(self.git.git_dir, self.CONFIG)))
-
        parser = argparse.ArgumentParser()
        self.define_args(parser)
        self._args, self._args_unkn = parser.parse_known_args()
@@ -66,6 +73,14 @@ class CmdBase(object):
    def skip_git_actions(self):
        return self.args.skip_git_actions

+    def commit_if_needed(self, message, error=False):
+        if error or self.skip_git_actions:
+            self.not_committed_changes_warning()
+            return 1
+        else:
+            self.git.commit_all_changes_and_log_status(message)
+            return 0
+
    @staticmethod
    def not_committed_changes_warning():
        Logger.warn('Warning: changes were not committed to git')
@@ -1,10 +1,11 @@
 import os
-from boto.s3.connection import S3Connection
+
 import fasteners
+from boto.s3.connection import S3Connection

 from dvc.cmd_base import CmdBase
-from dvc.logger import Logger
 from dvc.exceptions import NeatLynxException
+from dvc.logger import Logger


 class DataRemoveError(NeatLynxException):
@@ -13,8 +14,8 @@ class DataRemoveError(NeatLynxException):


 class CmdDataRemove(CmdBase):
-    def __init__(self):
-        CmdBase.__init__(self)
+    def __init__(self, parse_config=True, git_obj=None, config_obj=None):
+        super(CmdDataRemove, self).__init__(parse_config, git_obj, config_obj)
        pass

    def define_args(self, parser):
@@ -22,66 +23,82 @@ class CmdDataRemove(CmdBase):

        parser.add_argument('target', metavar='', help='Target to remove - file or directory', nargs='*')
        parser.add_argument('-r', '--recursive', action='store_true', help='Remove directory recursively')
-        parser.add_argument('-c', '--remove-from-cloud', action='store_true', help='Keep file in cloud')
+        parser.add_argument('-l', '--keep-in-cloud', action='store_false', default=False,
+                            help='Do not remove data from cloud')
+        parser.add_argument('-c', '--keep-in-cache', action='store_false', default=False,
+                            help='Do not remove data from cache')
        pass

    def run(self):
        lock = fasteners.InterProcessLock(self.git.lock_file)
        gotten = lock.acquire(timeout=5)
        if not gotten:
-            Logger.printing('Cannot perform the command since DVC is busy and locked. Please retry the command later.')
+            Logger.info('Cannot perform the command since DVC is busy and locked. Please retry the command later.')
            return 1

        try:
-            if not self.skip_git_actions and not self.git.is_ready_to_go():
-                return 1
-
-            for target in self.args.target:
-                self.remove_target(target)
-
-            if self.skip_git_actions:
-                self.not_committed_changes_warning()
-                return 0
-
-            message = 'DVC data remove: {}'.format(' '.join(self.args.target))
-            self.git.commit_all_changes_and_log_status(message)
+            return self.remove_all_targets()
        finally:
            lock.release()

-    def remove_target(self, target):
-        if os.path.isdir(target):
-            if not self.args.recursive:
-                raise DataRemoveError('Directory cannot be removed. Use --recurcive flag.')
+    def remove_all_targets(self):
+        if not self.skip_git_actions and not self.git.is_ready_to_go():
+            return 1

-            if os.path.realpath(target) == \
-                    os.path.realpath(os.path.join(self.git.git_dir_abs, self.config.data_dir)):
-                raise DataRemoveError('data directory cannot be removed')
+        error = False
+        for target in self.args.target:
+            try:
+                if os.path.isdir(target):
+                    self.remove_dir(target)
+                else:
+                    self.remove_data_instance(target)
+            except NeatLynxException as ex:
+                Logger.error('Unable to remove data file "{}": {}'.format(target, ex))
+                error = True

-            return self.remove_dir(target)
+        message = 'DVC data remove: {}'.format(' '.join(self.args.target))
+        self.commit_if_needed(message, error)

-        data_path = self.path_factory.existing_data_path(target)
-        if os.path.islink(data_path.data.relative):
-            return self.remove_symlink(data_path)
+        return 0 if error == 0 else 1

-        raise DataRemoveError('Cannot remove a regular file "{}"'.format(target))
+    def remove_dir(self, target):
+        if not self.args.recursive:
+            raise DataRemoveError('Directory "%s" cannot be removed. Use --recurcive flag.' % target)
+
+        data_path = self.path_factory.data_path(target)
+        if data_path.data_dvc_short == '':
+            raise DataRemoveError('Data directory "%s" cannot be removed' % target)
+
+        return self.remove_dir_file_by_file(target)

    @staticmethod
    def remove_dir_if_empty(file):
        dir = os.path.dirname(file)
        if dir != '' and not os.listdir(dir):
            os.rmdir(dir)
+        pass

-    def remove_symlink(self, data_path):
-        if os.path.isfile(data_path.cache.relative):
+    def remove_data_instance(self, target):
+        # it raises exception if not a symlink is provided
+        data_path = self.path_factory.existing_data_path(target)
+
+        if not self.args.keep_in_cache and os.path.isfile(data_path.cache.relative):
            os.remove(data_path.cache.relative)
-            self.remove_dir_if_empty(data_path.cache.relavive)
+            self.remove_dir_if_empty(data_path.cache.relative)
+        else:
+            if not self.args.keep_in_cache:
+                Logger.warn(u'Unable to find cache file for data instance %s' % data_path.data.relative)

        if os.path.isfile(data_path.state.relative):
            os.remove(data_path.state.relative)
            self.remove_dir_if_empty(data_path.state.relative)
+        else:
+            Logger.warn(u'State file {} for data instance {} does not exist'.format(
+                data_path.state.relative, data_path.data.relative))

-        if self.args.remove_from_cloud:
-            self.remove_from_cloud(data_path.cache_file_aws_key)
+        if not self.args.keep_in_cloud:
+            aws_key = self.cache_file_aws_key(data_path.cache.dvc)
+            self.remove_from_cloud(aws_key)

        os.remove(data_path.data.relative)
        pass
@@ -96,19 +113,18 @@ class CmdDataRemove(CmdBase):
                Logger.warn('S3 remove warning: file "{}" does not exist in S3'.format(aws_file_name))
            else:
                key.delete()
-                Logger.printing('File "{}" was removed from S3'.format(aws_file_name))
+                Logger.info('File "{}" was removed from S3'.format(aws_file_name))
+        pass

-    def remove_dir(self, data_dir):
-        for f in os.listdir(data_dir):
-            fname = os.path.join(data_dir, f)
-            if os.path.isdir(fname):
-                self.remove_dir(fname)
-            elif os.path.islink(fname):
-                self.remove_symlink(fname)
+    def remove_dir_file_by_file(self, target):
+        for f in os.listdir(target):
+            file = os.path.join(target, f)
+            if os.path.isdir(file):
+                self.remove_dir_file_by_file(file)
            else:
-                raise DataRemoveError('Unsupported file type "{}"'.format(fname))
+                self.remove_data_instance(file)

-        os.rmdir(data_dir)
+        os.rmdir(target)
        pass


@@ -28,6 +28,10 @@ class ConfigI(object):
    def state_dir(self):
        return self._state_dir

+    @property
+    def aws_storage_prefix(self):
+        return ''
+

 class Config(ConfigI):
    CONFIG = 'dvc.conf'
@@ -22,6 +22,10 @@ class GitWrapperI(object):
    def git_dir(self):
        return self._git_dir

+    @property
+    def lock_file(self):
+        return os.path.join(self.git_dir_abs, '.' + Config.CONFIG + '.lock')
+
    @property
    def git_dir_abs(self):
        return os.path.realpath(self.git_dir)
@@ -34,6 +38,9 @@ class GitWrapperI(object):
    def curr_commit(self):
        return self._commit

+    def is_ready_to_go(self):
+        return True
+

 class GitWrapper(GitWrapperI):
    def __init__(self):
@@ -157,10 +164,6 @@ class GitWrapper(GitWrapperI):
            Logger.printing('\t{} {}'.format(status, file))
        pass

-    @property
-    def lock_file(self):
-        return os.path.join(self.git_dir_abs, '.' + Config.CONFIG + '.lock')
-
    @staticmethod
    def abs_paths_to_relative(files):
        cur_dir = os.path.realpath(os.curdir)
@@ -0,0 +1,129 @@
+import os
+import shutil
+from unittest import TestCase
+
+from dvc.config import ConfigI
+from dvc.git_wrapper import GitWrapperI, GitWrapper
+from dvc.path.data_path import DataPath
+from dvc.path.path_factory import PathFactory
+
+
+class BasicEnvironment(TestCase):
+    def init_environment(self, test_dir=os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'), curr_dir=None):
+        self._test_dir = os.path.realpath(test_dir)
+        self._proj_dir = 'proj'
+        self._test_git_dir = os.path.join(self._test_dir, self._proj_dir)
+        self._old_curr_dir_abs = os.path.realpath(os.curdir)
+
+        shutil.rmtree(self._test_dir, ignore_errors=True)
+
+        if curr_dir:
+            self._curr_dir = os.path.realpath(curr_dir)
+        else:
+            self._curr_dir = self._test_git_dir
+
+        if not os.path.exists(self._curr_dir):
+            os.makedirs(self._curr_dir)
+
+        if not os.path.isdir(self._test_git_dir):
+            os.makedirs(self._test_git_dir)
+
+        data_dir = os.path.join(self._test_git_dir, 'data')
+        if not os.path.isdir(data_dir):
+            os.makedirs(data_dir)
+
+        os.chdir(self._curr_dir)
+        # GitWrapper.exec_cmd(['git', 'init'])
+        # self._commit = GitWrapper().curr_commit
+
+    def tearDown(self):
+        if self._old_curr_dir_abs:
+            os.chdir(self._old_curr_dir_abs)
+        if self._test_git_dir:
+            shutil.rmtree(self._test_dir, ignore_errors=True)
+        pass
+
+
+class DirHierarchyEnvironment(BasicEnvironment):
+    def init_environment(self,
+                         test_dir=os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'),
+                         curr_dir=None,
+                         commit='abc12345'):
+        ''' Creates data environment with data, cache and state dirs.
+        data/
+            file1.txt     --> ../cache/file1.txt_abc123
+            dir1/
+                file2.txt      --> ../../cache/dir1/file2.txt_abc123
+                file3.txt      --> ../../cache/dir1/file3.txt_abc123
+                dir11/
+                    file4.txt  --> ../../../cache/dir1/dir11/file4.txt_abc123
+            dir2/
+                file5.txt      --> ../../cache/dir2/file5.txt_abc123
+                file6.txt      --> an actual file
+        '''
+
+        BasicEnvironment.init_environment(self, test_dir, curr_dir)
+
+        self._commit = commit
+
+        self._git = GitWrapperI(git_dir=self._test_git_dir, commit=self._commit)
+        self._config = ConfigI('data', 'cache', 'state')
+        self.path_factory = PathFactory(self._git, self._config)
+
+        self.dir1 = os.path.join('dir1')
+        self.dir11 = os.path.join('dir1/dir11')
+        self.dir2 = os.path.join('dir2')
+
+        os.mkdir('cache')
+        os.mkdir('state')
+        self.create_dirs(self.dir1)
+        self.create_dirs(self.dir11)
+        self.create_dirs(self.dir2)
+
+        self.file1, self.cache1, self.state1 = self.crate_data_item('file1.txt')
+        self.file2, self.cache2, self.state2 = self.crate_data_item(os.path.join(self.dir1, 'file2.txt'))
+        self.file3, self.cache3, self.state3 = self.crate_data_item(os.path.join(self.dir1, 'file3.txt'))
+        self.file4, self.cache4, self.state4 = self.crate_data_item(os.path.join(self.dir11, 'file4.txt'))
+        self.file5, self.cache5, self.state5 = self.crate_data_item(os.path.join(self.dir2, 'file5.txt'))
+        self.file6, self.cache6, self.state6 = self.crate_data_item(os.path.join(self.dir2, 'file6.txt'),
+                                                                    cache_file=False)
+        pass
+
+    @staticmethod
+    def create_content_file(file, content='some test'):
+        # print(file)
+        fd = open(file, 'w+')
+        fd.write(content)
+        fd.close()
+
+    def crate_data_item(self, data_file, cache_file=True, content='random text'):
+        file_result = os.path.join('data', data_file)
+        state_result = os.path.join('state', data_file + DataPath.STATE_FILE_SUFFIX)
+
+        self.create_content_file(state_result, 'state content')
+
+        if cache_file:
+            cache_result = os.path.join('cache', data_file + DataPath.CACHE_FILE_SEP + self._commit)
+            self.create_content_file(cache_result, content)
+
+            relevant_dir = self.relevant_dir(data_file)
+            os.symlink(os.path.join(relevant_dir, cache_result), file_result)
+            # print(file_result)
+        else:
+            cache_result = None
+            self.create_content_file(file_result, content)
+
+        return file_result, cache_result, state_result
+
+    @staticmethod
+    def relevant_dir(data_file):
+        deepth = data_file.count(os.path.sep)
+        relevant_path = '..'
+        for i in range(deepth):
+            relevant_path = os.path.join(relevant_path, '..')
+        return relevant_path
+
+    def create_dirs(self, dir):
+        os.mkdir(os.path.join('data', dir))
+        os.mkdir(os.path.join('cache', dir))
+        os.mkdir(os.path.join('state', dir))
@@ -0,0 +1,183 @@
+from unittest import TestCase
+import os
+import shutil
+
+from dvc.cmd_data_remove import CmdDataRemove, DataRemoveError
+from dvc.config import ConfigI
+from dvc.path.data_path import NotInDataDirError, DataFilePathError, DataPath
+from dvc.git_wrapper import GitWrapperI
+from dvc.path.path_factory import PathFactory
+from tests.basic_env import BasicEnvironment, DirHierarchyEnvironment
+
+
+class TestCmdDataRemove(DirHierarchyEnvironment):
+    def setUp(self):
+        DirHierarchyEnvironment.init_environment(self)
+
+    def test_file_by_file_removal(self):
+        cmd = CmdDataRemove(False, git_obj=self._git, config_obj=self._config)
+        cmd.args.keep_in_cloud = True
+
+        dir1_dvc_name = os.path.join('data', self.dir1)
+        self.assertTrue(os.path.exists(dir1_dvc_name))
+
+        cmd.remove_dir_file_by_file(dir1_dvc_name)
+        self.assertFalse(os.path.exists(dir1_dvc_name))
+
+    def test_remove_deep_dir(self):
+        cmd = CmdDataRemove(False, git_obj=self._git, config_obj=self._config)
+        cmd.args.keep_in_cloud = True
+        cmd.args.recursive = True
+
+        dir1_dvc_name = os.path.join('data', self.dir1)
+        self.assertTrue(os.path.exists(dir1_dvc_name))
+
+        cmd.remove_dir(dir1_dvc_name)
+        self.assertFalse(os.path.exists(dir1_dvc_name))
+
+    def test_not_recursive_removal(self):
+        cmd = CmdDataRemove(False, git_obj=self._git, config_obj=self._config)
+        cmd.args.keep_in_cloud = True
+        cmd.args.recursive = False
+
+        dir1_dvc_name = os.path.join('data', self.dir1)
+        self.assertTrue(os.path.exists(dir1_dvc_name))
+
+        with self.assertRaises(DataRemoveError):
+            cmd.remove_dir(dir1_dvc_name)
+        pass
+
+    def test_data_dir_removal(self):
+        cmd = CmdDataRemove(False, git_obj=self._git, config_obj=self._config)
+        cmd.args.keep_in_cloud = True
+        cmd.args.recursive = True
+
+        data_dir = 'data'
+        self.assertTrue(os.path.exists(data_dir))
+
+        with self.assertRaises(DataRemoveError):
+            cmd.remove_dir(data_dir)
+        pass
+
+
+class TestRemoveDataItem(DirHierarchyEnvironment):
+    def setUp(self):
+        DirHierarchyEnvironment.init_environment(self)
+
+    def test_remove_data_instance(self):
+        cmd = CmdDataRemove(False, git_obj=self._git, config_obj=self._config)
+        cmd.args.keep_in_cloud = True
+
+        self.assertTrue(os.path.isfile(self.file1))
+        self.assertTrue(os.path.isfile(self.cache1))
+        self.assertTrue(os.path.isfile(self.state1))
+
+        cmd.remove_data_instance(self.file1)
+        self.assertFalse(os.path.exists(self.file1))
+        self.assertFalse(os.path.exists(self.cache1))
+        self.assertFalse(os.path.exists(self.state1))
+
+    def test_keep_cache(self):
+        cmd = CmdDataRemove(False, git_obj=self._git, config_obj=self._config)
+        cmd.args.keep_in_cloud = True
+        cmd.args.keep_in_cache = True
+
+        self.assertTrue(os.path.isfile(self.file1))
+        self.assertTrue(os.path.isfile(self.cache1))
+        self.assertTrue(os.path.isfile(self.state1))
+
+        cmd.remove_data_instance(self.file1)
+        self.assertFalse(os.path.exists(self.file1))
+        self.assertTrue(os.path.exists(self.cache1))
+        self.assertFalse(os.path.exists(self.state1))
+
+    def test_remove_data_instance_without_cache(self):
+        cmd = CmdDataRemove(False, git_obj=self._git, config_obj=self._config)
+        cmd.args.keep_in_cloud = True
+
+        self.assertTrue(os.path.isfile(self.file6))
+        self.assertIsNone(self.cache6)
+        self.assertTrue(os.path.isfile(self.state6))
+
+        with self.assertRaises(DataFilePathError):
+            cmd.remove_data_instance(self.file6)
+
+
+class TestRemoveEndToEnd(DirHierarchyEnvironment):
+    def setUp(self):
+        DirHierarchyEnvironment.init_environment(self)
+
+    def test_end_to_end_with_an_error(self):
+        cmd = CmdDataRemove(False, git_obj=self._git, config_obj=self._config)
+        cmd.args.keep_in_cloud = True
+
+        dir11_full = os.path.join('data', self.dir11)
+        dir2_full = os.path.join('data', self.dir2)
+        file6_full = os.path.join('data', self.file6)
+
+        cmd.args.target = [dir11_full, self.file5, file6_full]
+        cmd.args.recursive = True
+
+        cmd.args.skip_git_actions = True
+
+        self.assertTrue(os.path.exists(dir11_full))
+        self.assertTrue(os.path.exists(self.file5))
+        self.assertTrue(os.path.exists(dir2_full))
+        self.assertTrue(os.path.exists(self.file1))
+        self.assertTrue(os.path.exists(self.file6))
+
+        self.assertEqual(cmd.remove_all_targets(), 1)
+
+        self.assertFalse(os.path.exists(dir11_full))
+        self.assertFalse(os.path.exists(self.file5))
+        self.assertTrue(os.path.exists(dir2_full))
+        self.assertTrue(os.path.exists(self.file1))
+        self.assertTrue(os.path.exists(self.file6))
+
+    def test_end_to_end_with_no_error(self):
+        cmd = CmdDataRemove(False, git_obj=self._git, config_obj=self._config)
+        cmd.args.keep_in_cloud = True
+
+        dir11_full = os.path.join('data', self.dir11)
+        dir2_full = os.path.join('data', self.dir2)
+
+        cmd.args.target = [dir11_full, self.file5]
+        cmd.args.recursive = True
+
+        cmd.args.skip_git_actions = True
+
+        self.assertTrue(os.path.exists(dir11_full))
+        self.assertTrue(os.path.exists(self.file5))
+        self.assertTrue(os.path.exists(dir2_full))
+        self.assertTrue(os.path.exists(self.file1))
+
+        self.assertEqual(cmd.remove_all_targets(), 0)
+
+        self.assertFalse(os.path.exists(dir11_full))
+        self.assertFalse(os.path.exists(self.file5))
+        self.assertTrue(os.path.exists(dir2_full))
+        self.assertTrue(os.path.exists(self.file1))
+
+    def test_run(self):
+        cmd = CmdDataRemove(False, git_obj=self._git, config_obj=self._config)
+        cmd.args.keep_in_cloud = True
+
+        dir11_full = os.path.join('data', self.dir11)
+        dir2_full = os.path.join('data', self.dir2)
+
+        cmd.args.target = [dir11_full, self.file5]
+        cmd.args.recursive = True
+
+        cmd.args.skip_git_actions = True
+
+        self.assertTrue(os.path.exists(dir11_full))
+        self.assertTrue(os.path.exists(self.file5))
+        self.assertTrue(os.path.exists(dir2_full))
+        self.assertTrue(os.path.exists(self.file1))
+
+        self.assertEqual(cmd.run(), 0)
+
+        self.assertFalse(os.path.exists(dir11_full))
+        self.assertFalse(os.path.exists(self.file5))
+        self.assertTrue(os.path.exists(dir2_full))
+        self.assertTrue(os.path.exists(self.file1))
@@ -1,47 +1,15 @@
-from unittest import TestCase
 import os
-import shutil

 from dvc.config import ConfigI
-from dvc.path.data_path import NotInDataDirError, DataFilePathError, DataPath
 from dvc.git_wrapper import GitWrapperI
+from dvc.path.data_path import NotInDataDirError, DataFilePathError, DataPath
 from dvc.path.path_factory import PathFactory
+from tests.basic_env import BasicEnvironment


-class BasicDataDirTest(TestCase):
-    def setUp(self, test_dir, curr_dir=None):
-        self._test_dir = os.path.realpath(test_dir)
-        self._proj_dir = 'proj'
-        self._test_git_dir = os.path.join(self._test_dir, self._proj_dir)
-        self._old_curr_dir_abs = os.path.realpath(os.curdir)
-
-        shutil.rmtree(self._test_dir, ignore_errors=True)
-
-        if curr_dir:
-            self._curr_dir = os.path.realpath(curr_dir)
-        else:
-            self._curr_dir = self._test_git_dir
-
-        if not os.path.exists(self._curr_dir):
-            os.makedirs(self._curr_dir)
-
-        if not os.path.isdir(self._test_git_dir):
-            os.makedirs(self._test_git_dir)
-
-        data_dir = os.path.join(self._test_git_dir, 'data')
-        if not os.path.isdir(data_dir):
-            os.makedirs(data_dir)
-
-        os.chdir(self._curr_dir)
-
-    def tearDown(self):
-        os.chdir(self._old_curr_dir_abs)
-        shutil.rmtree(self._test_dir, ignore_errors=True)
-
-
-class TestDataFileObjBasic(BasicDataDirTest):
+class TestDataFileObjBasic(BasicEnvironment):
    def setUp(self):
-        BasicDataDirTest.setUp(self, test_dir=os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'))
+        BasicEnvironment.init_environment(self, test_dir=os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'))

        git = GitWrapperI(git_dir=self._test_git_dir, commit='ad45ba8')
        config = ConfigI('data', 'cache', 'state')
@@ -67,9 +35,9 @@ class TestDataFileObjBasic(BasicDataDirTest):
        pass


-class TestPathFactory(BasicDataDirTest):
+class TestPathFactory(BasicEnvironment):
    def setUp(self):
-        BasicDataDirTest.setUp(self, test_dir=os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'))
+        BasicEnvironment.init_environment(self, test_dir=os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'))

        git = GitWrapperI(git_dir=self._test_git_dir, commit='ad45ba8')
        config = ConfigI('data', 'cache', 'state')
@@ -111,17 +79,44 @@ class TestPathFactory(BasicDataDirTest):
            self.path_factory.existing_data_path(self.file)
        pass

+    def test_data_path(self):
+        file = os.path.join('data', 'file1')
+        path = self.path_factory.path(file)
+        self.assertEqual(path.dvc, file)
+        self.assertEqual(path.relative, file)
+        self.assertTrue(path.abs.endswith(file))

-class TestDataPathInDataDir(BasicDataDirTest):
+    def test_to_data_path(self):
+        exclude_file = os.path.join('cache', 'file2')
+        data_path_file1 = os.path.join('data', 'file1')
+        data_path_file2 = os.path.join('data', 'file2')
+        files = [
+            data_path_file1,
+            exclude_file,
+            data_path_file2
+        ]
+
+        data_path_list, exclude_file_list = self.path_factory.to_data_path(files)
+
+        self.assertEqual(len(data_path_list), 2)
+        self.assertEqual(len(exclude_file_list), 1)
+
+        self.assertEqual(exclude_file_list[0], exclude_file)
+        data_path_set = set(x.data.dvc for x in data_path_list)
+        self.assertEqual(data_path_set, {data_path_file1, data_path_file2})
+
+
+class TestDataPathInDataDir(BasicEnvironment):
    def setUp(self):
-        BasicDataDirTest.setUp(self, os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'))
+        BasicEnvironment.init_environment(self, os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'))

        git = GitWrapperI(git_dir=self._test_git_dir, commit='eeeff8f')
-        config = ConfigI('da', 'ca', 'st')
-        path_factory = PathFactory(git, config)
+        self.data_dir = 'da'
+        config = ConfigI(self.data_dir, 'ca', 'st')
+        self.path_factory = PathFactory(git, config)

-        deep_path = os.path.join('da', 'dir1', 'd2', 'file.txt')
-        self.data_path = path_factory.data_path(deep_path)
+        deep_path = os.path.join(self.data_dir, 'dir1', 'd2', 'file.txt')
+        self.data_path = self.path_factory.data_path(deep_path)
        pass

    def test_data_file(self):
@@ -139,14 +134,19 @@ class TestDataPathInDataDir(BasicDataDirTest):
    def test_symlink(self):
        self.assertEqual(self.data_path._symlink_file, '../../../ca/dir1/d2/file.txt_eeeff8f')

+    def test_data_dir(self):
+        data_path = self.path_factory.data_path(self.data_dir)
+        self.assertEqual(data_path.data.dvc, self.data_dir)
+        self.assertEqual(data_path.data_dvc_short, '')

-class TestDataFileObjLongPath(BasicDataDirTest):
+
+class TestDataFileObjLongPath(BasicEnvironment):
    def setUp(self):
        curr_dir = os.path.join(os.path.sep, 'tmp', 'ntx_unit_test', 'proj', 'mydir', 'dd3')

-        BasicDataDirTest.setUp(self,
-                               os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'),
-                               curr_dir)
+        BasicEnvironment.init_environment(self,
+                                          os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'),
+                                          curr_dir)

        self._git = GitWrapperI(git_dir=self._test_git_dir, commit='123ed8')
        self._config = ConfigI('data', 'cache', 'state')
@@ -185,13 +185,13 @@ class TestDataFileObjLongPath(BasicDataDirTest):
        pass


-class RunOutsideGitRepoTest(BasicDataDirTest):
+class RunOutsideGitRepoTest(BasicEnvironment):
    def setUp(self):
        curr_dir = os.path.join(os.path.sep, 'tmp')

-        BasicDataDirTest.setUp(self,
-                               os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'),
-                               curr_dir)
+        BasicEnvironment.init_environment(self,
+                                          os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'),
+                                          curr_dir)

    def test_outside_git_dir(self):
        git = GitWrapperI(git_dir=self._test_git_dir, commit='123ed8')