From df6abc7528627453c0592a0ac25d8fc4bf451e0a Mon Sep 17 00:00:00 2001 From: Dmitry Petrov Date: Mon, 19 Jun 2017 16:44:18 -0700 Subject: [PATCH 01/11] Small sync and import fixes --- dvc/command/data_sync.py | 6 +++++- dvc/command/import_file.py | 19 ++++++++++++++++++- dvc/system.py | 2 -- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/dvc/command/data_sync.py b/dvc/command/data_sync.py index 4474b2594..e25039c0a 100644 --- a/dvc/command/data_sync.py +++ b/dvc/command/data_sync.py @@ -7,13 +7,14 @@ from dvc.runtime import Runtime from dvc.system import System from dvc.data_cloud import DataCloud -#FIXME add cmdline argument and config option for POOL_SIZE POOL_SIZE = 4 + class DataSyncError(DvcException): def __init__(self, msg): super(DataSyncError, self).__init__('Data sync error: {}'.format(msg)) + class CmdDataSync(CmdBase): def __init__(self, settings): super(CmdDataSync, self).__init__(settings) @@ -30,6 +31,9 @@ class CmdDataSync(CmdBase): pool = ThreadPool(processes=POOL_SIZE) targets = [] + if len(self.parsed_args.targets) == 0: + raise DataSyncError('Sync target is not specified') + for target in self.parsed_args.targets: if System.islink(target): targets.append(target) diff --git a/dvc/command/import_file.py b/dvc/command/import_file.py index 59f2f8a2c..93461ae45 100644 --- a/dvc/command/import_file.py +++ b/dvc/command/import_file.py @@ -14,6 +14,7 @@ from dvc.system import System from dvc.command.data_sync import POOL_SIZE from dvc.progress import Progress + class ImportFileError(DvcException): def __init__(self, msg): DvcException.__init__(self, 'Import file: {}'.format(msg)) @@ -36,7 +37,10 @@ class CmdImportFile(CmdBase): def run(self): targets = [] with DvcLock(self.is_locker, self.git): - output = self.parsed_args.output + output = self.get_output() + if not output: + return 1 + for input in self.parsed_args.input: if not os.path.isdir(input): targets.append((input, output)) @@ -60,6 +64,19 @@ class CmdImportFile(CmdBase): self.commit_if_needed(message) pass + def get_output(self): + output = self.parsed_args.output + + if len(output) > 0 and output[-1] == os.path.sep and not os.path.isdir(output): + Logger.error(u'Import error: output directory {} does not exist'.format(output)) + return None + + if type(self.parsed_args.input) == list and len(self.parsed_args.input) > 1 and not os.path.isdir(output): + Logger.error(u'Import error: output {} has to be directory for multiple file import'.format(output)) + return None + + return output + def import_and_commit_if_needed(self, input, output, lock=False, check_if_ready=True): if check_if_ready and not self.no_git_actions and not self.git.is_ready_to_go(): return 1 diff --git a/dvc/system.py b/dvc/system.py index 752545d31..43c52053a 100644 --- a/dvc/system.py +++ b/dvc/system.py @@ -82,9 +82,7 @@ class System(object): def get_long_path(path): """Convert short path to a full path. It is needed for Windows.""" if System.is_unix(): - print('======================11111111111111111111') return path - print('======================222222222222222222222') buffer = create_unicode_buffer(System.LONG_PATH_BUFFER_SIZE) get_long_path_name = windll.kernel32.GetLongPathNameW From 17ffcea10fc12e7e92b8a21db2ce5634e729e93f Mon Sep 17 00:00:00 2001 From: Dmitry Petrov Date: Mon, 19 Jun 2017 18:13:55 -0700 Subject: [PATCH 02/11] Error when syncing data subfolders. Close #94 --- dvc/data_cloud.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index f0752574c..a14221bf6 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -1,20 +1,19 @@ import base64 import hashlib import os +import threading import configparser import shutil from boto.s3.connection import S3Connection from google.cloud import storage as gc -from dvc.command.base import CmdBase, DvcLock from dvc.logger import Logger from dvc.exceptions import DvcException -from dvc.runtime import Runtime -from dvc.system import System from dvc.config import ConfigError from dvc.settings import Settings + class DataCloudError(DvcException): def __init__(self, msg): super(DataCloudError, self).__init__('Data sync error: {}'.format(msg)) @@ -47,6 +46,7 @@ class DataCloudBase(object): self._settings = settings self._config = config self._cloud_config = cloud_config + self._lock = threading.Lock() @property def storage_path(self): @@ -97,8 +97,25 @@ class DataCloudBase(object): if os.path.isfile(item.resolved_cache.dvc): self.sync_to_cloud(item) else: + self.create_directory(fname, item) self.sync_from_cloud(item) + def create_directory(self, fname, item): + self._lock.acquire() + try: + dir = os.path.dirname(item.cache.relative) + if not os.path.exists(dir): + Logger.debug(u'Creating directory {}'.format(dir)) + try: + os.makedirs(dir) + except OSError as ex: + raise DataCloudError(u'Cannot create directory {}: {}'.format(dir, ex)) + elif not os.path.isdir(dir): + msg = u'File {} cannot be synced because {} is not a directory' + raise DataCloudError(msg.format(fname, dir)) + finally: + self._lock.release() + def remove_from_cloud(self, item): pass @@ -116,6 +133,7 @@ class DataCloudLOCAL(DataCloudBase): Logger.debug('rm from cloud ' + item.resolved_cache.dvc) os.remove(item.resolved_cache.dvc) + class DataCloudAWS(DataCloudBase): """ DataCloud class for Amazon Web Services """ @property From 0e3278641c0bd3bc590e713678fece4b12b04712 Mon Sep 17 00:00:00 2001 From: Dmitry Petrov Date: Mon, 19 Jun 2017 18:21:58 -0700 Subject: [PATCH 03/11] Small tests headers fixes --- tests/test_cmd_data_remove.py | 27 --------------------------- tests/test_cmd_gc.py | 3 ++- tests/test_config.py | 5 ----- tests/test_repro.py | 1 - 4 files changed, 2 insertions(+), 34 deletions(-) diff --git a/tests/test_cmd_data_remove.py b/tests/test_cmd_data_remove.py index 0d228fb86..3b603eefc 100644 --- a/tests/test_cmd_data_remove.py +++ b/tests/test_cmd_data_remove.py @@ -100,33 +100,6 @@ class TestRemoveEndToEnd(DirHierarchyEnvironment): def setUp(self): DirHierarchyEnvironment.init_environment(self) - # def test_end_to_end_with_an_error(self): - # cmd = CmdDataRemove(self.settings) - # cmd.parsed_args.keep_in_cloud = True - # - # dir11_full = self.dir11 - # dir2_full = self.dir2 - # file6_full = self.file6 - # - # cmd.parsed_args.target = [dir11_full, self.file5, file6_full] - # cmd.parsed_args.recursive = True - # - # cmd.parsed_args.skip_git_actions = True - # - # self.assertTrue(os.path.exists(dir11_full)) - # self.assertTrue(os.path.exists(self.file5)) - # self.assertTrue(os.path.exists(dir2_full)) - # self.assertTrue(os.path.exists(self.file1)) - # self.assertTrue(os.path.exists(self.file6)) - # - # self.assertFalse(cmd.remove_all_targets()) - # - # self.assertFalse(os.path.exists(dir11_full)) - # self.assertFalse(os.path.exists(self.file5)) - # self.assertTrue(os.path.exists(dir2_full)) - # self.assertTrue(os.path.exists(self.file1)) - # self.assertTrue(os.path.exists(self.file6)) - def test_end_to_end_with_no_error(self): cmd = CmdDataRemove(self.settings) cmd.parsed_args.keep_in_cloud = True diff --git a/tests/test_cmd_gc.py b/tests/test_cmd_gc.py index 15506d586..be8e9c13d 100644 --- a/tests/test_cmd_gc.py +++ b/tests/test_cmd_gc.py @@ -1,4 +1,5 @@ -import glob, os +import glob +import os from dvc.command.gc import CmdGC from dvc.config import ConfigI diff --git a/tests/test_config.py b/tests/test_config.py index 43e1ae8e4..b18f2c2a8 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,7 +1,4 @@ -import io import os -import shutil -import traceback from unittest import TestCase import mock @@ -18,8 +15,6 @@ except ImportError: # Python 3 builtin_module_name = 'builtins' -from dvc.path.path import Path -from dvc.system import System from dvc.config import Config from dvc.data_cloud import DataCloud from dvc.settings import Settings diff --git a/tests/test_repro.py b/tests/test_repro.py index 6483286fe..6bd78eaf4 100644 --- a/tests/test_repro.py +++ b/tests/test_repro.py @@ -4,7 +4,6 @@ from dvc.command.remove import CmdDataRemove from dvc.command.repro import CmdRepro from dvc.command.run import CmdRun from dvc.executor import Executor -from dvc.logger import Logger from tests.test_cmd_run import RunBasicTest From 05dfaa3d481442f7ac4fb39dc813e5b67573ff6f Mon Sep 17 00:00:00 2001 From: Dmitry Petrov Date: Mon, 19 Jun 2017 18:29:44 -0700 Subject: [PATCH 04/11] Rename file test_cmd_data_remove --- tests/{test_cmd_data_remove.py => test_cmd_remove.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{test_cmd_data_remove.py => test_cmd_remove.py} (100%) diff --git a/tests/test_cmd_data_remove.py b/tests/test_cmd_remove.py similarity index 100% rename from tests/test_cmd_data_remove.py rename to tests/test_cmd_remove.py From 71566b1a247a83b42bbb1b8be7da3e63d51c5847 Mon Sep 17 00:00:00 2001 From: Dmitry Petrov Date: Mon, 19 Jun 2017 18:32:30 -0700 Subject: [PATCH 05/11] Rename CmdDataRemove class --- dvc/command/remove.py | 8 ++++---- dvc/main.py | 4 ++-- tests/test_cmd_remove.py | 24 ++++++++++++------------ tests/test_repro.py | 4 ++-- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/dvc/command/remove.py b/dvc/command/remove.py index 75d859f61..db13860bd 100644 --- a/dvc/command/remove.py +++ b/dvc/command/remove.py @@ -6,12 +6,12 @@ from dvc.path.data_item import DataItemError from dvc.runtime import Runtime -class CmdDataRemove(Traverse): +class CmdRemove(Traverse): def __init__(self, settings): - super(CmdDataRemove, self).__init__(settings, "remove") + super(CmdRemove, self).__init__(settings, "remove") def define_args(self, parser): - super(CmdDataRemove, self).define_args(parser) + super(CmdRemove, self).define_args(parser) parser.add_argument('-r', '--recursive', action='store_true', help='Remove directory recursively.') parser.add_argument('-c', '--keep-in-cache', action='store_false', default=False, help='Do not remove data from cache.') @@ -86,4 +86,4 @@ class CmdDataRemove(Traverse): if __name__ == '__main__': - Runtime.run(CmdDataRemove) + Runtime.run(CmdRemove) diff --git a/dvc/main.py b/dvc/main.py index f4d691d48..c5dd4a5ae 100644 --- a/dvc/main.py +++ b/dvc/main.py @@ -10,7 +10,7 @@ import sys from dvc.runtime import Runtime from dvc.command.init import CmdInit -from dvc.command.remove import CmdDataRemove +from dvc.command.remove import CmdRemove from dvc.command.run import CmdRun from dvc.command.repro import CmdRepro from dvc.command.data_sync import CmdDataSync @@ -64,7 +64,7 @@ def main(): elif cmd == 'import': Runtime.run(CmdImportFile) elif cmd == 'remove': - Runtime.run(CmdDataRemove) + Runtime.run(CmdRemove) elif cmd == 'lock': Runtime.run(CmdLock) elif cmd == 'cloud-run': diff --git a/tests/test_cmd_remove.py b/tests/test_cmd_remove.py index 3b603eefc..9ccabac53 100644 --- a/tests/test_cmd_remove.py +++ b/tests/test_cmd_remove.py @@ -1,15 +1,15 @@ import os -from dvc.command.remove import CmdDataRemove +from dvc.command.remove import CmdRemove from tests.basic_env import DirHierarchyEnvironment -class TestCmdDataRemove(DirHierarchyEnvironment): +class TestCmdRemove(DirHierarchyEnvironment): def setUp(self): DirHierarchyEnvironment.init_environment(self) def test_file_by_file_removal(self): - cmd = CmdDataRemove(self.settings) + cmd = CmdRemove(self.settings) cmd.parsed_args.keep_in_cloud = True dir1_dvc_name = os.path.join('data', self.dir1) @@ -19,7 +19,7 @@ class TestCmdDataRemove(DirHierarchyEnvironment): self.assertFalse(os.path.exists(dir1_dvc_name)) def test_remove_deep_dir(self): - cmd = CmdDataRemove(self.settings) + cmd = CmdRemove(self.settings) cmd.parsed_args.keep_in_cloud = True cmd.parsed_args.recursive = True @@ -30,7 +30,7 @@ class TestCmdDataRemove(DirHierarchyEnvironment): self.assertFalse(os.path.exists(dir1_dvc_name)) def test_not_recursive_removal(self): - cmd = CmdDataRemove(self.settings) + cmd = CmdRemove(self.settings) cmd.parsed_args.keep_in_cloud = True cmd.parsed_args.recursive = False @@ -41,7 +41,7 @@ class TestCmdDataRemove(DirHierarchyEnvironment): pass def test_data_dir_removal(self): - cmd = CmdDataRemove(self.settings) + cmd = CmdRemove(self.settings) cmd.parsed_args.keep_in_cloud = True cmd.parsed_args.recursive = True @@ -57,7 +57,7 @@ class TestRemoveDataItem(DirHierarchyEnvironment): DirHierarchyEnvironment.init_environment(self) def test_remove_data_instance(self): - cmd = CmdDataRemove(self.settings) + cmd = CmdRemove(self.settings) cmd.parsed_args.keep_in_cloud = True self.assertTrue(os.path.isfile(self.file1)) @@ -70,7 +70,7 @@ class TestRemoveDataItem(DirHierarchyEnvironment): self.assertFalse(os.path.exists(self.state1)) def test_keep_cache(self): - cmd = CmdDataRemove(self.settings) + cmd = CmdRemove(self.settings) cmd.parsed_args.keep_in_cloud = True cmd.parsed_args.keep_in_cache = True @@ -84,7 +84,7 @@ class TestRemoveDataItem(DirHierarchyEnvironment): self.assertFalse(os.path.exists(self.state1)) def test_remove_data_instance_without_cache(self): - cmd = CmdDataRemove(self.settings) + cmd = CmdRemove(self.settings) cmd.parsed_args.keep_in_cloud = True self.assertTrue(os.path.isfile(self.file6)) @@ -101,7 +101,7 @@ class TestRemoveEndToEnd(DirHierarchyEnvironment): DirHierarchyEnvironment.init_environment(self) def test_end_to_end_with_no_error(self): - cmd = CmdDataRemove(self.settings) + cmd = CmdRemove(self.settings) cmd.parsed_args.keep_in_cloud = True dir11_full = os.path.join('data', self.dir11) @@ -125,7 +125,7 @@ class TestRemoveEndToEnd(DirHierarchyEnvironment): self.assertTrue(os.path.exists(self.file1)) def test_run(self): - cmd = CmdDataRemove(self.settings) + cmd = CmdRemove(self.settings) cmd.parsed_args.keep_in_cloud = True dir11_full = os.path.join('data', self.dir11) @@ -154,7 +154,7 @@ class TestRemoveEndToEndWithAnError(DirHierarchyEnvironment): DirHierarchyEnvironment.init_environment(self) def test(self): - cmd = CmdDataRemove(self.settings) + cmd = CmdRemove(self.settings) cmd.parsed_args.keep_in_cloud = True dir11_full = os.path.join(self._config.data_dir, self.dir11) diff --git a/tests/test_repro.py b/tests/test_repro.py index 6bd78eaf4..20d993e2b 100644 --- a/tests/test_repro.py +++ b/tests/test_repro.py @@ -1,6 +1,6 @@ import os -from dvc.command.remove import CmdDataRemove +from dvc.command.remove import CmdRemove from dvc.command.repro import CmdRepro from dvc.command.run import CmdRun from dvc.executor import Executor @@ -93,7 +93,7 @@ class ReproChangedDependency(ReproBasicEnv): def recreate_file1(self): self.settings._args = [self.file_name1, '--keep-in-cloud'] - CmdDataRemove(self.settings).run() + CmdRemove(self.settings).run() file1_code_file = 'file1_2.py' self.create_file_and_commit(file1_code_file, 'print("Goodbye")' + os.linesep + 'print("Jack")') From 15cadc44f6f3af2b6f4ac2f36b796305fcecdeae Mon Sep 17 00:00:00 2001 From: Dmitry Petrov Date: Mon, 19 Jun 2017 18:48:05 -0700 Subject: [PATCH 06/11] Simplify output validation logic --- dvc/command/import_file.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/dvc/command/import_file.py b/dvc/command/import_file.py index 93461ae45..515630729 100644 --- a/dvc/command/import_file.py +++ b/dvc/command/import_file.py @@ -37,9 +37,8 @@ class CmdImportFile(CmdBase): def run(self): targets = [] with DvcLock(self.is_locker, self.git): - output = self.get_output() - if not output: - return 1 + output = self.parsed_args.output + self.verify_output(output) for input in self.parsed_args.input: if not os.path.isdir(input): @@ -64,18 +63,14 @@ class CmdImportFile(CmdBase): self.commit_if_needed(message) pass - def get_output(self): - output = self.parsed_args.output - + def verify_output(self, output): if len(output) > 0 and output[-1] == os.path.sep and not os.path.isdir(output): - Logger.error(u'Import error: output directory {} does not exist'.format(output)) - return None + raise ImportFileError(u'Import error: output directory {} does not exist'.format(output)) if type(self.parsed_args.input) == list and len(self.parsed_args.input) > 1 and not os.path.isdir(output): - Logger.error(u'Import error: output {} has to be directory for multiple file import'.format(output)) - return None - - return output + msg = u'Import error: output {} has to be directory for multiple file import' + raise ImportFileError(msg.format(output)) + pass def import_and_commit_if_needed(self, input, output, lock=False, check_if_ready=True): if check_if_ready and not self.no_git_actions and not self.git.is_ready_to_go(): From c16f97a3cf6fd47efb8c410fcf415c14f415cd07 Mon Sep 17 00:00:00 2001 From: Dmitry Petrov Date: Mon, 19 Jun 2017 19:07:54 -0700 Subject: [PATCH 07/11] Verify output refactoring --- dvc/command/import_file.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dvc/command/import_file.py b/dvc/command/import_file.py index 515630729..7d6a6a611 100644 --- a/dvc/command/import_file.py +++ b/dvc/command/import_file.py @@ -38,7 +38,7 @@ class CmdImportFile(CmdBase): targets = [] with DvcLock(self.is_locker, self.git): output = self.parsed_args.output - self.verify_output(output) + self.verify_output(output, self.parsed_args.input) for input in self.parsed_args.input: if not os.path.isdir(input): @@ -63,11 +63,12 @@ class CmdImportFile(CmdBase): self.commit_if_needed(message) pass - def verify_output(self, output): + @staticmethod + def verify_output(output, input): if len(output) > 0 and output[-1] == os.path.sep and not os.path.isdir(output): raise ImportFileError(u'Import error: output directory {} does not exist'.format(output)) - if type(self.parsed_args.input) == list and len(self.parsed_args.input) > 1 and not os.path.isdir(output): + if len(input) > 1 and not os.path.isdir(output): msg = u'Import error: output {} has to be directory for multiple file import' raise ImportFileError(msg.format(output)) pass From 47e1b421f5d4cd98ea27fb18b6706f79604d4f74 Mon Sep 17 00:00:00 2001 From: Dmitry Petrov Date: Mon, 19 Jun 2017 19:20:04 -0700 Subject: [PATCH 08/11] Verify output refactoring. Attempt 2 --- dvc/command/import_file.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dvc/command/import_file.py b/dvc/command/import_file.py index 7d6a6a611..eca877bd4 100644 --- a/dvc/command/import_file.py +++ b/dvc/command/import_file.py @@ -65,7 +65,7 @@ class CmdImportFile(CmdBase): @staticmethod def verify_output(output, input): - if len(output) > 0 and output[-1] == os.path.sep and not os.path.isdir(output): + if output.is_dir_path(output) and not os.path.isdir(output): raise ImportFileError(u'Import error: output directory {} does not exist'.format(output)) if len(input) > 1 and not os.path.isdir(output): @@ -73,6 +73,10 @@ class CmdImportFile(CmdBase): raise ImportFileError(msg.format(output)) pass + @staticmethod + def is_dir_path(self, output): + return len(output) > 0 and output[-1] == os.path.sep + def import_and_commit_if_needed(self, input, output, lock=False, check_if_ready=True): if check_if_ready and not self.no_git_actions and not self.git.is_ready_to_go(): return 1 From 674f20c800f00f4f7ca909ba0e1a0d44a2faeea4 Mon Sep 17 00:00:00 2001 From: Dmitry Petrov Date: Mon, 19 Jun 2017 19:30:01 -0700 Subject: [PATCH 09/11] Verify output refactoring. Small fixes. --- dvc/command/import_file.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dvc/command/import_file.py b/dvc/command/import_file.py index eca877bd4..6f1b63427 100644 --- a/dvc/command/import_file.py +++ b/dvc/command/import_file.py @@ -65,7 +65,7 @@ class CmdImportFile(CmdBase): @staticmethod def verify_output(output, input): - if output.is_dir_path(output) and not os.path.isdir(output): + if CmdImportFile.is_dir_path(output) and not os.path.isdir(output): raise ImportFileError(u'Import error: output directory {} does not exist'.format(output)) if len(input) > 1 and not os.path.isdir(output): @@ -74,7 +74,7 @@ class CmdImportFile(CmdBase): pass @staticmethod - def is_dir_path(self, output): + def is_dir_path(output): return len(output) > 0 and output[-1] == os.path.sep def import_and_commit_if_needed(self, input, output, lock=False, check_if_ready=True): From ed923db203d028976a57e2bd82e6ef0697435d3e Mon Sep 17 00:00:00 2001 From: Dmitry Petrov Date: Mon, 19 Jun 2017 21:31:31 -0700 Subject: [PATCH 10/11] Import cmd unit-tests --- dvc/command/import_file.py | 6 +- dvc/git_wrapper.py | 3 + tests/test_import.py | 117 +++++++++++++++++++++++++++++++++++++ 3 files changed, 123 insertions(+), 3 deletions(-) create mode 100644 tests/test_import.py diff --git a/dvc/command/import_file.py b/dvc/command/import_file.py index 6f1b63427..5c6aa6eac 100644 --- a/dvc/command/import_file.py +++ b/dvc/command/import_file.py @@ -66,10 +66,10 @@ class CmdImportFile(CmdBase): @staticmethod def verify_output(output, input): if CmdImportFile.is_dir_path(output) and not os.path.isdir(output): - raise ImportFileError(u'Import error: output directory {} does not exist'.format(output)) + raise ImportFileError(u'output directory {} does not exist'.format(output)) if len(input) > 1 and not os.path.isdir(output): - msg = u'Import error: output {} has to be directory for multiple file import' + msg = u'output {} has to be directory for multiple file import' raise ImportFileError(msg.format(output)) pass @@ -230,4 +230,4 @@ class CmdImportFile(CmdBase): if __name__ == '__main__': - Runtime.run(CmdDataImport) + Runtime.run(CmdImportFile) diff --git a/dvc/git_wrapper.py b/dvc/git_wrapper.py index 7295ee106..f466db785 100644 --- a/dvc/git_wrapper.py +++ b/dvc/git_wrapper.py @@ -70,6 +70,9 @@ class GitWrapperI(object): return result + def commit_all_changes_and_log_status(self, message): + pass + class GitWrapper(GitWrapperI): def __init__(self): diff --git a/tests/test_import.py b/tests/test_import.py new file mode 100644 index 000000000..fa3e95395 --- /dev/null +++ b/tests/test_import.py @@ -0,0 +1,117 @@ +import os +import tempfile +from copy import copy + +from dvc.command.import_file import CmdImportFile, ImportFileError +from tests.basic_env import DirHierarchyEnvironment + + +class TestCmdRemove(DirHierarchyEnvironment): + def setUp(self): + DirHierarchyEnvironment.init_environment(self) + + def test_import_file_to_dir(self): + with tempfile.NamedTemporaryFile() as temp: + content = 'Some data' + temp.write(content) + temp.flush() + + dir = os.path.join('data', self.dir1) + settings = copy(self.settings) + settings.set_args([temp.name, dir]) + cmd = CmdImportFile(settings) + cmd.run() + + dvc_name = os.path.join(dir, os.path.basename(temp.name)) + data_item = self.path_factory.existing_data_item(dvc_name) + + self.assertTrue(os.path.exists(data_item.data.relative)) + self.assertTrue(os.path.islink(data_item.data.relative)) + self.assertEqual(open(data_item.data.relative).read(), content) + + self.assertTrue(os.path.exists(data_item.cache.relative)) + self.assertEqual(open(data_item.cache.relative).read(), content) + + self.assertTrue(os.path.exists(data_item.state.relative)) + pass + + def test_import_file_to_file(self): + with tempfile.NamedTemporaryFile() as temp: + content = 'Some data' + temp.write(content) + temp.flush() + + target_file_name = 'targ.txt' + target_file = os.path.join('data', self.dir1, target_file_name) + settings = copy(self.settings) + settings.set_args([temp.name, target_file]) + cmd = CmdImportFile(settings) + cmd.run() + + self.assertTrue(os.path.exists(target_file)) + self.assertTrue(open(target_file).read(), content) + pass + + def test_import_to_existing_dir(self): + with tempfile.NamedTemporaryFile() as temp: + content = 'Some data' + temp.write(content) + temp.flush() + + dir = os.path.join('data', self.dir11) + settings = copy(self.settings) + settings.set_args([temp.name, dir]) + cmd = CmdImportFile(settings) + cmd.run() + + dvc_name = os.path.join(dir, os.path.basename(temp.name)) + + self.assertTrue(os.path.exists(dvc_name)) + self.assertTrue(open(dvc_name).read(), content) + pass + + def test_import_to_not_existing_dir(self): + with tempfile.NamedTemporaryFile() as temp: + content = 'Some data' + temp.write(content) + temp.flush() + + dir = os.path.join('data', self.dir1, 'not_existing_dir/') + settings = copy(self.settings) + settings.set_args([temp.name, dir]) + cmd = CmdImportFile(settings) + + with self.assertRaises(ImportFileError): + cmd.run() + + dvc_name = os.path.join(dir, os.path.basename(temp.name)) + + self.assertFalse(os.path.exists(dir)) + self.assertFalse(os.path.exists(dvc_name)) + pass + + def test_multiple_import(self): + with tempfile.NamedTemporaryFile() as temp1, tempfile.NamedTemporaryFile() as temp2: + content = 'Some data' + + temp1.write(content) + temp1.flush() + + temp2.write(content) + temp2.flush() + + dir = os.path.join('data', self.dir11) + settings = copy(self.settings) + settings.set_args([temp1.name, temp2.name, dir]) + cmd = CmdImportFile(settings) + cmd.run() + + dvc_name1 = os.path.join(dir, os.path.basename(temp1.name)) + dvc_name2 = os.path.join(dir, os.path.basename(temp2.name)) + + self.assertTrue(os.path.exists(dvc_name1)) + self.assertTrue(open(dvc_name1).read(), content) + + self.assertTrue(os.path.exists(dvc_name2)) + self.assertTrue(open(dvc_name2).read(), content) + pass \ No newline at end of file From 3f962deb253977dda089835bba01580662a1cec8 Mon Sep 17 00:00:00 2001 From: Dmitry Petrov Date: Mon, 19 Jun 2017 21:59:26 -0700 Subject: [PATCH 11/11] Fix: NamedTemporaryFile(mode=w) --- tests/test_import.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/test_import.py b/tests/test_import.py index fa3e95395..4b1cee6d8 100644 --- a/tests/test_import.py +++ b/tests/test_import.py @@ -11,7 +11,7 @@ class TestCmdRemove(DirHierarchyEnvironment): DirHierarchyEnvironment.init_environment(self) def test_import_file_to_dir(self): - with tempfile.NamedTemporaryFile() as temp: + with tempfile.NamedTemporaryFile(mode='w') as temp: content = 'Some data' temp.write(content) temp.flush() @@ -36,7 +36,7 @@ class TestCmdRemove(DirHierarchyEnvironment): pass def test_import_file_to_file(self): - with tempfile.NamedTemporaryFile() as temp: + with tempfile.NamedTemporaryFile(mode='w') as temp: content = 'Some data' temp.write(content) temp.flush() @@ -53,7 +53,7 @@ class TestCmdRemove(DirHierarchyEnvironment): pass def test_import_to_existing_dir(self): - with tempfile.NamedTemporaryFile() as temp: + with tempfile.NamedTemporaryFile(mode='w') as temp: content = 'Some data' temp.write(content) temp.flush() @@ -71,7 +71,7 @@ class TestCmdRemove(DirHierarchyEnvironment): pass def test_import_to_not_existing_dir(self): - with tempfile.NamedTemporaryFile() as temp: + with tempfile.NamedTemporaryFile(mode='w') as temp: content = 'Some data' temp.write(content) temp.flush() @@ -91,7 +91,8 @@ class TestCmdRemove(DirHierarchyEnvironment): pass def test_multiple_import(self): - with tempfile.NamedTemporaryFile() as temp1, tempfile.NamedTemporaryFile() as temp2: + with tempfile.NamedTemporaryFile(mode='w') as temp1,\ + tempfile.NamedTemporaryFile(mode='w') as temp2: content = 'Some data' temp1.write(content)