From a67988a80ef5c8c3db16ca01526fc129319586c9 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Thu, 6 Dec 2018 06:40:55 +0200 Subject: [PATCH] test: enable flake8 checks for tests Signed-off-by: Ruslan Kuprieiev --- .travis.yml | 1 + tests/__main__.py | 6 ++- tests/basic_env.py | 8 ++-- tests/test_add.py | 8 ++-- tests/test_analytics.py | 1 - tests/test_cache.py | 15 ++++--- tests/test_checkout.py | 8 ++-- tests/test_cli.py | 1 - tests/test_config.py | 2 +- tests/test_data_cloud.py | 9 +++-- tests/test_gc.py | 5 --- tests/test_import.py | 6 +-- tests/test_init.py | 3 +- tests/test_logger.py | 14 +++---- tests/test_metrics.py | 51 ++++++++++++++---------- tests/test_move.py | 11 ++++-- tests/test_pipeline.py | 42 ++++++++++++-------- tests/test_remote.py | 31 +++++++++------ tests/test_remove.py | 4 +- tests/test_repro.py | 85 ++++++++++++++++++++++++---------------- tests/test_run.py | 23 +++++------ tests/test_stage.py | 14 +++---- tests/test_state.py | 9 +---- tests/test_system.py | 2 +- tests/test_unprotect.py | 15 ++++--- tests/test_utils.py | 7 ++-- 26 files changed, 210 insertions(+), 171 deletions(-) diff --git a/.travis.yml b/.travis.yml index fe70758a1..f4e755587 100644 --- a/.travis.yml +++ b/.travis.yml @@ -85,6 +85,7 @@ before_script: - python setup.py checkdocs # stop the build if there are any flake8 errors - flake8 dvc + - flake8 tests script: - bash ./scripts/ci/script.sh deploy: diff --git a/tests/__main__.py b/tests/__main__.py index 1c15c89df..b49dfcb25 100644 --- a/tests/__main__.py +++ b/tests/__main__.py @@ -4,9 +4,11 @@ from subprocess import check_call REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) os.chdir(REPO_ROOT) -os.putenv('PATH', '{}:{}'.format(os.path.join(REPO_ROOT, 'bin'), os.getenv('PATH'))) +os.putenv('PATH', '{}:{}'.format(os.path.join(REPO_ROOT, 'bin'), + os.getenv('PATH'))) os.putenv('DVC_HOME', REPO_ROOT) os.putenv('DVC_TEST', 'true') -cmd = 'nosetests -v --processes=-1 --process-timeout=500 --cover-inclusive --cover-erase --cover-package=dvc --with-coverage' +cmd = 'nosetests -v --processes=-1 --process-timeout=500 --cover-inclusive ' \ + '--cover-erase --cover-package=dvc --with-coverage' check_call(cmd, shell=True) diff --git a/tests/basic_env.py b/tests/basic_env.py index 978dba7d9..cc9b3d513 100644 --- a/tests/basic_env.py +++ b/tests/basic_env.py @@ -1,6 +1,5 @@ import os import uuid -import shutil import tempfile from git import Repo from unittest import TestCase @@ -9,7 +8,9 @@ from dvc.project import Project class TestDir(TestCase): - GCP_CREDS_FILE = os.path.abspath(os.path.join('scripts', 'ci', 'gcp-creds.json')) + GCP_CREDS_FILE = os.path.abspath(os.path.join('scripts', + 'ci', + 'gcp-creds.json')) DATA_DIR = 'data_dir' DATA_SUB_DIR = os.path.join(DATA_DIR, 'data_sub_dir') DATA = os.path.join(DATA_DIR, 'data') @@ -21,7 +22,8 @@ class TestDir(TestCase): BAR = 'bar' BAR_CONTENTS = BAR CODE = 'code.py' - CODE_CONTENTS = 'import sys\nimport shutil\nshutil.copyfile(sys.argv[1], sys.argv[2])' + CODE_CONTENTS = ('import sys\nimport shutil\n' + 'shutil.copyfile(sys.argv[1], sys.argv[2])') def _pushd(self, d): self._saved_dir = os.path.realpath(os.curdir) diff --git a/tests/test_add.py b/tests/test_add.py index d41d845e0..2c898a626 100644 --- a/tests/test_add.py +++ b/tests/test_add.py @@ -1,5 +1,4 @@ import os -import stat import time import shutil import filecmp @@ -9,8 +8,6 @@ from dvc.utils import file_md5 from dvc.stage import Stage from dvc.exceptions import DvcException from dvc.output.base import OutputAlreadyTrackedError -from dvc.output.base import OutputDoesNotExistError, OutputIsNotFileOrDirError -from dvc.command.add import CmdAdd from tests.basic_env import TestDvc @@ -34,7 +31,7 @@ class TestAdd(TestDvc): class TestAddUnupportedFile(TestDvc): def test(self): - with self.assertRaises(DvcException) as cx: + with self.assertRaises(DvcException): self.dvc.add('unsupported://unsupported') @@ -82,6 +79,7 @@ class TestAddDirectoryWithForwardSlash(TestDvc): self.assertTrue(stage is not None) self.assertEquals(os.path.abspath('directory.dvc'), stage.path) + class TestAddTrackedFile(TestDvc): def test(self): fname = 'tracked_file' @@ -89,7 +87,7 @@ class TestAddTrackedFile(TestDvc): self.dvc.scm.add([fname]) self.dvc.scm.commit('add {}'.format(fname)) - with self.assertRaises(OutputAlreadyTrackedError) as cx: + with self.assertRaises(OutputAlreadyTrackedError): self.dvc.add(fname) diff --git a/tests/test_analytics.py b/tests/test_analytics.py index 9d5c37b9f..a0666a5f7 100644 --- a/tests/test_analytics.py +++ b/tests/test_analytics.py @@ -1,7 +1,6 @@ import os import mock import requests -import tempfile from dvc.main import main from dvc.analytics import Analytics diff --git a/tests/test_cache.py b/tests/test_cache.py index 1705eedd3..7a9dd3a53 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -2,9 +2,7 @@ import os import shutil from dvc.cache import Cache -from dvc.system import System from dvc.main import main -from dvc.remote.local import RemoteLOCAL from tests.basic_env import TestDvc, TestDir @@ -97,11 +95,16 @@ class TestSharedCacheDir(TestDir): self.assertFalse(os.path.exists(os.path.join('dir1', '.dvc', 'cache'))) self.assertFalse(os.path.exists(os.path.join('dir2', '.dvc', 'cache'))) - subdirs = list(filter(lambda x: os.path.isdir(os.path.join(cache_dir, x)), os.listdir(cache_dir))) + subdirs = list(filter( + lambda x: os.path.isdir(os.path.join(cache_dir, x)), + os.listdir(cache_dir))) self.assertEqual(len(subdirs), 3) - self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[0]))), 1) - self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[1]))), 1) - self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[2]))), 1) + self.assertEqual(len(os.listdir(os.path.join(cache_dir, + subdirs[0]))), 1) + self.assertEqual(len(os.listdir(os.path.join(cache_dir, + subdirs[1]))), 1) + self.assertEqual(len(os.listdir(os.path.join(cache_dir, + subdirs[2]))), 1) class TestCacheLinkType(TestDvc): diff --git a/tests/test_checkout.py b/tests/test_checkout.py index 3684263ed..8c8cac816 100644 --- a/tests/test_checkout.py +++ b/tests/test_checkout.py @@ -1,7 +1,6 @@ import os import yaml import time -import stat import shutil import filecmp import collections @@ -17,11 +16,11 @@ from dvc.exceptions import DvcException from mock import patch + class TestCheckout(TestRepro): def setUp(self): super(TestCheckout, self).setUp() - stages = self.dvc.add(self.DATA_DIR) self.assertEqual(len(stages), 1) self.data_dir_stage = stages[0] @@ -118,7 +117,8 @@ class CheckoutBase(TestDvc): self.dvc.scm.commit('adding ' + fname) def read_ignored(self): - return list(map(lambda s: s.strip('\n'), open(self.GIT_IGNORE).readlines())) + return list(map(lambda s: s.strip('\n'), + open(self.GIT_IGNORE).readlines())) def outs_info(self, stage): FileInfo = collections.namedtuple('FileInfo', 'path inode') @@ -135,6 +135,7 @@ class CheckoutBase(TestDvc): for path in paths ] + class TestRemoveFilesWhenCheckout(CheckoutBase): def test(self): fname = 'file_in_a_branch' @@ -342,6 +343,7 @@ class TestCheckoutNotCachedFile(TestDvc): stage = self.dvc.run(cmd=cmd, deps=[self.FOO, self.CODE], outs_no_cache=['out']) + self.assertTrue(stage is not None) self.dvc.checkout(force=True) diff --git a/tests/test_cli.py b/tests/test_cli.py index c31c58124..7d19b4783 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,7 +1,6 @@ import os from dvc.cli import parse_args -from dvc.main import main from dvc.command.init import CmdInit from dvc.command.run import CmdRun from dvc.command.data_sync import CmdDataPull, CmdDataPush diff --git a/tests/test_config.py b/tests/test_config.py index 78be9edca..8d2c02bcf 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -67,7 +67,7 @@ class TestConfigCLI(TestDvc): self._do_test(True) def test_non_existing(self): - #FIXME check stdout/err + # FIXME check stdout/err ret = main(['config', 'non_existing_section.field']) self.assertEqual(ret, 1) diff --git a/tests/test_data_cloud.py b/tests/test_data_cloud.py index b54955fc1..35149a324 100644 --- a/tests/test_data_cloud.py +++ b/tests/test_data_cloud.py @@ -89,7 +89,9 @@ def _should_test_hdfs(): return False try: - check_output(['hadoop', 'version'], shell=True, executable=os.getenv('SHELL')) + check_output(['hadoop', 'version'], + shell=True, + executable=os.getenv('SHELL')) except (CalledProcessError, IOError): return False @@ -421,8 +423,9 @@ class TestDataCloudCLIBase(TestDvc): # NOTE: check if remote gc works correctly on directories self.main(['gc', '-c', '-f'] + args) - shutil.move(self.dvc.cache.local.cache_dir, self.dvc.cache.local.cache_dir + '.back') - + shutil.move(self.dvc.cache.local.cache_dir, + self.dvc.cache.local.cache_dir + '.back') + sleep() self.main(['fetch'] + args) diff --git a/tests/test_gc.py b/tests/test_gc.py index 4467f0986..bb8c961e7 100644 --- a/tests/test_gc.py +++ b/tests/test_gc.py @@ -1,15 +1,10 @@ import os -import shutil -import filecmp import configobj from git import Repo from dvc.main import main -from dvc.utils import file_md5 -from dvc.stage import Stage from dvc.project import Project -from dvc.command.gc import CmdGC from tests.basic_env import TestDvc from tests.basic_env import TestDir diff --git a/tests/test_import.py b/tests/test_import.py index 58b036199..c1e6cca7b 100644 --- a/tests/test_import.py +++ b/tests/test_import.py @@ -1,12 +1,7 @@ import os -import stat -import shutil -import tempfile from uuid import uuid4 from dvc.main import main -from dvc.command.imp import CmdImport -from dvc.exceptions import DvcException from tests.basic_env import TestDvc @@ -24,6 +19,7 @@ class TestCmdImport(TestDvc): ret = main(['import', 'unsupported://path', 'import_unsupported']) self.assertNotEqual(ret, 0) + class TestDefaultOutput(TestDvc): def test(self): tmpdir = self.mkdtemp() diff --git a/tests/test_init.py b/tests/test_init.py index ff2f1a021..8109deedd 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -1,7 +1,6 @@ import os from dvc.main import main -from dvc.command.init import CmdInit from dvc.project import Project, InitError from tests.basic_env import TestGit, TestDir @@ -14,7 +13,7 @@ class TestInit(TestGit): def test_api(self): Project.init() - + self._test_init() def test_cli(self): diff --git a/tests/test_logger.py b/tests/test_logger.py index 237f93792..c4865f6bd 100644 --- a/tests/test_logger.py +++ b/tests/test_logger.py @@ -5,13 +5,10 @@ try: except ImportError: from io import StringIO -import colorama import logging from dvc.command.base import CmdBase -from dvc.config import Config from dvc.logger import Logger -from dvc.main import main from tests.basic_env import TestDvc @@ -52,7 +49,6 @@ class TestLogger(TestDvc): Logger.init() self.assertEqual(len(Logger.logger().handlers), 2) - @patch('dvc.logger.Logger._already_initialized', return_value=False) @patch('sys.stderr', new_callable=StringIO) @patch('sys.stdout', new_callable=StringIO) @@ -62,9 +58,9 @@ class TestLogger(TestDvc): Logger.error(error_message) self.assertEqual('', mock_stdout.getvalue()) self.assertEqual('Error: {}\n\nHaving any troubles? ' - 'Hit us up at dvc.org/support, we ' - 'are always happy to help!\n'.format(error_message), - mock_stderr.getvalue()) + 'Hit us up at dvc.org/support, we ' + 'are always happy to help!\n'.format(error_message), + mock_stderr.getvalue()) @patch('dvc.logger.Logger._already_initialized', return_value=False) @patch('sys.stderr', new_callable=StringIO) @@ -75,7 +71,7 @@ class TestLogger(TestDvc): Logger.info(non_error_message) self.assertEqual('', mock_stderr.getvalue()) self.assertEqual('{}\n'.format(non_error_message), - mock_stdout.getvalue()) + mock_stdout.getvalue()) class TestLoggerException(TestDvc): @@ -132,9 +128,11 @@ class TestLoggerException(TestDvc): class TestLoggerQuiet(TestDvc): def setUp(self): super(TestLoggerQuiet, self).setUp() + class A(object): quiet = True verbose = False + CmdBase._set_loglevel(A()) @patch('dvc.logger.Logger._already_initialized', return_value=False) diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 02a7475cf..99e22fc44 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -35,7 +35,12 @@ class TestMetrics(TestDvc): fd.write('branch\n') fd.write(branch) - self.dvc.scm.add(['metric', 'metric_json', 'metric_tsv', 'metric_htsv', 'metric_csv', 'metric_hcsv']) + self.dvc.scm.add(['metric', + 'metric_json', + 'metric_tsv', + 'metric_htsv', + 'metric_csv', + 'metric_hcsv']) self.dvc.scm.commit('metric') self.dvc.scm.checkout('master') @@ -96,7 +101,9 @@ class TestMetrics(TestDvc): class TestMetricsReproCLI(TestDvc): def test(self): stage = self.dvc.run(metrics_no_cache=['metrics'], - cmd='python {} {} {}'.format(self.CODE, self.FOO, 'metrics')) + cmd='python {} {} {}'.format(self.CODE, + self.FOO, + 'metrics')) ret = main(['repro', '-m', stage.path]) self.assertEqual(ret, 0) @@ -132,53 +139,57 @@ class TestMetricsReproCLI(TestDvc): class TestMetricsCLICompat(TestMetrics): def test(self): - #FIXME check output + # FIXME check output ret = main(['metrics', 'show', '--all-branches', 'metric', '-v']) self.assertEqual(ret, 0) - ret = main(['metrics', 'show', '--all-branches', 'metric_json', '--json-path', 'branch']) + ret = main(['metrics', 'show', '--all-branches', 'metric_json', + '--json-path', 'branch']) self.assertEqual(ret, 0) - ret = main(['metrics', 'show', '--all-branches', 'metric_tsv', '--tsv-path', '0,0']) + ret = main(['metrics', 'show', '--all-branches', 'metric_tsv', + '--tsv-path', '0,0']) self.assertEqual(ret, 0) - ret = main(['metrics', 'show', '--all-branches', 'metric_htsv', '--htsv-path', 'branch,0']) + ret = main(['metrics', 'show', '--all-branches', 'metric_htsv', + '--htsv-path', 'branch,0']) self.assertEqual(ret, 0) - ret = main(['metrics', 'show', '--all-branches', 'metric_csv', '--csv-path', '0,0']) + ret = main(['metrics', 'show', '--all-branches', 'metric_csv', + '--csv-path', '0,0']) self.assertEqual(ret, 0) - ret = main(['metrics', 'show', '--all-branches', 'metric_hcsv', '--hcsv-path', 'branch,0']) + ret = main(['metrics', 'show', '--all-branches', 'metric_hcsv', + '--hcsv-path', 'branch,0']) self.assertEqual(ret, 0) class TestMetricsCLI(TestMetrics): def test(self): - #FIXME check output + # FIXME check output ret = main(['metrics', 'show', '-a', 'metric', '-v']) self.assertEqual(ret, 0) - ret = main(['metrics', 'show', '-a', 'metric_json', '-t', 'json', - '-x', 'branch']) + ret = main(['metrics', 'show', '-a', 'metric_json', + '-t', 'json', '-x', 'branch']) self.assertEqual(ret, 0) - ret = main(['metrics', 'show', '-a', 'metric_tsv', '-t', 'tsv', - '-x', '0,0']) + ret = main(['metrics', 'show', '-a', 'metric_tsv', + '-t', 'tsv', '-x', '0,0']) self.assertEqual(ret, 0) - ret = main(['metrics', 'show', '-a', 'metric_htsv', '-t', 'htsv', - '-x', 'branch,0']) + ret = main(['metrics', 'show', '-a', 'metric_htsv', + '-t', 'htsv', '-x', 'branch,0']) self.assertEqual(ret, 0) - ret = main(['metrics', 'show', '-a', 'metric_csv', '-t', 'csv', - '-x', '0,0']) + ret = main(['metrics', 'show', '-a', 'metric_csv', + '-t', 'csv', '-x', '0,0']) self.assertEqual(ret, 0) - ret = main(['metrics', 'show', '-a', 'metric_hcsv', '-t', 'hcsv', - '-x', 'branch,0']) + ret = main(['metrics', 'show', '-a', 'metric_hcsv', + '-t', 'hcsv', '-x', 'branch,0']) self.assertEqual(ret, 0) - def test_dir(self): os.mkdir('metrics_dir') diff --git a/tests/test_move.py b/tests/test_move.py index d12440068..9e719691b 100644 --- a/tests/test_move.py +++ b/tests/test_move.py @@ -19,14 +19,16 @@ class TestMove(TestDvc): class TestMoveNonExistentFile(TestDvc): def test(self): - with self.assertRaises(DvcException) as cx: + with self.assertRaises(DvcException): self.dvc.move('non_existent_file', 'dst') class TestMoveDirectory(TestDvc): def test(self): dst = 'dst' - stage_add = self.dvc.add(self.DATA_DIR) + stages = self.dvc.add(self.DATA_DIR) + self.assertEqual(len(stages), 1) + self.assertTrue(stages[0] is not None) self.dvc.move(self.DATA_DIR, dst) self.assertFalse(os.path.exists(self.DATA_DIR)) self.assertTrue(os.path.exists(dst)) @@ -34,7 +36,10 @@ class TestMoveDirectory(TestDvc): class TestCmdMove(TestDvc): def test(self): - stage = self.dvc.add(self.FOO) + stages = self.dvc.add(self.FOO) + self.assertEqual(len(stages), 1) + self.assertTrue(stages[0] is not None) + ret = main(['move', self.FOO, self.FOO + '1']) self.assertEqual(ret, 0) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 7fe287298..affdeb373 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -19,11 +19,11 @@ class TestPipelineShowSingle(TestDvc): def test_commands(self): ret = main(['pipeline', 'show', self.stage, '--commands']) - self.assertEqual(ret, 0) + self.assertEqual(ret, 0) def test_outs(self): ret = main(['pipeline', 'show', self.stage, '--outs']) - self.assertEqual(ret, 0) + self.assertEqual(ret, 0) def test_ascii(self): ret = main(['pipeline', 'show', '--ascii', self.stage]) @@ -43,12 +43,14 @@ class TestPipelineShowSingle(TestDvc): self.assertEqual(ret, 0) def test_dot_commands(self): - ret = main(['pipeline', 'show', '--dot', self.dotFile, self.stage, '--commands']) + ret = main(['pipeline', 'show', '--dot', self.dotFile, self.stage, + '--commands']) self.assertEqual(ret, 0) self.assertTrue(os.path.isfile(self.dotFile)) def test_dot_outs(self): - ret = main(['pipeline', 'show', '--dot', self.dotFile, self.stage, '--outs']) + ret = main(['pipeline', 'show', '--dot', self.dotFile, self.stage, + '--outs']) self.assertEqual(ret, 0) self.assertTrue(os.path.isfile(self.dotFile)) @@ -72,23 +74,25 @@ class TestPipelineShow(TestRepro): def test_commands(self): ret = main(['pipeline', 'show', self.file1_stage, '--commands']) - self.assertEqual(ret, 0) + self.assertEqual(ret, 0) def test_outs(self): ret = main(['pipeline', 'show', self.file1_stage, '--outs']) - self.assertEqual(ret, 0) + self.assertEqual(ret, 0) def test_ascii(self): ret = main(['pipeline', 'show', '--ascii', self.file1_stage]) self.assertEqual(ret, 0) def test_dot(self): - ret = main(['pipeline', 'show', '--dot', self.dotFile, self.file1_stage]) + ret = main(['pipeline', 'show', '--dot', self.dotFile, + self.file1_stage]) self.assertEqual(ret, 0) self.assertTrue(os.path.isfile(self.dotFile)) def test_ascii_commands(self): - ret = main(['pipeline', 'show', '--ascii', self.file1_stage, '--commands']) + ret = main(['pipeline', 'show', '--ascii', self.file1_stage, + '--commands']) self.assertEqual(ret, 0) def test_ascii_outs(self): @@ -96,12 +100,14 @@ class TestPipelineShow(TestRepro): self.assertEqual(ret, 0) def test_dot_commands(self): - ret = main(['pipeline', 'show', '--dot', self.dotFile, self.file1_stage, '--commands']) + ret = main(['pipeline', 'show', '--dot', self.dotFile, + self.file1_stage, '--commands']) self.assertEqual(ret, 0) self.assertTrue(os.path.isfile(self.dotFile)) def test_dot_outs(self): - ret = main(['pipeline', 'show', '--dot', self.dotFile, self.file1_stage, '--outs']) + ret = main(['pipeline', 'show', '--dot', self.dotFile, + self.file1_stage, '--outs']) self.assertEqual(ret, 0) self.assertTrue(os.path.isfile(self.dotFile)) @@ -125,23 +131,25 @@ class TestPipelineShowDeep(TestReproChangedDeepData): def test_commands(self): ret = main(['pipeline', 'show', self.file1_stage, '--commands']) - self.assertEqual(ret, 0) + self.assertEqual(ret, 0) def test_outs(self): ret = main(['pipeline', 'show', self.file1_stage, '--outs']) - self.assertEqual(ret, 0) + self.assertEqual(ret, 0) def test_ascii(self): ret = main(['pipeline', 'show', '--ascii', self.file1_stage]) self.assertEqual(ret, 0) def test_dot(self): - ret = main(['pipeline', 'show', '--dot', self.dotFile, self.file1_stage]) + ret = main(['pipeline', 'show', '--dot', self.dotFile, + self.file1_stage]) self.assertEqual(ret, 0) self.assertTrue(os.path.isfile(self.dotFile)) def test_ascii_commands(self): - ret = main(['pipeline', 'show', '--ascii', self.file1_stage, '--commands']) + ret = main(['pipeline', 'show', '--ascii', self.file1_stage, + '--commands']) self.assertEqual(ret, 0) def test_ascii_outs(self): @@ -149,12 +157,14 @@ class TestPipelineShowDeep(TestReproChangedDeepData): self.assertEqual(ret, 0) def test_dot_commands(self): - ret = main(['pipeline', 'show', '--dot', self.dotFile, self.file1_stage, '--commands']) + ret = main(['pipeline', 'show', '--dot', self.dotFile, + self.file1_stage, '--commands']) self.assertEqual(ret, 0) self.assertTrue(os.path.isfile(self.dotFile)) def test_dot_outs(self): - ret = main(['pipeline', 'show', '--dot', self.dotFile, self.file1_stage, '--outs']) + ret = main(['pipeline', 'show', '--dot', self.dotFile, + self.file1_stage, '--outs']) self.assertEqual(ret, 0) self.assertTrue(os.path.isfile(self.dotFile)) diff --git a/tests/test_remote.py b/tests/test_remote.py index 2e1d57ccd..ad479fbe7 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -2,7 +2,6 @@ import os import configobj from dvc.main import main -from dvc.project import Project from dvc.command.remote import CmdRemoteAdd from dvc.exceptions import UnsupportedRemoteError from dvc.remote import Remote @@ -19,12 +18,14 @@ class TestRemote(TestDvc): self.assertNotEqual(main(['remote', 'remove', remotes[0]]), 0) for r in remotes: - self.assertEqual(main(['remote', 'add', '--default', r, 's3://bucket/name']), 0) + self.assertEqual(main(['remote', 'add', '--default', r, + 's3://bucket/name']), 0) self.assertEqual(main(['remote', 'list']), 0) self.assertEqual(main(['remote', 'remove', remotes[0]]), 0) - self.assertEqual(main(['remote', 'modify', remotes[0], 'option', 'value']), 0) + self.assertEqual(main(['remote', 'modify', remotes[0], + 'option', 'value']), 0) self.assertEqual(main(['remote', 'list']), 0) @@ -52,18 +53,26 @@ class TestRemote(TestDvc): class TestRemoteRemoveDefault(TestDvc): def test(self): remote = 'mys3' - self.assertEqual(main(['remote', 'add', '--default', remote, 's3://bucket/name']), 0) - self.assertEqual(main(['remote', 'modify', remote, 'profile', 'default']), 0) + self.assertEqual(main(['remote', 'add', '--default', remote, + 's3://bucket/name']), 0) + self.assertEqual(main(['remote', 'modify', remote, 'profile', + 'default']), 0) self.assertEqual(main(['config', '--local', 'core.remote', remote]), 0) - config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir, Config.CONFIG)) - local_config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir, Config.CONFIG_LOCAL)) - self.assertEqual(config[Config.SECTION_CORE][Config.SECTION_CORE_REMOTE], remote) - self.assertEqual(local_config[Config.SECTION_CORE][Config.SECTION_CORE_REMOTE], remote) + config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir, + Config.CONFIG)) + local_config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir, + Config.CONFIG_LOCAL)) + self.assertEqual(config[Config.SECTION_CORE] + [Config.SECTION_CORE_REMOTE], remote) + self.assertEqual(local_config[Config.SECTION_CORE] + [Config.SECTION_CORE_REMOTE], remote) self.assertEqual(main(['remote', 'remove', remote]), 0) - config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir, Config.CONFIG)) - local_config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir, Config.CONFIG_LOCAL)) + config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir, + Config.CONFIG)) + local_config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir, + Config.CONFIG_LOCAL)) section = Config.SECTION_REMOTE_FMT.format(remote) self.assertTrue(section not in config.keys()) diff --git a/tests/test_remove.py b/tests/test_remove.py index 66729f15e..9b17c3544 100644 --- a/tests/test_remove.py +++ b/tests/test_remove.py @@ -3,7 +3,6 @@ import os from dvc.main import main from dvc.stage import Stage, StageFileDoesNotExistError from dvc.exceptions import DvcException -from dvc.command.remove import CmdRemove from tests.basic_env import TestDvc from mock import patch @@ -30,7 +29,7 @@ class TestRemove(TestDvc): class TestRemoveNonExistentFile(TestDvc): def test(self): - with self.assertRaises(StageFileDoesNotExistError) as cx: + with self.assertRaises(StageFileDoesNotExistError): self.dvc.remove('non_existent_dvc_file') @@ -60,6 +59,7 @@ class TestCmdRemove(TestDvc): 'non-existing-dvc-file']) self.assertNotEqual(ret, 0) + class TestRemovePurge(TestDvc): def test(self): dvcfile = self.dvc.add(self.FOO)[0].path diff --git a/tests/test_repro.py b/tests/test_repro.py index 0b8e8544e..29d227cdd 100644 --- a/tests/test_repro.py +++ b/tests/test_repro.py @@ -1,6 +1,5 @@ import os import yaml -import stat import shutil import filecmp import getpass @@ -17,7 +16,6 @@ import uuid from google.cloud import storage as gc from dvc.main import main -from dvc.command.repro import CmdRepro from dvc.project import Project, ReproductionError from dvc.utils import file_md5 from dvc.remote.local import RemoteLOCAL @@ -46,7 +44,9 @@ class TestRepro(TestDvc): self.dvc.run(fname=self.file1_stage, outs=[self.file1], deps=[self.FOO, self.CODE], - cmd='python {} {} {}'.format(self.CODE, self.FOO, self.file1)) + cmd='python {} {} {}'.format(self.CODE, + self.FOO, + self.file1)) class TestReproFail(TestRepro): @@ -71,7 +71,9 @@ class TestReproDepUnderDir(TestDvc): self.dvc.run(fname=self.file1_stage, outs=[self.file1], deps=[self.DATA, self.CODE], - cmd='python {} {} {}'.format(self.CODE, self.DATA, self.file1)) + cmd='python {} {} {}'.format(self.CODE, + self.DATA, + self.file1)) self.assertTrue(filecmp.cmp(self.file1, self.DATA, shallow=False)) @@ -131,7 +133,8 @@ class TestReproNoDeps(TestRepro): out = 'out' code_file = 'out.py' stage_file = 'out.dvc' - code = 'import uuid\nwith open("{}", "w+") as fd:\n\tfd.write(str(uuid.uuid4()))\n'.format(out) + code = 'import uuid\nwith open("{}", "w+") as fd:\n' \ + '\tfd.write(str(uuid.uuid4()))\n'.format(out) with open(code_file, 'w+') as fd: fd.write(code) self.dvc.run(fname=stage_file, @@ -160,7 +163,8 @@ class TestReproChangedCode(TestRepro): def swap_code(self): os.unlink(self.CODE) new_contents = self.CODE_CONTENTS - new_contents += "\nshutil.copyfile('{}', sys.argv[2])\n".format(self.BAR) + new_contents += "\nshutil.copyfile('{}', " \ + "sys.argv[2])\n".format(self.BAR) self.create(self.CODE, new_contents) @@ -214,11 +218,6 @@ class TestReproDryNoExec(TestDvc): with open(f, 'w+') as fobj: fobj.write(str(d)) - if os.name == 'nt': - cp = 'copy' - else: - cp = 'cp' - ret = main(['run', '--no-exec', '-d', idir, @@ -288,15 +287,18 @@ class TestReproPipelines(TestDvc): self.dvc.run(fname=self.file1_stage, outs=[self.file1], deps=[self.FOO, self.CODE], - cmd='python {} {} {}'.format(self.CODE, self.FOO, self.file1)) + cmd='python {} {} {}'.format(self.CODE, + self.FOO, + self.file1)) self.file2 = 'file2' self.file2_stage = self.file2 + '.dvc' self.dvc.run(fname=self.file2_stage, outs=[self.file2], deps=[self.BAR, self.CODE], - cmd='python {} {} {}'.format(self.CODE, self.BAR, self.file2)) - + cmd='python {} {} {}'.format(self.CODE, + self.BAR, + self.file2)) def test(self): stages = self.dvc.reproduce(all_pipelines=True, force=True) @@ -319,7 +321,9 @@ class TestReproLocked(TestReproChangedData): self.dvc.run(fname=file2_stage, outs=[file2], deps=[self.file1, self.CODE], - cmd='python {} {} {}'.format(self.CODE, self.file1, file2)) + cmd='python {} {} {}'.format(self.CODE, + self.file1, + file2)) self.swap_foo_with_bar() @@ -368,6 +372,7 @@ class TestReproLockedCallback(TestDvc): stages = self.dvc.reproduce(file1_stage) self.assertEqual(len(stages), 1) + class TestReproLockedUnchanged(TestRepro): def test(self): """ @@ -425,7 +430,7 @@ class TestNonExistingOutput(TestRepro): def test(self): os.unlink(self.FOO) - with self.assertRaises(ReproductionError) as cx: + with self.assertRaises(ReproductionError): self.dvc.reproduce(self.file1_stage) @@ -447,9 +452,11 @@ class TestReproChangedDir(TestDvc): stage_name = 'dir.dvc' dir_name = 'dir' dir_code = 'dir.py' + code = "import os; import shutil; os.mkdir(\"{}\"); " \ + "shutil.copyfile(\"{}\", os.path.join(\"{}\", \"{}\"))" with open(dir_code, 'w+') as fd: - fd.write("import os; import shutil; os.mkdir(\"{}\"); shutil.copyfile(\"{}\", os.path.join(\"{}\", \"{}\"))".format(dir_name, file_name, dir_name, file_name)) + fd.write(code.format(dir_name, file_name, dir_name, file_name)) self.dvc.run(fname=stage_name, outs=[dir_name], @@ -474,7 +481,8 @@ class TestReproChangedDirData(TestDvc): sleep() with open(dir_code, 'w+') as fd: - fd.write("import os; import sys; import shutil; shutil.copytree(sys.argv[1], sys.argv[2])") + fd.write("import os; import sys; import shutil; " + "shutil.copytree(sys.argv[1], sys.argv[2])") sleep() @@ -557,11 +565,11 @@ class TestCmdReproChdir(TestDvc): shutil.copyfile(self.CODE, code) ret = main(['run', - '-f', 'Dvcfile', - '-c', dname, - '-d', self.FOO, - '-o', self.BAR, - 'python {} {} {}'.format(self.CODE, self.FOO, self.BAR)]) + '-f', 'Dvcfile', + '-c', dname, + '-d', self.FOO, + '-o', self.BAR, + 'python {} {} {}'.format(self.CODE, self.FOO, self.BAR)]) self.assertEqual(ret, 0) self.assertTrue(os.path.isfile(foo)) self.assertTrue(os.path.isfile(bar)) @@ -601,7 +609,8 @@ class TestReproExternalBase(TestDvc): if not self.should_test(): return - cache = self.scheme + self.scheme_sep + self.bucket + self.sep + str(uuid.uuid4()) + cache = (self.scheme + self.scheme_sep + self.bucket + self.sep + + str(uuid.uuid4())) ret = main(['config', 'cache.' + self.cache_scheme, 'myrepo']) self.assertEqual(ret, 0) @@ -610,7 +619,8 @@ class TestReproExternalBase(TestDvc): remote_name = 'myremote' remote_key = str(uuid.uuid4()) - remote = self.scheme + self.scheme_sep + self.bucket + self.sep + remote_key + remote = (self.scheme + self.scheme_sep + self.bucket + self.sep + + remote_key) ret = main(['remote', 'add', remote_name, remote]) self.assertEqual(ret, 0) @@ -621,8 +631,10 @@ class TestReproExternalBase(TestDvc): foo_key = remote_key + self.sep + self.FOO bar_key = remote_key + self.sep + self.BAR - foo_path = self.scheme + self.scheme_sep + self.bucket + self.sep + foo_key - bar_path = self.scheme + self.scheme_sep + self.bucket + self.sep + bar_key + foo_path = (self.scheme + self.scheme_sep + self.bucket + self.sep + + foo_key) + bar_path = (self.scheme + self.scheme_sep + self.bucket + self.sep + + bar_key) # Using both plain and remote notation out_foo_path = 'remote://' + remote_name + '/' + self.FOO @@ -637,12 +649,13 @@ class TestReproExternalBase(TestDvc): self.assertTrue(filecmp.cmp('import', self.FOO, shallow=False)) self.assertEqual(self.dvc.status(import_stage.path), {}) - import_remote_stage = self.dvc.imp(out_foo_path, out_foo_path + '_imported') + import_remote_stage = self.dvc.imp(out_foo_path, + out_foo_path + '_imported') self.assertEqual(self.dvc.status(import_remote_stage.path), {}) cmd_stage = self.dvc.run(outs=[out_bar_path], - deps=[out_foo_path], - cmd=self.cmd(foo_path, bar_path)) + deps=[out_foo_path], + cmd=self.cmd(foo_path, bar_path)) self.assertEqual(self.dvc.status(cmd_stage.path), {}) self.assertEqual(self.dvc.status(), {}) @@ -816,7 +829,9 @@ class TestReproExternalSSH(TestReproExternalBase): print(err) self.assertEqual(p.returncode, 0) - p = Popen('echo "{}" | ssh {} "tr -d \'\n\' > {}"'.format(body, dest, path), + p = Popen('echo "{}" | ssh {} "tr -d \'\n\' > {}"'.format(body, + dest, + path), shell=True, executable=os.getenv('SHELL'), stdin=PIPE, @@ -904,7 +919,7 @@ class TestReproExternalHTTP(TestReproExternalBase): self.dvc = Project('.') # Import - with StaticFileServer() as server: + with StaticFileServer(): import_url = urljoin(self.remote, self.FOO) import_output = 'imported_file' import_stage = self.dvc.imp(import_url, import_output) @@ -913,7 +928,7 @@ class TestReproExternalHTTP(TestReproExternalBase): self.assertTrue(filecmp.cmp(import_output, self.FOO, shallow=False)) # Run --deps - with StaticFileServer() as server: + with StaticFileServer(): run_dependency = urljoin(self.remote, self.BAR) run_output = 'remote_file' cmd = 'open("{}", "w+")'.format(run_output) @@ -924,6 +939,7 @@ class TestReproExternalHTTP(TestReproExternalBase): run_stage = self.dvc.run(deps=[run_dependency], outs=[run_output], cmd='python create-output.py') + self.assertTrue(run_stage is not None) self.assertTrue(os.path.exists(run_output)) @@ -934,7 +950,7 @@ class TestReproExternalHTTP(TestReproExternalBase): shutil.move(self.local_cache, self.external_cache_id) self.assertFalse(os.path.exists(self.local_cache)) - with StaticFileServer() as server: + with StaticFileServer(): self.dvc.pull(import_stage.path, remote='mycache') self.assertTrue(os.path.exists(import_output)) @@ -962,6 +978,7 @@ class TestReproShell(TestDvc): with open(fname, 'r') as fd: self.assertEqual(os.getenv('SHELL'), fd.read().strip()) + class TestReproNoSCM(TestRepro): def test(self): shutil.rmtree(self.dvc.scm.dir) diff --git a/tests/test_run.py b/tests/test_run.py index ebfde3486..4cf05012a 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -1,17 +1,14 @@ import os import mock -import time import shutil import filecmp import subprocess -from dvc.project import Project from dvc.main import main from dvc.utils import file_md5 from dvc.system import System -from dvc.stage import Stage, StageFileBadNameError, MissingDep +from dvc.stage import StageFileBadNameError, MissingDep from dvc.stage import StageBadCwdError, StageFileAlreadyExistsError -from dvc.command.run import CmdRun from dvc.exceptions import (OutputDuplicationError, CircularDependencyError, WorkingDirectoryAsOutputError) @@ -42,7 +39,7 @@ class TestRun(TestDvc): self.assertEqual(len(stage.deps), len(deps)) self.assertEqual(len(stage.outs), len(outs + outs_no_cache)) self.assertEqual(stage.outs[0].path, outs[0]) - self.assertEqual(stage.outs[0].md5, file_md5(self.FOO)[0]) + self.assertEqual(stage.outs[0].md5, file_md5(self.FOO)[0]) self.assertTrue(stage.path, fname) with self.assertRaises(OutputDuplicationError): @@ -94,7 +91,6 @@ class TestRunBadStageFilename(TestDvc): cwd=os.curdir) - class TestRunNoExec(TestDvc): def test(self): self.dvc.run(cmd='python {} {} {}'.format(self.CODE, self.FOO, 'out'), @@ -165,7 +161,8 @@ class TestCmdRun(TestDvc): def test_run_args_with_spaces(self): with open(self.CODE, 'w') as fobj: - fobj.write("import sys\nopen(sys.argv[1], 'w+').write(sys.argv[2])") + fobj.write( + "import sys\nopen(sys.argv[1], 'w+').write(sys.argv[2])") arg = 'arg1 arg2' log = 'log' @@ -412,28 +409,28 @@ class TestRunDeterministicChangedDep(TestRunDeterministicBase): os.unlink(self.FOO) shutil.copy(self.BAR, self.FOO) with self.assertRaises(StageFileAlreadyExistsError): - self._run() + self._run() class TestRunDeterministicChangedDepsList(TestRunDeterministicBase): def test(self): self.deps = [self.BAR, self.CODE] with self.assertRaises(StageFileAlreadyExistsError): - self._run() + self._run() class TestRunDeterministicNewDep(TestRunDeterministicBase): def test(self): self.deps = [self.FOO, self.BAR, self.CODE] with self.assertRaises(StageFileAlreadyExistsError): - self._run() + self._run() class TestRunDeterministicRemoveDep(TestRunDeterministicBase): def test(self): self.deps = [self.CODE] with self.assertRaises(StageFileAlreadyExistsError): - self._run() + self._run() class TestRunDeterministicChangedOut(TestRunDeterministicBase): @@ -441,11 +438,11 @@ class TestRunDeterministicChangedOut(TestRunDeterministicBase): os.unlink(self.out_file) self.out_file_mtime = None with self.assertRaises(StageFileAlreadyExistsError): - self._run() + self._run() class TestRunDeterministicChangedCmd(TestRunDeterministicBase): def test(self): self.cmd += ' arg' with self.assertRaises(StageFileAlreadyExistsError): - self._run() + self._run() diff --git a/tests/test_stage.py b/tests/test_stage.py index 0daae8eff..f439641c1 100644 --- a/tests/test_stage.py +++ b/tests/test_stage.py @@ -52,15 +52,15 @@ class TestSchemaDepsOuts(TestSchema): Stage.validate(d) def test_list(self): - l = [{OutputLOCAL.PARAM_PATH: 'foo', RemoteLOCAL.PARAM_MD5: '123'}, - {OutputLOCAL.PARAM_PATH: 'bar', RemoteLOCAL.PARAM_MD5: None}, - {OutputLOCAL.PARAM_PATH: 'baz'}] - d = {Stage.PARAM_DEPS: l} + lst = [{OutputLOCAL.PARAM_PATH: 'foo', RemoteLOCAL.PARAM_MD5: '123'}, + {OutputLOCAL.PARAM_PATH: 'bar', RemoteLOCAL.PARAM_MD5: None}, + {OutputLOCAL.PARAM_PATH: 'baz'}] + d = {Stage.PARAM_DEPS: lst} Stage.validate(d) - l[0][OutputLOCAL.PARAM_CACHE] = True - l[1][OutputLOCAL.PARAM_CACHE] = False - d = {Stage.PARAM_OUTS: l} + lst[0][OutputLOCAL.PARAM_CACHE] = True + lst[1][OutputLOCAL.PARAM_CACHE] = False + d = {Stage.PARAM_OUTS: lst} Stage.validate(d) diff --git a/tests/test_state.py b/tests/test_state.py index 94e90c19d..212464f25 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -1,8 +1,6 @@ import os -import stat import time -from dvc.system import System from dvc.state import State from dvc.utils import file_md5 from dvc.main import main @@ -14,8 +12,6 @@ class TestState(TestDvc): def test_update(self): path = os.path.join(self.dvc.root_dir, self.FOO) md5 = file_md5(path)[0] - mtime = os.path.getmtime(path) - inode = System.inode(path) state = State(self.dvc, self.dvc.config._config) @@ -28,16 +24,13 @@ class TestState(TestDvc): # mtime(i.e. 1sec for HFS) that cause problems with # our 'state' system not being able to distinguish # files that were modified within that delta. - time.sleep(1) + time.sleep(1) os.unlink(path) with open(path, 'w+') as fd: fd.write('1') md5 = file_md5(path)[0] - mtime = os.path.getmtime(path) - inode = System.inode(path) - entry_md5 = state.update(path) self.assertEqual(entry_md5, md5) diff --git a/tests/test_system.py b/tests/test_system.py index 3a56bfc3a..ef47177a2 100644 --- a/tests/test_system.py +++ b/tests/test_system.py @@ -14,6 +14,6 @@ class TestSystem(TestDir): fd = os.open(locked, os.O_WRONLY | os.O_CREAT | os.O_EXCL) # should not raise WinError - info = System.getdirinfo(locked) + System.getdirinfo(locked) os.close(fd) diff --git a/tests/test_unprotect.py b/tests/test_unprotect.py index b62769357..d51990a60 100644 --- a/tests/test_unprotect.py +++ b/tests/test_unprotect.py @@ -2,7 +2,6 @@ import os import stat from dvc.main import main -from dvc.project import Project from tests.basic_env import TestDvc @@ -15,17 +14,17 @@ class TestUnprotect(TestDvc): ret = main(['add', self.FOO]) self.assertEqual(ret, 0) - self.assertNotEqual(os.stat(self.FOO).st_mode & (stat.S_IREAD| - stat.S_IRGRP| + self.assertNotEqual(os.stat(self.FOO).st_mode & (stat.S_IREAD | + stat.S_IRGRP | stat.S_IROTH), 0) - self.assertEqual(os.stat(self.FOO).st_mode & (stat.S_IWRITE| - stat.S_IWGRP| + self.assertEqual(os.stat(self.FOO).st_mode & (stat.S_IWRITE | + stat.S_IWGRP | stat.S_IWOTH), 0) ret = main(['unprotect', self.FOO]) self.assertEqual(ret, 0) - self.assertNotEqual(os.stat(self.FOO).st_mode & (stat.S_IREAD| - stat.S_IRGRP| - stat.S_IROTH| + self.assertNotEqual(os.stat(self.FOO).st_mode & (stat.S_IREAD | + stat.S_IRGRP | + stat.S_IROTH | stat.S_IWRITE), 0) diff --git a/tests/test_utils.py b/tests/test_utils.py index 9b503fbf2..929d19da5 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -21,7 +21,9 @@ class TestUtils(TestDvc): self.assertTrue(filecmp.cmp(src, dest, shallow=False)) utils.copyfile(src, dest_dir) - self.assertTrue(filecmp.cmp(src, '{}/{}'.format(dest_dir, src), shallow=False)) + self.assertTrue(filecmp.cmp(src, + '{}/{}'.format(dest_dir, src), + shallow=False)) shutil.rmtree(dest_dir) os.remove(src) @@ -41,13 +43,12 @@ class TestUtils(TestDvc): 'outs': [{'path': 'file1', 'metric': {'type': 'raw'}, 'cache': False, - 'md5':'acbd18db4cc2f85cedef654fccc4a4d8'}], + 'md5': 'acbd18db4cc2f85cedef654fccc4a4d8'}], 'deps': [{'path': 'foo', 'md5': 'acbd18db4cc2f85cedef654fccc4a4d8'}, {'path': 'code.py', 'md5': 'd05447644b89960913c7eee5fd776adb'}]} - md5 = '8b263fa05ede6c3145c164829be694b4' self.assertEqual(md5, utils.dict_md5(d, exclude=['metric', 'locked']))