Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-08 15:28:26 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1418 from efiop/master

Browse Source 
test: enable flake8 checks for tests
This commit is contained in:
Ruslan Kuprieiev
2018-12-06 07:07:19 +02:00
committed by GitHub
parent ea2caf9dfd a67988a80e
commit 8574f37217
26 changed files with 210 additions and 171 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.travis.yml
+1
View File
@@ -85,6 +85,7 @@ before_script:
- python setup.py checkdocs
# stop the build if there are any flake8 errors
- flake8 dvc
- flake8 tests
script:
- bash ./scripts/ci/script.sh
deploy:
tests/__main__.py
+4 -2
View File
@@ -4,9 +4,11 @@ from subprocess import check_call
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
os.chdir(REPO_ROOT)
os.putenv('PATH', '{}:{}'.format(os.path.join(REPO_ROOT, 'bin'), os.getenv('PATH')))
os.putenv('PATH', '{}:{}'.format(os.path.join(REPO_ROOT, 'bin'),
os.getenv('PATH')))
os.putenv('DVC_HOME', REPO_ROOT)
os.putenv('DVC_TEST', 'true')
cmd = 'nosetests -v --processes=-1 --process-timeout=500 --cover-inclusive --cover-erase --cover-package=dvc --with-coverage'
cmd = 'nosetests -v --processes=-1 --process-timeout=500 --cover-inclusive ' \
'--cover-erase --cover-package=dvc --with-coverage'
check_call(cmd, shell=True)
tests/basic_env.py
+5 -3
View File
@@ -1,6 +1,5 @@
import os
import uuid
import shutil
import tempfile
from git import Repo
from unittest import TestCase
@@ -9,7 +8,9 @@ from dvc.project import Project
class TestDir(TestCase):
GCP_CREDS_FILE = os.path.abspath(os.path.join('scripts', 'ci', 'gcp-creds.json'))
GCP_CREDS_FILE = os.path.abspath(os.path.join('scripts',
'ci',
'gcp-creds.json'))
DATA_DIR = 'data_dir'
DATA_SUB_DIR = os.path.join(DATA_DIR, 'data_sub_dir')
DATA = os.path.join(DATA_DIR, 'data')
@@ -21,7 +22,8 @@ class TestDir(TestCase):
BAR = 'bar'
BAR_CONTENTS = BAR
CODE = 'code.py'
CODE_CONTENTS = 'import sys\nimport shutil\nshutil.copyfile(sys.argv[1], sys.argv[2])'
CODE_CONTENTS = ('import sys\nimport shutil\n'
'shutil.copyfile(sys.argv[1], sys.argv[2])')
def _pushd(self, d):
self._saved_dir = os.path.realpath(os.curdir)
tests/test_add.py
+3 -5
View File
@@ -1,5 +1,4 @@
import os
import stat
import time
import shutil
import filecmp
@@ -9,8 +8,6 @@ from dvc.utils import file_md5
from dvc.stage import Stage
from dvc.exceptions import DvcException
from dvc.output.base import OutputAlreadyTrackedError
from dvc.output.base import OutputDoesNotExistError, OutputIsNotFileOrDirError
from dvc.command.add import CmdAdd
from tests.basic_env import TestDvc
@@ -34,7 +31,7 @@ class TestAdd(TestDvc):
class TestAddUnupportedFile(TestDvc):
def test(self):
with self.assertRaises(DvcException) as cx:
with self.assertRaises(DvcException):
self.dvc.add('unsupported://unsupported')
@@ -82,6 +79,7 @@ class TestAddDirectoryWithForwardSlash(TestDvc):
self.assertTrue(stage is not None)
self.assertEquals(os.path.abspath('directory.dvc'), stage.path)
class TestAddTrackedFile(TestDvc):
def test(self):
fname = 'tracked_file'
@@ -89,7 +87,7 @@ class TestAddTrackedFile(TestDvc):
self.dvc.scm.add([fname])
self.dvc.scm.commit('add {}'.format(fname))
with self.assertRaises(OutputAlreadyTrackedError) as cx:
with self.assertRaises(OutputAlreadyTrackedError):
self.dvc.add(fname)
tests/test_analytics.py
-1
View File
@@ -1,7 +1,6 @@
import os
import mock
import requests
import tempfile
from dvc.main import main
from dvc.analytics import Analytics
tests/test_cache.py
+9 -6
View File
@@ -2,9 +2,7 @@ import os
import shutil
from dvc.cache import Cache
from dvc.system import System
from dvc.main import main
from dvc.remote.local import RemoteLOCAL
from tests.basic_env import TestDvc, TestDir
@@ -97,11 +95,16 @@ class TestSharedCacheDir(TestDir):
self.assertFalse(os.path.exists(os.path.join('dir1', '.dvc', 'cache')))
self.assertFalse(os.path.exists(os.path.join('dir2', '.dvc', 'cache')))
subdirs = list(filter(lambda x: os.path.isdir(os.path.join(cache_dir, x)), os.listdir(cache_dir)))
subdirs = list(filter(
lambda x: os.path.isdir(os.path.join(cache_dir, x)),
os.listdir(cache_dir)))
self.assertEqual(len(subdirs), 3)
self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[0]))), 1)
self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[1]))), 1)
self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[2]))), 1)
self.assertEqual(len(os.listdir(os.path.join(cache_dir,
subdirs[0]))), 1)
self.assertEqual(len(os.listdir(os.path.join(cache_dir,
subdirs[1]))), 1)
self.assertEqual(len(os.listdir(os.path.join(cache_dir,
subdirs[2]))), 1)
class TestCacheLinkType(TestDvc):
tests/test_checkout.py
+5 -3
View File
@@ -1,7 +1,6 @@
import os
import yaml
import time
import stat
import shutil
import filecmp
import collections
@@ -17,11 +16,11 @@ from dvc.exceptions import DvcException
from mock import patch
class TestCheckout(TestRepro):
def setUp(self):
super(TestCheckout, self).setUp()
stages = self.dvc.add(self.DATA_DIR)
self.assertEqual(len(stages), 1)
self.data_dir_stage = stages[0]
@@ -118,7 +117,8 @@ class CheckoutBase(TestDvc):
self.dvc.scm.commit('adding ' + fname)
def read_ignored(self):
return list(map(lambda s: s.strip('\n'), open(self.GIT_IGNORE).readlines()))
return list(map(lambda s: s.strip('\n'),
open(self.GIT_IGNORE).readlines()))
def outs_info(self, stage):
FileInfo = collections.namedtuple('FileInfo', 'path inode')
@@ -135,6 +135,7 @@ class CheckoutBase(TestDvc):
for path in paths
]
class TestRemoveFilesWhenCheckout(CheckoutBase):
def test(self):
fname = 'file_in_a_branch'
@@ -342,6 +343,7 @@ class TestCheckoutNotCachedFile(TestDvc):
stage = self.dvc.run(cmd=cmd,
deps=[self.FOO, self.CODE],
outs_no_cache=['out'])
self.assertTrue(stage is not None)
self.dvc.checkout(force=True)
tests/test_cli.py
-1
View File
@@ -1,7 +1,6 @@
import os
from dvc.cli import parse_args
from dvc.main import main
from dvc.command.init import CmdInit
from dvc.command.run import CmdRun
from dvc.command.data_sync import CmdDataPull, CmdDataPush
tests/test_config.py
+1 -1
View File
@@ -67,7 +67,7 @@ class TestConfigCLI(TestDvc):
self._do_test(True)
def test_non_existing(self):
#FIXME check stdout/err
# FIXME check stdout/err
ret = main(['config', 'non_existing_section.field'])
self.assertEqual(ret, 1)
tests/test_data_cloud.py
+6 -3
View File
@@ -89,7 +89,9 @@ def _should_test_hdfs():
return False
try:
check_output(['hadoop', 'version'], shell=True, executable=os.getenv('SHELL'))
check_output(['hadoop', 'version'],
shell=True,
executable=os.getenv('SHELL'))
except (CalledProcessError, IOError):
return False
@@ -421,8 +423,9 @@ class TestDataCloudCLIBase(TestDvc):
# NOTE: check if remote gc works correctly on directories
self.main(['gc', '-c', '-f'] + args)
shutil.move(self.dvc.cache.local.cache_dir, self.dvc.cache.local.cache_dir + '.back')
shutil.move(self.dvc.cache.local.cache_dir,
self.dvc.cache.local.cache_dir + '.back')
sleep()
self.main(['fetch'] + args)
tests/test_gc.py
-5
View File
@@ -1,15 +1,10 @@
import os
import shutil
import filecmp
import configobj
from git import Repo
from dvc.main import main
from dvc.utils import file_md5
from dvc.stage import Stage
from dvc.project import Project
from dvc.command.gc import CmdGC
from tests.basic_env import TestDvc
from tests.basic_env import TestDir
tests/test_import.py
+1 -5
View File
@@ -1,12 +1,7 @@
import os
import stat
import shutil
import tempfile
from uuid import uuid4
from dvc.main import main
from dvc.command.imp import CmdImport
from dvc.exceptions import DvcException
from tests.basic_env import TestDvc
@@ -24,6 +19,7 @@ class TestCmdImport(TestDvc):
ret = main(['import', 'unsupported://path', 'import_unsupported'])
self.assertNotEqual(ret, 0)
class TestDefaultOutput(TestDvc):
def test(self):
tmpdir = self.mkdtemp()
tests/test_init.py
+1 -2
View File
@@ -1,7 +1,6 @@
import os
from dvc.main import main
from dvc.command.init import CmdInit
from dvc.project import Project, InitError
from tests.basic_env import TestGit, TestDir
@@ -14,7 +13,7 @@ class TestInit(TestGit):
def test_api(self):
Project.init()
self._test_init()
def test_cli(self):
tests/test_logger.py
+6 -8
View File
@@ -5,13 +5,10 @@ try:
except ImportError:
from io import StringIO
import colorama
import logging
from dvc.command.base import CmdBase
from dvc.config import Config
from dvc.logger import Logger
from dvc.main import main
from tests.basic_env import TestDvc
@@ -52,7 +49,6 @@ class TestLogger(TestDvc):
Logger.init()
self.assertEqual(len(Logger.logger().handlers), 2)
@patch('dvc.logger.Logger._already_initialized', return_value=False)
@patch('sys.stderr', new_callable=StringIO)
@patch('sys.stdout', new_callable=StringIO)
@@ -62,9 +58,9 @@ class TestLogger(TestDvc):
Logger.error(error_message)
self.assertEqual('', mock_stdout.getvalue())
self.assertEqual('Error: {}\n\nHaving any troubles? '
'Hit us up at dvc.org/support, we '
'are always happy to help!\n'.format(error_message),
mock_stderr.getvalue())
'Hit us up at dvc.org/support, we '
'are always happy to help!\n'.format(error_message),
mock_stderr.getvalue())
@patch('dvc.logger.Logger._already_initialized', return_value=False)
@patch('sys.stderr', new_callable=StringIO)
@@ -75,7 +71,7 @@ class TestLogger(TestDvc):
Logger.info(non_error_message)
self.assertEqual('', mock_stderr.getvalue())
self.assertEqual('{}\n'.format(non_error_message),
mock_stdout.getvalue())
mock_stdout.getvalue())
class TestLoggerException(TestDvc):
@@ -132,9 +128,11 @@ class TestLoggerException(TestDvc):
class TestLoggerQuiet(TestDvc):
def setUp(self):
super(TestLoggerQuiet, self).setUp()
class A(object):
quiet = True
verbose = False
CmdBase._set_loglevel(A())
@patch('dvc.logger.Logger._already_initialized', return_value=False)
tests/test_metrics.py
+31 -20
View File
@@ -35,7 +35,12 @@ class TestMetrics(TestDvc):
fd.write('branch\n')
fd.write(branch)
self.dvc.scm.add(['metric', 'metric_json', 'metric_tsv', 'metric_htsv', 'metric_csv', 'metric_hcsv'])
self.dvc.scm.add(['metric',
'metric_json',
'metric_tsv',
'metric_htsv',
'metric_csv',
'metric_hcsv'])
self.dvc.scm.commit('metric')
self.dvc.scm.checkout('master')
@@ -96,7 +101,9 @@ class TestMetrics(TestDvc):
class TestMetricsReproCLI(TestDvc):
def test(self):
stage = self.dvc.run(metrics_no_cache=['metrics'],
cmd='python {} {} {}'.format(self.CODE, self.FOO, 'metrics'))
cmd='python {} {} {}'.format(self.CODE,
self.FOO,
'metrics'))
ret = main(['repro', '-m', stage.path])
self.assertEqual(ret, 0)
@@ -132,53 +139,57 @@ class TestMetricsReproCLI(TestDvc):
class TestMetricsCLICompat(TestMetrics):
def test(self):
#FIXME check output
# FIXME check output
ret = main(['metrics', 'show', '--all-branches', 'metric', '-v'])
self.assertEqual(ret, 0)
ret = main(['metrics', 'show', '--all-branches', 'metric_json', '--json-path', 'branch'])
ret = main(['metrics', 'show', '--all-branches', 'metric_json',
'--json-path', 'branch'])
self.assertEqual(ret, 0)
ret = main(['metrics', 'show', '--all-branches', 'metric_tsv', '--tsv-path', '0,0'])
ret = main(['metrics', 'show', '--all-branches', 'metric_tsv',
'--tsv-path', '0,0'])
self.assertEqual(ret, 0)
ret = main(['metrics', 'show', '--all-branches', 'metric_htsv', '--htsv-path', 'branch,0'])
ret = main(['metrics', 'show', '--all-branches', 'metric_htsv',
'--htsv-path', 'branch,0'])
self.assertEqual(ret, 0)
ret = main(['metrics', 'show', '--all-branches', 'metric_csv', '--csv-path', '0,0'])
ret = main(['metrics', 'show', '--all-branches', 'metric_csv',
'--csv-path', '0,0'])
self.assertEqual(ret, 0)
ret = main(['metrics', 'show', '--all-branches', 'metric_hcsv', '--hcsv-path', 'branch,0'])
ret = main(['metrics', 'show', '--all-branches', 'metric_hcsv',
'--hcsv-path', 'branch,0'])
self.assertEqual(ret, 0)
class TestMetricsCLI(TestMetrics):
def test(self):
#FIXME check output
# FIXME check output
ret = main(['metrics', 'show', '-a', 'metric', '-v'])
self.assertEqual(ret, 0)
ret = main(['metrics', 'show', '-a', 'metric_json', '-t', 'json',
'-x', 'branch'])
ret = main(['metrics', 'show', '-a', 'metric_json',
'-t', 'json', '-x', 'branch'])
self.assertEqual(ret, 0)
ret = main(['metrics', 'show', '-a', 'metric_tsv', '-t', 'tsv',
'-x', '0,0'])
ret = main(['metrics', 'show', '-a', 'metric_tsv',
'-t', 'tsv', '-x', '0,0'])
self.assertEqual(ret, 0)
ret = main(['metrics', 'show', '-a', 'metric_htsv', '-t', 'htsv',
'-x', 'branch,0'])
ret = main(['metrics', 'show', '-a', 'metric_htsv',
'-t', 'htsv', '-x', 'branch,0'])
self.assertEqual(ret, 0)
ret = main(['metrics', 'show', '-a', 'metric_csv', '-t', 'csv',
'-x', '0,0'])
ret = main(['metrics', 'show', '-a', 'metric_csv',
'-t', 'csv', '-x', '0,0'])
self.assertEqual(ret, 0)
ret = main(['metrics', 'show', '-a', 'metric_hcsv', '-t', 'hcsv',
'-x', 'branch,0'])
ret = main(['metrics', 'show', '-a', 'metric_hcsv',
'-t', 'hcsv', '-x', 'branch,0'])
self.assertEqual(ret, 0)
def test_dir(self):
os.mkdir('metrics_dir')
tests/test_move.py
+8 -3
View File
@@ -19,14 +19,16 @@ class TestMove(TestDvc):
class TestMoveNonExistentFile(TestDvc):
def test(self):
with self.assertRaises(DvcException) as cx:
with self.assertRaises(DvcException):
self.dvc.move('non_existent_file', 'dst')
class TestMoveDirectory(TestDvc):
def test(self):
dst = 'dst'
stage_add = self.dvc.add(self.DATA_DIR)
stages = self.dvc.add(self.DATA_DIR)
self.assertEqual(len(stages), 1)
self.assertTrue(stages[0] is not None)
self.dvc.move(self.DATA_DIR, dst)
self.assertFalse(os.path.exists(self.DATA_DIR))
self.assertTrue(os.path.exists(dst))
@@ -34,7 +36,10 @@ class TestMoveDirectory(TestDvc):
class TestCmdMove(TestDvc):
def test(self):
stage = self.dvc.add(self.FOO)
stages = self.dvc.add(self.FOO)
self.assertEqual(len(stages), 1)
self.assertTrue(stages[0] is not None)
ret = main(['move', self.FOO, self.FOO + '1'])
self.assertEqual(ret, 0)
tests/test_pipeline.py
+26 -16
View File
@@ -19,11 +19,11 @@ class TestPipelineShowSingle(TestDvc):
def test_commands(self):
ret = main(['pipeline', 'show', self.stage, '--commands'])
self.assertEqual(ret, 0)
self.assertEqual(ret, 0)
def test_outs(self):
ret = main(['pipeline', 'show', self.stage, '--outs'])
self.assertEqual(ret, 0)
self.assertEqual(ret, 0)
def test_ascii(self):
ret = main(['pipeline', 'show', '--ascii', self.stage])
@@ -43,12 +43,14 @@ class TestPipelineShowSingle(TestDvc):
self.assertEqual(ret, 0)
def test_dot_commands(self):
ret = main(['pipeline', 'show', '--dot', self.dotFile, self.stage, '--commands'])
ret = main(['pipeline', 'show', '--dot', self.dotFile, self.stage,
'--commands'])
self.assertEqual(ret, 0)
self.assertTrue(os.path.isfile(self.dotFile))
def test_dot_outs(self):
ret = main(['pipeline', 'show', '--dot', self.dotFile, self.stage, '--outs'])
ret = main(['pipeline', 'show', '--dot', self.dotFile, self.stage,
'--outs'])
self.assertEqual(ret, 0)
self.assertTrue(os.path.isfile(self.dotFile))
@@ -72,23 +74,25 @@ class TestPipelineShow(TestRepro):
def test_commands(self):
ret = main(['pipeline', 'show', self.file1_stage, '--commands'])
self.assertEqual(ret, 0)
self.assertEqual(ret, 0)
def test_outs(self):
ret = main(['pipeline', 'show', self.file1_stage, '--outs'])
self.assertEqual(ret, 0)
self.assertEqual(ret, 0)
def test_ascii(self):
ret = main(['pipeline', 'show', '--ascii', self.file1_stage])
self.assertEqual(ret, 0)
def test_dot(self):
ret = main(['pipeline', 'show', '--dot', self.dotFile, self.file1_stage])
ret = main(['pipeline', 'show', '--dot', self.dotFile,
self.file1_stage])
self.assertEqual(ret, 0)
self.assertTrue(os.path.isfile(self.dotFile))
def test_ascii_commands(self):
ret = main(['pipeline', 'show', '--ascii', self.file1_stage, '--commands'])
ret = main(['pipeline', 'show', '--ascii', self.file1_stage,
'--commands'])
self.assertEqual(ret, 0)
def test_ascii_outs(self):
@@ -96,12 +100,14 @@ class TestPipelineShow(TestRepro):
self.assertEqual(ret, 0)
def test_dot_commands(self):
ret = main(['pipeline', 'show', '--dot', self.dotFile, self.file1_stage, '--commands'])
ret = main(['pipeline', 'show', '--dot', self.dotFile,
self.file1_stage, '--commands'])
self.assertEqual(ret, 0)
self.assertTrue(os.path.isfile(self.dotFile))
def test_dot_outs(self):
ret = main(['pipeline', 'show', '--dot', self.dotFile, self.file1_stage, '--outs'])
ret = main(['pipeline', 'show', '--dot', self.dotFile,
self.file1_stage, '--outs'])
self.assertEqual(ret, 0)
self.assertTrue(os.path.isfile(self.dotFile))
@@ -125,23 +131,25 @@ class TestPipelineShowDeep(TestReproChangedDeepData):
def test_commands(self):
ret = main(['pipeline', 'show', self.file1_stage, '--commands'])
self.assertEqual(ret, 0)
self.assertEqual(ret, 0)
def test_outs(self):
ret = main(['pipeline', 'show', self.file1_stage, '--outs'])
self.assertEqual(ret, 0)
self.assertEqual(ret, 0)
def test_ascii(self):
ret = main(['pipeline', 'show', '--ascii', self.file1_stage])
self.assertEqual(ret, 0)
def test_dot(self):
ret = main(['pipeline', 'show', '--dot', self.dotFile, self.file1_stage])
ret = main(['pipeline', 'show', '--dot', self.dotFile,
self.file1_stage])
self.assertEqual(ret, 0)
self.assertTrue(os.path.isfile(self.dotFile))
def test_ascii_commands(self):
ret = main(['pipeline', 'show', '--ascii', self.file1_stage, '--commands'])
ret = main(['pipeline', 'show', '--ascii', self.file1_stage,
'--commands'])
self.assertEqual(ret, 0)
def test_ascii_outs(self):
@@ -149,12 +157,14 @@ class TestPipelineShowDeep(TestReproChangedDeepData):
self.assertEqual(ret, 0)
def test_dot_commands(self):
ret = main(['pipeline', 'show', '--dot', self.dotFile, self.file1_stage, '--commands'])
ret = main(['pipeline', 'show', '--dot', self.dotFile,
self.file1_stage, '--commands'])
self.assertEqual(ret, 0)
self.assertTrue(os.path.isfile(self.dotFile))
def test_dot_outs(self):
ret = main(['pipeline', 'show', '--dot', self.dotFile, self.file1_stage, '--outs'])
ret = main(['pipeline', 'show', '--dot', self.dotFile,
self.file1_stage, '--outs'])
self.assertEqual(ret, 0)
self.assertTrue(os.path.isfile(self.dotFile))
tests/test_remote.py
+20 -11
View File
@@ -2,7 +2,6 @@ import os
import configobj
from dvc.main import main
from dvc.project import Project
from dvc.command.remote import CmdRemoteAdd
from dvc.exceptions import UnsupportedRemoteError
from dvc.remote import Remote
@@ -19,12 +18,14 @@ class TestRemote(TestDvc):
self.assertNotEqual(main(['remote', 'remove', remotes[0]]), 0)
for r in remotes:
self.assertEqual(main(['remote', 'add', '--default', r, 's3://bucket/name']), 0)
self.assertEqual(main(['remote', 'add', '--default', r,
's3://bucket/name']), 0)
self.assertEqual(main(['remote', 'list']), 0)
self.assertEqual(main(['remote', 'remove', remotes[0]]), 0)
self.assertEqual(main(['remote', 'modify', remotes[0], 'option', 'value']), 0)
self.assertEqual(main(['remote', 'modify', remotes[0],
'option', 'value']), 0)
self.assertEqual(main(['remote', 'list']), 0)
@@ -52,18 +53,26 @@ class TestRemote(TestDvc):
class TestRemoteRemoveDefault(TestDvc):
def test(self):
remote = 'mys3'
self.assertEqual(main(['remote', 'add', '--default', remote, 's3://bucket/name']), 0)
self.assertEqual(main(['remote', 'modify', remote, 'profile', 'default']), 0)
self.assertEqual(main(['remote', 'add', '--default', remote,
's3://bucket/name']), 0)
self.assertEqual(main(['remote', 'modify', remote, 'profile',
'default']), 0)
self.assertEqual(main(['config', '--local', 'core.remote', remote]), 0)
config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir, Config.CONFIG))
local_config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir, Config.CONFIG_LOCAL))
self.assertEqual(config[Config.SECTION_CORE][Config.SECTION_CORE_REMOTE], remote)
self.assertEqual(local_config[Config.SECTION_CORE][Config.SECTION_CORE_REMOTE], remote)
config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir,
Config.CONFIG))
local_config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir,
Config.CONFIG_LOCAL))
self.assertEqual(config[Config.SECTION_CORE]
[Config.SECTION_CORE_REMOTE], remote)
self.assertEqual(local_config[Config.SECTION_CORE]
[Config.SECTION_CORE_REMOTE], remote)
self.assertEqual(main(['remote', 'remove', remote]), 0)
config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir, Config.CONFIG))
local_config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir, Config.CONFIG_LOCAL))
config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir,
Config.CONFIG))
local_config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir,
Config.CONFIG_LOCAL))
section = Config.SECTION_REMOTE_FMT.format(remote)
self.assertTrue(section not in config.keys())
tests/test_remove.py
+2 -2
View File
@@ -3,7 +3,6 @@ import os
from dvc.main import main
from dvc.stage import Stage, StageFileDoesNotExistError
from dvc.exceptions import DvcException
from dvc.command.remove import CmdRemove
from tests.basic_env import TestDvc
from mock import patch
@@ -30,7 +29,7 @@ class TestRemove(TestDvc):
class TestRemoveNonExistentFile(TestDvc):
def test(self):
with self.assertRaises(StageFileDoesNotExistError) as cx:
with self.assertRaises(StageFileDoesNotExistError):
self.dvc.remove('non_existent_dvc_file')
@@ -60,6 +59,7 @@ class TestCmdRemove(TestDvc):
'non-existing-dvc-file'])
self.assertNotEqual(ret, 0)
class TestRemovePurge(TestDvc):
def test(self):
dvcfile = self.dvc.add(self.FOO)[0].path
tests/test_repro.py
+51 -34
View File
@@ -1,6 +1,5 @@
import os
import yaml
import stat
import shutil
import filecmp
import getpass
@@ -17,7 +16,6 @@ import uuid
from google.cloud import storage as gc
from dvc.main import main
from dvc.command.repro import CmdRepro
from dvc.project import Project, ReproductionError
from dvc.utils import file_md5
from dvc.remote.local import RemoteLOCAL
@@ -46,7 +44,9 @@ class TestRepro(TestDvc):
self.dvc.run(fname=self.file1_stage,
outs=[self.file1],
deps=[self.FOO, self.CODE],
cmd='python {} {} {}'.format(self.CODE, self.FOO, self.file1))
cmd='python {} {} {}'.format(self.CODE,
self.FOO,
self.file1))
class TestReproFail(TestRepro):
@@ -71,7 +71,9 @@ class TestReproDepUnderDir(TestDvc):
self.dvc.run(fname=self.file1_stage,
outs=[self.file1],
deps=[self.DATA, self.CODE],
cmd='python {} {} {}'.format(self.CODE, self.DATA, self.file1))
cmd='python {} {} {}'.format(self.CODE,
self.DATA,
self.file1))
self.assertTrue(filecmp.cmp(self.file1, self.DATA, shallow=False))
@@ -131,7 +133,8 @@ class TestReproNoDeps(TestRepro):
out = 'out'
code_file = 'out.py'
stage_file = 'out.dvc'
code = 'import uuid\nwith open("{}", "w+") as fd:\n\tfd.write(str(uuid.uuid4()))\n'.format(out)
code = 'import uuid\nwith open("{}", "w+") as fd:\n' \
'\tfd.write(str(uuid.uuid4()))\n'.format(out)
with open(code_file, 'w+') as fd:
fd.write(code)
self.dvc.run(fname=stage_file,
@@ -160,7 +163,8 @@ class TestReproChangedCode(TestRepro):
def swap_code(self):
os.unlink(self.CODE)
new_contents = self.CODE_CONTENTS
new_contents += "\nshutil.copyfile('{}', sys.argv[2])\n".format(self.BAR)
new_contents += "\nshutil.copyfile('{}', " \
"sys.argv[2])\n".format(self.BAR)
self.create(self.CODE, new_contents)
@@ -214,11 +218,6 @@ class TestReproDryNoExec(TestDvc):
with open(f, 'w+') as fobj:
fobj.write(str(d))
if os.name == 'nt':
cp = 'copy'
else:
cp = 'cp'
ret = main(['run',
'--no-exec',
'-d', idir,
@@ -288,15 +287,18 @@ class TestReproPipelines(TestDvc):
self.dvc.run(fname=self.file1_stage,
outs=[self.file1],
deps=[self.FOO, self.CODE],
cmd='python {} {} {}'.format(self.CODE, self.FOO, self.file1))
cmd='python {} {} {}'.format(self.CODE,
self.FOO,
self.file1))
self.file2 = 'file2'
self.file2_stage = self.file2 + '.dvc'
self.dvc.run(fname=self.file2_stage,
outs=[self.file2],
deps=[self.BAR, self.CODE],
cmd='python {} {} {}'.format(self.CODE, self.BAR, self.file2))
cmd='python {} {} {}'.format(self.CODE,
self.BAR,
self.file2))
def test(self):
stages = self.dvc.reproduce(all_pipelines=True, force=True)
@@ -319,7 +321,9 @@ class TestReproLocked(TestReproChangedData):
self.dvc.run(fname=file2_stage,
outs=[file2],
deps=[self.file1, self.CODE],
cmd='python {} {} {}'.format(self.CODE, self.file1, file2))
cmd='python {} {} {}'.format(self.CODE,
self.file1,
file2))
self.swap_foo_with_bar()
@@ -368,6 +372,7 @@ class TestReproLockedCallback(TestDvc):
stages = self.dvc.reproduce(file1_stage)
self.assertEqual(len(stages), 1)
class TestReproLockedUnchanged(TestRepro):
def test(self):
"""
@@ -425,7 +430,7 @@ class TestNonExistingOutput(TestRepro):
def test(self):
os.unlink(self.FOO)
with self.assertRaises(ReproductionError) as cx:
with self.assertRaises(ReproductionError):
self.dvc.reproduce(self.file1_stage)
@@ -447,9 +452,11 @@ class TestReproChangedDir(TestDvc):
stage_name = 'dir.dvc'
dir_name = 'dir'
dir_code = 'dir.py'
code = "import os; import shutil; os.mkdir(\"{}\"); " \
"shutil.copyfile(\"{}\", os.path.join(\"{}\", \"{}\"))"
with open(dir_code, 'w+') as fd:
fd.write("import os; import shutil; os.mkdir(\"{}\"); shutil.copyfile(\"{}\", os.path.join(\"{}\", \"{}\"))".format(dir_name, file_name, dir_name, file_name))
fd.write(code.format(dir_name, file_name, dir_name, file_name))
self.dvc.run(fname=stage_name,
outs=[dir_name],
@@ -474,7 +481,8 @@ class TestReproChangedDirData(TestDvc):
sleep()
with open(dir_code, 'w+') as fd:
fd.write("import os; import sys; import shutil; shutil.copytree(sys.argv[1], sys.argv[2])")
fd.write("import os; import sys; import shutil; "
"shutil.copytree(sys.argv[1], sys.argv[2])")
sleep()
@@ -557,11 +565,11 @@ class TestCmdReproChdir(TestDvc):
shutil.copyfile(self.CODE, code)
ret = main(['run',
'-f', 'Dvcfile',
'-c', dname,
'-d', self.FOO,
'-o', self.BAR,
'python {} {} {}'.format(self.CODE, self.FOO, self.BAR)])
'-f', 'Dvcfile',
'-c', dname,
'-d', self.FOO,
'-o', self.BAR,
'python {} {} {}'.format(self.CODE, self.FOO, self.BAR)])
self.assertEqual(ret, 0)
self.assertTrue(os.path.isfile(foo))
self.assertTrue(os.path.isfile(bar))
@@ -601,7 +609,8 @@ class TestReproExternalBase(TestDvc):
if not self.should_test():
return
cache = self.scheme + self.scheme_sep + self.bucket + self.sep + str(uuid.uuid4())
cache = (self.scheme + self.scheme_sep + self.bucket + self.sep
+ str(uuid.uuid4()))
ret = main(['config', 'cache.' + self.cache_scheme, 'myrepo'])
self.assertEqual(ret, 0)
@@ -610,7 +619,8 @@ class TestReproExternalBase(TestDvc):
remote_name = 'myremote'
remote_key = str(uuid.uuid4())
remote = self.scheme + self.scheme_sep + self.bucket + self.sep + remote_key
remote = (self.scheme + self.scheme_sep + self.bucket + self.sep
+ remote_key)
ret = main(['remote', 'add', remote_name, remote])
self.assertEqual(ret, 0)
@@ -621,8 +631,10 @@ class TestReproExternalBase(TestDvc):
foo_key = remote_key + self.sep + self.FOO
bar_key = remote_key + self.sep + self.BAR
foo_path = self.scheme + self.scheme_sep + self.bucket + self.sep + foo_key
bar_path = self.scheme + self.scheme_sep + self.bucket + self.sep + bar_key
foo_path = (self.scheme + self.scheme_sep + self.bucket + self.sep
+ foo_key)
bar_path = (self.scheme + self.scheme_sep + self.bucket + self.sep
+ bar_key)
# Using both plain and remote notation
out_foo_path = 'remote://' + remote_name + '/' + self.FOO
@@ -637,12 +649,13 @@ class TestReproExternalBase(TestDvc):
self.assertTrue(filecmp.cmp('import', self.FOO, shallow=False))
self.assertEqual(self.dvc.status(import_stage.path), {})
import_remote_stage = self.dvc.imp(out_foo_path, out_foo_path + '_imported')
import_remote_stage = self.dvc.imp(out_foo_path,
out_foo_path + '_imported')
self.assertEqual(self.dvc.status(import_remote_stage.path), {})
cmd_stage = self.dvc.run(outs=[out_bar_path],
deps=[out_foo_path],
cmd=self.cmd(foo_path, bar_path))
deps=[out_foo_path],
cmd=self.cmd(foo_path, bar_path))
self.assertEqual(self.dvc.status(cmd_stage.path), {})
self.assertEqual(self.dvc.status(), {})
@@ -816,7 +829,9 @@ class TestReproExternalSSH(TestReproExternalBase):
print(err)
self.assertEqual(p.returncode, 0)
p = Popen('echo "{}" | ssh {} "tr -d \'\n\' > {}"'.format(body, dest, path),
p = Popen('echo "{}" | ssh {} "tr -d \'\n\' > {}"'.format(body,
dest,
path),
shell=True,
executable=os.getenv('SHELL'),
stdin=PIPE,
@@ -904,7 +919,7 @@ class TestReproExternalHTTP(TestReproExternalBase):
self.dvc = Project('.')
# Import
with StaticFileServer() as server:
with StaticFileServer():
import_url = urljoin(self.remote, self.FOO)
import_output = 'imported_file'
import_stage = self.dvc.imp(import_url, import_output)
@@ -913,7 +928,7 @@ class TestReproExternalHTTP(TestReproExternalBase):
self.assertTrue(filecmp.cmp(import_output, self.FOO, shallow=False))
# Run --deps
with StaticFileServer() as server:
with StaticFileServer():
run_dependency = urljoin(self.remote, self.BAR)
run_output = 'remote_file'
cmd = 'open("{}", "w+")'.format(run_output)
@@ -924,6 +939,7 @@ class TestReproExternalHTTP(TestReproExternalBase):
run_stage = self.dvc.run(deps=[run_dependency],
outs=[run_output],
cmd='python create-output.py')
self.assertTrue(run_stage is not None)
self.assertTrue(os.path.exists(run_output))
@@ -934,7 +950,7 @@ class TestReproExternalHTTP(TestReproExternalBase):
shutil.move(self.local_cache, self.external_cache_id)
self.assertFalse(os.path.exists(self.local_cache))
with StaticFileServer() as server:
with StaticFileServer():
self.dvc.pull(import_stage.path, remote='mycache')
self.assertTrue(os.path.exists(import_output))
@@ -962,6 +978,7 @@ class TestReproShell(TestDvc):
with open(fname, 'r') as fd:
self.assertEqual(os.getenv('SHELL'), fd.read().strip())
class TestReproNoSCM(TestRepro):
def test(self):
shutil.rmtree(self.dvc.scm.dir)
tests/test_run.py
+10 -13
View File
@@ -1,17 +1,14 @@
import os
import mock
import time
import shutil
import filecmp
import subprocess
from dvc.project import Project
from dvc.main import main
from dvc.utils import file_md5
from dvc.system import System
from dvc.stage import Stage, StageFileBadNameError, MissingDep
from dvc.stage import StageFileBadNameError, MissingDep
from dvc.stage import StageBadCwdError, StageFileAlreadyExistsError
from dvc.command.run import CmdRun
from dvc.exceptions import (OutputDuplicationError,
CircularDependencyError,
WorkingDirectoryAsOutputError)
@@ -42,7 +39,7 @@ class TestRun(TestDvc):
self.assertEqual(len(stage.deps), len(deps))
self.assertEqual(len(stage.outs), len(outs + outs_no_cache))
self.assertEqual(stage.outs[0].path, outs[0])
self.assertEqual(stage.outs[0].md5, file_md5(self.FOO)[0])
self.assertEqual(stage.outs[0].md5, file_md5(self.FOO)[0])
self.assertTrue(stage.path, fname)
with self.assertRaises(OutputDuplicationError):
@@ -94,7 +91,6 @@ class TestRunBadStageFilename(TestDvc):
cwd=os.curdir)
class TestRunNoExec(TestDvc):
def test(self):
self.dvc.run(cmd='python {} {} {}'.format(self.CODE, self.FOO, 'out'),
@@ -165,7 +161,8 @@ class TestCmdRun(TestDvc):
def test_run_args_with_spaces(self):
with open(self.CODE, 'w') as fobj:
fobj.write("import sys\nopen(sys.argv[1], 'w+').write(sys.argv[2])")
fobj.write(
"import sys\nopen(sys.argv[1], 'w+').write(sys.argv[2])")
arg = 'arg1 arg2'
log = 'log'
@@ -412,28 +409,28 @@ class TestRunDeterministicChangedDep(TestRunDeterministicBase):
os.unlink(self.FOO)
shutil.copy(self.BAR, self.FOO)
with self.assertRaises(StageFileAlreadyExistsError):
self._run()
self._run()
class TestRunDeterministicChangedDepsList(TestRunDeterministicBase):
def test(self):
self.deps = [self.BAR, self.CODE]
with self.assertRaises(StageFileAlreadyExistsError):
self._run()
self._run()
class TestRunDeterministicNewDep(TestRunDeterministicBase):
def test(self):
self.deps = [self.FOO, self.BAR, self.CODE]
with self.assertRaises(StageFileAlreadyExistsError):
self._run()
self._run()
class TestRunDeterministicRemoveDep(TestRunDeterministicBase):
def test(self):
self.deps = [self.CODE]
with self.assertRaises(StageFileAlreadyExistsError):
self._run()
self._run()
class TestRunDeterministicChangedOut(TestRunDeterministicBase):
@@ -441,11 +438,11 @@ class TestRunDeterministicChangedOut(TestRunDeterministicBase):
os.unlink(self.out_file)
self.out_file_mtime = None
with self.assertRaises(StageFileAlreadyExistsError):
self._run()
self._run()
class TestRunDeterministicChangedCmd(TestRunDeterministicBase):
def test(self):
self.cmd += ' arg'
with self.assertRaises(StageFileAlreadyExistsError):
self._run()
self._run()
tests/test_stage.py
+7 -7
View File
@@ -52,15 +52,15 @@ class TestSchemaDepsOuts(TestSchema):
Stage.validate(d)
def test_list(self):
l = [{OutputLOCAL.PARAM_PATH: 'foo', RemoteLOCAL.PARAM_MD5: '123'},
{OutputLOCAL.PARAM_PATH: 'bar', RemoteLOCAL.PARAM_MD5: None},
{OutputLOCAL.PARAM_PATH: 'baz'}]
d = {Stage.PARAM_DEPS: l}
lst = [{OutputLOCAL.PARAM_PATH: 'foo', RemoteLOCAL.PARAM_MD5: '123'},
{OutputLOCAL.PARAM_PATH: 'bar', RemoteLOCAL.PARAM_MD5: None},
{OutputLOCAL.PARAM_PATH: 'baz'}]
d = {Stage.PARAM_DEPS: lst}
Stage.validate(d)
l[0][OutputLOCAL.PARAM_CACHE] = True
l[1][OutputLOCAL.PARAM_CACHE] = False
d = {Stage.PARAM_OUTS: l}
lst[0][OutputLOCAL.PARAM_CACHE] = True
lst[1][OutputLOCAL.PARAM_CACHE] = False
d = {Stage.PARAM_OUTS: lst}
Stage.validate(d)
tests/test_state.py
+1 -8
View File
@@ -1,8 +1,6 @@
import os
import stat
import time
from dvc.system import System
from dvc.state import State
from dvc.utils import file_md5
from dvc.main import main
@@ -14,8 +12,6 @@ class TestState(TestDvc):
def test_update(self):
path = os.path.join(self.dvc.root_dir, self.FOO)
md5 = file_md5(path)[0]
mtime = os.path.getmtime(path)
inode = System.inode(path)
state = State(self.dvc, self.dvc.config._config)
@@ -28,16 +24,13 @@ class TestState(TestDvc):
# mtime(i.e. 1sec for HFS) that cause problems with
# our 'state' system not being able to distinguish
# files that were modified within that delta.
time.sleep(1)
time.sleep(1)
os.unlink(path)
with open(path, 'w+') as fd:
fd.write('1')
md5 = file_md5(path)[0]
mtime = os.path.getmtime(path)
inode = System.inode(path)
entry_md5 = state.update(path)
self.assertEqual(entry_md5, md5)
tests/test_system.py
+1 -1
View File
@@ -14,6 +14,6 @@ class TestSystem(TestDir):
fd = os.open(locked, os.O_WRONLY | os.O_CREAT | os.O_EXCL)
# should not raise WinError
info = System.getdirinfo(locked)
System.getdirinfo(locked)
os.close(fd)
tests/test_unprotect.py
+7 -8
View File
@@ -2,7 +2,6 @@ import os
import stat
from dvc.main import main
from dvc.project import Project
from tests.basic_env import TestDvc
@@ -15,17 +14,17 @@ class TestUnprotect(TestDvc):
ret = main(['add', self.FOO])
self.assertEqual(ret, 0)
self.assertNotEqual(os.stat(self.FOO).st_mode & (stat.S_IREAD|
stat.S_IRGRP|
self.assertNotEqual(os.stat(self.FOO).st_mode & (stat.S_IREAD |
stat.S_IRGRP |
stat.S_IROTH), 0)
self.assertEqual(os.stat(self.FOO).st_mode & (stat.S_IWRITE|
stat.S_IWGRP|
self.assertEqual(os.stat(self.FOO).st_mode & (stat.S_IWRITE |
stat.S_IWGRP |
stat.S_IWOTH), 0)
ret = main(['unprotect', self.FOO])
self.assertEqual(ret, 0)
self.assertNotEqual(os.stat(self.FOO).st_mode & (stat.S_IREAD|
stat.S_IRGRP|
stat.S_IROTH|
self.assertNotEqual(os.stat(self.FOO).st_mode & (stat.S_IREAD |
stat.S_IRGRP |
stat.S_IROTH |
stat.S_IWRITE), 0)
tests/test_utils.py
+4 -3
View File
@@ -21,7 +21,9 @@ class TestUtils(TestDvc):
self.assertTrue(filecmp.cmp(src, dest, shallow=False))
utils.copyfile(src, dest_dir)
self.assertTrue(filecmp.cmp(src, '{}/{}'.format(dest_dir, src), shallow=False))
self.assertTrue(filecmp.cmp(src,
'{}/{}'.format(dest_dir, src),
shallow=False))
shutil.rmtree(dest_dir)
os.remove(src)
@@ -41,13 +43,12 @@ class TestUtils(TestDvc):
'outs': [{'path': 'file1',
'metric': {'type': 'raw'},
'cache': False,
'md5':'acbd18db4cc2f85cedef654fccc4a4d8'}],
'md5': 'acbd18db4cc2f85cedef654fccc4a4d8'}],
'deps': [{'path': 'foo',
'md5': 'acbd18db4cc2f85cedef654fccc4a4d8'},
{'path': 'code.py',
'md5': 'd05447644b89960913c7eee5fd776adb'}]}
md5 = '8b263fa05ede6c3145c164829be694b4'
self.assertEqual(md5, utils.dict_md5(d, exclude=['metric', 'locked']))