diff --git a/dvc/__init__.py b/dvc/__init__.py index eaa4b1fbf..00afe0a8a 100644 --- a/dvc/__init__.py +++ b/dvc/__init__.py @@ -3,6 +3,9 @@ DVC ---- Make your data science projects reproducible and shareable. """ + +from __future__ import unicode_literals + import os import warnings diff --git a/dvc/__main__.py b/dvc/__main__.py index 10bdd5b51..b338cad48 100644 --- a/dvc/__main__.py +++ b/dvc/__main__.py @@ -1,5 +1,7 @@ """Main entry point for dvc command line tool.""" +from __future__ import unicode_literals + import sys from dvc.main import main diff --git a/dvc/analytics.py b/dvc/analytics.py index 620947ce5..1a14a9c88 100644 --- a/dvc/analytics.py +++ b/dvc/analytics.py @@ -1,5 +1,9 @@ """Collect and send usage analytics""" +from __future__ import unicode_literals + +from dvc.utils.compat import str + import os import json import errno diff --git a/dvc/cache.py b/dvc/cache.py index 6d3cfd964..b66f638f2 100644 --- a/dvc/cache.py +++ b/dvc/cache.py @@ -1,5 +1,7 @@ """Manages cache of a dvc project.""" +from __future__ import unicode_literals + import os from dvc.config import Config diff --git a/dvc/cli.py b/dvc/cli.py index afa8990d8..627aa4a2d 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -1,6 +1,7 @@ """Command line interface for dvc.""" from __future__ import print_function +from __future__ import unicode_literals import sys import argparse diff --git a/dvc/command/add.py b/dvc/command/add.py index cd3b6dac1..7c6a66650 100644 --- a/dvc/command/add.py +++ b/dvc/command/add.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import dvc.logger as logger from dvc.exceptions import DvcException from dvc.command.base import CmdBase diff --git a/dvc/command/base.py b/dvc/command/base.py index 5beff9809..7ca1e413e 100644 --- a/dvc/command/base.py +++ b/dvc/command/base.py @@ -1,4 +1,4 @@ -import sys +from __future__ import unicode_literals import dvc.logger as logger @@ -11,7 +11,9 @@ def fix_subparsers(subparsers): Args: subparsers: subparsers to fix. """ - if sys.version_info[0] == 3: # pragma: no cover + from dvc.utils.compat import is_py3 + + if is_py3: # pragma: no cover subparsers.required = True subparsers.dest = 'cmd' diff --git a/dvc/command/cache.py b/dvc/command/cache.py index dee60f046..8464e745a 100644 --- a/dvc/command/cache.py +++ b/dvc/command/cache.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import argparse from dvc.command.base import fix_subparsers diff --git a/dvc/command/checkout.py b/dvc/command/checkout.py index 89673ac8c..61cabd15c 100644 --- a/dvc/command/checkout.py +++ b/dvc/command/checkout.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + from dvc.command.base import CmdBase diff --git a/dvc/command/config.py b/dvc/command/config.py index cc06b22fa..9d4c058e8 100644 --- a/dvc/command/config.py +++ b/dvc/command/config.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import os import argparse diff --git a/dvc/command/daemon.py b/dvc/command/daemon.py index 01228880a..a5c864313 100644 --- a/dvc/command/daemon.py +++ b/dvc/command/daemon.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + from dvc.command.base import CmdBase, fix_subparsers diff --git a/dvc/command/data_sync.py b/dvc/command/data_sync.py index 7fe0f919a..e1e3eec61 100644 --- a/dvc/command/data_sync.py +++ b/dvc/command/data_sync.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import argparse import dvc.logger as logger diff --git a/dvc/command/destroy.py b/dvc/command/destroy.py index c7676b53d..6003decde 100644 --- a/dvc/command/destroy.py +++ b/dvc/command/destroy.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import dvc.prompt as prompt import dvc.logger as logger from dvc.command.base import CmdBase diff --git a/dvc/command/gc.py b/dvc/command/gc.py index 697024420..3e74f66dd 100644 --- a/dvc/command/gc.py +++ b/dvc/command/gc.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import os import dvc.prompt as prompt import dvc.logger as logger diff --git a/dvc/command/imp.py b/dvc/command/imp.py index c4eaac4b7..35f8b93c5 100644 --- a/dvc/command/imp.py +++ b/dvc/command/imp.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import os import dvc.logger as logger diff --git a/dvc/command/init.py b/dvc/command/init.py index 952f05573..b09daf440 100644 --- a/dvc/command/init.py +++ b/dvc/command/init.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import dvc.logger as logger diff --git a/dvc/command/install.py b/dvc/command/install.py index 47be9eb31..89f2ee7e5 100644 --- a/dvc/command/install.py +++ b/dvc/command/install.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import dvc.logger as logger from dvc.command.base import CmdBase diff --git a/dvc/command/lock.py b/dvc/command/lock.py index 9ae847b01..29109f973 100644 --- a/dvc/command/lock.py +++ b/dvc/command/lock.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import dvc.logger as logger from dvc.exceptions import DvcException from dvc.command.base import CmdBase diff --git a/dvc/command/metrics.py b/dvc/command/metrics.py index 36a5b4be7..a748d0b00 100644 --- a/dvc/command/metrics.py +++ b/dvc/command/metrics.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import dvc.logger as logger from dvc.exceptions import DvcException from dvc.command.base import CmdBase, fix_subparsers diff --git a/dvc/command/move.py b/dvc/command/move.py index f738e69bc..606ed2430 100644 --- a/dvc/command/move.py +++ b/dvc/command/move.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import dvc.logger as logger from dvc.exceptions import DvcException from dvc.command.base import CmdBase diff --git a/dvc/command/pipeline.py b/dvc/command/pipeline.py index e15f664c9..3c2bfd0fd 100644 --- a/dvc/command/pipeline.py +++ b/dvc/command/pipeline.py @@ -1,3 +1,7 @@ +from __future__ import unicode_literals + +from dvc.utils.compat import str + import os import dvc.logger as logger diff --git a/dvc/command/remote.py b/dvc/command/remote.py index 9bf83b03d..48079cbd0 100644 --- a/dvc/command/remote.py +++ b/dvc/command/remote.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import os import re diff --git a/dvc/command/remove.py b/dvc/command/remove.py index 2f0a5594a..833d2134c 100644 --- a/dvc/command/remove.py +++ b/dvc/command/remove.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import dvc.prompt as prompt import dvc.logger as logger from dvc.exceptions import DvcException @@ -13,7 +15,7 @@ class CmdRemove(CmdBase): return False msg = ( - u'Are you sure you want to remove {} with its outputs?' + 'Are you sure you want to remove {} with its outputs?' .format(target) ) @@ -21,8 +23,8 @@ class CmdRemove(CmdBase): return False raise DvcException( - u'Cannot purge without a confirmation from the user.' - u" Use '-f' to force." + 'Cannot purge without a confirmation from the user.' + " Use '-f' to force." ) def run(self): diff --git a/dvc/command/repro.py b/dvc/command/repro.py index b7afe0452..64be612ac 100644 --- a/dvc/command/repro.py +++ b/dvc/command/repro.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import os import dvc.logger as logger diff --git a/dvc/command/root.py b/dvc/command/root.py index 46d177f51..ef8abedf6 100644 --- a/dvc/command/root.py +++ b/dvc/command/root.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import os import dvc.logger as logger diff --git a/dvc/command/run.py b/dvc/command/run.py index 983a26bd5..5f3a7b08d 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import os import argparse diff --git a/dvc/command/status.py b/dvc/command/status.py index fb8cce33f..a8506f3c3 100644 --- a/dvc/command/status.py +++ b/dvc/command/status.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import dvc.logger as logger from dvc.command.data_sync import CmdDataBase diff --git a/dvc/command/unprotect.py b/dvc/command/unprotect.py index 98e910c87..3f4cd8705 100644 --- a/dvc/command/unprotect.py +++ b/dvc/command/unprotect.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import dvc.logger as logger from dvc.exceptions import DvcException from dvc.command.base import CmdBase diff --git a/dvc/config.py b/dvc/config.py index aed75bebe..c6941fcff 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -1,6 +1,9 @@ -""" -DVC config objects. -""" +"""DVC config objects.""" + +from __future__ import unicode_literals + +from dvc.utils.compat import str + import os import errno import configobj @@ -18,8 +21,9 @@ class ConfigError(DvcException): ex (Exception): optional exception that has caused this error. """ def __init__(self, msg, ex=None): - super(ConfigError, self).__init__('config file error: {}'.format(msg), - ex) + super(ConfigError, self).__init__( + 'config file error: {}'.format(msg), ex + ) def supported_cache_type(types): @@ -531,6 +535,6 @@ class Config(object): # pylint: disable=too-many-instance-attributes for s_key, s_value in config.items(): new_s = {} for key, value in s_value.items(): - new_s[key.lower()] = value + new_s[key.lower()] = str(value) new_config[s_key.lower()] = new_s return new_config diff --git a/dvc/daemon.py b/dvc/daemon.py index 1d67a37b9..737af3953 100644 --- a/dvc/daemon.py +++ b/dvc/daemon.py @@ -1,5 +1,7 @@ """Launch `dvc daemon` command in a separate detached process.""" +from __future__ import unicode_literals + import os import sys import inspect diff --git a/dvc/dagascii.py b/dvc/dagascii.py index 6f0c458e5..927a6acce 100644 --- a/dvc/dagascii.py +++ b/dvc/dagascii.py @@ -1,4 +1,8 @@ """Draws DAG in ASCII.""" + +from __future__ import unicode_literals +from __future__ import print_function + import sys import math diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index cd37fcae5..d04a36f9c 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -1,5 +1,7 @@ """Manages dvc remotes that user can use with push/pull/status comamnds.""" +from __future__ import unicode_literals + import dvc.logger as logger from dvc.config import Config, ConfigError diff --git a/dvc/dependency/__init__.py b/dvc/dependency/__init__.py index 0025060ec..9c84388dc 100644 --- a/dvc/dependency/__init__.py +++ b/dvc/dependency/__init__.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import schema from dvc.config import Config diff --git a/dvc/dependency/base.py b/dvc/dependency/base.py index 4c860cee3..2859ee496 100644 --- a/dvc/dependency/base.py +++ b/dvc/dependency/base.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + from dvc.exceptions import DvcException diff --git a/dvc/dependency/gs.py b/dvc/dependency/gs.py index 4f26b05ef..7c41dede7 100644 --- a/dvc/dependency/gs.py +++ b/dvc/dependency/gs.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + from dvc.output.gs import OutputGS from dvc.dependency.base import DependencyBase diff --git a/dvc/dependency/hdfs.py b/dvc/dependency/hdfs.py index 3f5659334..c67899f72 100644 --- a/dvc/dependency/hdfs.py +++ b/dvc/dependency/hdfs.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + from dvc.output.hdfs import OutputHDFS from dvc.dependency.base import DependencyBase diff --git a/dvc/dependency/http.py b/dvc/dependency/http.py index 562e690c0..84691ca3f 100644 --- a/dvc/dependency/http.py +++ b/dvc/dependency/http.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + from dvc.utils.compat import urlparse, urljoin from dvc.output.base import OutputBase from dvc.remote.http import RemoteHTTP diff --git a/dvc/dependency/local.py b/dvc/dependency/local.py index 02c77176a..60fb124c0 100644 --- a/dvc/dependency/local.py +++ b/dvc/dependency/local.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + from dvc.dependency.base import DependencyBase from dvc.output.local import OutputLOCAL diff --git a/dvc/dependency/s3.py b/dvc/dependency/s3.py index 5a40e647e..382e77a7d 100644 --- a/dvc/dependency/s3.py +++ b/dvc/dependency/s3.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + from dvc.output.s3 import OutputS3 from dvc.dependency.base import DependencyBase diff --git a/dvc/dependency/ssh.py b/dvc/dependency/ssh.py index 1e62ab5e1..b48c5e302 100644 --- a/dvc/dependency/ssh.py +++ b/dvc/dependency/ssh.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + from dvc.output.ssh import OutputSSH from dvc.dependency.base import DependencyBase diff --git a/dvc/exceptions.py b/dvc/exceptions.py index 565ce4023..cc4751b12 100644 --- a/dvc/exceptions.py +++ b/dvc/exceptions.py @@ -1,5 +1,9 @@ """Exceptions raised by the dvc.""" +from __future__ import unicode_literals + +from dvc.utils.compat import str, builtin_str + import traceback @@ -32,9 +36,12 @@ class OutputDuplicationError(DvcException): stages (list): list of paths to stages. """ def __init__(self, output, stages): - assert isinstance(output, str) + assert isinstance(output, str) or isinstance(output, builtin_str) assert isinstance(stages, list) - assert all(isinstance(stage, str) for stage in stages) + assert all( + isinstance(stage, str) or isinstance(stage, builtin_str) + for stage in stages + ) msg = ("file/directory '{}' is specified as an output in more than one" "stage: {}").format(output, "\n ".join(stages)) super(OutputDuplicationError, self).__init__(msg) @@ -50,8 +57,8 @@ class WorkingDirectoryAsOutputError(DvcException): output. """ def __init__(self, cwd, fname): - assert isinstance(cwd, str) - assert isinstance(fname, str) + assert isinstance(cwd, str) or isinstance(cwd, builtin_str) + assert isinstance(fname, str) or isinstance(fname, builtin_str) msg = ( "current working directory '{cwd}' is specified as an output in" " '{fname}'. Use another CWD to prevent any data removal." @@ -68,7 +75,9 @@ class CircularDependencyError(DvcException): dependency (str): path to the dependency. """ def __init__(self, dependency): - assert isinstance(dependency, str) + assert (isinstance(dependency, str) or + isinstance(dependency, builtin_str)) + msg = ("file/directory '{}' is specified as an output and as a " "dependency.") super(CircularDependencyError, self).__init__(msg.format(dependency)) @@ -82,7 +91,7 @@ class ArgumentDuplicationError(DvcException): path (str): path to the file/directory. """ def __init__(self, path): - assert isinstance(path, str) + assert isinstance(path, str) or isinstance(path, builtin_str) msg = "file '{}' is specified more than once." super(ArgumentDuplicationError, self).__init__(msg.format(path)) diff --git a/dvc/istextfile.py b/dvc/istextfile.py index 6f0f46fae..8bd48ed06 100644 --- a/dvc/istextfile.py +++ b/dvc/istextfile.py @@ -1,18 +1,19 @@ """Use heuristics to guess if it is a text file or a binary file.""" +from __future__ import unicode_literals + +from dvc.utils.compat import is_py3 + # Based on https://eli.thegreenplace.net/2011/10/19/ # perls-guess-if-file-is-text-or-binary-implemented-in-python -import sys -PY3 = sys.version_info[0] == 3 - # A function that takes an integer in the 8-bit range and returns # a single-character byte object in py3 / a single-character string # in py2. # def _int2byte(i): - if PY3: + if is_py3: return bytes((i,)) return chr(i) diff --git a/dvc/lock.py b/dvc/lock.py index 473f9d7ff..f66b162e8 100644 --- a/dvc/lock.py +++ b/dvc/lock.py @@ -1,5 +1,7 @@ """Manages dvc lock file.""" +from __future__ import unicode_literals + import os import time import zc.lockfile diff --git a/dvc/logger.py b/dvc/logger.py index 8ea17d304..11d24d8df 100644 --- a/dvc/logger.py +++ b/dvc/logger.py @@ -1,5 +1,9 @@ """Manages logger for dvc project.""" +from __future__ import unicode_literals + +from dvc.utils.compat import str + import re import sys import logging @@ -184,7 +188,7 @@ def colorize(message, color=None): 'red': colorama.Fore.RED, } - return u'{color}{message}{nc}'.format( + return '{color}{message}{nc}'.format( color=colors.get(color, ''), message=message, nc=colorama.Fore.RESET, diff --git a/dvc/main.py b/dvc/main.py index 789350425..1fdd0943d 100644 --- a/dvc/main.py +++ b/dvc/main.py @@ -1,5 +1,7 @@ """Main entry point for dvc CLI.""" +from __future__ import unicode_literals + import dvc.logger as logger from dvc.cli import parse_args from dvc.command.base import CmdBase diff --git a/dvc/output/__init__.py b/dvc/output/__init__.py index aa283de20..82f569141 100644 --- a/dvc/output/__init__.py +++ b/dvc/output/__init__.py @@ -1,7 +1,9 @@ +from __future__ import unicode_literals + import schema from dvc.config import Config -from dvc.utils.compat import urlparse +from dvc.utils.compat import urlparse, str from dvc.output.base import OutputBase from dvc.output.s3 import OutputS3 diff --git a/dvc/output/base.py b/dvc/output/base.py index 3b8b2b445..440f0d137 100644 --- a/dvc/output/base.py +++ b/dvc/output/base.py @@ -1,3 +1,6 @@ +from __future__ import unicode_literals +from dvc.utils.compat import str + import re from schema import Or, Optional diff --git a/dvc/output/gs.py b/dvc/output/gs.py index 74bb41218..29a75e4ad 100644 --- a/dvc/output/gs.py +++ b/dvc/output/gs.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + from dvc.remote.gs import RemoteGS from dvc.output.s3 import OutputS3 diff --git a/dvc/output/hdfs.py b/dvc/output/hdfs.py index 4759f4e82..9e35dd4a5 100644 --- a/dvc/output/hdfs.py +++ b/dvc/output/hdfs.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import posixpath from dvc.utils.compat import urlparse diff --git a/dvc/output/local.py b/dvc/output/local.py index 17eb1d927..5cf988359 100644 --- a/dvc/output/local.py +++ b/dvc/output/local.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import os import dvc.logger as logger diff --git a/dvc/output/s3.py b/dvc/output/s3.py index 7c46e11e3..2434d2fc5 100644 --- a/dvc/output/s3.py +++ b/dvc/output/s3.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import posixpath from dvc.remote.s3 import RemoteS3 diff --git a/dvc/output/ssh.py b/dvc/output/ssh.py index 6953ae50c..fd02fdc73 100644 --- a/dvc/output/ssh.py +++ b/dvc/output/ssh.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import getpass import posixpath diff --git a/dvc/progress.py b/dvc/progress.py index 432ceaeec..4ac34132a 100644 --- a/dvc/progress.py +++ b/dvc/progress.py @@ -1,6 +1,10 @@ """Manages progress bars for dvc project.""" from __future__ import print_function +from __future__ import unicode_literals + +from dvc.utils.compat import str + import sys import threading import dvc.logger as logger diff --git a/dvc/project.py b/dvc/project.py index 95399e1d5..11bae9d14 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -1,3 +1,7 @@ +from __future__ import unicode_literals + +from dvc.utils.compat import str, builtin_str + import collections import os import dvc.prompt as prompt @@ -1001,7 +1005,7 @@ class Project(object): col, row = hxsv_path.split(',') row = int(row) - reader = list(csv.DictReader(fd, delimiter=delimiter)) + reader = list(csv.DictReader(fd, delimiter=builtin_str(delimiter))) return self._do_read_metric_xsv(reader, row, col) def _read_metric_xsv(self, fd, xsv_path, delimiter): @@ -1010,7 +1014,7 @@ class Project(object): col, row = xsv_path.split(',') row = int(row) col = int(col) - reader = list(csv.reader(fd, delimiter=delimiter)) + reader = list(csv.reader(fd, delimiter=builtin_str(delimiter))) return self._do_read_metric_xsv(reader, row, col) def _read_metric(self, path, typ=None, xpath=None): diff --git a/dvc/prompt.py b/dvc/prompt.py index 9ae6a302a..e88acd24d 100644 --- a/dvc/prompt.py +++ b/dvc/prompt.py @@ -1,13 +1,12 @@ """Manages user prompts.""" +from __future__ import unicode_literals +from __future__ import print_function + import sys from getpass import getpass -try: - # NOTE: in Python3 raw_input() was renamed to input() - input = raw_input # pylint: disable=redefined-builtin, invalid-name -except NameError: - pass +from dvc.utils.compat import input def _ask(prompt, limited_to=None): diff --git a/dvc/remote/__init__.py b/dvc/remote/__init__.py index e57665fb7..7b1bdcb92 100644 --- a/dvc/remote/__init__.py +++ b/dvc/remote/__init__.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + from dvc.remote.azure import RemoteAzure from dvc.remote.gs import RemoteGS from dvc.remote.hdfs import RemoteHDFS diff --git a/dvc/remote/azure.py b/dvc/remote/azure.py index 749fb4d63..9b3b1ac4e 100644 --- a/dvc/remote/azure.py +++ b/dvc/remote/azure.py @@ -1,4 +1,6 @@ from __future__ import absolute_import +from __future__ import unicode_literals + import os import re diff --git a/dvc/remote/base.py b/dvc/remote/base.py index 1dec91615..977f6bb0e 100644 --- a/dvc/remote/base.py +++ b/dvc/remote/base.py @@ -1,3 +1,7 @@ +from __future__ import unicode_literals + +from dvc.utils.compat import str + import os import re import errno diff --git a/dvc/remote/gs.py b/dvc/remote/gs.py index e2abf722f..f486e5752 100644 --- a/dvc/remote/gs.py +++ b/dvc/remote/gs.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import os try: diff --git a/dvc/remote/hdfs.py b/dvc/remote/hdfs.py index fb8c75bd4..f9c5acc65 100644 --- a/dvc/remote/hdfs.py +++ b/dvc/remote/hdfs.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import os import re import getpass diff --git a/dvc/remote/http.py b/dvc/remote/http.py index 2faf871a8..fc502fc46 100644 --- a/dvc/remote/http.py +++ b/dvc/remote/http.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import os import threading import requests diff --git a/dvc/remote/local.py b/dvc/remote/local.py index 99e6884a4..4815d5aba 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -1,3 +1,7 @@ +from __future__ import unicode_literals + +from dvc.utils.compat import str + import os import stat import uuid @@ -239,12 +243,12 @@ class RemoteLOCAL(RemoteBase): with open(path, 'r') as fd: d = json.load(fd) except Exception: - msg = u"Failed to load dir cache '{}'" + msg = "Failed to load dir cache '{}'" logger.error(msg.format(os.path.relpath(path))) return [] if not isinstance(d, list): - msg = u"dir cache file format error '{}' [skipping the file]" + msg = "dir cache file format error '{}' [skipping the file]" logger.error(msg.format(os.path.relpath(path))) return [] diff --git a/dvc/remote/s3.py b/dvc/remote/s3.py index 75dc3f6aa..ee0c5e127 100644 --- a/dvc/remote/s3.py +++ b/dvc/remote/s3.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import os import threading diff --git a/dvc/remote/ssh.py b/dvc/remote/ssh.py index 54030105d..bd6ecf778 100644 --- a/dvc/remote/ssh.py +++ b/dvc/remote/ssh.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import os import getpass import posixpath diff --git a/dvc/scm/__init__.py b/dvc/scm/__init__.py index 14b66c44a..ef5d66b77 100644 --- a/dvc/scm/__init__.py +++ b/dvc/scm/__init__.py @@ -1,5 +1,7 @@ """Manages source control systems(e.g. Git).""" +from __future__ import unicode_literals + from dvc.scm.base import Base from dvc.scm.git import Git diff --git a/dvc/scm/base.py b/dvc/scm/base.py index d00a11e5e..d7c83e2bf 100644 --- a/dvc/scm/base.py +++ b/dvc/scm/base.py @@ -1,5 +1,7 @@ """Manages source control systems(e.g. Git) in dvc.""" +from __future__ import unicode_literals + import os from dvc.exceptions import DvcException diff --git a/dvc/scm/git.py b/dvc/scm/git.py index 63bb35137..0b38aeabc 100644 --- a/dvc/scm/git.py +++ b/dvc/scm/git.py @@ -1,9 +1,13 @@ """Manages Git.""" +from __future__ import unicode_literals + # NOTE: need this one because python 2 would re-import this module # (called git.py) instead of importing gitpython's git module. from __future__ import absolute_import +from dvc.utils.compat import str + import os import dvc.logger as logger diff --git a/dvc/stage.py b/dvc/stage.py index 196b22a43..426376a88 100644 --- a/dvc/stage.py +++ b/dvc/stage.py @@ -1,3 +1,7 @@ +from __future__ import unicode_literals + +from dvc.utils.compat import str + import os import yaml import posixpath @@ -14,7 +18,7 @@ from dvc.utils import dict_md5, fix_env class StageCmdFailedError(DvcException): def __init__(self, stage): - msg = u"stage '{}' cmd {} failed".format(stage.relpath, stage.cmd) + msg = "stage '{}' cmd {} failed".format(stage.relpath, stage.cmd) super(StageCmdFailedError, self).__init__(msg) @@ -72,7 +76,7 @@ class MissingDep(DvcException): else: dep = 'dependency' - msg = u'missing {}: {}'.format(dep, ', '.join(map(str, deps))) + msg = 'missing {}: {}'.format(dep, ', '.join(map(str, deps))) super(MissingDep, self).__init__(msg) @@ -84,7 +88,7 @@ class MissingDataSource(DvcException): if len(missing_files) > 1: source += 's' - msg = u'missing data {}: {}'.format(source, ', '.join(missing_files)) + msg = 'missing data {}: {}'.format(source, ', '.join(missing_files)) super(MissingDataSource, self).__init__(msg) @@ -251,7 +255,7 @@ class Stage(object): if interactive and not prompt.confirm(msg): raise DvcException('reproduction aborted by the user') - logger.info(u"Reproducing '{stage}'".format(stage=self.relpath)) + logger.info("Reproducing '{stage}'".format(stage=self.relpath)) self.run(dry=dry) @@ -261,8 +265,10 @@ class Stage(object): @staticmethod def validate(d, fname=None): + from dvc.utils import convert_to_unicode + try: - Schema(Stage.SCHEMA).validate(d) + Schema(Stage.SCHEMA).validate(convert_to_unicode(d)) except SchemaError as exc: raise StageFileFormatError(fname, exc) @@ -556,7 +562,7 @@ class Stage(object): self.deps[0].download(self.outs[0].path_info) elif self.is_data_source: - msg = u"Verifying data sources in '{}'".format(self.relpath) + msg = "Verifying data sources in '{}'".format(self.relpath) logger.info(msg) if not dry: self.check_missing_outputs() diff --git a/dvc/state.py b/dvc/state.py index 16a00ed9c..4ed5a20f1 100644 --- a/dvc/state.py +++ b/dvc/state.py @@ -1,5 +1,9 @@ """Manages state database used for checksum caching.""" +from __future__ import unicode_literals + +from dvc.utils.compat import str + import os import time import sqlite3 diff --git a/dvc/system.py b/dvc/system.py index 9c584aa37..72b5a2d62 100644 --- a/dvc/system.py +++ b/dvc/system.py @@ -1,3 +1,7 @@ +from __future__ import unicode_literals + +from dvc.utils.compat import str + import os diff --git a/dvc/updater.py b/dvc/updater.py index 24227a45e..62343977b 100644 --- a/dvc/updater.py +++ b/dvc/updater.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import sys import os import time diff --git a/dvc/utils/__init__.py b/dvc/utils/__init__.py index 120fff3ef..7d8a243fd 100644 --- a/dvc/utils/__init__.py +++ b/dvc/utils/__init__.py @@ -1,6 +1,8 @@ -""" -Helpers for other modules. -""" +"""Helpers for other modules.""" + +from __future__ import unicode_literals +from dvc.utils.compat import str, builtin_str + import os import sys import stat @@ -9,11 +11,6 @@ import json import shutil import hashlib -import dvc.logger as logger -from dvc.progress import progress -from dvc.istextfile import istextfile - - LOCAL_CHUNK_SIZE = 1024*1024 LARGE_FILE_SIZE = 1024*1024*1024 LARGE_DIR_SIZE = 100 @@ -25,6 +22,10 @@ def dos2unix(data): def file_md5(fname): """ get the (md5 hexdigest, md5 digest) of a file """ + import dvc.logger as logger + from dvc.progress import progress + from dvc.istextfile import istextfile + if os.path.exists(fname): hash_md5 = hashlib.md5() binary = not istextfile(fname) @@ -81,6 +82,9 @@ def dict_filter(d, exclude=[]): elif isinstance(d, dict): ret = {} for k, v in d.items(): + if isinstance(k, builtin_str): + k = str(k) + assert isinstance(k, str) if k in exclude: continue @@ -97,7 +101,9 @@ def dict_md5(d, exclude=[]): def copyfile(src, dest, no_progress_bar=False, name=None): - '''Copy file with progress bar''' + """Copy file with progress bar""" + from dvc.progress import progress + copied = 0 name = name if name else os.path.basename(dest) total = os.stat(src).st_size @@ -140,10 +146,12 @@ def move(src, dst): def remove(path): + import dvc.logger as logger + if not os.path.exists(path): return - logger.debug(u"Removing '{}'".format(os.path.relpath(path))) + logger.debug("Removing '{}'".format(os.path.relpath(path))) def _chmod(func, p, excinfo): perm = os.stat(p).st_mode @@ -192,3 +200,14 @@ def fix_env(env=None): env.pop(lp_key, None) return env + + +def convert_to_unicode(data): + if isinstance(data, builtin_str): + return str(data) + elif isinstance(data, dict): + return dict(map(convert_to_unicode, data.items())) + elif isinstance(data, list) or isinstance(data, tuple): + return type(data)(map(convert_to_unicode, data)) + else: + return data diff --git a/dvc/utils/compat.py b/dvc/utils/compat.py index 3059d41bb..01b80d1a1 100644 --- a/dvc/utils/compat.py +++ b/dvc/utils/compat.py @@ -1,6 +1,45 @@ """Handle import compatibility between Python 2 and Python 3""" -try: - from urlparse import urlparse, urljoin # noqa: F401 -except ImportError: +import sys + + +# Syntax sugar. +_ver = sys.version_info + +#: Python 2.x? +is_py2 = (_ver[0] == 2) + +#: Python 3.x? +is_py3 = (_ver[0] == 3) + +if is_py2: + from urlparse import urlparse, urljoin # noqa: F401 + from StringIO import StringIO # noqa: F401 + from BaseHTTPServer import HTTPServer # noqa: F401 + from SimpleHTTPServer import SimpleHTTPRequestHandler # noqa: F401 + import ConfigParser # noqa: F401 + + builtin_str = str # noqa: F821 + bytes = str # noqa: F821 + str = unicode # noqa: F821 + basestring = basestring # noqa: F821 + numeric_types = (int, long, float) # noqa: F821 + integer_types = (int, long) # noqa: F821 + input = raw_input # noqa: F821 + +elif is_py3: from urllib.parse import urlparse, urljoin # noqa: F401 + from io import StringIO # noqa: F401 + from http.server import ( # noqa: F401 + HTTPServer, # noqa: F401 + SimpleHTTPRequestHandler # noqa: F401 + ) # noqa: F401 + import configparser as ConfigParser # noqa: F401 + + builtin_str = str # noqa: F821 + str = str # noqa: F821 + bytes = bytes # noqa: F821 + basestring = (str, bytes) # noqa: F821 + numeric_types = (int, float) # noqa: F821 + integer_types = (int,) # noqa: F821 + input = input # noqa: F821 diff --git a/scripts/innosetup/config_gen.py b/scripts/innosetup/config_gen.py index ab7601ba9..2e82e58ae 100644 --- a/scripts/innosetup/config_gen.py +++ b/scripts/innosetup/config_gen.py @@ -1,10 +1,6 @@ # This script generates config.ini for setup.iss script from dvc import VERSION - -try: - import configparser as ConfigParser -except ImportError: - import ConfigParser +from dvc.utils.compat import ConfigParser config = ConfigParser.ConfigParser() config.add_section('Version') diff --git a/tests/test_add.py b/tests/test_add.py index e7621b116..3fe7e7a4a 100644 --- a/tests/test_add.py +++ b/tests/test_add.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import os import yaml import time diff --git a/tests/test_data_cloud.py b/tests/test_data_cloud.py index f9cdb8415..8aae99606 100644 --- a/tests/test_data_cloud.py +++ b/tests/test_data_cloud.py @@ -11,6 +11,7 @@ from dvc.state import State from mock import patch import dvc.logger as logger +from dvc.utils.compat import str from dvc.main import main from dvc.config import Config from dvc.data_cloud import (DataCloud, RemoteS3, RemoteGS, RemoteAzure, @@ -21,10 +22,7 @@ from dvc.utils import file_md5 from tests.basic_env import TestDvc from tests.utils import spy -try: - from StringIO import StringIO -except ImportError: - from io import StringIO +from dvc.utils.compat import StringIO TEST_REMOTE = 'upstream' TEST_SECTION = 'remote "{}"'.format(TEST_REMOTE) diff --git a/tests/test_import.py b/tests/test_import.py index 0ea0b6fa1..ae30e4150 100644 --- a/tests/test_import.py +++ b/tests/test_import.py @@ -1,3 +1,5 @@ +from dvc.utils.compat import str + import os from uuid import uuid4 @@ -7,10 +9,7 @@ from dvc.main import main from mock import patch from tests.basic_env import TestDvc -try: - from StringIO import StringIO -except ImportError: - from io import StringIO +from dvc.utils.compat import StringIO class TestCmdImport(TestDvc): diff --git a/tests/test_logger.py b/tests/test_logger.py index 25ed6cdc4..7d2ba5872 100644 --- a/tests/test_logger.py +++ b/tests/test_logger.py @@ -2,15 +2,11 @@ import traceback from mock import patch from unittest import TestCase -try: - from StringIO import StringIO -except ImportError: - from io import StringIO - import logging import dvc.logger as logger from dvc.command.base import CmdBase from dvc.exceptions import DvcException +from dvc.utils.compat import StringIO class TestLogger(TestCase): diff --git a/tests/test_repro.py b/tests/test_repro.py index 2a10475b1..dc23db423 100644 --- a/tests/test_repro.py +++ b/tests/test_repro.py @@ -1,3 +1,5 @@ +from dvc.utils.compat import str + import os import yaml import shutil diff --git a/tests/test_state.py b/tests/test_state.py index 5846e7a04..713009198 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -1,5 +1,7 @@ import os +from dvc.utils.compat import str + from dvc.state import State from dvc.utils import file_md5 from dvc.main import main diff --git a/tests/unit/state.py b/tests/unit/state.py index 956c0b11f..e967da717 100644 --- a/tests/unit/state.py +++ b/tests/unit/state.py @@ -1,3 +1,5 @@ +from dvc.utils.compat import str + import os from dvc.state import State diff --git a/tests/utils/httpd.py b/tests/utils/httpd.py index 3ea180446..078004116 100644 --- a/tests/utils/httpd.py +++ b/tests/utils/httpd.py @@ -2,11 +2,7 @@ import hashlib import os import threading -try: - from http.server import HTTPServer, SimpleHTTPRequestHandler -except ImportError: - from BaseHTTPServer import HTTPServer - from SimpleHTTPServer import SimpleHTTPRequestHandler +from dvc.utils.compat import HTTPServer, SimpleHTTPRequestHandler class ETagHandler(SimpleHTTPRequestHandler):