Compare commits

..
48 Commits
Author SHA1 Message Date
Ruslan Kuprieiev c6811ae005 dvc: bump to 0.19.10 2018-10-12 04:22:26 +03:00
Ruslan Kuprieiev 6ad674ca88 Merge pull request #1213 from efiop/master
state: use UNSIGNED INTEGER for inodes
2018-10-12 04:22:03 +03:00
Ruslan Kuprieiev 45afd36d76 state: use UNSIGNED INTEGER for inodes
Fixes #1207

Signed-off-by: Ruslan Kuprieiev <[email protected]>
2018-10-12 03:51:29 +03:00
Ruslan Kuprieiev 0aac01ab88 state: add debug msg
Signed-off-by: Ruslan Kuprieiev <[email protected]>
2018-10-12 03:51:29 +03:00
Ruslan Kuprieiev ce3bcc4af9 Merge pull request #1211 from efiop/master
dagascii: import asciimatics only when needed
2018-10-12 00:11:06 +03:00
Ruslan Kuprieiev 06064061d6 dagascii: import asciimatics only when needed
Related to #1210

Signed-off-by: Ruslan Kuprieiev <[email protected]>
2018-10-11 23:53:56 +03:00
Ruslan Kuprieiev cae6e712c6 dvc: bump to 0.19.9 2018-10-11 16:11:17 +03:00
Ruslan Kuprieiev 4b525fe1fc system: fix typo in _reflink_darwin 2018-10-11 16:05:27 +03:00
Ruslan Kuprieiev 13a2e4692d Merge pull request #1201 from efiop/master
repro: show which dependencies have changed
2018-10-11 00:10:39 +03:00
Ruslan Kuprieiev 1de87bc91c repro: show which dependencies have changed
Fixes #1164

Signed-off-by: Ruslan Kuprieiev <[email protected]>
2018-10-10 23:40:26 +03:00
Ruslan Kuprieiev 82b10a8256 Merge pull request #1200 from efiop/master
import: add a note about supported schemes
2018-10-10 02:59:09 +03:00
Ruslan Kuprieiev a736082595 import: add a note about supported schemes
Fixes #1199

Signed-off-by: Ruslan Kuprieiev <[email protected]>
2018-10-10 02:47:20 +03:00
Ruslan Kuprieiev 4888ab82a7 dvc: bump to 0.19.8 2018-10-09 21:01:17 +03:00
Ruslan Kuprieiev bc93cc0bf6 Merge pull request #1198 from efiop/master
hdfs: don't use close_fds on Windows
2018-10-09 21:00:55 +03:00
Ruslan Kuprieiev 971f3259f1 hdfs: don't use close_fds on Windows
Fixes #1197

Signed-off-by: Ruslan Kuprieiev <[email protected]>
2018-10-09 20:42:18 +03:00
Ruslan Kuprieiev a4f9789aa4 travis: upload coverage on python 2 2018-10-08 03:30:02 +03:00
Ruslan Kuprieiev 584972ba0a travis: upload coverage on every build 2018-10-08 03:08:35 +03:00
Ruslan Kuprieiev b2b76b84d6 Merge pull request #1195 from efiop/master
dvc: stop using ntfsutils module
2018-10-07 18:07:51 +03:00
Ruslan Kuprieiev a6ea2f3db9 dvc: stop using ntfsutils module
Fixes #1162

Signed-off-by: Ruslan Kuprieiev <[email protected]>
2018-10-07 17:49:29 +03:00
Ruslan Kuprieiev 881f188016 tests: run in parallel mode 2018-10-07 03:15:56 +03:00
Ruslan Kuprieiev c3f145591b Merge pull request #1193 from efiop/master
test: upload coverage only on CRON
2018-10-07 02:19:27 +03:00
Ruslan Kuprieiev 1a8842d8c6 test: upload coverage only on CRON
Signed-off-by: Ruslan Kuprieiev <[email protected]>
2018-10-07 02:09:46 +03:00
Ruslan Kuprieiev 143a7f79a1 test: add version requirements for codecov and coverage 2018-10-06 23:24:55 +03:00
Ruslan Kuprieiev 7bff8c6f89 dagascii: don't try to cover tty code 2018-10-06 16:46:01 +03:00
Ruslan Kuprieiev 2daa694bf2 Merge pull request #1192 from efiop/master
dagascii: add more keys
2018-10-06 15:52:52 +03:00
Ruslan Kuprieiev 27a96ef814 dagascii: add more keys
Signed-off-by: Ruslan Kuprieiev <[email protected]>
2018-10-06 15:41:36 +03:00
Ruslan Kuprieiev 315bbfde3b Merge pull request #1191 from efiop/master
show: use asciimatics to provide scrollable graph
2018-10-06 14:31:43 +03:00
Ruslan Kuprieiev 53b37bb266 show: use asciimatics to provide scrollable graph
Fixes #1177

Signed-off-by: Ruslan Kuprieiev <[email protected]>
2018-10-06 01:04:00 +03:00
Ruslan Kuprieiev 005db9bb3a Merge pull request #1190 from efiop/master
dvc: add a note about dvc.org/support to error msgs
2018-10-05 22:24:20 +03:00
Ruslan Kuprieiev 7b36fd007c dvc: add a note about dvc.org/support to error msgs
Fixes #1152

Signed-off-by: Ruslan Kuprieiev <[email protected]>
2018-10-05 21:26:55 +03:00
Ruslan Kuprieiev 2c9b08a89b Merge pull request #1189 from mroutis/update-networkx
Update networkx package (== 2.1, >= 2.1)
2018-10-05 03:07:54 +03:00
Ruslan Kuprieiev 66a962976e Merge pull request #1188 from mroutis/add-aur-repo
Add AUR package
2018-10-05 03:07:28 +03:00
Mr. Outis 4eff31dd5e Update README noticing that the AUR package is not officially maintained 2018-10-04 18:55:26 -05:00
Mr. Outis 377be75dd0 Update networkx package in setup.py 2018-10-04 18:44:10 -05:00
Ruslan Kuprieiev e015b9a167 readme: add a new line 2018-10-05 02:14:12 +03:00
Mr. Outis 2a7ec2d2df Update networkx package (== 2.1, >= 2.1)
The latest version is 2.2 (https://pypi.org/project/networkx).

Updating it to prevent collisions with some package managers already
using the latest one.
2018-10-04 18:08:27 -05:00
Mr. Outis 3d34f4e8c7 Add arch linux repository 2018-10-04 18:03:56 -05:00
Ruslan Kuprieiev c10bd344ea Merge pull request #1185 from efiop/master
stage: properly report StageFileFormatError
2018-10-04 13:30:11 +03:00
Ruslan Kuprieiev 6a8cd31ac9 stage: properly report StageFileFormatError
Fixes #1186

Signed-off-by: Ruslan Kuprieiev <[email protected]>
2018-10-04 13:21:01 +03:00
Ruslan Kuprieiev 2c49d7d004 Merge pull request #1182 from efiop/master
gc: add a comfirmation prompt
2018-10-03 00:00:15 +03:00
Ruslan Kuprieiev 990bbe32c1 gc: add a comfirmation prompt
Signed-off-by: Ruslan Kuprieiev <[email protected]>
2018-10-02 23:39:45 +03:00
Ruslan Kuprieiev f3668b0a4e cli: status: fix typo 2018-10-02 01:41:45 +03:00
Ruslan Kuprieiev 73eaa2a611 dvc: bump to 0.19.7 2018-10-01 21:35:42 +03:00
Ruslan Kuprieiev e63ae3d676 Merge pull request #1181 from jeepkd/fix-aws-env
s3: fix dvc not loading AWS settings from environment variables
2018-10-01 21:32:22 +03:00
Jeep Kiddee 9528ad6a1d Fix dvc not loading AWS settings from environment variables 2018-10-02 00:59:59 +07:00
Ruslan Kuprieiev aea92e0bdf Merge pull request #1180 from efiop/master
Misc test improvements
2018-10-01 18:26:42 +03:00
Ruslan Kuprieiev d1077c4457 test: improve coverage
Signed-off-by: Ruslan Kuprieiev <[email protected]>
2018-10-01 18:11:17 +03:00
Ruslan Kuprieiev edf73dc52f test: skip hdfs test if can't connect
Signed-off-by: Ruslan Kuprieiev <[email protected]>
2018-10-01 16:29:55 +03:00
26 changed files with 387 additions and 127 deletions
+11
View File
@@ -112,6 +112,17 @@ or:
$ brew cask install iterative/homebrew-dvc/dvc
AUR
---
*Unofficial package*, any inquiries regarding the AUR package,
`refer to the maintainer <https://github.com/mroutis/pkgbuilds>`_.
.. code-block:: bash
# https://aur.archlinux.org/packages/dvc
$ yay -S dvc
Related technologies
====================
+4 -4
View File
@@ -7,7 +7,7 @@ import os
import warnings
VERSION_BASE = '0.19.6'
VERSION_BASE = '0.19.10'
__version__ = VERSION_BASE
PACKAGEPATH = os.path.abspath(os.path.dirname(__file__))
@@ -33,13 +33,13 @@ if os.path.exists(os.path.join(HOMEPATH, 'setup.py')):
with open(VERSIONPATH, 'w+') as fd:
fd.write('# AUTOGENERATED by dvc/__init__.py\n')
fd.write('version = "{}"\n'.format(__version__))
except Exception:
except Exception: # pragma: no cover
pass
else:
else: # pragma: no cover
# Remove version.py so that it doesn't get into the release
if os.path.exists(VERSIONPATH):
os.unlink(VERSIONPATH)
else:
else: # pragma: no cover
# dvc was installed with pip or something. Hopefully we have our
# auto-generated version.py to help us provide a true version
try:
+19 -5
View File
@@ -46,7 +46,7 @@ class DvcParser(argparse.ArgumentParser):
sys.exit(2)
class VersionAction(argparse.Action):
class VersionAction(argparse.Action): # pragma: no cover
def __call__(self, parser, namespace, values, option_string=None):
print(VERSION)
sys.exit(0)
@@ -160,7 +160,15 @@ def parse_args(argv=None):
help=IMPORT_HELP)
import_parser.add_argument(
'url',
help='URL.')
help="URL. Supported urls: "
"'/path/to/file', "
"'C:\\\\path\\to\\file', "
"'s3://bucket/path/to/file', "
"'gs://bucket/path/to/file', "
"'hdfs://example.com/path/to/file', "
"'ssh://example.com:/path/to/file', "
"'remote://myremote/path/to/file'(see "
"`dvc remote` commands). ")
import_parser.add_argument(
'out',
help='Output.')
@@ -402,7 +410,7 @@ def parse_args(argv=None):
'--with-deps',
action='store_true',
default=False,
help='Show statusof a local cache compared to a '
help='Show status of a local cache compared to a '
'remote repository for all dependencies of the '
'specified target.')
status_parser.set_defaults(func=CmdDataStatus)
@@ -541,6 +549,12 @@ def parse_args(argv=None):
'-r',
'--remote',
help='Remote repository to collect garbage in.')
gc_parser.add_argument(
'-f',
'--force',
action='store_true',
default=False,
help='Force garbage collection.')
gc_parser.set_defaults(func=CmdGC)
# Config
@@ -882,7 +896,7 @@ def parse_args(argv=None):
and hasattr(args, 'targets') \
and len(args.targets) == 0 \
and not (hasattr(args, 'all_pipelines')
and args.all_pipelines):
and args.all_pipelines): # pragma: no cover
if hasattr(args, 'cwd'):
cwd = args.cwd
else:
@@ -900,7 +914,7 @@ def parse_args(argv=None):
and len(args.deps) == 0 \
and len(args.outs) == 0 \
and len(args.outs_no_cache) == 0 \
and len(args.command) == 0:
and len(args.command) == 0: # pragma: no cover
run_parser.error("Too few arguments. Specify at least one: "
"'-d', '-o', '-O', 'command'.")
+16
View File
@@ -3,6 +3,22 @@ from dvc.command.base import CmdBase
class CmdGC(CmdBase):
def run(self):
msg = 'This will remove all cache except the cache that is used in '
if not self.args.all_branches and not self.args.all_tags:
msg += 'the current git branch.'
elif self.args.all_branches and not self.args.all_tags:
msg += 'all git branches.'
elif not self.args.all_branches and self.args.all_tags:
msg += 'all git tags.'
else:
msg += 'all git branches and all git tags.'
self.project.logger.warn(msg)
msg = 'Are you sure you want to proceed?'
if not self.args.force and not self.project.prompt.prompt(msg):
return 1
self.project.gc(all_branches=self.args.all_branches,
all_tags=self.args.all_tags,
cloud=self.args.cloud,
+84 -2
View File
@@ -1,4 +1,6 @@
import sys
import math
import select
from grandalf.graphs import Vertex, Edge, Graph
from grandalf.layouts import SugiyamaLayout
@@ -12,11 +14,91 @@ class AsciiCanvas(object):
self.cols = cols
self.lines = lines
self.canvas = [[' '] * cols for l in range(lines)]
def draw(self):
for line in self.canvas:
print(''.join(line))
if sys.stdout.isatty(): # pragma: no cover
from asciimatics.screen import Screen
Screen.wrapper(self._do_draw)
else:
for line in self.canvas:
print(''.join(line))
def _do_draw(self, screen): # pragma: no cover
from asciimatics.event import KeyboardEvent
offset_x = 0
offset_y = 0
smaxrow, smaxcol = screen.dimensions
assert smaxrow > 1
assert smaxcol > 1
smaxrow -= 1
smaxcol -= 1
if self.lines + 1 > smaxrow:
max_y = self.lines + 1 - smaxrow
else:
max_y = 0
if self.cols + 1 > smaxcol:
max_x = self.cols + 1 - smaxcol
else:
max_x = 0
while True:
for y in range(smaxrow + 1):
y_index = offset_y + y
line = []
for x in range(smaxcol + 1):
x_index = offset_x + x
if len(self.canvas) > y_index \
and len(self.canvas[y_index]) > x_index:
line.append(self.canvas[y_index][x_index])
else:
line.append(' ')
assert len(line) == (smaxcol + 1)
screen.print_at(''.join(line), 0, y)
screen.refresh()
# NOTE: get_event() doesn't block by itself,
# so we have to do the blocking ourselves.
select.select([sys.stdin], [], [], None)
event = screen.get_event()
if not isinstance(event, KeyboardEvent):
continue
k = event.key_code
if k == screen.KEY_DOWN or k == ord('s'):
offset_y += 1
elif k == screen.KEY_PAGE_DOWN or k == ord('S'):
offset_y += smaxrow
elif k == screen.KEY_UP or k == ord('w'):
offset_y -= 1
elif k == screen.KEY_PAGE_UP or k == ord('W'):
offset_y -= smaxrow
elif k == screen.KEY_RIGHT or k == ord('d'):
offset_x += 1
elif k == ord('D'):
offset_x += smaxcol
elif k == screen.KEY_LEFT or k == ord('a'):
offset_x -= 1
elif k == ord('A'):
offset_x -= smaxcol
elif k == ord('q') or k == ord('Q'):
break
if offset_y > max_y:
offset_y = max_y
elif offset_y < 0:
offset_y = 0
if offset_x > max_x:
offset_x = max_x
elif offset_x < 0:
offset_x = 0
def point(self, x, y, char):
assert x >= 0
+2 -1
View File
@@ -67,7 +67,8 @@ class DependencyLOCAL(DependencyBase):
if not self.exists:
raise self.DoesNotExistError(self.rel_path)
if not os.path.isfile(self.path) and not os.path.isdir(self.path):
if not os.path.isfile(self.path) \
and not os.path.isdir(self.path): # pragma: no cover
raise self.IsNotFileOrDirError(self.rel_path)
if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \
+1 -1
View File
@@ -8,7 +8,7 @@ class DvcException(Exception):
if cause:
try:
self.cause_tb = traceback.format_exc()
except Exception:
except Exception: # pragma: no cover
pass
super(DvcException, self).__init__(msg)
+17 -7
View File
@@ -19,9 +19,16 @@ class Logger(object):
}
COLOR_MAP = {
'debug': colorama.Fore.BLUE,
'warn': colorama.Fore.YELLOW,
'error': colorama.Fore.RED
'green': colorama.Fore.GREEN,
'yellow': colorama.Fore.YELLOW,
'blue': colorama.Fore.BLUE,
'red': colorama.Fore.RED,
}
LEVEL_COLOR_MAP = {
'debug': 'blue',
'warn': 'yellow',
'error': 'red',
}
def __init__(self, loglevel=None):
@@ -58,12 +65,12 @@ class Logger(object):
Logger.logger().setLevel(logging.DEBUG)
@staticmethod
def colorize(msg, typ):
def colorize(msg, color):
header = ''
footer = ''
if sys.stdout.isatty(): # pragma: no cover
header = Logger.COLOR_MAP.get(typ.lower(), '')
header = Logger.COLOR_MAP.get(color.lower(), '')
footer = colorama.Style.RESET_ALL
return u'{}{}{}'.format(header, msg, footer)
@@ -84,7 +91,8 @@ class Logger(object):
@staticmethod
def _prefix(msg, typ):
return Logger.colorize('{}: '.format(msg), typ)
color = Logger.LEVEL_COLOR_MAP.get(typ.lower(), '')
return Logger.colorize('{}: '.format(msg), color)
@staticmethod
def error_prefix():
@@ -105,7 +113,9 @@ class Logger(object):
if Logger.logger().getEffectiveLevel() == logging.DEBUG and exc:
str_tb = str_tb if str_tb else traceback.format_exc()
Logger.logger().error(prefix + str_tb)
return Logger.logger().error(prefix + msg + str_exc)
chat = "\n\nHaving any troubles? Hit us up at dvc.org/support, " \
"we are always happy to help!"
return Logger.logger().error(prefix + msg + str_exc + chat)
@staticmethod
def warn(msg):
+2 -1
View File
@@ -89,7 +89,8 @@ class OutputLOCAL(DependencyLOCAL):
if not os.path.exists(self.path):
raise self.DoesNotExistError(self.rel_path)
if not os.path.isfile(self.path) and not os.path.isdir(self.path):
if not os.path.isfile(self.path) \
and not os.path.isdir(self.path): # pragma: no cover
raise self.IsNotFileOrDirError(self.rel_path)
if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \
+6 -1
View File
@@ -29,9 +29,14 @@ class RemoteHDFS(RemoteBase):
cmd = 'hadoop fs -' + cmd
if user:
cmd = 'HADOOP_USER_NAME={} '.format(user) + cmd
# NOTE: close_fds doesn't work with redirected stdin/stdout/stderr.
# See https://github.com/iterative/dvc/issues/1197.
close_fds = (os.name != 'nt')
p = Popen(cmd,
shell=True,
close_fds=True,
close_fds=close_fds,
executable=os.getenv('SHELL'),
env=fix_env(os.environ),
stdin=PIPE,
+9 -7
View File
@@ -57,19 +57,13 @@ class RemoteS3(RemoteBase):
creds_conf = configobj.ConfigObj(credentialpath)
creds = creds_conf.get(self.profile, {})
else:
creds = {}
creds = self._get_aws_config_from_env()
self.creds = creds
self.region = creds.get('region', self.region)
self.aws_access_key_id = creds.get('aws_access_key_id', None)
self.aws_secret_access_key = creds.get('aws_secret_access_key', None)
self.region = os.getenv('AWS_DEFAULT_REGION', self.region)
self.aws_access_key_id = os.getenv('AWS_ACCESS_KEY_ID',
self.aws_access_key_id)
self.aws_secret_access_key = os.getenv('AWS_SECRET_ACCESS_KEY',
self.aws_secret_access_key)
@property
def bucket(self):
return urlparse(self.url).netloc
@@ -89,6 +83,14 @@ class RemoteS3(RemoteBase):
region_name=self.region)
return session.client('s3', endpoint_url=self.endpoint_url)
def _get_aws_config_from_env(self):
creds = {
'region': os.getenv('AWS_DEFAULT_REGION'),
'aws_access_key_id': os.getenv('AWS_ACCESS_KEY_ID'),
'aws_secret_access_key': os.getenv('AWS_SECRET_ACCESS_KEY'),
}
return {k: v for k, v in creds.items() if v is not None}
def get_etag(self, bucket, key):
try:
obj = self.s3.head_object(Bucket=bucket, Key=key)
+39 -22
View File
@@ -1,6 +1,5 @@
import os
import yaml
import itertools
import posixpath
import subprocess
from schema import Schema, SchemaError, Optional, Or, And
@@ -19,8 +18,9 @@ class StageCmdFailedError(DvcException):
class StageFileFormatError(DvcException):
def __init__(self):
super(StageFileFormatError, self).__init__('Stage file format error')
def __init__(self, fname, e):
msg = "Stage file '{}' format error: {}".format(fname, str(e))
super(StageFileFormatError, self).__init__(msg)
class StageFileDoesNotExistError(DvcException):
@@ -134,8 +134,6 @@ class Stage(object):
if self.md5 and md5 and self.md5 == md5:
return False
msg = "Dvc file '{}' md5 changed(expected '{}', actual '{}')"
self.project.logger.debug(msg.format(self.relpath, self.md5, md5))
return True
@property
@@ -148,30 +146,51 @@ class Stage(object):
len(self.deps) == 1 and \
len(self.outs) == 1
def changed(self):
def changed(self, print_info=False):
ret = False
if print_info:
log = self.project.logger.info
else:
log = self.project.logger.debug
if self.is_callback:
msg = "Dvc file '{}' is a 'callback' stage (has a command and " \
"no dependencies) and thus always considered as changed."
self.project.logger.warn(msg.format(self.relpath))
ret = True
if self.locked:
entries = self.outs
else:
entries = itertools.chain(self.outs, self.deps)
for entry in entries:
if entry.changed():
if not self.locked:
for dep in self.deps:
if not dep.changed():
continue
if print_info:
msg = "Dependency '{}' of '{}' changed."
log(msg.format(dep, self.relpath))
ret = True
for out in self.outs:
if not out.changed():
continue
if print_info:
msg = "Output '{}' of '{}' changed."
log(msg.format(out, self.relpath))
ret = True
if self.changed_md5():
if print_info:
msg = "Dvc file '{}' changed."
log(msg.format(self.relpath))
ret = True
if ret:
msg = u'Dvc file \'{}\' changed'.format(self.relpath)
msg = "Stage '{}' changed.".format(self.relpath)
color = 'yellow'
else:
msg = u'Dvc file \'{}\' didn\'t change'.format(self.relpath)
msg = "Stage '{}' didn't change.".format(self.relpath)
color = 'green'
self.project.logger.debug(msg)
log(Logger.colorize(msg, color))
return ret
@@ -184,7 +203,7 @@ class Stage(object):
os.unlink(self.path)
def reproduce(self, force=False, dry=False, interactive=False):
if not self.changed() and not force:
if not self.changed(print_info=True) and not force:
return None
if (self.cmd or self.is_import) and not self.locked and not dry:
@@ -207,17 +226,15 @@ class Stage(object):
return self
@staticmethod
def validate(d):
def validate(d, fname=None):
try:
Schema(Stage.SCHEMA).validate(d)
except SchemaError as exc:
Logger.debug(str(exc))
raise StageFileFormatError()
raise StageFileFormatError(fname, exc)
@staticmethod
def loadd(project, d, path):
Stage.validate(d)
Stage.validate(d, fname=os.path.relpath(path))
path = os.path.abspath(path)
cwd = os.path.dirname(path)
cmd = d.get(Stage.PARAM_CMD, None)
+47 -43
View File
@@ -15,10 +15,10 @@ class StateDuplicateError(DvcException):
class State(object):
VERSION = 0
VERSION = 1
STATE_FILE = 'state'
STATE_TABLE = 'state'
STATE_TABLE_LAYOUT = "inode INTEGER PRIMARY KEY, " \
STATE_TABLE_LAYOUT = "inode UNSIGNED INTEGER PRIMARY KEY, " \
"mtime TEXT NOT NULL, " \
"md5 TEXT NOT NULL, " \
"timestamp TEXT NOT NULL"
@@ -29,7 +29,7 @@ class State(object):
LINK_STATE_TABLE = 'link_state'
LINK_STATE_TABLE_LAYOUT = "path TEXT PRIMARY KEY, " \
"inode INTEGER NOT NULL, " \
"inode UNSIGNED INTEGER NOT NULL, " \
"mtime TEXT NOT NULL"
STATE_ROW_LIMIT = 10000000
@@ -82,11 +82,15 @@ class State(object):
return actual.split('.')[0] != md5.split('.')[0]
def _execute(self, cmd):
Logger.debug(cmd)
return self.c.execute(cmd)
def _load(self):
from dvc import VERSION
cmd = "PRAGMA user_version;"
self.c.execute(cmd)
self._execute(cmd)
ret = self.c.fetchall()
assert len(ret) == 1
assert len(ret[0]) == 1
@@ -106,26 +110,26 @@ class State(object):
"Reformatting to the current version '{}'."
self.project.logger.warn(msg.format(self.version, self.VERSION))
cmd = "DROP TABLE IF EXISTS {};"
self.c.execute(cmd.format(self.STATE_TABLE))
self.c.execute(cmd.format(self.STATE_INFO_TABLE))
self.c.execute(cmd.format(self.LINK_STATE_TABLE))
self._execute(cmd.format(self.STATE_TABLE))
self._execute(cmd.format(self.STATE_INFO_TABLE))
self._execute(cmd.format(self.LINK_STATE_TABLE))
# Check that the state file is indeed a database
cmd = "CREATE TABLE IF NOT EXISTS {} ({})"
self.c.execute(cmd.format(self.STATE_TABLE,
self.STATE_TABLE_LAYOUT))
self.c.execute(cmd.format(self.STATE_INFO_TABLE,
self.STATE_INFO_TABLE_LAYOUT))
self.c.execute(cmd.format(self.LINK_STATE_TABLE,
self.LINK_STATE_TABLE_LAYOUT))
self._execute(cmd.format(self.STATE_TABLE,
self.STATE_TABLE_LAYOUT))
self._execute(cmd.format(self.STATE_INFO_TABLE,
self.STATE_INFO_TABLE_LAYOUT))
self._execute(cmd.format(self.LINK_STATE_TABLE,
self.LINK_STATE_TABLE_LAYOUT))
cmd = "INSERT OR IGNORE INTO {} (count) SELECT 0 " \
"WHERE NOT EXISTS (SELECT * FROM {})"
self.c.execute(cmd.format(self.STATE_INFO_TABLE,
self.STATE_INFO_TABLE))
self._execute(cmd.format(self.STATE_INFO_TABLE,
self.STATE_INFO_TABLE))
cmd = "PRAGMA user_version = {};"
self.c.execute(cmd.format(self.VERSION))
self._execute(cmd.format(self.VERSION))
def load(self):
retries = 1
@@ -157,8 +161,8 @@ class State(object):
assert self.db is not None
cmd = "SELECT count from {} WHERE rowid={}"
self.c.execute(cmd.format(self.STATE_INFO_TABLE,
self.STATE_INFO_ROW))
self._execute(cmd.format(self.STATE_INFO_TABLE,
self.STATE_INFO_ROW))
ret = self.c.fetchall()
assert len(ret) == 1
assert len(ret[0]) == 1
@@ -172,24 +176,24 @@ class State(object):
delete += int(self.row_limit * (self.row_cleanup_quota/100.))
cmd = "DELETE FROM {} WHERE timestamp IN (" \
"SELECT timestamp FROM {} ORDER BY timestamp ASC LIMIT {});"
self.c.execute(cmd.format(self.STATE_TABLE,
self.STATE_TABLE,
delete))
self._execute(cmd.format(self.STATE_TABLE,
self.STATE_TABLE,
delete))
self.c.execute("VACUUM")
self._execute("VACUUM")
cmd = "SELECT COUNT(*) FROM {}"
self.c.execute(cmd.format(self.STATE_TABLE))
self._execute(cmd.format(self.STATE_TABLE))
ret = self.c.fetchall()
assert len(ret) == 1
assert len(ret[0]) == 1
count = ret[0][0]
cmd = "UPDATE {} SET count = {} WHERE rowid = {}"
self.c.execute(cmd.format(self.STATE_INFO_TABLE,
count,
self.STATE_INFO_ROW))
self._execute(cmd.format(self.STATE_INFO_TABLE,
count,
self.STATE_INFO_ROW))
self.db.commit()
self.c.close()
@@ -225,17 +229,17 @@ class State(object):
cmd = 'SELECT * from {} WHERE inode={}'.format(self.STATE_TABLE,
inode)
self.c.execute(cmd)
self._execute(cmd)
ret = self.c.fetchall()
if len(ret) == 0:
md5, info = self._collect(path)
cmd = 'INSERT INTO {}(inode, mtime, md5, timestamp) ' \
'VALUES ({}, "{}", "{}", "{}")'
self.c.execute(cmd.format(self.STATE_TABLE,
inode,
mtime,
md5,
int(nanotime.timestamp(time.time()))))
self._execute(cmd.format(self.STATE_TABLE,
inode,
mtime,
md5,
int(nanotime.timestamp(time.time()))))
self.inserts += 1
else:
assert len(ret) == 1
@@ -247,17 +251,17 @@ class State(object):
cmd = 'UPDATE {} SET ' \
'mtime = "{}", md5 = "{}", timestamp = "{}" ' \
'WHERE inode = {}'
self.c.execute(cmd.format(self.STATE_TABLE,
mtime,
md5,
int(nanotime.timestamp(time.time())),
inode))
self._execute(cmd.format(self.STATE_TABLE,
mtime,
md5,
int(nanotime.timestamp(time.time())),
inode))
else:
info = None
cmd = 'UPDATE {} SET timestamp = "{}" WHERE inode = {}'
self.c.execute(cmd.format(self.STATE_TABLE,
int(nanotime.timestamp(time.time())),
inode))
self._execute(cmd.format(self.STATE_TABLE,
int(nanotime.timestamp(time.time())),
inode))
return (md5, info)
@@ -283,12 +287,12 @@ class State(object):
relpath,
inode,
mtime)
self.c.execute(cmd)
self._execute(cmd)
def remove_unused_links(self, used):
unused = []
self.c.execute('SELECT * FROM {}'.format(self.LINK_STATE_TABLE))
self._execute('SELECT * FROM {}'.format(self.LINK_STATE_TABLE))
for row in self.c:
p, i, m = row
path = os.path.join(self.root_dir, p)
@@ -309,4 +313,4 @@ class State(object):
for p in unused:
cmd = 'DELETE FROM {} WHERE path = "{}"'
self.c.execute(cmd.format(self.LINK_STATE_TABLE, p))
self._execute(cmd.format(self.LINK_STATE_TABLE, p))
+69 -22
View File
@@ -1,8 +1,5 @@
import os
if os.name == 'nt':
import ntfsutils.hardlink as winlink
class System(object):
@staticmethod
@@ -11,10 +8,20 @@ class System(object):
@staticmethod
def hardlink(source, link_name):
import ctypes
if System.is_unix():
return os.link(source, link_name)
os.link(source, link_name)
return
return winlink.create(source, link_name)
CreateHardLink = ctypes.windll.kernel32.CreateHardLinkW
CreateHardLink.argtypes = [ctypes.c_wchar_p,
ctypes.c_wchar_p,
ctypes.c_void_p]
CreateHardLink.restype = ctypes.wintypes.BOOL
res = CreateHardLink(link_name, source, None)
if res == 0:
raise ctypes.WinError()
@staticmethod
def symlink(source, link_name):
@@ -48,7 +55,7 @@ class System(object):
clonefile.restype = ctypes.c_int
return clonefile(ctypes.c_char_p(src.encode('utf-8')),
ctypes.c_char_p(src.encode('utf-8')),
ctypes.c_char_p(dst.encode('utf-8')),
ctypes.c_int(0))
@staticmethod
@@ -102,30 +109,70 @@ class System(object):
if ret != 0:
raise DvcException('Reflink is not supported')
# FIXME
# Temporary fix while waiting for the PR to be merged and released:
# https://github.com/sunshowers/ntfs/pull/11
@staticmethod
def getdirinfo(path):
from ntfsutils.fs import FILE_FLAG_BACKUP_SEMANTICS, FILE_SHARE_READ
from ntfsutils.fs import OPEN_EXISTING, BY_HANDLE_FILE_INFORMATION
from ntfsutils.fs import CreateFile, WinError
from ntfsutils.fs import GetFileInformationByHandle, CloseHandle
import ctypes
from ctypes import c_void_p, c_wchar_p, Structure, WinError
from ctypes.wintypes import DWORD, HANDLE, POINTER, BOOL
FILE_FLAG_BACKUP_SEMANTICS = 0x02000000
FILE_SHARE_READ = 0x00000001
OPEN_EXISTING = 3
class FILETIME(Structure):
_fields_ = [("dwLowDateTime", DWORD),
("dwHighDateTime", DWORD)]
class BY_HANDLE_FILE_INFORMATION(Structure):
_fields_ = [("dwFileAttributes", DWORD),
("ftCreationTime", FILETIME),
("ftLastAccessTime", FILETIME),
("ftLastWriteTime", FILETIME),
("dwVolumeSerialNumber", DWORD),
("nFileSizeHigh", DWORD),
("nFileSizeLow", DWORD),
("nNumberOfLinks", DWORD),
("nFileIndexHigh", DWORD),
("nFileIndexLow", DWORD)]
flags = FILE_FLAG_BACKUP_SEMANTICS
hfile = CreateFile(path,
0,
FILE_SHARE_READ,
None,
OPEN_EXISTING,
flags,
None)
func = ctypes.windll.kernel32.CreateFileW
func.argtypes = [c_wchar_p,
DWORD,
DWORD,
c_void_p,
DWORD,
DWORD,
HANDLE]
func.restype = HANDLE
hfile = func(path,
0,
FILE_SHARE_READ,
None,
OPEN_EXISTING,
flags,
None)
if hfile is None:
raise WinError()
func = ctypes.windll.kernel32.GetFileInformationByHandle
func.argtypes = [HANDLE, POINTER(BY_HANDLE_FILE_INFORMATION)]
func.restype = BOOL
info = BY_HANDLE_FILE_INFORMATION()
rv = GetFileInformationByHandle(hfile, info)
CloseHandle(hfile)
rv = func(hfile, info)
func = ctypes.windll.kernel32.CloseHandle
func.argtypes = [HANDLE]
func.restype = BOOL
func(hfile)
if rv == 0:
raise WinError()
return info
@staticmethod
+1 -1
View File
@@ -6,7 +6,7 @@ from dvc import VERSION_BASE
from dvc.logger import Logger
class Updater(object):
class Updater(object): # pragma: no cover
URL = 'https://4ki8820rsf.execute-api.us-east-2.amazonaws.com/' \
'prod/latest-version'
UPDATER_FILE = 'updater'
+2 -2
View File
@@ -8,10 +8,9 @@ google-cloud-storage==1.12.0
PyInstaller==3.3.1
colorama>=0.3.9
configobj>=5.0.6
networkx==2.1
networkx>=2.1
pyyaml>=3.12
gitpython>=2.1.8
ntfsutils>=0.1.4
setuptools>=34.0.0
nanotime>=0.5.2
pyasn1>=0.4.1
@@ -23,3 +22,4 @@ wheel>=0.31.1
futures>=3.2.0; python_version == "2.7"
grandalf==0.6
pydot>=1.2.4
asciimatics>=1.10.0
+2 -1
View File
@@ -6,7 +6,8 @@ set -e
python -mtests
if [[ "$TRAVIS_PULL_REQUEST" == "false" && \
"$TRAVIS_SECURE_ENV_VARS" == "true" ]]; then
"$TRAVIS_SECURE_ENV_VARS" == "true" && \
"$(python -c 'import sys; print(sys.version_info[0])')" == '2' ]]; then
codecov
fi
+2 -2
View File
@@ -12,10 +12,9 @@ install_requires = [
"future>=0.16.0",
"colorama>=0.3.9",
"configobj>=5.0.6",
"networkx==2.1",
"networkx>=2.1",
"pyyaml>=3.12",
"gitpython>=2.1.8",
"ntfsutils>=0.1.4",
"setuptools>=34.0.0",
"nanotime>=0.5.2",
"pyasn1>=0.4.1",
@@ -23,6 +22,7 @@ install_requires = [
"jsonpath-rw==1.4.0",
"requests>=2.18.4",
"grandalf==0.6",
"asciimatics>=1.10.0",
]
# Extra dependencies for remote integrations
+2 -2
View File
@@ -1,8 +1,8 @@
pytest>=3.3.1
nose>=1.3.7
mock>=2.0.0
coverage
codecov
coverage>=4.5.1
codecov>=2.0.15
xmltodict>=0.11.0
awscli>=1.14.18
google-compute-engine
+2 -1
View File
@@ -7,4 +7,5 @@ os.chdir(REPO_ROOT)
os.putenv('PATH', '{}:{}'.format(os.path.join(REPO_ROOT, 'bin'), os.getenv('PATH')))
os.putenv('DVC_HOME', REPO_ROOT)
check_call('nosetests -v --processes=-1 --process-timeout=200 --cover-inclusive --cover-erase --cover-package=dvc --with-coverage', shell=True)
cmd = 'nosetests -v --processes=-1 --process-timeout=200 --cover-inclusive --cover-erase --cover-package=dvc --with-coverage'
check_call(cmd, shell=True)
+9
View File
@@ -103,3 +103,12 @@ class TestSharedCacheDir(TestDir):
self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[0]))), 1)
self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[1]))), 1)
self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[2]))), 1)
class TestCacheLinkType(TestDvc):
def test(self):
ret = main(['config', 'cache.type', 'reflink,copy'])
self.assertEqual(ret, 0)
ret = main(['add', self.FOO])
self.assertEqual(ret, 0)
+12
View File
@@ -205,3 +205,15 @@ class TestCheckoutEmptyDir(TestDvc):
self.assertTrue(os.path.isdir(dname))
self.assertEqual(len(os.listdir(dname)), 0)
class TestCheckoutNotCachedFile(TestDvc):
def test(self):
cmd = 'python {} {} {}'.format(self.CODE, self.FOO, 'out')
self.dvc.add(self.FOO)
stage = self.dvc.run(cmd=cmd,
deps=[self.FOO, self.CODE],
outs_no_cache=['out'])
self.dvc.checkout()
+8 -1
View File
@@ -1,5 +1,5 @@
from subprocess import CalledProcessError
from subprocess import check_output
from subprocess import check_output, Popen
from unittest import SkipTest
import os
import time
@@ -94,6 +94,13 @@ def _should_test_hdfs():
except (CalledProcessError, IOError):
return False
p = Popen('hadoop fs -ls hdfs://127.0.0.1/',
shell=True,
executable=os.getenv('SHELL'))
p.communicate()
if p.returncode != 0:
return False
return True
+2 -1
View File
@@ -29,7 +29,8 @@ class TestGC(TestDvc):
self._test_gc()
def test_cli(self):
ret = main(['gc'])
ret = main(['gc', '-f'])
self.assertEqual(ret, 0)
self._test_gc()
def _test_gc(self):
+13
View File
@@ -72,3 +72,16 @@ class TestRemoteRemoveDefault(TestDvc):
core = local_config.get(Config.SECTION_CORE, None)
if core is not None:
self.assertTrue(Config.SECTION_CORE_REMOTE not in core.keys())
class TestRemoteRemove(TestDvc):
def test(self):
ret = main(['config', 'core.jobs', '1'])
self.assertEqual(ret, 0)
remote = 'mys3'
ret = main(['remote', 'add', remote, 's3://bucket/name'])
self.assertEqual(ret, 0)
ret = main(['remote', 'remove', remote])
self.assertEqual(ret, 0)
+6
View File
@@ -146,6 +146,12 @@ class TestReproDry(TestReproChangedData):
self.assertFalse(filecmp.cmp(self.file1, self.BAR, shallow=False))
class TestReproUpToDate(TestRepro):
def test(self):
ret = main(['repro', self.file1_stage])
self.assertEqual(ret, 0)
class TestReproDryNoExec(TestDvc):
def test(self):
deps = []