Compare commits

..
12 Commits
Author SHA1 Message Date
Ruslan Kuprieiev c6811ae005 dvc: bump to 0.19.10 2018-10-12 04:22:26 +03:00
Ruslan Kuprieiev 6ad674ca88 Merge pull request #1213 from efiop/master
state: use UNSIGNED INTEGER for inodes
2018-10-12 04:22:03 +03:00
Ruslan Kuprieiev 45afd36d76 state: use UNSIGNED INTEGER for inodes
Fixes #1207

Signed-off-by: Ruslan Kuprieiev <[email protected]>
2018-10-12 03:51:29 +03:00
Ruslan Kuprieiev 0aac01ab88 state: add debug msg
Signed-off-by: Ruslan Kuprieiev <[email protected]>
2018-10-12 03:51:29 +03:00
Ruslan Kuprieiev ce3bcc4af9 Merge pull request #1211 from efiop/master
dagascii: import asciimatics only when needed
2018-10-12 00:11:06 +03:00
Ruslan Kuprieiev 06064061d6 dagascii: import asciimatics only when needed
Related to #1210

Signed-off-by: Ruslan Kuprieiev <[email protected]>
2018-10-11 23:53:56 +03:00
Ruslan Kuprieiev cae6e712c6 dvc: bump to 0.19.9 2018-10-11 16:11:17 +03:00
Ruslan Kuprieiev 4b525fe1fc system: fix typo in _reflink_darwin 2018-10-11 16:05:27 +03:00
Ruslan Kuprieiev 13a2e4692d Merge pull request #1201 from efiop/master
repro: show which dependencies have changed
2018-10-11 00:10:39 +03:00
Ruslan Kuprieiev 1de87bc91c repro: show which dependencies have changed
Fixes #1164

Signed-off-by: Ruslan Kuprieiev <[email protected]>
2018-10-10 23:40:26 +03:00
Ruslan Kuprieiev 82b10a8256 Merge pull request #1200 from efiop/master
import: add a note about supported schemes
2018-10-10 02:59:09 +03:00
Ruslan Kuprieiev a736082595 import: add a note about supported schemes
Fixes #1199

Signed-off-by: Ruslan Kuprieiev <[email protected]>
2018-10-10 02:47:20 +03:00
7 changed files with 108 additions and 70 deletions
+1 -1
View File
@@ -7,7 +7,7 @@ import os
import warnings
VERSION_BASE = '0.19.8'
VERSION_BASE = '0.19.10'
__version__ = VERSION_BASE
PACKAGEPATH = os.path.abspath(os.path.dirname(__file__))
+9 -1
View File
@@ -160,7 +160,15 @@ def parse_args(argv=None):
help=IMPORT_HELP)
import_parser.add_argument(
'url',
help='URL.')
help="URL. Supported urls: "
"'/path/to/file', "
"'C:\\\\path\\to\\file', "
"'s3://bucket/path/to/file', "
"'gs://bucket/path/to/file', "
"'hdfs://example.com/path/to/file', "
"'ssh://example.com:/path/to/file', "
"'remote://myremote/path/to/file'(see "
"`dvc remote` commands). ")
import_parser.add_argument(
'out',
help='Output.')
+3 -3
View File
@@ -2,9 +2,6 @@ import sys
import math
import select
from asciimatics.screen import Screen
from asciimatics.event import KeyboardEvent
from grandalf.graphs import Vertex, Edge, Graph
from grandalf.layouts import SugiyamaLayout
from grandalf.routing import route_with_lines, EdgeViewer
@@ -22,12 +19,15 @@ class AsciiCanvas(object):
def draw(self):
if sys.stdout.isatty(): # pragma: no cover
from asciimatics.screen import Screen
Screen.wrapper(self._do_draw)
else:
for line in self.canvas:
print(''.join(line))
def _do_draw(self, screen): # pragma: no cover
from asciimatics.event import KeyboardEvent
offset_x = 0
offset_y = 0
smaxrow, smaxcol = screen.dimensions
+14 -6
View File
@@ -19,9 +19,16 @@ class Logger(object):
}
COLOR_MAP = {
'debug': colorama.Fore.BLUE,
'warn': colorama.Fore.YELLOW,
'error': colorama.Fore.RED
'green': colorama.Fore.GREEN,
'yellow': colorama.Fore.YELLOW,
'blue': colorama.Fore.BLUE,
'red': colorama.Fore.RED,
}
LEVEL_COLOR_MAP = {
'debug': 'blue',
'warn': 'yellow',
'error': 'red',
}
def __init__(self, loglevel=None):
@@ -58,12 +65,12 @@ class Logger(object):
Logger.logger().setLevel(logging.DEBUG)
@staticmethod
def colorize(msg, typ):
def colorize(msg, color):
header = ''
footer = ''
if sys.stdout.isatty(): # pragma: no cover
header = Logger.COLOR_MAP.get(typ.lower(), '')
header = Logger.COLOR_MAP.get(color.lower(), '')
footer = colorama.Style.RESET_ALL
return u'{}{}{}'.format(header, msg, footer)
@@ -84,7 +91,8 @@ class Logger(object):
@staticmethod
def _prefix(msg, typ):
return Logger.colorize('{}: '.format(msg), typ)
color = Logger.LEVEL_COLOR_MAP.get(typ.lower(), '')
return Logger.colorize('{}: '.format(msg), color)
@staticmethod
def error_prefix():
+33 -15
View File
@@ -1,6 +1,5 @@
import os
import yaml
import itertools
import posixpath
import subprocess
from schema import Schema, SchemaError, Optional, Or, And
@@ -135,8 +134,6 @@ class Stage(object):
if self.md5 and md5 and self.md5 == md5:
return False
msg = "Dvc file '{}' md5 changed(expected '{}', actual '{}')"
self.project.logger.debug(msg.format(self.relpath, self.md5, md5))
return True
@property
@@ -149,30 +146,51 @@ class Stage(object):
len(self.deps) == 1 and \
len(self.outs) == 1
def changed(self):
def changed(self, print_info=False):
ret = False
if print_info:
log = self.project.logger.info
else:
log = self.project.logger.debug
if self.is_callback:
msg = "Dvc file '{}' is a 'callback' stage (has a command and " \
"no dependencies) and thus always considered as changed."
self.project.logger.warn(msg.format(self.relpath))
ret = True
if self.locked:
entries = self.outs
else:
entries = itertools.chain(self.outs, self.deps)
for entry in entries:
if entry.changed():
if not self.locked:
for dep in self.deps:
if not dep.changed():
continue
if print_info:
msg = "Dependency '{}' of '{}' changed."
log(msg.format(dep, self.relpath))
ret = True
for out in self.outs:
if not out.changed():
continue
if print_info:
msg = "Output '{}' of '{}' changed."
log(msg.format(out, self.relpath))
ret = True
if self.changed_md5():
if print_info:
msg = "Dvc file '{}' changed."
log(msg.format(self.relpath))
ret = True
if ret:
msg = u'Dvc file \'{}\' changed'.format(self.relpath)
msg = "Stage '{}' changed.".format(self.relpath)
color = 'yellow'
else:
msg = u'Dvc file \'{}\' didn\'t change'.format(self.relpath)
msg = "Stage '{}' didn't change.".format(self.relpath)
color = 'green'
self.project.logger.debug(msg)
log(Logger.colorize(msg, color))
return ret
@@ -185,7 +203,7 @@ class Stage(object):
os.unlink(self.path)
def reproduce(self, force=False, dry=False, interactive=False):
if not self.changed() and not force:
if not self.changed(print_info=True) and not force:
return None
if (self.cmd or self.is_import) and not self.locked and not dry:
+47 -43
View File
@@ -15,10 +15,10 @@ class StateDuplicateError(DvcException):
class State(object):
VERSION = 0
VERSION = 1
STATE_FILE = 'state'
STATE_TABLE = 'state'
STATE_TABLE_LAYOUT = "inode INTEGER PRIMARY KEY, " \
STATE_TABLE_LAYOUT = "inode UNSIGNED INTEGER PRIMARY KEY, " \
"mtime TEXT NOT NULL, " \
"md5 TEXT NOT NULL, " \
"timestamp TEXT NOT NULL"
@@ -29,7 +29,7 @@ class State(object):
LINK_STATE_TABLE = 'link_state'
LINK_STATE_TABLE_LAYOUT = "path TEXT PRIMARY KEY, " \
"inode INTEGER NOT NULL, " \
"inode UNSIGNED INTEGER NOT NULL, " \
"mtime TEXT NOT NULL"
STATE_ROW_LIMIT = 10000000
@@ -82,11 +82,15 @@ class State(object):
return actual.split('.')[0] != md5.split('.')[0]
def _execute(self, cmd):
Logger.debug(cmd)
return self.c.execute(cmd)
def _load(self):
from dvc import VERSION
cmd = "PRAGMA user_version;"
self.c.execute(cmd)
self._execute(cmd)
ret = self.c.fetchall()
assert len(ret) == 1
assert len(ret[0]) == 1
@@ -106,26 +110,26 @@ class State(object):
"Reformatting to the current version '{}'."
self.project.logger.warn(msg.format(self.version, self.VERSION))
cmd = "DROP TABLE IF EXISTS {};"
self.c.execute(cmd.format(self.STATE_TABLE))
self.c.execute(cmd.format(self.STATE_INFO_TABLE))
self.c.execute(cmd.format(self.LINK_STATE_TABLE))
self._execute(cmd.format(self.STATE_TABLE))
self._execute(cmd.format(self.STATE_INFO_TABLE))
self._execute(cmd.format(self.LINK_STATE_TABLE))
# Check that the state file is indeed a database
cmd = "CREATE TABLE IF NOT EXISTS {} ({})"
self.c.execute(cmd.format(self.STATE_TABLE,
self.STATE_TABLE_LAYOUT))
self.c.execute(cmd.format(self.STATE_INFO_TABLE,
self.STATE_INFO_TABLE_LAYOUT))
self.c.execute(cmd.format(self.LINK_STATE_TABLE,
self.LINK_STATE_TABLE_LAYOUT))
self._execute(cmd.format(self.STATE_TABLE,
self.STATE_TABLE_LAYOUT))
self._execute(cmd.format(self.STATE_INFO_TABLE,
self.STATE_INFO_TABLE_LAYOUT))
self._execute(cmd.format(self.LINK_STATE_TABLE,
self.LINK_STATE_TABLE_LAYOUT))
cmd = "INSERT OR IGNORE INTO {} (count) SELECT 0 " \
"WHERE NOT EXISTS (SELECT * FROM {})"
self.c.execute(cmd.format(self.STATE_INFO_TABLE,
self.STATE_INFO_TABLE))
self._execute(cmd.format(self.STATE_INFO_TABLE,
self.STATE_INFO_TABLE))
cmd = "PRAGMA user_version = {};"
self.c.execute(cmd.format(self.VERSION))
self._execute(cmd.format(self.VERSION))
def load(self):
retries = 1
@@ -157,8 +161,8 @@ class State(object):
assert self.db is not None
cmd = "SELECT count from {} WHERE rowid={}"
self.c.execute(cmd.format(self.STATE_INFO_TABLE,
self.STATE_INFO_ROW))
self._execute(cmd.format(self.STATE_INFO_TABLE,
self.STATE_INFO_ROW))
ret = self.c.fetchall()
assert len(ret) == 1
assert len(ret[0]) == 1
@@ -172,24 +176,24 @@ class State(object):
delete += int(self.row_limit * (self.row_cleanup_quota/100.))
cmd = "DELETE FROM {} WHERE timestamp IN (" \
"SELECT timestamp FROM {} ORDER BY timestamp ASC LIMIT {});"
self.c.execute(cmd.format(self.STATE_TABLE,
self.STATE_TABLE,
delete))
self._execute(cmd.format(self.STATE_TABLE,
self.STATE_TABLE,
delete))
self.c.execute("VACUUM")
self._execute("VACUUM")
cmd = "SELECT COUNT(*) FROM {}"
self.c.execute(cmd.format(self.STATE_TABLE))
self._execute(cmd.format(self.STATE_TABLE))
ret = self.c.fetchall()
assert len(ret) == 1
assert len(ret[0]) == 1
count = ret[0][0]
cmd = "UPDATE {} SET count = {} WHERE rowid = {}"
self.c.execute(cmd.format(self.STATE_INFO_TABLE,
count,
self.STATE_INFO_ROW))
self._execute(cmd.format(self.STATE_INFO_TABLE,
count,
self.STATE_INFO_ROW))
self.db.commit()
self.c.close()
@@ -225,17 +229,17 @@ class State(object):
cmd = 'SELECT * from {} WHERE inode={}'.format(self.STATE_TABLE,
inode)
self.c.execute(cmd)
self._execute(cmd)
ret = self.c.fetchall()
if len(ret) == 0:
md5, info = self._collect(path)
cmd = 'INSERT INTO {}(inode, mtime, md5, timestamp) ' \
'VALUES ({}, "{}", "{}", "{}")'
self.c.execute(cmd.format(self.STATE_TABLE,
inode,
mtime,
md5,
int(nanotime.timestamp(time.time()))))
self._execute(cmd.format(self.STATE_TABLE,
inode,
mtime,
md5,
int(nanotime.timestamp(time.time()))))
self.inserts += 1
else:
assert len(ret) == 1
@@ -247,17 +251,17 @@ class State(object):
cmd = 'UPDATE {} SET ' \
'mtime = "{}", md5 = "{}", timestamp = "{}" ' \
'WHERE inode = {}'
self.c.execute(cmd.format(self.STATE_TABLE,
mtime,
md5,
int(nanotime.timestamp(time.time())),
inode))
self._execute(cmd.format(self.STATE_TABLE,
mtime,
md5,
int(nanotime.timestamp(time.time())),
inode))
else:
info = None
cmd = 'UPDATE {} SET timestamp = "{}" WHERE inode = {}'
self.c.execute(cmd.format(self.STATE_TABLE,
int(nanotime.timestamp(time.time())),
inode))
self._execute(cmd.format(self.STATE_TABLE,
int(nanotime.timestamp(time.time())),
inode))
return (md5, info)
@@ -283,12 +287,12 @@ class State(object):
relpath,
inode,
mtime)
self.c.execute(cmd)
self._execute(cmd)
def remove_unused_links(self, used):
unused = []
self.c.execute('SELECT * FROM {}'.format(self.LINK_STATE_TABLE))
self._execute('SELECT * FROM {}'.format(self.LINK_STATE_TABLE))
for row in self.c:
p, i, m = row
path = os.path.join(self.root_dir, p)
@@ -309,4 +313,4 @@ class State(object):
for p in unused:
cmd = 'DELETE FROM {} WHERE path = "{}"'
self.c.execute(cmd.format(self.LINK_STATE_TABLE, p))
self._execute(cmd.format(self.LINK_STATE_TABLE, p))
+1 -1
View File
@@ -55,7 +55,7 @@ class System(object):
clonefile.restype = ctypes.c_int
return clonefile(ctypes.c_char_p(src.encode('utf-8')),
ctypes.c_char_p(src.encode('utf-8')),
ctypes.c_char_p(dst.encode('utf-8')),
ctypes.c_int(0))
@staticmethod