mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-18 23:10:14 +01:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c6811ae005 | ||
|
|
6ad674ca88 | ||
|
|
45afd36d76 | ||
|
|
0aac01ab88 | ||
|
|
ce3bcc4af9 | ||
|
|
06064061d6 | ||
|
|
cae6e712c6 | ||
|
|
4b525fe1fc | ||
|
|
13a2e4692d | ||
|
|
1de87bc91c | ||
|
|
82b10a8256 | ||
|
|
a736082595 |
+1
-1
@@ -7,7 +7,7 @@ import os
|
||||
import warnings
|
||||
|
||||
|
||||
VERSION_BASE = '0.19.8'
|
||||
VERSION_BASE = '0.19.10'
|
||||
__version__ = VERSION_BASE
|
||||
|
||||
PACKAGEPATH = os.path.abspath(os.path.dirname(__file__))
|
||||
|
||||
+9
-1
@@ -160,7 +160,15 @@ def parse_args(argv=None):
|
||||
help=IMPORT_HELP)
|
||||
import_parser.add_argument(
|
||||
'url',
|
||||
help='URL.')
|
||||
help="URL. Supported urls: "
|
||||
"'/path/to/file', "
|
||||
"'C:\\\\path\\to\\file', "
|
||||
"'s3://bucket/path/to/file', "
|
||||
"'gs://bucket/path/to/file', "
|
||||
"'hdfs://example.com/path/to/file', "
|
||||
"'ssh://example.com:/path/to/file', "
|
||||
"'remote://myremote/path/to/file'(see "
|
||||
"`dvc remote` commands). ")
|
||||
import_parser.add_argument(
|
||||
'out',
|
||||
help='Output.')
|
||||
|
||||
+3
-3
@@ -2,9 +2,6 @@ import sys
|
||||
import math
|
||||
import select
|
||||
|
||||
from asciimatics.screen import Screen
|
||||
from asciimatics.event import KeyboardEvent
|
||||
|
||||
from grandalf.graphs import Vertex, Edge, Graph
|
||||
from grandalf.layouts import SugiyamaLayout
|
||||
from grandalf.routing import route_with_lines, EdgeViewer
|
||||
@@ -22,12 +19,15 @@ class AsciiCanvas(object):
|
||||
|
||||
def draw(self):
|
||||
if sys.stdout.isatty(): # pragma: no cover
|
||||
from asciimatics.screen import Screen
|
||||
Screen.wrapper(self._do_draw)
|
||||
else:
|
||||
for line in self.canvas:
|
||||
print(''.join(line))
|
||||
|
||||
def _do_draw(self, screen): # pragma: no cover
|
||||
from asciimatics.event import KeyboardEvent
|
||||
|
||||
offset_x = 0
|
||||
offset_y = 0
|
||||
smaxrow, smaxcol = screen.dimensions
|
||||
|
||||
+14
-6
@@ -19,9 +19,16 @@ class Logger(object):
|
||||
}
|
||||
|
||||
COLOR_MAP = {
|
||||
'debug': colorama.Fore.BLUE,
|
||||
'warn': colorama.Fore.YELLOW,
|
||||
'error': colorama.Fore.RED
|
||||
'green': colorama.Fore.GREEN,
|
||||
'yellow': colorama.Fore.YELLOW,
|
||||
'blue': colorama.Fore.BLUE,
|
||||
'red': colorama.Fore.RED,
|
||||
}
|
||||
|
||||
LEVEL_COLOR_MAP = {
|
||||
'debug': 'blue',
|
||||
'warn': 'yellow',
|
||||
'error': 'red',
|
||||
}
|
||||
|
||||
def __init__(self, loglevel=None):
|
||||
@@ -58,12 +65,12 @@ class Logger(object):
|
||||
Logger.logger().setLevel(logging.DEBUG)
|
||||
|
||||
@staticmethod
|
||||
def colorize(msg, typ):
|
||||
def colorize(msg, color):
|
||||
header = ''
|
||||
footer = ''
|
||||
|
||||
if sys.stdout.isatty(): # pragma: no cover
|
||||
header = Logger.COLOR_MAP.get(typ.lower(), '')
|
||||
header = Logger.COLOR_MAP.get(color.lower(), '')
|
||||
footer = colorama.Style.RESET_ALL
|
||||
|
||||
return u'{}{}{}'.format(header, msg, footer)
|
||||
@@ -84,7 +91,8 @@ class Logger(object):
|
||||
|
||||
@staticmethod
|
||||
def _prefix(msg, typ):
|
||||
return Logger.colorize('{}: '.format(msg), typ)
|
||||
color = Logger.LEVEL_COLOR_MAP.get(typ.lower(), '')
|
||||
return Logger.colorize('{}: '.format(msg), color)
|
||||
|
||||
@staticmethod
|
||||
def error_prefix():
|
||||
|
||||
+33
-15
@@ -1,6 +1,5 @@
|
||||
import os
|
||||
import yaml
|
||||
import itertools
|
||||
import posixpath
|
||||
import subprocess
|
||||
from schema import Schema, SchemaError, Optional, Or, And
|
||||
@@ -135,8 +134,6 @@ class Stage(object):
|
||||
if self.md5 and md5 and self.md5 == md5:
|
||||
return False
|
||||
|
||||
msg = "Dvc file '{}' md5 changed(expected '{}', actual '{}')"
|
||||
self.project.logger.debug(msg.format(self.relpath, self.md5, md5))
|
||||
return True
|
||||
|
||||
@property
|
||||
@@ -149,30 +146,51 @@ class Stage(object):
|
||||
len(self.deps) == 1 and \
|
||||
len(self.outs) == 1
|
||||
|
||||
def changed(self):
|
||||
def changed(self, print_info=False):
|
||||
ret = False
|
||||
|
||||
if print_info:
|
||||
log = self.project.logger.info
|
||||
else:
|
||||
log = self.project.logger.debug
|
||||
|
||||
if self.is_callback:
|
||||
msg = "Dvc file '{}' is a 'callback' stage (has a command and " \
|
||||
"no dependencies) and thus always considered as changed."
|
||||
self.project.logger.warn(msg.format(self.relpath))
|
||||
ret = True
|
||||
|
||||
if self.locked:
|
||||
entries = self.outs
|
||||
else:
|
||||
entries = itertools.chain(self.outs, self.deps)
|
||||
|
||||
for entry in entries:
|
||||
if entry.changed():
|
||||
if not self.locked:
|
||||
for dep in self.deps:
|
||||
if not dep.changed():
|
||||
continue
|
||||
if print_info:
|
||||
msg = "Dependency '{}' of '{}' changed."
|
||||
log(msg.format(dep, self.relpath))
|
||||
ret = True
|
||||
|
||||
for out in self.outs:
|
||||
if not out.changed():
|
||||
continue
|
||||
if print_info:
|
||||
msg = "Output '{}' of '{}' changed."
|
||||
log(msg.format(out, self.relpath))
|
||||
ret = True
|
||||
|
||||
if self.changed_md5():
|
||||
if print_info:
|
||||
msg = "Dvc file '{}' changed."
|
||||
log(msg.format(self.relpath))
|
||||
ret = True
|
||||
|
||||
if ret:
|
||||
msg = u'Dvc file \'{}\' changed'.format(self.relpath)
|
||||
msg = "Stage '{}' changed.".format(self.relpath)
|
||||
color = 'yellow'
|
||||
else:
|
||||
msg = u'Dvc file \'{}\' didn\'t change'.format(self.relpath)
|
||||
msg = "Stage '{}' didn't change.".format(self.relpath)
|
||||
color = 'green'
|
||||
|
||||
self.project.logger.debug(msg)
|
||||
log(Logger.colorize(msg, color))
|
||||
|
||||
return ret
|
||||
|
||||
@@ -185,7 +203,7 @@ class Stage(object):
|
||||
os.unlink(self.path)
|
||||
|
||||
def reproduce(self, force=False, dry=False, interactive=False):
|
||||
if not self.changed() and not force:
|
||||
if not self.changed(print_info=True) and not force:
|
||||
return None
|
||||
|
||||
if (self.cmd or self.is_import) and not self.locked and not dry:
|
||||
|
||||
+47
-43
@@ -15,10 +15,10 @@ class StateDuplicateError(DvcException):
|
||||
|
||||
|
||||
class State(object):
|
||||
VERSION = 0
|
||||
VERSION = 1
|
||||
STATE_FILE = 'state'
|
||||
STATE_TABLE = 'state'
|
||||
STATE_TABLE_LAYOUT = "inode INTEGER PRIMARY KEY, " \
|
||||
STATE_TABLE_LAYOUT = "inode UNSIGNED INTEGER PRIMARY KEY, " \
|
||||
"mtime TEXT NOT NULL, " \
|
||||
"md5 TEXT NOT NULL, " \
|
||||
"timestamp TEXT NOT NULL"
|
||||
@@ -29,7 +29,7 @@ class State(object):
|
||||
|
||||
LINK_STATE_TABLE = 'link_state'
|
||||
LINK_STATE_TABLE_LAYOUT = "path TEXT PRIMARY KEY, " \
|
||||
"inode INTEGER NOT NULL, " \
|
||||
"inode UNSIGNED INTEGER NOT NULL, " \
|
||||
"mtime TEXT NOT NULL"
|
||||
|
||||
STATE_ROW_LIMIT = 10000000
|
||||
@@ -82,11 +82,15 @@ class State(object):
|
||||
|
||||
return actual.split('.')[0] != md5.split('.')[0]
|
||||
|
||||
def _execute(self, cmd):
|
||||
Logger.debug(cmd)
|
||||
return self.c.execute(cmd)
|
||||
|
||||
def _load(self):
|
||||
from dvc import VERSION
|
||||
|
||||
cmd = "PRAGMA user_version;"
|
||||
self.c.execute(cmd)
|
||||
self._execute(cmd)
|
||||
ret = self.c.fetchall()
|
||||
assert len(ret) == 1
|
||||
assert len(ret[0]) == 1
|
||||
@@ -106,26 +110,26 @@ class State(object):
|
||||
"Reformatting to the current version '{}'."
|
||||
self.project.logger.warn(msg.format(self.version, self.VERSION))
|
||||
cmd = "DROP TABLE IF EXISTS {};"
|
||||
self.c.execute(cmd.format(self.STATE_TABLE))
|
||||
self.c.execute(cmd.format(self.STATE_INFO_TABLE))
|
||||
self.c.execute(cmd.format(self.LINK_STATE_TABLE))
|
||||
self._execute(cmd.format(self.STATE_TABLE))
|
||||
self._execute(cmd.format(self.STATE_INFO_TABLE))
|
||||
self._execute(cmd.format(self.LINK_STATE_TABLE))
|
||||
|
||||
# Check that the state file is indeed a database
|
||||
cmd = "CREATE TABLE IF NOT EXISTS {} ({})"
|
||||
self.c.execute(cmd.format(self.STATE_TABLE,
|
||||
self.STATE_TABLE_LAYOUT))
|
||||
self.c.execute(cmd.format(self.STATE_INFO_TABLE,
|
||||
self.STATE_INFO_TABLE_LAYOUT))
|
||||
self.c.execute(cmd.format(self.LINK_STATE_TABLE,
|
||||
self.LINK_STATE_TABLE_LAYOUT))
|
||||
self._execute(cmd.format(self.STATE_TABLE,
|
||||
self.STATE_TABLE_LAYOUT))
|
||||
self._execute(cmd.format(self.STATE_INFO_TABLE,
|
||||
self.STATE_INFO_TABLE_LAYOUT))
|
||||
self._execute(cmd.format(self.LINK_STATE_TABLE,
|
||||
self.LINK_STATE_TABLE_LAYOUT))
|
||||
|
||||
cmd = "INSERT OR IGNORE INTO {} (count) SELECT 0 " \
|
||||
"WHERE NOT EXISTS (SELECT * FROM {})"
|
||||
self.c.execute(cmd.format(self.STATE_INFO_TABLE,
|
||||
self.STATE_INFO_TABLE))
|
||||
self._execute(cmd.format(self.STATE_INFO_TABLE,
|
||||
self.STATE_INFO_TABLE))
|
||||
|
||||
cmd = "PRAGMA user_version = {};"
|
||||
self.c.execute(cmd.format(self.VERSION))
|
||||
self._execute(cmd.format(self.VERSION))
|
||||
|
||||
def load(self):
|
||||
retries = 1
|
||||
@@ -157,8 +161,8 @@ class State(object):
|
||||
assert self.db is not None
|
||||
|
||||
cmd = "SELECT count from {} WHERE rowid={}"
|
||||
self.c.execute(cmd.format(self.STATE_INFO_TABLE,
|
||||
self.STATE_INFO_ROW))
|
||||
self._execute(cmd.format(self.STATE_INFO_TABLE,
|
||||
self.STATE_INFO_ROW))
|
||||
ret = self.c.fetchall()
|
||||
assert len(ret) == 1
|
||||
assert len(ret[0]) == 1
|
||||
@@ -172,24 +176,24 @@ class State(object):
|
||||
delete += int(self.row_limit * (self.row_cleanup_quota/100.))
|
||||
cmd = "DELETE FROM {} WHERE timestamp IN (" \
|
||||
"SELECT timestamp FROM {} ORDER BY timestamp ASC LIMIT {});"
|
||||
self.c.execute(cmd.format(self.STATE_TABLE,
|
||||
self.STATE_TABLE,
|
||||
delete))
|
||||
self._execute(cmd.format(self.STATE_TABLE,
|
||||
self.STATE_TABLE,
|
||||
delete))
|
||||
|
||||
self.c.execute("VACUUM")
|
||||
self._execute("VACUUM")
|
||||
|
||||
cmd = "SELECT COUNT(*) FROM {}"
|
||||
|
||||
self.c.execute(cmd.format(self.STATE_TABLE))
|
||||
self._execute(cmd.format(self.STATE_TABLE))
|
||||
ret = self.c.fetchall()
|
||||
assert len(ret) == 1
|
||||
assert len(ret[0]) == 1
|
||||
count = ret[0][0]
|
||||
|
||||
cmd = "UPDATE {} SET count = {} WHERE rowid = {}"
|
||||
self.c.execute(cmd.format(self.STATE_INFO_TABLE,
|
||||
count,
|
||||
self.STATE_INFO_ROW))
|
||||
self._execute(cmd.format(self.STATE_INFO_TABLE,
|
||||
count,
|
||||
self.STATE_INFO_ROW))
|
||||
|
||||
self.db.commit()
|
||||
self.c.close()
|
||||
@@ -225,17 +229,17 @@ class State(object):
|
||||
cmd = 'SELECT * from {} WHERE inode={}'.format(self.STATE_TABLE,
|
||||
inode)
|
||||
|
||||
self.c.execute(cmd)
|
||||
self._execute(cmd)
|
||||
ret = self.c.fetchall()
|
||||
if len(ret) == 0:
|
||||
md5, info = self._collect(path)
|
||||
cmd = 'INSERT INTO {}(inode, mtime, md5, timestamp) ' \
|
||||
'VALUES ({}, "{}", "{}", "{}")'
|
||||
self.c.execute(cmd.format(self.STATE_TABLE,
|
||||
inode,
|
||||
mtime,
|
||||
md5,
|
||||
int(nanotime.timestamp(time.time()))))
|
||||
self._execute(cmd.format(self.STATE_TABLE,
|
||||
inode,
|
||||
mtime,
|
||||
md5,
|
||||
int(nanotime.timestamp(time.time()))))
|
||||
self.inserts += 1
|
||||
else:
|
||||
assert len(ret) == 1
|
||||
@@ -247,17 +251,17 @@ class State(object):
|
||||
cmd = 'UPDATE {} SET ' \
|
||||
'mtime = "{}", md5 = "{}", timestamp = "{}" ' \
|
||||
'WHERE inode = {}'
|
||||
self.c.execute(cmd.format(self.STATE_TABLE,
|
||||
mtime,
|
||||
md5,
|
||||
int(nanotime.timestamp(time.time())),
|
||||
inode))
|
||||
self._execute(cmd.format(self.STATE_TABLE,
|
||||
mtime,
|
||||
md5,
|
||||
int(nanotime.timestamp(time.time())),
|
||||
inode))
|
||||
else:
|
||||
info = None
|
||||
cmd = 'UPDATE {} SET timestamp = "{}" WHERE inode = {}'
|
||||
self.c.execute(cmd.format(self.STATE_TABLE,
|
||||
int(nanotime.timestamp(time.time())),
|
||||
inode))
|
||||
self._execute(cmd.format(self.STATE_TABLE,
|
||||
int(nanotime.timestamp(time.time())),
|
||||
inode))
|
||||
|
||||
return (md5, info)
|
||||
|
||||
@@ -283,12 +287,12 @@ class State(object):
|
||||
relpath,
|
||||
inode,
|
||||
mtime)
|
||||
self.c.execute(cmd)
|
||||
self._execute(cmd)
|
||||
|
||||
def remove_unused_links(self, used):
|
||||
unused = []
|
||||
|
||||
self.c.execute('SELECT * FROM {}'.format(self.LINK_STATE_TABLE))
|
||||
self._execute('SELECT * FROM {}'.format(self.LINK_STATE_TABLE))
|
||||
for row in self.c:
|
||||
p, i, m = row
|
||||
path = os.path.join(self.root_dir, p)
|
||||
@@ -309,4 +313,4 @@ class State(object):
|
||||
|
||||
for p in unused:
|
||||
cmd = 'DELETE FROM {} WHERE path = "{}"'
|
||||
self.c.execute(cmd.format(self.LINK_STATE_TABLE, p))
|
||||
self._execute(cmd.format(self.LINK_STATE_TABLE, p))
|
||||
|
||||
+1
-1
@@ -55,7 +55,7 @@ class System(object):
|
||||
clonefile.restype = ctypes.c_int
|
||||
|
||||
return clonefile(ctypes.c_char_p(src.encode('utf-8')),
|
||||
ctypes.c_char_p(src.encode('utf-8')),
|
||||
ctypes.c_char_p(dst.encode('utf-8')),
|
||||
ctypes.c_int(0))
|
||||
|
||||
@staticmethod
|
||||
|
||||
Reference in New Issue
Block a user