mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-18 23:10:14 +01:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c6811ae005 | ||
|
|
6ad674ca88 | ||
|
|
45afd36d76 | ||
|
|
0aac01ab88 | ||
|
|
ce3bcc4af9 | ||
|
|
06064061d6 | ||
|
|
cae6e712c6 | ||
|
|
4b525fe1fc | ||
|
|
13a2e4692d | ||
|
|
1de87bc91c | ||
|
|
82b10a8256 | ||
|
|
a736082595 | ||
|
|
4888ab82a7 | ||
|
|
bc93cc0bf6 | ||
|
|
971f3259f1 | ||
|
|
a4f9789aa4 | ||
|
|
584972ba0a | ||
|
|
b2b76b84d6 | ||
|
|
a6ea2f3db9 | ||
|
|
881f188016 | ||
|
|
c3f145591b | ||
|
|
1a8842d8c6 | ||
|
|
143a7f79a1 | ||
|
|
7bff8c6f89 | ||
|
|
2daa694bf2 | ||
|
|
27a96ef814 | ||
|
|
315bbfde3b | ||
|
|
53b37bb266 | ||
|
|
005db9bb3a | ||
|
|
7b36fd007c | ||
|
|
2c9b08a89b | ||
|
|
66a962976e | ||
|
|
4eff31dd5e | ||
|
|
377be75dd0 | ||
|
|
e015b9a167 | ||
|
|
2a7ec2d2df | ||
|
|
3d34f4e8c7 | ||
|
|
c10bd344ea | ||
|
|
6a8cd31ac9 | ||
|
|
2c49d7d004 | ||
|
|
990bbe32c1 | ||
|
|
f3668b0a4e | ||
|
|
73eaa2a611 | ||
|
|
e63ae3d676 | ||
|
|
9528ad6a1d | ||
|
|
aea92e0bdf | ||
|
|
d1077c4457 | ||
|
|
edf73dc52f |
+11
@@ -112,6 +112,17 @@ or:
|
||||
|
||||
$ brew cask install iterative/homebrew-dvc/dvc
|
||||
|
||||
AUR
|
||||
---
|
||||
|
||||
*Unofficial package*, any inquiries regarding the AUR package,
|
||||
`refer to the maintainer <https://github.com/mroutis/pkgbuilds>`_.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
# https://aur.archlinux.org/packages/dvc
|
||||
$ yay -S dvc
|
||||
|
||||
Related technologies
|
||||
====================
|
||||
|
||||
|
||||
+4
-4
@@ -7,7 +7,7 @@ import os
|
||||
import warnings
|
||||
|
||||
|
||||
VERSION_BASE = '0.19.6'
|
||||
VERSION_BASE = '0.19.10'
|
||||
__version__ = VERSION_BASE
|
||||
|
||||
PACKAGEPATH = os.path.abspath(os.path.dirname(__file__))
|
||||
@@ -33,13 +33,13 @@ if os.path.exists(os.path.join(HOMEPATH, 'setup.py')):
|
||||
with open(VERSIONPATH, 'w+') as fd:
|
||||
fd.write('# AUTOGENERATED by dvc/__init__.py\n')
|
||||
fd.write('version = "{}"\n'.format(__version__))
|
||||
except Exception:
|
||||
except Exception: # pragma: no cover
|
||||
pass
|
||||
else:
|
||||
else: # pragma: no cover
|
||||
# Remove version.py so that it doesn't get into the release
|
||||
if os.path.exists(VERSIONPATH):
|
||||
os.unlink(VERSIONPATH)
|
||||
else:
|
||||
else: # pragma: no cover
|
||||
# dvc was installed with pip or something. Hopefully we have our
|
||||
# auto-generated version.py to help us provide a true version
|
||||
try:
|
||||
|
||||
+19
-5
@@ -46,7 +46,7 @@ class DvcParser(argparse.ArgumentParser):
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
class VersionAction(argparse.Action):
|
||||
class VersionAction(argparse.Action): # pragma: no cover
|
||||
def __call__(self, parser, namespace, values, option_string=None):
|
||||
print(VERSION)
|
||||
sys.exit(0)
|
||||
@@ -160,7 +160,15 @@ def parse_args(argv=None):
|
||||
help=IMPORT_HELP)
|
||||
import_parser.add_argument(
|
||||
'url',
|
||||
help='URL.')
|
||||
help="URL. Supported urls: "
|
||||
"'/path/to/file', "
|
||||
"'C:\\\\path\\to\\file', "
|
||||
"'s3://bucket/path/to/file', "
|
||||
"'gs://bucket/path/to/file', "
|
||||
"'hdfs://example.com/path/to/file', "
|
||||
"'ssh://example.com:/path/to/file', "
|
||||
"'remote://myremote/path/to/file'(see "
|
||||
"`dvc remote` commands). ")
|
||||
import_parser.add_argument(
|
||||
'out',
|
||||
help='Output.')
|
||||
@@ -402,7 +410,7 @@ def parse_args(argv=None):
|
||||
'--with-deps',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Show statusof a local cache compared to a '
|
||||
help='Show status of a local cache compared to a '
|
||||
'remote repository for all dependencies of the '
|
||||
'specified target.')
|
||||
status_parser.set_defaults(func=CmdDataStatus)
|
||||
@@ -541,6 +549,12 @@ def parse_args(argv=None):
|
||||
'-r',
|
||||
'--remote',
|
||||
help='Remote repository to collect garbage in.')
|
||||
gc_parser.add_argument(
|
||||
'-f',
|
||||
'--force',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Force garbage collection.')
|
||||
gc_parser.set_defaults(func=CmdGC)
|
||||
|
||||
# Config
|
||||
@@ -882,7 +896,7 @@ def parse_args(argv=None):
|
||||
and hasattr(args, 'targets') \
|
||||
and len(args.targets) == 0 \
|
||||
and not (hasattr(args, 'all_pipelines')
|
||||
and args.all_pipelines):
|
||||
and args.all_pipelines): # pragma: no cover
|
||||
if hasattr(args, 'cwd'):
|
||||
cwd = args.cwd
|
||||
else:
|
||||
@@ -900,7 +914,7 @@ def parse_args(argv=None):
|
||||
and len(args.deps) == 0 \
|
||||
and len(args.outs) == 0 \
|
||||
and len(args.outs_no_cache) == 0 \
|
||||
and len(args.command) == 0:
|
||||
and len(args.command) == 0: # pragma: no cover
|
||||
run_parser.error("Too few arguments. Specify at least one: "
|
||||
"'-d', '-o', '-O', 'command'.")
|
||||
|
||||
|
||||
@@ -3,6 +3,22 @@ from dvc.command.base import CmdBase
|
||||
|
||||
class CmdGC(CmdBase):
|
||||
def run(self):
|
||||
msg = 'This will remove all cache except the cache that is used in '
|
||||
if not self.args.all_branches and not self.args.all_tags:
|
||||
msg += 'the current git branch.'
|
||||
elif self.args.all_branches and not self.args.all_tags:
|
||||
msg += 'all git branches.'
|
||||
elif not self.args.all_branches and self.args.all_tags:
|
||||
msg += 'all git tags.'
|
||||
else:
|
||||
msg += 'all git branches and all git tags.'
|
||||
|
||||
self.project.logger.warn(msg)
|
||||
|
||||
msg = 'Are you sure you want to proceed?'
|
||||
if not self.args.force and not self.project.prompt.prompt(msg):
|
||||
return 1
|
||||
|
||||
self.project.gc(all_branches=self.args.all_branches,
|
||||
all_tags=self.args.all_tags,
|
||||
cloud=self.args.cloud,
|
||||
|
||||
+84
-2
@@ -1,4 +1,6 @@
|
||||
import sys
|
||||
import math
|
||||
import select
|
||||
|
||||
from grandalf.graphs import Vertex, Edge, Graph
|
||||
from grandalf.layouts import SugiyamaLayout
|
||||
@@ -12,11 +14,91 @@ class AsciiCanvas(object):
|
||||
|
||||
self.cols = cols
|
||||
self.lines = lines
|
||||
|
||||
self.canvas = [[' '] * cols for l in range(lines)]
|
||||
|
||||
def draw(self):
|
||||
for line in self.canvas:
|
||||
print(''.join(line))
|
||||
if sys.stdout.isatty(): # pragma: no cover
|
||||
from asciimatics.screen import Screen
|
||||
Screen.wrapper(self._do_draw)
|
||||
else:
|
||||
for line in self.canvas:
|
||||
print(''.join(line))
|
||||
|
||||
def _do_draw(self, screen): # pragma: no cover
|
||||
from asciimatics.event import KeyboardEvent
|
||||
|
||||
offset_x = 0
|
||||
offset_y = 0
|
||||
smaxrow, smaxcol = screen.dimensions
|
||||
assert smaxrow > 1
|
||||
assert smaxcol > 1
|
||||
smaxrow -= 1
|
||||
smaxcol -= 1
|
||||
|
||||
if self.lines + 1 > smaxrow:
|
||||
max_y = self.lines + 1 - smaxrow
|
||||
else:
|
||||
max_y = 0
|
||||
|
||||
if self.cols + 1 > smaxcol:
|
||||
max_x = self.cols + 1 - smaxcol
|
||||
else:
|
||||
max_x = 0
|
||||
|
||||
while True:
|
||||
for y in range(smaxrow + 1):
|
||||
y_index = offset_y + y
|
||||
line = []
|
||||
for x in range(smaxcol + 1):
|
||||
x_index = offset_x + x
|
||||
if len(self.canvas) > y_index \
|
||||
and len(self.canvas[y_index]) > x_index:
|
||||
line.append(self.canvas[y_index][x_index])
|
||||
else:
|
||||
line.append(' ')
|
||||
assert len(line) == (smaxcol + 1)
|
||||
screen.print_at(''.join(line), 0, y)
|
||||
|
||||
screen.refresh()
|
||||
|
||||
# NOTE: get_event() doesn't block by itself,
|
||||
# so we have to do the blocking ourselves.
|
||||
select.select([sys.stdin], [], [], None)
|
||||
|
||||
event = screen.get_event()
|
||||
if not isinstance(event, KeyboardEvent):
|
||||
continue
|
||||
|
||||
k = event.key_code
|
||||
if k == screen.KEY_DOWN or k == ord('s'):
|
||||
offset_y += 1
|
||||
elif k == screen.KEY_PAGE_DOWN or k == ord('S'):
|
||||
offset_y += smaxrow
|
||||
elif k == screen.KEY_UP or k == ord('w'):
|
||||
offset_y -= 1
|
||||
elif k == screen.KEY_PAGE_UP or k == ord('W'):
|
||||
offset_y -= smaxrow
|
||||
elif k == screen.KEY_RIGHT or k == ord('d'):
|
||||
offset_x += 1
|
||||
elif k == ord('D'):
|
||||
offset_x += smaxcol
|
||||
elif k == screen.KEY_LEFT or k == ord('a'):
|
||||
offset_x -= 1
|
||||
elif k == ord('A'):
|
||||
offset_x -= smaxcol
|
||||
elif k == ord('q') or k == ord('Q'):
|
||||
break
|
||||
|
||||
if offset_y > max_y:
|
||||
offset_y = max_y
|
||||
elif offset_y < 0:
|
||||
offset_y = 0
|
||||
|
||||
if offset_x > max_x:
|
||||
offset_x = max_x
|
||||
elif offset_x < 0:
|
||||
offset_x = 0
|
||||
|
||||
def point(self, x, y, char):
|
||||
assert x >= 0
|
||||
|
||||
@@ -67,7 +67,8 @@ class DependencyLOCAL(DependencyBase):
|
||||
if not self.exists:
|
||||
raise self.DoesNotExistError(self.rel_path)
|
||||
|
||||
if not os.path.isfile(self.path) and not os.path.isdir(self.path):
|
||||
if not os.path.isfile(self.path) \
|
||||
and not os.path.isdir(self.path): # pragma: no cover
|
||||
raise self.IsNotFileOrDirError(self.rel_path)
|
||||
|
||||
if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \
|
||||
|
||||
+1
-1
@@ -8,7 +8,7 @@ class DvcException(Exception):
|
||||
if cause:
|
||||
try:
|
||||
self.cause_tb = traceback.format_exc()
|
||||
except Exception:
|
||||
except Exception: # pragma: no cover
|
||||
pass
|
||||
super(DvcException, self).__init__(msg)
|
||||
|
||||
|
||||
+17
-7
@@ -19,9 +19,16 @@ class Logger(object):
|
||||
}
|
||||
|
||||
COLOR_MAP = {
|
||||
'debug': colorama.Fore.BLUE,
|
||||
'warn': colorama.Fore.YELLOW,
|
||||
'error': colorama.Fore.RED
|
||||
'green': colorama.Fore.GREEN,
|
||||
'yellow': colorama.Fore.YELLOW,
|
||||
'blue': colorama.Fore.BLUE,
|
||||
'red': colorama.Fore.RED,
|
||||
}
|
||||
|
||||
LEVEL_COLOR_MAP = {
|
||||
'debug': 'blue',
|
||||
'warn': 'yellow',
|
||||
'error': 'red',
|
||||
}
|
||||
|
||||
def __init__(self, loglevel=None):
|
||||
@@ -58,12 +65,12 @@ class Logger(object):
|
||||
Logger.logger().setLevel(logging.DEBUG)
|
||||
|
||||
@staticmethod
|
||||
def colorize(msg, typ):
|
||||
def colorize(msg, color):
|
||||
header = ''
|
||||
footer = ''
|
||||
|
||||
if sys.stdout.isatty(): # pragma: no cover
|
||||
header = Logger.COLOR_MAP.get(typ.lower(), '')
|
||||
header = Logger.COLOR_MAP.get(color.lower(), '')
|
||||
footer = colorama.Style.RESET_ALL
|
||||
|
||||
return u'{}{}{}'.format(header, msg, footer)
|
||||
@@ -84,7 +91,8 @@ class Logger(object):
|
||||
|
||||
@staticmethod
|
||||
def _prefix(msg, typ):
|
||||
return Logger.colorize('{}: '.format(msg), typ)
|
||||
color = Logger.LEVEL_COLOR_MAP.get(typ.lower(), '')
|
||||
return Logger.colorize('{}: '.format(msg), color)
|
||||
|
||||
@staticmethod
|
||||
def error_prefix():
|
||||
@@ -105,7 +113,9 @@ class Logger(object):
|
||||
if Logger.logger().getEffectiveLevel() == logging.DEBUG and exc:
|
||||
str_tb = str_tb if str_tb else traceback.format_exc()
|
||||
Logger.logger().error(prefix + str_tb)
|
||||
return Logger.logger().error(prefix + msg + str_exc)
|
||||
chat = "\n\nHaving any troubles? Hit us up at dvc.org/support, " \
|
||||
"we are always happy to help!"
|
||||
return Logger.logger().error(prefix + msg + str_exc + chat)
|
||||
|
||||
@staticmethod
|
||||
def warn(msg):
|
||||
|
||||
+2
-1
@@ -89,7 +89,8 @@ class OutputLOCAL(DependencyLOCAL):
|
||||
if not os.path.exists(self.path):
|
||||
raise self.DoesNotExistError(self.rel_path)
|
||||
|
||||
if not os.path.isfile(self.path) and not os.path.isdir(self.path):
|
||||
if not os.path.isfile(self.path) \
|
||||
and not os.path.isdir(self.path): # pragma: no cover
|
||||
raise self.IsNotFileOrDirError(self.rel_path)
|
||||
|
||||
if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \
|
||||
|
||||
+6
-1
@@ -29,9 +29,14 @@ class RemoteHDFS(RemoteBase):
|
||||
cmd = 'hadoop fs -' + cmd
|
||||
if user:
|
||||
cmd = 'HADOOP_USER_NAME={} '.format(user) + cmd
|
||||
|
||||
# NOTE: close_fds doesn't work with redirected stdin/stdout/stderr.
|
||||
# See https://github.com/iterative/dvc/issues/1197.
|
||||
close_fds = (os.name != 'nt')
|
||||
|
||||
p = Popen(cmd,
|
||||
shell=True,
|
||||
close_fds=True,
|
||||
close_fds=close_fds,
|
||||
executable=os.getenv('SHELL'),
|
||||
env=fix_env(os.environ),
|
||||
stdin=PIPE,
|
||||
|
||||
+9
-7
@@ -57,19 +57,13 @@ class RemoteS3(RemoteBase):
|
||||
creds_conf = configobj.ConfigObj(credentialpath)
|
||||
creds = creds_conf.get(self.profile, {})
|
||||
else:
|
||||
creds = {}
|
||||
creds = self._get_aws_config_from_env()
|
||||
|
||||
self.creds = creds
|
||||
self.region = creds.get('region', self.region)
|
||||
self.aws_access_key_id = creds.get('aws_access_key_id', None)
|
||||
self.aws_secret_access_key = creds.get('aws_secret_access_key', None)
|
||||
|
||||
self.region = os.getenv('AWS_DEFAULT_REGION', self.region)
|
||||
self.aws_access_key_id = os.getenv('AWS_ACCESS_KEY_ID',
|
||||
self.aws_access_key_id)
|
||||
self.aws_secret_access_key = os.getenv('AWS_SECRET_ACCESS_KEY',
|
||||
self.aws_secret_access_key)
|
||||
|
||||
@property
|
||||
def bucket(self):
|
||||
return urlparse(self.url).netloc
|
||||
@@ -89,6 +83,14 @@ class RemoteS3(RemoteBase):
|
||||
region_name=self.region)
|
||||
return session.client('s3', endpoint_url=self.endpoint_url)
|
||||
|
||||
def _get_aws_config_from_env(self):
|
||||
creds = {
|
||||
'region': os.getenv('AWS_DEFAULT_REGION'),
|
||||
'aws_access_key_id': os.getenv('AWS_ACCESS_KEY_ID'),
|
||||
'aws_secret_access_key': os.getenv('AWS_SECRET_ACCESS_KEY'),
|
||||
}
|
||||
return {k: v for k, v in creds.items() if v is not None}
|
||||
|
||||
def get_etag(self, bucket, key):
|
||||
try:
|
||||
obj = self.s3.head_object(Bucket=bucket, Key=key)
|
||||
|
||||
+39
-22
@@ -1,6 +1,5 @@
|
||||
import os
|
||||
import yaml
|
||||
import itertools
|
||||
import posixpath
|
||||
import subprocess
|
||||
from schema import Schema, SchemaError, Optional, Or, And
|
||||
@@ -19,8 +18,9 @@ class StageCmdFailedError(DvcException):
|
||||
|
||||
|
||||
class StageFileFormatError(DvcException):
|
||||
def __init__(self):
|
||||
super(StageFileFormatError, self).__init__('Stage file format error')
|
||||
def __init__(self, fname, e):
|
||||
msg = "Stage file '{}' format error: {}".format(fname, str(e))
|
||||
super(StageFileFormatError, self).__init__(msg)
|
||||
|
||||
|
||||
class StageFileDoesNotExistError(DvcException):
|
||||
@@ -134,8 +134,6 @@ class Stage(object):
|
||||
if self.md5 and md5 and self.md5 == md5:
|
||||
return False
|
||||
|
||||
msg = "Dvc file '{}' md5 changed(expected '{}', actual '{}')"
|
||||
self.project.logger.debug(msg.format(self.relpath, self.md5, md5))
|
||||
return True
|
||||
|
||||
@property
|
||||
@@ -148,30 +146,51 @@ class Stage(object):
|
||||
len(self.deps) == 1 and \
|
||||
len(self.outs) == 1
|
||||
|
||||
def changed(self):
|
||||
def changed(self, print_info=False):
|
||||
ret = False
|
||||
|
||||
if print_info:
|
||||
log = self.project.logger.info
|
||||
else:
|
||||
log = self.project.logger.debug
|
||||
|
||||
if self.is_callback:
|
||||
msg = "Dvc file '{}' is a 'callback' stage (has a command and " \
|
||||
"no dependencies) and thus always considered as changed."
|
||||
self.project.logger.warn(msg.format(self.relpath))
|
||||
ret = True
|
||||
|
||||
if self.locked:
|
||||
entries = self.outs
|
||||
else:
|
||||
entries = itertools.chain(self.outs, self.deps)
|
||||
|
||||
for entry in entries:
|
||||
if entry.changed():
|
||||
if not self.locked:
|
||||
for dep in self.deps:
|
||||
if not dep.changed():
|
||||
continue
|
||||
if print_info:
|
||||
msg = "Dependency '{}' of '{}' changed."
|
||||
log(msg.format(dep, self.relpath))
|
||||
ret = True
|
||||
|
||||
for out in self.outs:
|
||||
if not out.changed():
|
||||
continue
|
||||
if print_info:
|
||||
msg = "Output '{}' of '{}' changed."
|
||||
log(msg.format(out, self.relpath))
|
||||
ret = True
|
||||
|
||||
if self.changed_md5():
|
||||
if print_info:
|
||||
msg = "Dvc file '{}' changed."
|
||||
log(msg.format(self.relpath))
|
||||
ret = True
|
||||
|
||||
if ret:
|
||||
msg = u'Dvc file \'{}\' changed'.format(self.relpath)
|
||||
msg = "Stage '{}' changed.".format(self.relpath)
|
||||
color = 'yellow'
|
||||
else:
|
||||
msg = u'Dvc file \'{}\' didn\'t change'.format(self.relpath)
|
||||
msg = "Stage '{}' didn't change.".format(self.relpath)
|
||||
color = 'green'
|
||||
|
||||
self.project.logger.debug(msg)
|
||||
log(Logger.colorize(msg, color))
|
||||
|
||||
return ret
|
||||
|
||||
@@ -184,7 +203,7 @@ class Stage(object):
|
||||
os.unlink(self.path)
|
||||
|
||||
def reproduce(self, force=False, dry=False, interactive=False):
|
||||
if not self.changed() and not force:
|
||||
if not self.changed(print_info=True) and not force:
|
||||
return None
|
||||
|
||||
if (self.cmd or self.is_import) and not self.locked and not dry:
|
||||
@@ -207,17 +226,15 @@ class Stage(object):
|
||||
return self
|
||||
|
||||
@staticmethod
|
||||
def validate(d):
|
||||
def validate(d, fname=None):
|
||||
try:
|
||||
Schema(Stage.SCHEMA).validate(d)
|
||||
except SchemaError as exc:
|
||||
Logger.debug(str(exc))
|
||||
raise StageFileFormatError()
|
||||
raise StageFileFormatError(fname, exc)
|
||||
|
||||
@staticmethod
|
||||
def loadd(project, d, path):
|
||||
Stage.validate(d)
|
||||
|
||||
Stage.validate(d, fname=os.path.relpath(path))
|
||||
path = os.path.abspath(path)
|
||||
cwd = os.path.dirname(path)
|
||||
cmd = d.get(Stage.PARAM_CMD, None)
|
||||
|
||||
+47
-43
@@ -15,10 +15,10 @@ class StateDuplicateError(DvcException):
|
||||
|
||||
|
||||
class State(object):
|
||||
VERSION = 0
|
||||
VERSION = 1
|
||||
STATE_FILE = 'state'
|
||||
STATE_TABLE = 'state'
|
||||
STATE_TABLE_LAYOUT = "inode INTEGER PRIMARY KEY, " \
|
||||
STATE_TABLE_LAYOUT = "inode UNSIGNED INTEGER PRIMARY KEY, " \
|
||||
"mtime TEXT NOT NULL, " \
|
||||
"md5 TEXT NOT NULL, " \
|
||||
"timestamp TEXT NOT NULL"
|
||||
@@ -29,7 +29,7 @@ class State(object):
|
||||
|
||||
LINK_STATE_TABLE = 'link_state'
|
||||
LINK_STATE_TABLE_LAYOUT = "path TEXT PRIMARY KEY, " \
|
||||
"inode INTEGER NOT NULL, " \
|
||||
"inode UNSIGNED INTEGER NOT NULL, " \
|
||||
"mtime TEXT NOT NULL"
|
||||
|
||||
STATE_ROW_LIMIT = 10000000
|
||||
@@ -82,11 +82,15 @@ class State(object):
|
||||
|
||||
return actual.split('.')[0] != md5.split('.')[0]
|
||||
|
||||
def _execute(self, cmd):
|
||||
Logger.debug(cmd)
|
||||
return self.c.execute(cmd)
|
||||
|
||||
def _load(self):
|
||||
from dvc import VERSION
|
||||
|
||||
cmd = "PRAGMA user_version;"
|
||||
self.c.execute(cmd)
|
||||
self._execute(cmd)
|
||||
ret = self.c.fetchall()
|
||||
assert len(ret) == 1
|
||||
assert len(ret[0]) == 1
|
||||
@@ -106,26 +110,26 @@ class State(object):
|
||||
"Reformatting to the current version '{}'."
|
||||
self.project.logger.warn(msg.format(self.version, self.VERSION))
|
||||
cmd = "DROP TABLE IF EXISTS {};"
|
||||
self.c.execute(cmd.format(self.STATE_TABLE))
|
||||
self.c.execute(cmd.format(self.STATE_INFO_TABLE))
|
||||
self.c.execute(cmd.format(self.LINK_STATE_TABLE))
|
||||
self._execute(cmd.format(self.STATE_TABLE))
|
||||
self._execute(cmd.format(self.STATE_INFO_TABLE))
|
||||
self._execute(cmd.format(self.LINK_STATE_TABLE))
|
||||
|
||||
# Check that the state file is indeed a database
|
||||
cmd = "CREATE TABLE IF NOT EXISTS {} ({})"
|
||||
self.c.execute(cmd.format(self.STATE_TABLE,
|
||||
self.STATE_TABLE_LAYOUT))
|
||||
self.c.execute(cmd.format(self.STATE_INFO_TABLE,
|
||||
self.STATE_INFO_TABLE_LAYOUT))
|
||||
self.c.execute(cmd.format(self.LINK_STATE_TABLE,
|
||||
self.LINK_STATE_TABLE_LAYOUT))
|
||||
self._execute(cmd.format(self.STATE_TABLE,
|
||||
self.STATE_TABLE_LAYOUT))
|
||||
self._execute(cmd.format(self.STATE_INFO_TABLE,
|
||||
self.STATE_INFO_TABLE_LAYOUT))
|
||||
self._execute(cmd.format(self.LINK_STATE_TABLE,
|
||||
self.LINK_STATE_TABLE_LAYOUT))
|
||||
|
||||
cmd = "INSERT OR IGNORE INTO {} (count) SELECT 0 " \
|
||||
"WHERE NOT EXISTS (SELECT * FROM {})"
|
||||
self.c.execute(cmd.format(self.STATE_INFO_TABLE,
|
||||
self.STATE_INFO_TABLE))
|
||||
self._execute(cmd.format(self.STATE_INFO_TABLE,
|
||||
self.STATE_INFO_TABLE))
|
||||
|
||||
cmd = "PRAGMA user_version = {};"
|
||||
self.c.execute(cmd.format(self.VERSION))
|
||||
self._execute(cmd.format(self.VERSION))
|
||||
|
||||
def load(self):
|
||||
retries = 1
|
||||
@@ -157,8 +161,8 @@ class State(object):
|
||||
assert self.db is not None
|
||||
|
||||
cmd = "SELECT count from {} WHERE rowid={}"
|
||||
self.c.execute(cmd.format(self.STATE_INFO_TABLE,
|
||||
self.STATE_INFO_ROW))
|
||||
self._execute(cmd.format(self.STATE_INFO_TABLE,
|
||||
self.STATE_INFO_ROW))
|
||||
ret = self.c.fetchall()
|
||||
assert len(ret) == 1
|
||||
assert len(ret[0]) == 1
|
||||
@@ -172,24 +176,24 @@ class State(object):
|
||||
delete += int(self.row_limit * (self.row_cleanup_quota/100.))
|
||||
cmd = "DELETE FROM {} WHERE timestamp IN (" \
|
||||
"SELECT timestamp FROM {} ORDER BY timestamp ASC LIMIT {});"
|
||||
self.c.execute(cmd.format(self.STATE_TABLE,
|
||||
self.STATE_TABLE,
|
||||
delete))
|
||||
self._execute(cmd.format(self.STATE_TABLE,
|
||||
self.STATE_TABLE,
|
||||
delete))
|
||||
|
||||
self.c.execute("VACUUM")
|
||||
self._execute("VACUUM")
|
||||
|
||||
cmd = "SELECT COUNT(*) FROM {}"
|
||||
|
||||
self.c.execute(cmd.format(self.STATE_TABLE))
|
||||
self._execute(cmd.format(self.STATE_TABLE))
|
||||
ret = self.c.fetchall()
|
||||
assert len(ret) == 1
|
||||
assert len(ret[0]) == 1
|
||||
count = ret[0][0]
|
||||
|
||||
cmd = "UPDATE {} SET count = {} WHERE rowid = {}"
|
||||
self.c.execute(cmd.format(self.STATE_INFO_TABLE,
|
||||
count,
|
||||
self.STATE_INFO_ROW))
|
||||
self._execute(cmd.format(self.STATE_INFO_TABLE,
|
||||
count,
|
||||
self.STATE_INFO_ROW))
|
||||
|
||||
self.db.commit()
|
||||
self.c.close()
|
||||
@@ -225,17 +229,17 @@ class State(object):
|
||||
cmd = 'SELECT * from {} WHERE inode={}'.format(self.STATE_TABLE,
|
||||
inode)
|
||||
|
||||
self.c.execute(cmd)
|
||||
self._execute(cmd)
|
||||
ret = self.c.fetchall()
|
||||
if len(ret) == 0:
|
||||
md5, info = self._collect(path)
|
||||
cmd = 'INSERT INTO {}(inode, mtime, md5, timestamp) ' \
|
||||
'VALUES ({}, "{}", "{}", "{}")'
|
||||
self.c.execute(cmd.format(self.STATE_TABLE,
|
||||
inode,
|
||||
mtime,
|
||||
md5,
|
||||
int(nanotime.timestamp(time.time()))))
|
||||
self._execute(cmd.format(self.STATE_TABLE,
|
||||
inode,
|
||||
mtime,
|
||||
md5,
|
||||
int(nanotime.timestamp(time.time()))))
|
||||
self.inserts += 1
|
||||
else:
|
||||
assert len(ret) == 1
|
||||
@@ -247,17 +251,17 @@ class State(object):
|
||||
cmd = 'UPDATE {} SET ' \
|
||||
'mtime = "{}", md5 = "{}", timestamp = "{}" ' \
|
||||
'WHERE inode = {}'
|
||||
self.c.execute(cmd.format(self.STATE_TABLE,
|
||||
mtime,
|
||||
md5,
|
||||
int(nanotime.timestamp(time.time())),
|
||||
inode))
|
||||
self._execute(cmd.format(self.STATE_TABLE,
|
||||
mtime,
|
||||
md5,
|
||||
int(nanotime.timestamp(time.time())),
|
||||
inode))
|
||||
else:
|
||||
info = None
|
||||
cmd = 'UPDATE {} SET timestamp = "{}" WHERE inode = {}'
|
||||
self.c.execute(cmd.format(self.STATE_TABLE,
|
||||
int(nanotime.timestamp(time.time())),
|
||||
inode))
|
||||
self._execute(cmd.format(self.STATE_TABLE,
|
||||
int(nanotime.timestamp(time.time())),
|
||||
inode))
|
||||
|
||||
return (md5, info)
|
||||
|
||||
@@ -283,12 +287,12 @@ class State(object):
|
||||
relpath,
|
||||
inode,
|
||||
mtime)
|
||||
self.c.execute(cmd)
|
||||
self._execute(cmd)
|
||||
|
||||
def remove_unused_links(self, used):
|
||||
unused = []
|
||||
|
||||
self.c.execute('SELECT * FROM {}'.format(self.LINK_STATE_TABLE))
|
||||
self._execute('SELECT * FROM {}'.format(self.LINK_STATE_TABLE))
|
||||
for row in self.c:
|
||||
p, i, m = row
|
||||
path = os.path.join(self.root_dir, p)
|
||||
@@ -309,4 +313,4 @@ class State(object):
|
||||
|
||||
for p in unused:
|
||||
cmd = 'DELETE FROM {} WHERE path = "{}"'
|
||||
self.c.execute(cmd.format(self.LINK_STATE_TABLE, p))
|
||||
self._execute(cmd.format(self.LINK_STATE_TABLE, p))
|
||||
|
||||
+69
-22
@@ -1,8 +1,5 @@
|
||||
import os
|
||||
|
||||
if os.name == 'nt':
|
||||
import ntfsutils.hardlink as winlink
|
||||
|
||||
|
||||
class System(object):
|
||||
@staticmethod
|
||||
@@ -11,10 +8,20 @@ class System(object):
|
||||
|
||||
@staticmethod
|
||||
def hardlink(source, link_name):
|
||||
import ctypes
|
||||
if System.is_unix():
|
||||
return os.link(source, link_name)
|
||||
os.link(source, link_name)
|
||||
return
|
||||
|
||||
return winlink.create(source, link_name)
|
||||
CreateHardLink = ctypes.windll.kernel32.CreateHardLinkW
|
||||
CreateHardLink.argtypes = [ctypes.c_wchar_p,
|
||||
ctypes.c_wchar_p,
|
||||
ctypes.c_void_p]
|
||||
CreateHardLink.restype = ctypes.wintypes.BOOL
|
||||
|
||||
res = CreateHardLink(link_name, source, None)
|
||||
if res == 0:
|
||||
raise ctypes.WinError()
|
||||
|
||||
@staticmethod
|
||||
def symlink(source, link_name):
|
||||
@@ -48,7 +55,7 @@ class System(object):
|
||||
clonefile.restype = ctypes.c_int
|
||||
|
||||
return clonefile(ctypes.c_char_p(src.encode('utf-8')),
|
||||
ctypes.c_char_p(src.encode('utf-8')),
|
||||
ctypes.c_char_p(dst.encode('utf-8')),
|
||||
ctypes.c_int(0))
|
||||
|
||||
@staticmethod
|
||||
@@ -102,30 +109,70 @@ class System(object):
|
||||
if ret != 0:
|
||||
raise DvcException('Reflink is not supported')
|
||||
|
||||
# FIXME
|
||||
# Temporary fix while waiting for the PR to be merged and released:
|
||||
# https://github.com/sunshowers/ntfs/pull/11
|
||||
@staticmethod
|
||||
def getdirinfo(path):
|
||||
from ntfsutils.fs import FILE_FLAG_BACKUP_SEMANTICS, FILE_SHARE_READ
|
||||
from ntfsutils.fs import OPEN_EXISTING, BY_HANDLE_FILE_INFORMATION
|
||||
from ntfsutils.fs import CreateFile, WinError
|
||||
from ntfsutils.fs import GetFileInformationByHandle, CloseHandle
|
||||
import ctypes
|
||||
from ctypes import c_void_p, c_wchar_p, Structure, WinError
|
||||
from ctypes.wintypes import DWORD, HANDLE, POINTER, BOOL
|
||||
|
||||
FILE_FLAG_BACKUP_SEMANTICS = 0x02000000
|
||||
FILE_SHARE_READ = 0x00000001
|
||||
OPEN_EXISTING = 3
|
||||
|
||||
class FILETIME(Structure):
|
||||
_fields_ = [("dwLowDateTime", DWORD),
|
||||
("dwHighDateTime", DWORD)]
|
||||
|
||||
class BY_HANDLE_FILE_INFORMATION(Structure):
|
||||
_fields_ = [("dwFileAttributes", DWORD),
|
||||
("ftCreationTime", FILETIME),
|
||||
("ftLastAccessTime", FILETIME),
|
||||
("ftLastWriteTime", FILETIME),
|
||||
("dwVolumeSerialNumber", DWORD),
|
||||
("nFileSizeHigh", DWORD),
|
||||
("nFileSizeLow", DWORD),
|
||||
("nNumberOfLinks", DWORD),
|
||||
("nFileIndexHigh", DWORD),
|
||||
("nFileIndexLow", DWORD)]
|
||||
|
||||
flags = FILE_FLAG_BACKUP_SEMANTICS
|
||||
hfile = CreateFile(path,
|
||||
0,
|
||||
FILE_SHARE_READ,
|
||||
None,
|
||||
OPEN_EXISTING,
|
||||
flags,
|
||||
None)
|
||||
|
||||
func = ctypes.windll.kernel32.CreateFileW
|
||||
func.argtypes = [c_wchar_p,
|
||||
DWORD,
|
||||
DWORD,
|
||||
c_void_p,
|
||||
DWORD,
|
||||
DWORD,
|
||||
HANDLE]
|
||||
func.restype = HANDLE
|
||||
|
||||
hfile = func(path,
|
||||
0,
|
||||
FILE_SHARE_READ,
|
||||
None,
|
||||
OPEN_EXISTING,
|
||||
flags,
|
||||
None)
|
||||
if hfile is None:
|
||||
raise WinError()
|
||||
|
||||
func = ctypes.windll.kernel32.GetFileInformationByHandle
|
||||
func.argtypes = [HANDLE, POINTER(BY_HANDLE_FILE_INFORMATION)]
|
||||
func.restype = BOOL
|
||||
|
||||
info = BY_HANDLE_FILE_INFORMATION()
|
||||
rv = GetFileInformationByHandle(hfile, info)
|
||||
CloseHandle(hfile)
|
||||
rv = func(hfile, info)
|
||||
|
||||
func = ctypes.windll.kernel32.CloseHandle
|
||||
func.argtypes = [HANDLE]
|
||||
func.restype = BOOL
|
||||
|
||||
func(hfile)
|
||||
|
||||
if rv == 0:
|
||||
raise WinError()
|
||||
|
||||
return info
|
||||
|
||||
@staticmethod
|
||||
|
||||
+1
-1
@@ -6,7 +6,7 @@ from dvc import VERSION_BASE
|
||||
from dvc.logger import Logger
|
||||
|
||||
|
||||
class Updater(object):
|
||||
class Updater(object): # pragma: no cover
|
||||
URL = 'https://4ki8820rsf.execute-api.us-east-2.amazonaws.com/' \
|
||||
'prod/latest-version'
|
||||
UPDATER_FILE = 'updater'
|
||||
|
||||
+2
-2
@@ -8,10 +8,9 @@ google-cloud-storage==1.12.0
|
||||
PyInstaller==3.3.1
|
||||
colorama>=0.3.9
|
||||
configobj>=5.0.6
|
||||
networkx==2.1
|
||||
networkx>=2.1
|
||||
pyyaml>=3.12
|
||||
gitpython>=2.1.8
|
||||
ntfsutils>=0.1.4
|
||||
setuptools>=34.0.0
|
||||
nanotime>=0.5.2
|
||||
pyasn1>=0.4.1
|
||||
@@ -23,3 +22,4 @@ wheel>=0.31.1
|
||||
futures>=3.2.0; python_version == "2.7"
|
||||
grandalf==0.6
|
||||
pydot>=1.2.4
|
||||
asciimatics>=1.10.0
|
||||
|
||||
@@ -6,7 +6,8 @@ set -e
|
||||
python -mtests
|
||||
|
||||
if [[ "$TRAVIS_PULL_REQUEST" == "false" && \
|
||||
"$TRAVIS_SECURE_ENV_VARS" == "true" ]]; then
|
||||
"$TRAVIS_SECURE_ENV_VARS" == "true" && \
|
||||
"$(python -c 'import sys; print(sys.version_info[0])')" == '2' ]]; then
|
||||
codecov
|
||||
fi
|
||||
|
||||
|
||||
@@ -12,10 +12,9 @@ install_requires = [
|
||||
"future>=0.16.0",
|
||||
"colorama>=0.3.9",
|
||||
"configobj>=5.0.6",
|
||||
"networkx==2.1",
|
||||
"networkx>=2.1",
|
||||
"pyyaml>=3.12",
|
||||
"gitpython>=2.1.8",
|
||||
"ntfsutils>=0.1.4",
|
||||
"setuptools>=34.0.0",
|
||||
"nanotime>=0.5.2",
|
||||
"pyasn1>=0.4.1",
|
||||
@@ -23,6 +22,7 @@ install_requires = [
|
||||
"jsonpath-rw==1.4.0",
|
||||
"requests>=2.18.4",
|
||||
"grandalf==0.6",
|
||||
"asciimatics>=1.10.0",
|
||||
]
|
||||
|
||||
# Extra dependencies for remote integrations
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
pytest>=3.3.1
|
||||
nose>=1.3.7
|
||||
mock>=2.0.0
|
||||
coverage
|
||||
codecov
|
||||
coverage>=4.5.1
|
||||
codecov>=2.0.15
|
||||
xmltodict>=0.11.0
|
||||
awscli>=1.14.18
|
||||
google-compute-engine
|
||||
|
||||
+2
-1
@@ -7,4 +7,5 @@ os.chdir(REPO_ROOT)
|
||||
os.putenv('PATH', '{}:{}'.format(os.path.join(REPO_ROOT, 'bin'), os.getenv('PATH')))
|
||||
os.putenv('DVC_HOME', REPO_ROOT)
|
||||
|
||||
check_call('nosetests -v --processes=-1 --process-timeout=200 --cover-inclusive --cover-erase --cover-package=dvc --with-coverage', shell=True)
|
||||
cmd = 'nosetests -v --processes=-1 --process-timeout=200 --cover-inclusive --cover-erase --cover-package=dvc --with-coverage'
|
||||
check_call(cmd, shell=True)
|
||||
|
||||
@@ -103,3 +103,12 @@ class TestSharedCacheDir(TestDir):
|
||||
self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[0]))), 1)
|
||||
self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[1]))), 1)
|
||||
self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[2]))), 1)
|
||||
|
||||
|
||||
class TestCacheLinkType(TestDvc):
|
||||
def test(self):
|
||||
ret = main(['config', 'cache.type', 'reflink,copy'])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
ret = main(['add', self.FOO])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
@@ -205,3 +205,15 @@ class TestCheckoutEmptyDir(TestDvc):
|
||||
|
||||
self.assertTrue(os.path.isdir(dname))
|
||||
self.assertEqual(len(os.listdir(dname)), 0)
|
||||
|
||||
|
||||
class TestCheckoutNotCachedFile(TestDvc):
|
||||
def test(self):
|
||||
cmd = 'python {} {} {}'.format(self.CODE, self.FOO, 'out')
|
||||
|
||||
self.dvc.add(self.FOO)
|
||||
stage = self.dvc.run(cmd=cmd,
|
||||
deps=[self.FOO, self.CODE],
|
||||
outs_no_cache=['out'])
|
||||
|
||||
self.dvc.checkout()
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
from subprocess import CalledProcessError
|
||||
from subprocess import check_output
|
||||
from subprocess import check_output, Popen
|
||||
from unittest import SkipTest
|
||||
import os
|
||||
import time
|
||||
@@ -94,6 +94,13 @@ def _should_test_hdfs():
|
||||
except (CalledProcessError, IOError):
|
||||
return False
|
||||
|
||||
p = Popen('hadoop fs -ls hdfs://127.0.0.1/',
|
||||
shell=True,
|
||||
executable=os.getenv('SHELL'))
|
||||
p.communicate()
|
||||
if p.returncode != 0:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
|
||||
+2
-1
@@ -29,7 +29,8 @@ class TestGC(TestDvc):
|
||||
self._test_gc()
|
||||
|
||||
def test_cli(self):
|
||||
ret = main(['gc'])
|
||||
ret = main(['gc', '-f'])
|
||||
self.assertEqual(ret, 0)
|
||||
self._test_gc()
|
||||
|
||||
def _test_gc(self):
|
||||
|
||||
@@ -72,3 +72,16 @@ class TestRemoteRemoveDefault(TestDvc):
|
||||
core = local_config.get(Config.SECTION_CORE, None)
|
||||
if core is not None:
|
||||
self.assertTrue(Config.SECTION_CORE_REMOTE not in core.keys())
|
||||
|
||||
|
||||
class TestRemoteRemove(TestDvc):
|
||||
def test(self):
|
||||
ret = main(['config', 'core.jobs', '1'])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
remote = 'mys3'
|
||||
ret = main(['remote', 'add', remote, 's3://bucket/name'])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
ret = main(['remote', 'remove', remote])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
@@ -146,6 +146,12 @@ class TestReproDry(TestReproChangedData):
|
||||
self.assertFalse(filecmp.cmp(self.file1, self.BAR, shallow=False))
|
||||
|
||||
|
||||
class TestReproUpToDate(TestRepro):
|
||||
def test(self):
|
||||
ret = main(['repro', self.file1_stage])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
|
||||
class TestReproDryNoExec(TestDvc):
|
||||
def test(self):
|
||||
deps = []
|
||||
|
||||
Reference in New Issue
Block a user