mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-18 23:10:14 +01:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c6811ae005 | ||
|
|
6ad674ca88 | ||
|
|
45afd36d76 | ||
|
|
0aac01ab88 | ||
|
|
ce3bcc4af9 | ||
|
|
06064061d6 | ||
|
|
cae6e712c6 | ||
|
|
4b525fe1fc | ||
|
|
13a2e4692d | ||
|
|
1de87bc91c | ||
|
|
82b10a8256 | ||
|
|
a736082595 | ||
|
|
4888ab82a7 | ||
|
|
bc93cc0bf6 | ||
|
|
971f3259f1 | ||
|
|
a4f9789aa4 | ||
|
|
584972ba0a | ||
|
|
b2b76b84d6 | ||
|
|
a6ea2f3db9 | ||
|
|
881f188016 | ||
|
|
c3f145591b | ||
|
|
1a8842d8c6 | ||
|
|
143a7f79a1 | ||
|
|
7bff8c6f89 | ||
|
|
2daa694bf2 | ||
|
|
27a96ef814 | ||
|
|
315bbfde3b | ||
|
|
53b37bb266 | ||
|
|
005db9bb3a | ||
|
|
7b36fd007c | ||
|
|
2c9b08a89b | ||
|
|
66a962976e | ||
|
|
4eff31dd5e | ||
|
|
377be75dd0 | ||
|
|
e015b9a167 | ||
|
|
2a7ec2d2df | ||
|
|
3d34f4e8c7 | ||
|
|
c10bd344ea | ||
|
|
6a8cd31ac9 | ||
|
|
2c49d7d004 | ||
|
|
990bbe32c1 | ||
|
|
f3668b0a4e | ||
|
|
73eaa2a611 | ||
|
|
e63ae3d676 | ||
|
|
9528ad6a1d | ||
|
|
aea92e0bdf | ||
|
|
d1077c4457 | ||
|
|
edf73dc52f | ||
|
|
b587f7833a | ||
|
|
998918c690 | ||
|
|
8aa53a5e71 | ||
|
|
cc2ebb2099 | ||
|
|
b9439f2e5b | ||
|
|
d112d2a829 | ||
|
|
f4fc3f47e4 | ||
|
|
417ad8706e | ||
|
|
2e09dbb592 | ||
|
|
67d63a4a5b | ||
|
|
8eb6fd5285 | ||
|
|
669b2ec303 | ||
|
|
04b3cd40e0 | ||
|
|
12d6292ed6 | ||
|
|
27edc4f027 | ||
|
|
5b0abe61ab | ||
|
|
1e1daeaae1 |
+11
@@ -112,6 +112,17 @@ or:
|
||||
|
||||
$ brew cask install iterative/homebrew-dvc/dvc
|
||||
|
||||
AUR
|
||||
---
|
||||
|
||||
*Unofficial package*, any inquiries regarding the AUR package,
|
||||
`refer to the maintainer <https://github.com/mroutis/pkgbuilds>`_.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
# https://aur.archlinux.org/packages/dvc
|
||||
$ yay -S dvc
|
||||
|
||||
Related technologies
|
||||
====================
|
||||
|
||||
|
||||
+4
-4
@@ -7,7 +7,7 @@ import os
|
||||
import warnings
|
||||
|
||||
|
||||
VERSION_BASE = '0.19.3'
|
||||
VERSION_BASE = '0.19.10'
|
||||
__version__ = VERSION_BASE
|
||||
|
||||
PACKAGEPATH = os.path.abspath(os.path.dirname(__file__))
|
||||
@@ -33,13 +33,13 @@ if os.path.exists(os.path.join(HOMEPATH, 'setup.py')):
|
||||
with open(VERSIONPATH, 'w+') as fd:
|
||||
fd.write('# AUTOGENERATED by dvc/__init__.py\n')
|
||||
fd.write('version = "{}"\n'.format(__version__))
|
||||
except Exception:
|
||||
except Exception: # pragma: no cover
|
||||
pass
|
||||
else:
|
||||
else: # pragma: no cover
|
||||
# Remove version.py so that it doesn't get into the release
|
||||
if os.path.exists(VERSIONPATH):
|
||||
os.unlink(VERSIONPATH)
|
||||
else:
|
||||
else: # pragma: no cover
|
||||
# dvc was installed with pip or something. Hopefully we have our
|
||||
# auto-generated version.py to help us provide a true version
|
||||
try:
|
||||
|
||||
+65
-7
@@ -3,7 +3,6 @@ from __future__ import print_function
|
||||
import os
|
||||
import sys
|
||||
import argparse
|
||||
from multiprocessing import cpu_count
|
||||
|
||||
from dvc.command.init import CmdInit
|
||||
from dvc.command.destroy import CmdDestroy
|
||||
@@ -47,7 +46,7 @@ class DvcParser(argparse.ArgumentParser):
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
class VersionAction(argparse.Action):
|
||||
class VersionAction(argparse.Action): # pragma: no cover
|
||||
def __call__(self, parser, namespace, values, option_string=None):
|
||||
print(VERSION)
|
||||
sys.exit(0)
|
||||
@@ -161,7 +160,15 @@ def parse_args(argv=None):
|
||||
help=IMPORT_HELP)
|
||||
import_parser.add_argument(
|
||||
'url',
|
||||
help='URL.')
|
||||
help="URL. Supported urls: "
|
||||
"'/path/to/file', "
|
||||
"'C:\\\\path\\to\\file', "
|
||||
"'s3://bucket/path/to/file', "
|
||||
"'gs://bucket/path/to/file', "
|
||||
"'hdfs://example.com/path/to/file', "
|
||||
"'ssh://example.com:/path/to/file', "
|
||||
"'remote://myremote/path/to/file'(see "
|
||||
"`dvc remote` commands). ")
|
||||
import_parser.add_argument(
|
||||
'out',
|
||||
help='Output.')
|
||||
@@ -257,7 +264,7 @@ def parse_args(argv=None):
|
||||
'-j',
|
||||
'--jobs',
|
||||
type=int,
|
||||
default=4*cpu_count(),
|
||||
default=None,
|
||||
help='Number of jobs to run simultaneously.')
|
||||
parent_cache_parser.add_argument(
|
||||
'--show-checksums',
|
||||
@@ -287,6 +294,12 @@ def parse_args(argv=None):
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Fetch cache for all branches.')
|
||||
pull_parser.add_argument(
|
||||
'-T',
|
||||
'--all-tags',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Fetch cache for all tags.')
|
||||
pull_parser.add_argument(
|
||||
'-d',
|
||||
'--with-deps',
|
||||
@@ -313,6 +326,12 @@ def parse_args(argv=None):
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Push cache for all branches.')
|
||||
push_parser.add_argument(
|
||||
'-T',
|
||||
'--all-tags',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Push cache for all tags.')
|
||||
push_parser.add_argument(
|
||||
'-d',
|
||||
'--with-deps',
|
||||
@@ -339,6 +358,12 @@ def parse_args(argv=None):
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Fetch cache for all branches.')
|
||||
fetch_parser.add_argument(
|
||||
'-T',
|
||||
'--all-tags',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Fetch cache for all tags.')
|
||||
fetch_parser.add_argument(
|
||||
'-d',
|
||||
'--with-deps',
|
||||
@@ -373,12 +398,19 @@ def parse_args(argv=None):
|
||||
default=False,
|
||||
help='Show status of a local cache compared to a '
|
||||
'remote repository for all branches.')
|
||||
status_parser.add_argument(
|
||||
'-T',
|
||||
'--all-tags',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Show status of a local cache compared to a '
|
||||
'remote repository for all tags.')
|
||||
status_parser.add_argument(
|
||||
'-d',
|
||||
'--with-deps',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Show statusof a local cache compared to a '
|
||||
help='Show status of a local cache compared to a '
|
||||
'remote repository for all dependencies of the '
|
||||
'specified target.')
|
||||
status_parser.set_defaults(func=CmdDataStatus)
|
||||
@@ -439,6 +471,12 @@ def parse_args(argv=None):
|
||||
default=False,
|
||||
help='Reproduce the whole pipeline that the '
|
||||
'specified stage file belongs to.')
|
||||
repro_parser.add_argument(
|
||||
'-P',
|
||||
'--all-pipelines',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Reproduce all pipelines in the project.')
|
||||
repro_parser.set_defaults(func=CmdRepro)
|
||||
|
||||
# Remove
|
||||
@@ -495,6 +533,12 @@ def parse_args(argv=None):
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Collect garbage for all branches.')
|
||||
gc_parser.add_argument(
|
||||
'-T',
|
||||
'--all-tags',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Collect garbage for all tags.')
|
||||
gc_parser.add_argument(
|
||||
'-c',
|
||||
'--cloud',
|
||||
@@ -505,6 +549,12 @@ def parse_args(argv=None):
|
||||
'-r',
|
||||
'--remote',
|
||||
help='Remote repository to collect garbage in.')
|
||||
gc_parser.add_argument(
|
||||
'-f',
|
||||
'--force',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Force garbage collection.')
|
||||
gc_parser.set_defaults(func=CmdGC)
|
||||
|
||||
# Config
|
||||
@@ -688,6 +738,12 @@ def parse_args(argv=None):
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Show metrics for all branches.')
|
||||
metrics_show_parser.add_argument(
|
||||
'-T',
|
||||
'--all-tags',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Show metrics for all tags.')
|
||||
metrics_show_parser.set_defaults(func=CmdMetricsShow)
|
||||
|
||||
METRICS_ADD_HELP = 'Add metrics.'
|
||||
@@ -838,7 +894,9 @@ def parse_args(argv=None):
|
||||
if (issubclass(args.func, CmdRepro)
|
||||
or issubclass(args.func, CmdPipelineShow)) \
|
||||
and hasattr(args, 'targets') \
|
||||
and len(args.targets) == 0:
|
||||
and len(args.targets) == 0 \
|
||||
and not (hasattr(args, 'all_pipelines')
|
||||
and args.all_pipelines): # pragma: no cover
|
||||
if hasattr(args, 'cwd'):
|
||||
cwd = args.cwd
|
||||
else:
|
||||
@@ -856,7 +914,7 @@ def parse_args(argv=None):
|
||||
and len(args.deps) == 0 \
|
||||
and len(args.outs) == 0 \
|
||||
and len(args.outs_no_cache) == 0 \
|
||||
and len(args.command) == 0:
|
||||
and len(args.command) == 0: # pragma: no cover
|
||||
run_parser.error("Too few arguments. Specify at least one: "
|
||||
"'-d', '-o', '-O', 'command'.")
|
||||
|
||||
|
||||
@@ -24,6 +24,7 @@ class CmdDataPull(CmdDataBase):
|
||||
remote=self.args.remote,
|
||||
show_checksums=self.args.show_checksums,
|
||||
all_branches=self.args.all_branches,
|
||||
all_tags=self.args.all_tags,
|
||||
with_deps=self.args.with_deps)
|
||||
except Exception as exc:
|
||||
self.project.logger.error('Failed to pull data from the cloud',
|
||||
@@ -40,6 +41,7 @@ class CmdDataPush(CmdDataBase):
|
||||
remote=self.args.remote,
|
||||
show_checksums=self.args.show_checksums,
|
||||
all_branches=self.args.all_branches,
|
||||
all_tags=self.args.all_tags,
|
||||
with_deps=self.args.with_deps)
|
||||
except Exception as exc:
|
||||
self.project.logger.error('Failed to push data to the cloud', exc)
|
||||
@@ -55,6 +57,7 @@ class CmdDataFetch(CmdDataBase):
|
||||
remote=self.args.remote,
|
||||
show_checksums=self.args.show_checksums,
|
||||
all_branches=self.args.all_branches,
|
||||
all_tags=self.args.all_tags,
|
||||
with_deps=self.args.with_deps)
|
||||
except Exception as exc:
|
||||
self.project.logger.error('Failed to fetch data from the cloud',
|
||||
|
||||
@@ -3,7 +3,24 @@ from dvc.command.base import CmdBase
|
||||
|
||||
class CmdGC(CmdBase):
|
||||
def run(self):
|
||||
msg = 'This will remove all cache except the cache that is used in '
|
||||
if not self.args.all_branches and not self.args.all_tags:
|
||||
msg += 'the current git branch.'
|
||||
elif self.args.all_branches and not self.args.all_tags:
|
||||
msg += 'all git branches.'
|
||||
elif not self.args.all_branches and self.args.all_tags:
|
||||
msg += 'all git tags.'
|
||||
else:
|
||||
msg += 'all git branches and all git tags.'
|
||||
|
||||
self.project.logger.warn(msg)
|
||||
|
||||
msg = 'Are you sure you want to proceed?'
|
||||
if not self.args.force and not self.project.prompt.prompt(msg):
|
||||
return 1
|
||||
|
||||
self.project.gc(all_branches=self.args.all_branches,
|
||||
all_tags=self.args.all_tags,
|
||||
cloud=self.args.cloud,
|
||||
remote=self.args.remote)
|
||||
return 0
|
||||
|
||||
@@ -28,7 +28,8 @@ class CmdMetricsShow(CmdBase):
|
||||
self.project.metrics_show(self.args.path,
|
||||
typ=typ,
|
||||
xpath=xpath,
|
||||
all_branches=self.args.all_branches)
|
||||
all_branches=self.args.all_branches,
|
||||
all_tags=self.args.all_tags)
|
||||
except DvcException as exc:
|
||||
self.project.logger.error('Failed to show metrics', exc)
|
||||
return 1
|
||||
|
||||
@@ -21,7 +21,8 @@ class CmdRepro(CmdBase):
|
||||
force=self.args.force,
|
||||
dry=self.args.dry,
|
||||
interactive=self.args.interactive,
|
||||
pipeline=self.args.pipeline)
|
||||
pipeline=self.args.pipeline,
|
||||
all_pipelines=self.args.all_pipelines)
|
||||
|
||||
if len(stages) == 0:
|
||||
self.project.logger.info(CmdDataStatus.UP_TO_DATE_MSG)
|
||||
|
||||
@@ -31,6 +31,7 @@ class CmdDataStatus(CmdDataBase):
|
||||
show_checksums=self.args.show_checksums,
|
||||
remote=self.args.remote,
|
||||
all_branches=self.args.all_branches,
|
||||
all_tags=self.args.all_tags,
|
||||
with_deps=self.args.with_deps)
|
||||
if st:
|
||||
self._show(st, indent)
|
||||
|
||||
+171
-14
@@ -1,13 +1,170 @@
|
||||
import sys
|
||||
import math
|
||||
|
||||
from asciicanvas.asciicanvas import AsciiCanvas
|
||||
from asciicanvas.style import Style
|
||||
import select
|
||||
|
||||
from grandalf.graphs import Vertex, Edge, Graph
|
||||
from grandalf.layouts import SugiyamaLayout
|
||||
from grandalf.routing import route_with_lines, EdgeViewer
|
||||
|
||||
|
||||
class AsciiCanvas(object):
|
||||
def __init__(self, cols, lines):
|
||||
assert cols > 1
|
||||
assert lines > 1
|
||||
|
||||
self.cols = cols
|
||||
self.lines = lines
|
||||
|
||||
self.canvas = [[' '] * cols for l in range(lines)]
|
||||
|
||||
def draw(self):
|
||||
if sys.stdout.isatty(): # pragma: no cover
|
||||
from asciimatics.screen import Screen
|
||||
Screen.wrapper(self._do_draw)
|
||||
else:
|
||||
for line in self.canvas:
|
||||
print(''.join(line))
|
||||
|
||||
def _do_draw(self, screen): # pragma: no cover
|
||||
from asciimatics.event import KeyboardEvent
|
||||
|
||||
offset_x = 0
|
||||
offset_y = 0
|
||||
smaxrow, smaxcol = screen.dimensions
|
||||
assert smaxrow > 1
|
||||
assert smaxcol > 1
|
||||
smaxrow -= 1
|
||||
smaxcol -= 1
|
||||
|
||||
if self.lines + 1 > smaxrow:
|
||||
max_y = self.lines + 1 - smaxrow
|
||||
else:
|
||||
max_y = 0
|
||||
|
||||
if self.cols + 1 > smaxcol:
|
||||
max_x = self.cols + 1 - smaxcol
|
||||
else:
|
||||
max_x = 0
|
||||
|
||||
while True:
|
||||
for y in range(smaxrow + 1):
|
||||
y_index = offset_y + y
|
||||
line = []
|
||||
for x in range(smaxcol + 1):
|
||||
x_index = offset_x + x
|
||||
if len(self.canvas) > y_index \
|
||||
and len(self.canvas[y_index]) > x_index:
|
||||
line.append(self.canvas[y_index][x_index])
|
||||
else:
|
||||
line.append(' ')
|
||||
assert len(line) == (smaxcol + 1)
|
||||
screen.print_at(''.join(line), 0, y)
|
||||
|
||||
screen.refresh()
|
||||
|
||||
# NOTE: get_event() doesn't block by itself,
|
||||
# so we have to do the blocking ourselves.
|
||||
select.select([sys.stdin], [], [], None)
|
||||
|
||||
event = screen.get_event()
|
||||
if not isinstance(event, KeyboardEvent):
|
||||
continue
|
||||
|
||||
k = event.key_code
|
||||
if k == screen.KEY_DOWN or k == ord('s'):
|
||||
offset_y += 1
|
||||
elif k == screen.KEY_PAGE_DOWN or k == ord('S'):
|
||||
offset_y += smaxrow
|
||||
elif k == screen.KEY_UP or k == ord('w'):
|
||||
offset_y -= 1
|
||||
elif k == screen.KEY_PAGE_UP or k == ord('W'):
|
||||
offset_y -= smaxrow
|
||||
elif k == screen.KEY_RIGHT or k == ord('d'):
|
||||
offset_x += 1
|
||||
elif k == ord('D'):
|
||||
offset_x += smaxcol
|
||||
elif k == screen.KEY_LEFT or k == ord('a'):
|
||||
offset_x -= 1
|
||||
elif k == ord('A'):
|
||||
offset_x -= smaxcol
|
||||
elif k == ord('q') or k == ord('Q'):
|
||||
break
|
||||
|
||||
if offset_y > max_y:
|
||||
offset_y = max_y
|
||||
elif offset_y < 0:
|
||||
offset_y = 0
|
||||
|
||||
if offset_x > max_x:
|
||||
offset_x = max_x
|
||||
elif offset_x < 0:
|
||||
offset_x = 0
|
||||
|
||||
def point(self, x, y, char):
|
||||
assert x >= 0
|
||||
assert x < self.cols
|
||||
assert y >= 0
|
||||
assert y < self.lines
|
||||
|
||||
self.canvas[y][x] = char
|
||||
|
||||
def line(self, x0, y0, x1, y1, char):
|
||||
if x0 > x1:
|
||||
x1, x0 = x0, x1
|
||||
y1, y0 = y0, y1
|
||||
|
||||
dx = x1 - x0
|
||||
dy = y1 - y0
|
||||
|
||||
if dx == 0 and dy == 0:
|
||||
self.point(x0, y0, char)
|
||||
elif abs(dx) >= abs(dy):
|
||||
for x in range(x0, x1 + 1):
|
||||
if dx == 0:
|
||||
y = y0
|
||||
else:
|
||||
y = y0 + int(round((x - x0) * dy / float((dx))))
|
||||
self.point(x, y, char)
|
||||
elif y0 < y1:
|
||||
for y in range(y0, y1 + 1):
|
||||
if dy == 0:
|
||||
x = x0
|
||||
else:
|
||||
x = x0 + int(round((y - y0) * dx / float((dy))))
|
||||
self.point(x, y, char)
|
||||
else:
|
||||
for y in range(y1, y0 + 1):
|
||||
if dy == 0:
|
||||
x = x0
|
||||
else:
|
||||
x = x1 + int(round((y - y1) * dx / float((dy))))
|
||||
self.point(x, y, char)
|
||||
|
||||
def text(self, x, y, text):
|
||||
for i, c in enumerate(text):
|
||||
self.point(x + i, y, c)
|
||||
|
||||
def box(self, x0, y0, w, h):
|
||||
assert w > 1
|
||||
assert h > 1
|
||||
|
||||
w -= 1
|
||||
h -= 1
|
||||
|
||||
for x in range(x0, x0 + w):
|
||||
self.point(x, y0, '-')
|
||||
self.point(x, y0 + h, '-')
|
||||
|
||||
for y in range(y0, y0 + h):
|
||||
self.point(x0, y, '|')
|
||||
self.point(x0 + w, y, '|')
|
||||
|
||||
self.point(x0, y0, '+')
|
||||
self.point(x0 + w, y0, '+')
|
||||
self.point(x0, y0 + h, '+')
|
||||
self.point(x0 + w, y0 + h, '+')
|
||||
|
||||
|
||||
class Dagascii(object):
|
||||
def __init__(self, vertexes, edges):
|
||||
#
|
||||
@@ -100,22 +257,22 @@ class Dagascii(object):
|
||||
assert end_x >= 0
|
||||
assert end_y >= 0
|
||||
|
||||
canvas.add_line(start_x, start_y,
|
||||
end_x, end_y,
|
||||
style=Style('*'))
|
||||
canvas.line(start_x, start_y,
|
||||
end_x, end_y,
|
||||
'*')
|
||||
|
||||
for v in self.sug.g.sV:
|
||||
# NOTE: moving boxes w/2 to the left
|
||||
x = v.view.xy[0] - v.view.w/2.0
|
||||
y = v.view.xy[1]
|
||||
|
||||
canvas.add_nine_patch_rect(int(round(x - minx)),
|
||||
int(round(y - miny)),
|
||||
v.view.w,
|
||||
v.view.h)
|
||||
canvas.box(int(round(x - minx)),
|
||||
int(round(y - miny)),
|
||||
v.view.w,
|
||||
v.view.h)
|
||||
|
||||
canvas.add_text(int(round(x - minx)) + 1,
|
||||
int(round(y - miny)) + 1,
|
||||
v.data)
|
||||
canvas.text(int(round(x - minx)) + 1,
|
||||
int(round(y - miny)) + 1,
|
||||
v.data)
|
||||
|
||||
canvas.print_out()
|
||||
canvas.draw()
|
||||
|
||||
@@ -67,7 +67,8 @@ class DependencyLOCAL(DependencyBase):
|
||||
if not self.exists:
|
||||
raise self.DoesNotExistError(self.rel_path)
|
||||
|
||||
if not os.path.isfile(self.path) and not os.path.isdir(self.path):
|
||||
if not os.path.isfile(self.path) \
|
||||
and not os.path.isdir(self.path): # pragma: no cover
|
||||
raise self.IsNotFileOrDirError(self.rel_path)
|
||||
|
||||
if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \
|
||||
|
||||
+10
-1
@@ -8,7 +8,7 @@ class DvcException(Exception):
|
||||
if cause:
|
||||
try:
|
||||
self.cause_tb = traceback.format_exc()
|
||||
except Exception:
|
||||
except Exception: # pragma: no cover
|
||||
pass
|
||||
super(DvcException, self).__init__(msg)
|
||||
|
||||
@@ -29,3 +29,12 @@ class OutputDuplicationError(DvcException):
|
||||
for stage in stages:
|
||||
s += "\n " + stage
|
||||
super(OutputDuplicationError, self).__init__(msg.format(output, s))
|
||||
|
||||
|
||||
class MoveNotDataSourceError(DvcException):
|
||||
def __init__(self, path):
|
||||
msg = "Move is not permitted for stages that are not data sources. " \
|
||||
"You need to either move '{path}' to a new location and edit " \
|
||||
"it by hand, or remove '{path}' and create a new one at the " \
|
||||
"desired location."
|
||||
super(MoveNotDataSourceError, self).__init__(msg.format(path=path))
|
||||
|
||||
+17
-7
@@ -19,9 +19,16 @@ class Logger(object):
|
||||
}
|
||||
|
||||
COLOR_MAP = {
|
||||
'debug': colorama.Fore.BLUE,
|
||||
'warn': colorama.Fore.YELLOW,
|
||||
'error': colorama.Fore.RED
|
||||
'green': colorama.Fore.GREEN,
|
||||
'yellow': colorama.Fore.YELLOW,
|
||||
'blue': colorama.Fore.BLUE,
|
||||
'red': colorama.Fore.RED,
|
||||
}
|
||||
|
||||
LEVEL_COLOR_MAP = {
|
||||
'debug': 'blue',
|
||||
'warn': 'yellow',
|
||||
'error': 'red',
|
||||
}
|
||||
|
||||
def __init__(self, loglevel=None):
|
||||
@@ -58,12 +65,12 @@ class Logger(object):
|
||||
Logger.logger().setLevel(logging.DEBUG)
|
||||
|
||||
@staticmethod
|
||||
def colorize(msg, typ):
|
||||
def colorize(msg, color):
|
||||
header = ''
|
||||
footer = ''
|
||||
|
||||
if sys.stdout.isatty(): # pragma: no cover
|
||||
header = Logger.COLOR_MAP.get(typ.lower(), '')
|
||||
header = Logger.COLOR_MAP.get(color.lower(), '')
|
||||
footer = colorama.Style.RESET_ALL
|
||||
|
||||
return u'{}{}{}'.format(header, msg, footer)
|
||||
@@ -84,7 +91,8 @@ class Logger(object):
|
||||
|
||||
@staticmethod
|
||||
def _prefix(msg, typ):
|
||||
return Logger.colorize('{}: '.format(msg), typ)
|
||||
color = Logger.LEVEL_COLOR_MAP.get(typ.lower(), '')
|
||||
return Logger.colorize('{}: '.format(msg), color)
|
||||
|
||||
@staticmethod
|
||||
def error_prefix():
|
||||
@@ -105,7 +113,9 @@ class Logger(object):
|
||||
if Logger.logger().getEffectiveLevel() == logging.DEBUG and exc:
|
||||
str_tb = str_tb if str_tb else traceback.format_exc()
|
||||
Logger.logger().error(prefix + str_tb)
|
||||
return Logger.logger().error(prefix + msg + str_exc)
|
||||
chat = "\n\nHaving any troubles? Hit us up at dvc.org/support, " \
|
||||
"we are always happy to help!"
|
||||
return Logger.logger().error(prefix + msg + str_exc + chat)
|
||||
|
||||
@staticmethod
|
||||
def warn(msg):
|
||||
|
||||
+4
-3
@@ -83,13 +83,14 @@ class OutputLOCAL(DependencyLOCAL):
|
||||
super(OutputLOCAL, self).save()
|
||||
self._verify_metric()
|
||||
msg = 'Output \'{}\' doesn\'t use cache. Skipping saving.'
|
||||
self.project.logger.debug(msg.format(self.rel_path))
|
||||
self.project.logger.info(msg.format(self.rel_path))
|
||||
return
|
||||
|
||||
if not os.path.exists(self.path):
|
||||
raise self.DoesNotExistError(self.rel_path)
|
||||
|
||||
if not os.path.isfile(self.path) and not os.path.isdir(self.path):
|
||||
if not os.path.isfile(self.path) \
|
||||
and not os.path.isdir(self.path): # pragma: no cover
|
||||
raise self.IsNotFileOrDirError(self.rel_path)
|
||||
|
||||
if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \
|
||||
@@ -99,7 +100,7 @@ class OutputLOCAL(DependencyLOCAL):
|
||||
|
||||
if not self.changed():
|
||||
msg = 'Output \'{}\' didn\'t change. Skipping saving.'
|
||||
self.project.logger.debug(msg.format(self.rel_path))
|
||||
self.project.logger.info(msg.format(self.rel_path))
|
||||
return
|
||||
|
||||
if self.is_local:
|
||||
|
||||
+49
-25
@@ -1,6 +1,6 @@
|
||||
import os
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.exceptions import DvcException, MoveNotDataSourceError
|
||||
from dvc.stage import Stage
|
||||
|
||||
|
||||
@@ -209,8 +209,7 @@ class Project(object):
|
||||
continue
|
||||
|
||||
if not stage.is_data_source:
|
||||
msg = 'Dvcfile \'{}\' is not a data source.'
|
||||
raise DvcException(msg.format(stage.rel_path))
|
||||
raise MoveNotDataSourceError(stage.relpath)
|
||||
|
||||
found = True
|
||||
to_out = Output.loads_from(out.stage,
|
||||
@@ -307,12 +306,16 @@ class Project(object):
|
||||
return [stage]
|
||||
|
||||
def reproduce(self,
|
||||
target,
|
||||
target=None,
|
||||
recursive=True,
|
||||
force=False,
|
||||
dry=False,
|
||||
interactive=False,
|
||||
pipeline=False):
|
||||
pipeline=False,
|
||||
all_pipelines=False):
|
||||
|
||||
if target is None and not all_pipelines:
|
||||
raise ValueError()
|
||||
|
||||
if not interactive:
|
||||
config = self.config
|
||||
@@ -320,13 +323,18 @@ class Project(object):
|
||||
interactive = core.get(config.SECTION_CORE_INTERACTIVE, False)
|
||||
|
||||
targets = []
|
||||
if pipeline:
|
||||
stage = Stage.load(self, target)
|
||||
node = os.path.relpath(stage.path, self.root_dir)
|
||||
G = self._get_pipeline(node)
|
||||
for node in G.nodes():
|
||||
if G.in_degree(node) == 0:
|
||||
targets.append(os.path.join(self.root_dir, node))
|
||||
if pipeline or all_pipelines:
|
||||
if pipeline:
|
||||
stage = Stage.load(self, target)
|
||||
node = os.path.relpath(stage.path, self.root_dir)
|
||||
pipelines = [self._get_pipeline(node)]
|
||||
else:
|
||||
pipelines = self.pipelines()
|
||||
|
||||
for G in pipelines:
|
||||
for node in G.nodes():
|
||||
if G.in_degree(node) == 0:
|
||||
targets.append(os.path.join(self.root_dir, node))
|
||||
else:
|
||||
targets.append(target)
|
||||
|
||||
@@ -443,7 +451,8 @@ class Project(object):
|
||||
target=None,
|
||||
all_branches=False,
|
||||
active=True,
|
||||
with_deps=False):
|
||||
with_deps=False,
|
||||
all_tags=False):
|
||||
cache = {}
|
||||
cache['local'] = []
|
||||
cache['s3'] = []
|
||||
@@ -451,7 +460,8 @@ class Project(object):
|
||||
cache['hdfs'] = []
|
||||
cache['ssh'] = []
|
||||
|
||||
for branch in self.scm.brancher(all_branches=all_branches):
|
||||
for branch in self.scm.brancher(all_branches=all_branches,
|
||||
all_tags=all_tags):
|
||||
if target:
|
||||
stages = self._collect(target,
|
||||
with_deps=with_deps)
|
||||
@@ -486,12 +496,14 @@ class Project(object):
|
||||
all_branches=False,
|
||||
cloud=False,
|
||||
remote=None,
|
||||
with_deps=False):
|
||||
with_deps=False,
|
||||
all_tags=False):
|
||||
with self.state:
|
||||
clist = self._used_cache(target=None,
|
||||
all_branches=all_branches,
|
||||
active=False,
|
||||
with_deps=with_deps)
|
||||
with_deps=with_deps,
|
||||
all_tags=all_tags)
|
||||
self._do_gc('local', self.cache.local.gc, clist)
|
||||
|
||||
if self.cache.s3:
|
||||
@@ -519,10 +531,12 @@ class Project(object):
|
||||
remote=None,
|
||||
all_branches=False,
|
||||
show_checksums=False,
|
||||
with_deps=False):
|
||||
with_deps=False,
|
||||
all_tags=False):
|
||||
with self.state:
|
||||
used = self._used_cache(target,
|
||||
all_branches=all_branches,
|
||||
all_tags=all_tags,
|
||||
with_deps=with_deps)['local']
|
||||
self.cloud.push(used,
|
||||
jobs,
|
||||
@@ -535,10 +549,12 @@ class Project(object):
|
||||
remote=None,
|
||||
all_branches=False,
|
||||
show_checksums=False,
|
||||
with_deps=False):
|
||||
with_deps=False,
|
||||
all_tags=False):
|
||||
with self.state:
|
||||
used = self._used_cache(target,
|
||||
all_branches=all_branches,
|
||||
all_tags=all_tags,
|
||||
with_deps=with_deps)['local']
|
||||
self.cloud.pull(used,
|
||||
jobs,
|
||||
@@ -551,11 +567,13 @@ class Project(object):
|
||||
remote=None,
|
||||
all_branches=False,
|
||||
show_checksums=False,
|
||||
with_deps=False):
|
||||
with_deps=False,
|
||||
all_tags=False):
|
||||
self.fetch(target,
|
||||
jobs,
|
||||
remote=remote,
|
||||
all_branches=all_branches,
|
||||
all_tags=all_tags,
|
||||
show_checksums=show_checksums,
|
||||
with_deps=with_deps)
|
||||
self.checkout(target=target)
|
||||
@@ -584,11 +602,13 @@ class Project(object):
|
||||
remote=None,
|
||||
show_checksums=False,
|
||||
all_branches=False,
|
||||
with_deps=False):
|
||||
with_deps=False,
|
||||
all_tags=False):
|
||||
import dvc.remote.base as cloud
|
||||
|
||||
used = self._used_cache(target,
|
||||
all_branches=all_branches,
|
||||
all_tags=all_tags,
|
||||
with_deps=with_deps)['local']
|
||||
|
||||
status = {}
|
||||
@@ -615,7 +635,8 @@ class Project(object):
|
||||
remote=None,
|
||||
show_checksums=False,
|
||||
all_branches=False,
|
||||
with_deps=False):
|
||||
with_deps=False,
|
||||
all_tags=False):
|
||||
with self.state:
|
||||
if cloud:
|
||||
return self._cloud_status(target,
|
||||
@@ -623,7 +644,8 @@ class Project(object):
|
||||
remote=remote,
|
||||
show_checksums=show_checksums,
|
||||
all_branches=all_branches,
|
||||
with_deps=with_deps)
|
||||
with_deps=with_deps,
|
||||
all_tags=all_tags)
|
||||
return self._local_status(target)
|
||||
|
||||
def _read_metric_json(self, fd, json_path):
|
||||
@@ -704,9 +726,11 @@ class Project(object):
|
||||
path=None,
|
||||
typ=None,
|
||||
xpath=None,
|
||||
all_branches=False):
|
||||
all_branches=False,
|
||||
all_tags=False):
|
||||
res = {}
|
||||
for branch in self.scm.brancher(all_branches=all_branches):
|
||||
for branch in self.scm.brancher(all_branches=all_branches,
|
||||
all_tags=all_tags):
|
||||
astages = self.active_stages()
|
||||
outs = [out for stage in astages for out in stage.outs]
|
||||
|
||||
@@ -751,7 +775,7 @@ class Project(object):
|
||||
res[branch][rel] = metric
|
||||
|
||||
for branch, val in res.items():
|
||||
if all_branches:
|
||||
if all_branches or all_tags:
|
||||
self.logger.info('{}:'.format(branch))
|
||||
for fname, metric in val.items():
|
||||
self.logger.info('\t{}: {}'.format(fname, metric))
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import os
|
||||
import re
|
||||
import errno
|
||||
from multiprocessing import cpu_count
|
||||
|
||||
from dvc.config import Config
|
||||
from dvc.logger import Logger
|
||||
@@ -30,6 +31,7 @@ class DataCloudError(DvcException):
|
||||
class RemoteBase(object):
|
||||
REGEX = None
|
||||
REQUIRES = {}
|
||||
JOBS = 4 * cpu_count()
|
||||
|
||||
def __init__(self, project, config):
|
||||
pass
|
||||
|
||||
+6
-1
@@ -29,9 +29,14 @@ class RemoteHDFS(RemoteBase):
|
||||
cmd = 'hadoop fs -' + cmd
|
||||
if user:
|
||||
cmd = 'HADOOP_USER_NAME={} '.format(user) + cmd
|
||||
|
||||
# NOTE: close_fds doesn't work with redirected stdin/stdout/stderr.
|
||||
# See https://github.com/iterative/dvc/issues/1197.
|
||||
close_fds = (os.name != 'nt')
|
||||
|
||||
p = Popen(cmd,
|
||||
shell=True,
|
||||
close_fds=True,
|
||||
close_fds=close_fds,
|
||||
executable=os.getenv('SHELL'),
|
||||
env=fix_env(os.environ),
|
||||
stdin=PIPE,
|
||||
|
||||
+10
-4
@@ -487,7 +487,7 @@ class RemoteLOCAL(RemoteBase):
|
||||
|
||||
return removed
|
||||
|
||||
def status(self, checksum_infos, remote, jobs=1, show_checksums=False):
|
||||
def status(self, checksum_infos, remote, jobs=None, show_checksums=False):
|
||||
Logger.info("Preparing to pull data from {}".format(remote.url))
|
||||
title = "Collecting information"
|
||||
|
||||
@@ -523,7 +523,7 @@ class RemoteLOCAL(RemoteBase):
|
||||
def _do_pull(self,
|
||||
checksum_infos,
|
||||
remote,
|
||||
jobs=1,
|
||||
jobs=None,
|
||||
show_checksums=False):
|
||||
title = "Collecting information"
|
||||
|
||||
@@ -561,6 +561,9 @@ class RemoteLOCAL(RemoteBase):
|
||||
|
||||
assert len(path_infos) == len(cache) == len(md5s) == len(names)
|
||||
|
||||
if jobs is None:
|
||||
jobs = remote.JOBS
|
||||
|
||||
chunks = list(zip(to_chunks(path_infos, jobs),
|
||||
to_chunks(cache, jobs),
|
||||
to_chunks(names, jobs)))
|
||||
@@ -584,7 +587,7 @@ class RemoteLOCAL(RemoteBase):
|
||||
for f in futures:
|
||||
f.result()
|
||||
|
||||
def pull(self, checksum_infos, remote, jobs=1, show_checksums=False):
|
||||
def pull(self, checksum_infos, remote, jobs=None, show_checksums=False):
|
||||
Logger.info("Preparing to pull data from {}".format(remote.url))
|
||||
|
||||
# NOTE: try fetching missing dir info
|
||||
@@ -601,7 +604,7 @@ class RemoteLOCAL(RemoteBase):
|
||||
jobs,
|
||||
show_checksums=show_checksums)
|
||||
|
||||
def push(self, checksum_infos, remote, jobs=1, show_checksums=False):
|
||||
def push(self, checksum_infos, remote, jobs=None, show_checksums=False):
|
||||
Logger.info("Preparing to push data to {}".format(remote.url))
|
||||
title = "Collecting information"
|
||||
|
||||
@@ -646,6 +649,9 @@ class RemoteLOCAL(RemoteBase):
|
||||
|
||||
progress.update_target(title, 90, 100)
|
||||
|
||||
if jobs is None:
|
||||
jobs = remote.JOBS
|
||||
|
||||
chunks = list(zip(to_chunks(path_infos, jobs),
|
||||
to_chunks(cache, jobs),
|
||||
to_chunks(names, jobs)))
|
||||
|
||||
+12
-2
@@ -47,7 +47,9 @@ class RemoteS3(RemoteBase):
|
||||
'').lstrip('/')
|
||||
self.url = config.get(Config.SECTION_REMOTE_URL, storagepath)
|
||||
self.region = config.get(Config.SECTION_AWS_REGION, None)
|
||||
self.profile = config.get(Config.SECTION_AWS_PROFILE, 'default')
|
||||
self.profile = os.getenv('AWS_PROFILE',
|
||||
config.get(Config.SECTION_AWS_PROFILE,
|
||||
'default'))
|
||||
self.endpoint_url = config.get(Config.SECTION_AWS_ENDPOINT_URL, None)
|
||||
|
||||
credentialpath = config.get(Config.SECTION_AWS_CREDENTIALPATH, None)
|
||||
@@ -55,7 +57,7 @@ class RemoteS3(RemoteBase):
|
||||
creds_conf = configobj.ConfigObj(credentialpath)
|
||||
creds = creds_conf.get(self.profile, {})
|
||||
else:
|
||||
creds = {}
|
||||
creds = self._get_aws_config_from_env()
|
||||
|
||||
self.creds = creds
|
||||
self.region = creds.get('region', self.region)
|
||||
@@ -81,6 +83,14 @@ class RemoteS3(RemoteBase):
|
||||
region_name=self.region)
|
||||
return session.client('s3', endpoint_url=self.endpoint_url)
|
||||
|
||||
def _get_aws_config_from_env(self):
|
||||
creds = {
|
||||
'region': os.getenv('AWS_DEFAULT_REGION'),
|
||||
'aws_access_key_id': os.getenv('AWS_ACCESS_KEY_ID'),
|
||||
'aws_secret_access_key': os.getenv('AWS_SECRET_ACCESS_KEY'),
|
||||
}
|
||||
return {k: v for k, v in creds.items() if v is not None}
|
||||
|
||||
def get_etag(self, bucket, key):
|
||||
try:
|
||||
obj = self.s3.head_object(Bucket=bucket, Key=key)
|
||||
|
||||
@@ -44,6 +44,9 @@ class RemoteSSH(RemoteBase):
|
||||
REGEX = r'^ssh://((?P<user>.*)@)?(?P<host>[^/]*):(?P<path>/.*)$'
|
||||
|
||||
REQUIRES = {'paramiko': paramiko}
|
||||
|
||||
JOBS = 4
|
||||
|
||||
PARAM_MD5 = 'md5'
|
||||
|
||||
DEFAULT_PORT = 22
|
||||
|
||||
+30
-7
@@ -50,17 +50,34 @@ class Base(object):
|
||||
def branch(self, branch):
|
||||
pass
|
||||
|
||||
def brancher(self, branches=None, all_branches=False):
|
||||
if not branches and not all_branches:
|
||||
def brancher(self,
|
||||
branches=None,
|
||||
all_branches=False,
|
||||
tags=None,
|
||||
all_tags=False):
|
||||
if not branches and not all_branches \
|
||||
and not tags and not all_tags:
|
||||
yield ''
|
||||
return
|
||||
|
||||
saved = self.active_branch()
|
||||
if not branches:
|
||||
branches = self.list_branches() if all_branches else [saved]
|
||||
for branch in branches:
|
||||
self.checkout(branch)
|
||||
yield branch
|
||||
revs = []
|
||||
|
||||
if branches is not None:
|
||||
revs.extend(branches)
|
||||
elif all_branches:
|
||||
revs.extend(self.list_branches())
|
||||
elif tags is not None:
|
||||
revs.extend(tags)
|
||||
elif all_tags:
|
||||
revs.extend(self.list_tags())
|
||||
else:
|
||||
revs.extend([saved])
|
||||
|
||||
for rev in revs:
|
||||
self.checkout(rev)
|
||||
yield rev
|
||||
|
||||
self.checkout(saved)
|
||||
|
||||
def untracked_files(self):
|
||||
@@ -75,6 +92,9 @@ class Base(object):
|
||||
def list_branches(self):
|
||||
pass
|
||||
|
||||
def list_tags(self):
|
||||
pass
|
||||
|
||||
def install(self):
|
||||
pass
|
||||
|
||||
@@ -208,6 +228,9 @@ class Git(Base):
|
||||
def list_branches(self):
|
||||
return [h.name for h in self.repo.heads]
|
||||
|
||||
def list_tags(self):
|
||||
return [t.name for t in self.repo.tags]
|
||||
|
||||
def install(self):
|
||||
hook = os.path.join(self.root_dir,
|
||||
self.GIT_DIR,
|
||||
|
||||
+39
-22
@@ -1,6 +1,5 @@
|
||||
import os
|
||||
import yaml
|
||||
import itertools
|
||||
import posixpath
|
||||
import subprocess
|
||||
from schema import Schema, SchemaError, Optional, Or, And
|
||||
@@ -19,8 +18,9 @@ class StageCmdFailedError(DvcException):
|
||||
|
||||
|
||||
class StageFileFormatError(DvcException):
|
||||
def __init__(self):
|
||||
super(StageFileFormatError, self).__init__('Stage file format error')
|
||||
def __init__(self, fname, e):
|
||||
msg = "Stage file '{}' format error: {}".format(fname, str(e))
|
||||
super(StageFileFormatError, self).__init__(msg)
|
||||
|
||||
|
||||
class StageFileDoesNotExistError(DvcException):
|
||||
@@ -134,8 +134,6 @@ class Stage(object):
|
||||
if self.md5 and md5 and self.md5 == md5:
|
||||
return False
|
||||
|
||||
msg = "Dvc file '{}' md5 changed(expected '{}', actual '{}')"
|
||||
self.project.logger.debug(msg.format(self.relpath, self.md5, md5))
|
||||
return True
|
||||
|
||||
@property
|
||||
@@ -148,30 +146,51 @@ class Stage(object):
|
||||
len(self.deps) == 1 and \
|
||||
len(self.outs) == 1
|
||||
|
||||
def changed(self):
|
||||
def changed(self, print_info=False):
|
||||
ret = False
|
||||
|
||||
if print_info:
|
||||
log = self.project.logger.info
|
||||
else:
|
||||
log = self.project.logger.debug
|
||||
|
||||
if self.is_callback:
|
||||
msg = "Dvc file '{}' is a 'callback' stage (has a command and " \
|
||||
"no dependencies) and thus always considered as changed."
|
||||
self.project.logger.warn(msg.format(self.relpath))
|
||||
ret = True
|
||||
|
||||
if self.locked:
|
||||
entries = self.outs
|
||||
else:
|
||||
entries = itertools.chain(self.outs, self.deps)
|
||||
|
||||
for entry in entries:
|
||||
if entry.changed():
|
||||
if not self.locked:
|
||||
for dep in self.deps:
|
||||
if not dep.changed():
|
||||
continue
|
||||
if print_info:
|
||||
msg = "Dependency '{}' of '{}' changed."
|
||||
log(msg.format(dep, self.relpath))
|
||||
ret = True
|
||||
|
||||
for out in self.outs:
|
||||
if not out.changed():
|
||||
continue
|
||||
if print_info:
|
||||
msg = "Output '{}' of '{}' changed."
|
||||
log(msg.format(out, self.relpath))
|
||||
ret = True
|
||||
|
||||
if self.changed_md5():
|
||||
if print_info:
|
||||
msg = "Dvc file '{}' changed."
|
||||
log(msg.format(self.relpath))
|
||||
ret = True
|
||||
|
||||
if ret:
|
||||
msg = u'Dvc file \'{}\' changed'.format(self.relpath)
|
||||
msg = "Stage '{}' changed.".format(self.relpath)
|
||||
color = 'yellow'
|
||||
else:
|
||||
msg = u'Dvc file \'{}\' didn\'t change'.format(self.relpath)
|
||||
msg = "Stage '{}' didn't change.".format(self.relpath)
|
||||
color = 'green'
|
||||
|
||||
self.project.logger.debug(msg)
|
||||
log(Logger.colorize(msg, color))
|
||||
|
||||
return ret
|
||||
|
||||
@@ -184,7 +203,7 @@ class Stage(object):
|
||||
os.unlink(self.path)
|
||||
|
||||
def reproduce(self, force=False, dry=False, interactive=False):
|
||||
if not self.changed() and not force:
|
||||
if not self.changed(print_info=True) and not force:
|
||||
return None
|
||||
|
||||
if (self.cmd or self.is_import) and not self.locked and not dry:
|
||||
@@ -207,17 +226,15 @@ class Stage(object):
|
||||
return self
|
||||
|
||||
@staticmethod
|
||||
def validate(d):
|
||||
def validate(d, fname=None):
|
||||
try:
|
||||
Schema(Stage.SCHEMA).validate(d)
|
||||
except SchemaError as exc:
|
||||
Logger.debug(str(exc))
|
||||
raise StageFileFormatError()
|
||||
raise StageFileFormatError(fname, exc)
|
||||
|
||||
@staticmethod
|
||||
def loadd(project, d, path):
|
||||
Stage.validate(d)
|
||||
|
||||
Stage.validate(d, fname=os.path.relpath(path))
|
||||
path = os.path.abspath(path)
|
||||
cwd = os.path.dirname(path)
|
||||
cmd = d.get(Stage.PARAM_CMD, None)
|
||||
|
||||
+80
-43
@@ -15,9 +15,10 @@ class StateDuplicateError(DvcException):
|
||||
|
||||
|
||||
class State(object):
|
||||
VERSION = 1
|
||||
STATE_FILE = 'state'
|
||||
STATE_TABLE = 'state'
|
||||
STATE_TABLE_LAYOUT = "inode INTEGER PRIMARY KEY, " \
|
||||
STATE_TABLE_LAYOUT = "inode UNSIGNED INTEGER PRIMARY KEY, " \
|
||||
"mtime TEXT NOT NULL, " \
|
||||
"md5 TEXT NOT NULL, " \
|
||||
"timestamp TEXT NOT NULL"
|
||||
@@ -28,7 +29,7 @@ class State(object):
|
||||
|
||||
LINK_STATE_TABLE = 'link_state'
|
||||
LINK_STATE_TABLE_LAYOUT = "path TEXT PRIMARY KEY, " \
|
||||
"inode INTEGER NOT NULL, " \
|
||||
"inode UNSIGNED INTEGER NOT NULL, " \
|
||||
"mtime TEXT NOT NULL"
|
||||
|
||||
STATE_ROW_LIMIT = 10000000
|
||||
@@ -81,6 +82,55 @@ class State(object):
|
||||
|
||||
return actual.split('.')[0] != md5.split('.')[0]
|
||||
|
||||
def _execute(self, cmd):
|
||||
Logger.debug(cmd)
|
||||
return self.c.execute(cmd)
|
||||
|
||||
def _load(self):
|
||||
from dvc import VERSION
|
||||
|
||||
cmd = "PRAGMA user_version;"
|
||||
self._execute(cmd)
|
||||
ret = self.c.fetchall()
|
||||
assert len(ret) == 1
|
||||
assert len(ret[0]) == 1
|
||||
assert isinstance(ret[0][0], int)
|
||||
self.version = ret[0][0]
|
||||
|
||||
if self.version > self.VERSION:
|
||||
msg = "You are using an old version '{}' of dvc that is using " \
|
||||
"state file version '{}' which is not compatible with the " \
|
||||
"state file version '{}' that is used in this projet. " \
|
||||
"Please upgrade right now!"
|
||||
raise DvcException(msg.format(VERSION,
|
||||
self.VERSION,
|
||||
self.version))
|
||||
elif self.version < self.VERSION:
|
||||
msg = "State file version '{}' is too old. " \
|
||||
"Reformatting to the current version '{}'."
|
||||
self.project.logger.warn(msg.format(self.version, self.VERSION))
|
||||
cmd = "DROP TABLE IF EXISTS {};"
|
||||
self._execute(cmd.format(self.STATE_TABLE))
|
||||
self._execute(cmd.format(self.STATE_INFO_TABLE))
|
||||
self._execute(cmd.format(self.LINK_STATE_TABLE))
|
||||
|
||||
# Check that the state file is indeed a database
|
||||
cmd = "CREATE TABLE IF NOT EXISTS {} ({})"
|
||||
self._execute(cmd.format(self.STATE_TABLE,
|
||||
self.STATE_TABLE_LAYOUT))
|
||||
self._execute(cmd.format(self.STATE_INFO_TABLE,
|
||||
self.STATE_INFO_TABLE_LAYOUT))
|
||||
self._execute(cmd.format(self.LINK_STATE_TABLE,
|
||||
self.LINK_STATE_TABLE_LAYOUT))
|
||||
|
||||
cmd = "INSERT OR IGNORE INTO {} (count) SELECT 0 " \
|
||||
"WHERE NOT EXISTS (SELECT * FROM {})"
|
||||
self._execute(cmd.format(self.STATE_INFO_TABLE,
|
||||
self.STATE_INFO_TABLE))
|
||||
|
||||
cmd = "PRAGMA user_version = {};"
|
||||
self._execute(cmd.format(self.VERSION))
|
||||
|
||||
def load(self):
|
||||
retries = 1
|
||||
while True:
|
||||
@@ -93,20 +143,7 @@ class State(object):
|
||||
# Try loading once to check that the file is indeed a database
|
||||
# and reformat it if it is not.
|
||||
try:
|
||||
# Check that the state file is indeed a database
|
||||
cmd = "CREATE TABLE IF NOT EXISTS {} ({})"
|
||||
self.c.execute(cmd.format(self.STATE_TABLE,
|
||||
self.STATE_TABLE_LAYOUT))
|
||||
self.c.execute(cmd.format(self.STATE_INFO_TABLE,
|
||||
self.STATE_INFO_TABLE_LAYOUT))
|
||||
self.c.execute(cmd.format(self.LINK_STATE_TABLE,
|
||||
self.LINK_STATE_TABLE_LAYOUT))
|
||||
|
||||
cmd = "INSERT OR IGNORE INTO {} (count) SELECT 0 " \
|
||||
"WHERE NOT EXISTS (SELECT * FROM {})"
|
||||
self.c.execute(cmd.format(self.STATE_INFO_TABLE,
|
||||
self.STATE_INFO_TABLE))
|
||||
|
||||
self._load()
|
||||
return
|
||||
except sqlite3.DatabaseError:
|
||||
self.c.close()
|
||||
@@ -124,8 +161,8 @@ class State(object):
|
||||
assert self.db is not None
|
||||
|
||||
cmd = "SELECT count from {} WHERE rowid={}"
|
||||
self.c.execute(cmd.format(self.STATE_INFO_TABLE,
|
||||
self.STATE_INFO_ROW))
|
||||
self._execute(cmd.format(self.STATE_INFO_TABLE,
|
||||
self.STATE_INFO_ROW))
|
||||
ret = self.c.fetchall()
|
||||
assert len(ret) == 1
|
||||
assert len(ret[0]) == 1
|
||||
@@ -139,24 +176,24 @@ class State(object):
|
||||
delete += int(self.row_limit * (self.row_cleanup_quota/100.))
|
||||
cmd = "DELETE FROM {} WHERE timestamp IN (" \
|
||||
"SELECT timestamp FROM {} ORDER BY timestamp ASC LIMIT {});"
|
||||
self.c.execute(cmd.format(self.STATE_TABLE,
|
||||
self.STATE_TABLE,
|
||||
delete))
|
||||
self._execute(cmd.format(self.STATE_TABLE,
|
||||
self.STATE_TABLE,
|
||||
delete))
|
||||
|
||||
self.c.execute("VACUUM")
|
||||
self._execute("VACUUM")
|
||||
|
||||
cmd = "SELECT COUNT(*) FROM {}"
|
||||
|
||||
self.c.execute(cmd.format(self.STATE_TABLE))
|
||||
self._execute(cmd.format(self.STATE_TABLE))
|
||||
ret = self.c.fetchall()
|
||||
assert len(ret) == 1
|
||||
assert len(ret[0]) == 1
|
||||
count = ret[0][0]
|
||||
|
||||
cmd = "UPDATE {} SET count = {} WHERE rowid = {}"
|
||||
self.c.execute(cmd.format(self.STATE_INFO_TABLE,
|
||||
count,
|
||||
self.STATE_INFO_ROW))
|
||||
self._execute(cmd.format(self.STATE_INFO_TABLE,
|
||||
count,
|
||||
self.STATE_INFO_ROW))
|
||||
|
||||
self.db.commit()
|
||||
self.c.close()
|
||||
@@ -192,17 +229,17 @@ class State(object):
|
||||
cmd = 'SELECT * from {} WHERE inode={}'.format(self.STATE_TABLE,
|
||||
inode)
|
||||
|
||||
self.c.execute(cmd)
|
||||
self._execute(cmd)
|
||||
ret = self.c.fetchall()
|
||||
if len(ret) == 0:
|
||||
md5, info = self._collect(path)
|
||||
cmd = 'INSERT INTO {}(inode, mtime, md5, timestamp) ' \
|
||||
'VALUES ({}, "{}", "{}", "{}")'
|
||||
self.c.execute(cmd.format(self.STATE_TABLE,
|
||||
inode,
|
||||
mtime,
|
||||
md5,
|
||||
int(nanotime.timestamp(time.time()))))
|
||||
self._execute(cmd.format(self.STATE_TABLE,
|
||||
inode,
|
||||
mtime,
|
||||
md5,
|
||||
int(nanotime.timestamp(time.time()))))
|
||||
self.inserts += 1
|
||||
else:
|
||||
assert len(ret) == 1
|
||||
@@ -214,17 +251,17 @@ class State(object):
|
||||
cmd = 'UPDATE {} SET ' \
|
||||
'mtime = "{}", md5 = "{}", timestamp = "{}" ' \
|
||||
'WHERE inode = {}'
|
||||
self.c.execute(cmd.format(self.STATE_TABLE,
|
||||
mtime,
|
||||
md5,
|
||||
int(nanotime.timestamp(time.time())),
|
||||
inode))
|
||||
self._execute(cmd.format(self.STATE_TABLE,
|
||||
mtime,
|
||||
md5,
|
||||
int(nanotime.timestamp(time.time())),
|
||||
inode))
|
||||
else:
|
||||
info = None
|
||||
cmd = 'UPDATE {} SET timestamp = "{}" WHERE inode = {}'
|
||||
self.c.execute(cmd.format(self.STATE_TABLE,
|
||||
int(nanotime.timestamp(time.time())),
|
||||
inode))
|
||||
self._execute(cmd.format(self.STATE_TABLE,
|
||||
int(nanotime.timestamp(time.time())),
|
||||
inode))
|
||||
|
||||
return (md5, info)
|
||||
|
||||
@@ -250,12 +287,12 @@ class State(object):
|
||||
relpath,
|
||||
inode,
|
||||
mtime)
|
||||
self.c.execute(cmd)
|
||||
self._execute(cmd)
|
||||
|
||||
def remove_unused_links(self, used):
|
||||
unused = []
|
||||
|
||||
self.c.execute('SELECT * FROM {}'.format(self.LINK_STATE_TABLE))
|
||||
self._execute('SELECT * FROM {}'.format(self.LINK_STATE_TABLE))
|
||||
for row in self.c:
|
||||
p, i, m = row
|
||||
path = os.path.join(self.root_dir, p)
|
||||
@@ -276,4 +313,4 @@ class State(object):
|
||||
|
||||
for p in unused:
|
||||
cmd = 'DELETE FROM {} WHERE path = "{}"'
|
||||
self.c.execute(cmd.format(self.LINK_STATE_TABLE, p))
|
||||
self._execute(cmd.format(self.LINK_STATE_TABLE, p))
|
||||
|
||||
+69
-22
@@ -1,8 +1,5 @@
|
||||
import os
|
||||
|
||||
if os.name == 'nt':
|
||||
import ntfsutils.hardlink as winlink
|
||||
|
||||
|
||||
class System(object):
|
||||
@staticmethod
|
||||
@@ -11,10 +8,20 @@ class System(object):
|
||||
|
||||
@staticmethod
|
||||
def hardlink(source, link_name):
|
||||
import ctypes
|
||||
if System.is_unix():
|
||||
return os.link(source, link_name)
|
||||
os.link(source, link_name)
|
||||
return
|
||||
|
||||
return winlink.create(source, link_name)
|
||||
CreateHardLink = ctypes.windll.kernel32.CreateHardLinkW
|
||||
CreateHardLink.argtypes = [ctypes.c_wchar_p,
|
||||
ctypes.c_wchar_p,
|
||||
ctypes.c_void_p]
|
||||
CreateHardLink.restype = ctypes.wintypes.BOOL
|
||||
|
||||
res = CreateHardLink(link_name, source, None)
|
||||
if res == 0:
|
||||
raise ctypes.WinError()
|
||||
|
||||
@staticmethod
|
||||
def symlink(source, link_name):
|
||||
@@ -48,7 +55,7 @@ class System(object):
|
||||
clonefile.restype = ctypes.c_int
|
||||
|
||||
return clonefile(ctypes.c_char_p(src.encode('utf-8')),
|
||||
ctypes.c_char_p(src.encode('utf-8')),
|
||||
ctypes.c_char_p(dst.encode('utf-8')),
|
||||
ctypes.c_int(0))
|
||||
|
||||
@staticmethod
|
||||
@@ -102,30 +109,70 @@ class System(object):
|
||||
if ret != 0:
|
||||
raise DvcException('Reflink is not supported')
|
||||
|
||||
# FIXME
|
||||
# Temporary fix while waiting for the PR to be merged and released:
|
||||
# https://github.com/sunshowers/ntfs/pull/11
|
||||
@staticmethod
|
||||
def getdirinfo(path):
|
||||
from ntfsutils.fs import FILE_FLAG_BACKUP_SEMANTICS, FILE_SHARE_READ
|
||||
from ntfsutils.fs import OPEN_EXISTING, BY_HANDLE_FILE_INFORMATION
|
||||
from ntfsutils.fs import CreateFile, WinError
|
||||
from ntfsutils.fs import GetFileInformationByHandle, CloseHandle
|
||||
import ctypes
|
||||
from ctypes import c_void_p, c_wchar_p, Structure, WinError
|
||||
from ctypes.wintypes import DWORD, HANDLE, POINTER, BOOL
|
||||
|
||||
FILE_FLAG_BACKUP_SEMANTICS = 0x02000000
|
||||
FILE_SHARE_READ = 0x00000001
|
||||
OPEN_EXISTING = 3
|
||||
|
||||
class FILETIME(Structure):
|
||||
_fields_ = [("dwLowDateTime", DWORD),
|
||||
("dwHighDateTime", DWORD)]
|
||||
|
||||
class BY_HANDLE_FILE_INFORMATION(Structure):
|
||||
_fields_ = [("dwFileAttributes", DWORD),
|
||||
("ftCreationTime", FILETIME),
|
||||
("ftLastAccessTime", FILETIME),
|
||||
("ftLastWriteTime", FILETIME),
|
||||
("dwVolumeSerialNumber", DWORD),
|
||||
("nFileSizeHigh", DWORD),
|
||||
("nFileSizeLow", DWORD),
|
||||
("nNumberOfLinks", DWORD),
|
||||
("nFileIndexHigh", DWORD),
|
||||
("nFileIndexLow", DWORD)]
|
||||
|
||||
flags = FILE_FLAG_BACKUP_SEMANTICS
|
||||
hfile = CreateFile(path,
|
||||
0,
|
||||
FILE_SHARE_READ,
|
||||
None,
|
||||
OPEN_EXISTING,
|
||||
flags,
|
||||
None)
|
||||
|
||||
func = ctypes.windll.kernel32.CreateFileW
|
||||
func.argtypes = [c_wchar_p,
|
||||
DWORD,
|
||||
DWORD,
|
||||
c_void_p,
|
||||
DWORD,
|
||||
DWORD,
|
||||
HANDLE]
|
||||
func.restype = HANDLE
|
||||
|
||||
hfile = func(path,
|
||||
0,
|
||||
FILE_SHARE_READ,
|
||||
None,
|
||||
OPEN_EXISTING,
|
||||
flags,
|
||||
None)
|
||||
if hfile is None:
|
||||
raise WinError()
|
||||
|
||||
func = ctypes.windll.kernel32.GetFileInformationByHandle
|
||||
func.argtypes = [HANDLE, POINTER(BY_HANDLE_FILE_INFORMATION)]
|
||||
func.restype = BOOL
|
||||
|
||||
info = BY_HANDLE_FILE_INFORMATION()
|
||||
rv = GetFileInformationByHandle(hfile, info)
|
||||
CloseHandle(hfile)
|
||||
rv = func(hfile, info)
|
||||
|
||||
func = ctypes.windll.kernel32.CloseHandle
|
||||
func.argtypes = [HANDLE]
|
||||
func.restype = BOOL
|
||||
|
||||
func(hfile)
|
||||
|
||||
if rv == 0:
|
||||
raise WinError()
|
||||
|
||||
return info
|
||||
|
||||
@staticmethod
|
||||
|
||||
+2
-2
@@ -6,11 +6,11 @@ from dvc import VERSION_BASE
|
||||
from dvc.logger import Logger
|
||||
|
||||
|
||||
class Updater(object):
|
||||
class Updater(object): # pragma: no cover
|
||||
URL = 'https://4ki8820rsf.execute-api.us-east-2.amazonaws.com/' \
|
||||
'prod/latest-version'
|
||||
UPDATER_FILE = 'updater'
|
||||
TIMEOUT = 7 * 24 * 60 * 60 # every week
|
||||
TIMEOUT = 24 * 60 * 60 # every day
|
||||
TIMEOUT_GET = 10
|
||||
|
||||
def __init__(self, dvc_dir):
|
||||
|
||||
+2
-3
@@ -8,10 +8,9 @@ google-cloud-storage==1.12.0
|
||||
PyInstaller==3.3.1
|
||||
colorama>=0.3.9
|
||||
configobj>=5.0.6
|
||||
networkx==2.1
|
||||
networkx>=2.1
|
||||
pyyaml>=3.12
|
||||
gitpython>=2.1.8
|
||||
ntfsutils>=0.1.4
|
||||
setuptools>=34.0.0
|
||||
nanotime>=0.5.2
|
||||
pyasn1>=0.4.1
|
||||
@@ -22,5 +21,5 @@ requests>=2.18.4
|
||||
wheel>=0.31.1
|
||||
futures>=3.2.0; python_version == "2.7"
|
||||
grandalf==0.6
|
||||
asciicanvas==0.0.3
|
||||
pydot>=1.2.4
|
||||
asciimatics>=1.10.0
|
||||
|
||||
@@ -6,7 +6,8 @@ set -e
|
||||
python -mtests
|
||||
|
||||
if [[ "$TRAVIS_PULL_REQUEST" == "false" && \
|
||||
"$TRAVIS_SECURE_ENV_VARS" == "true" ]]; then
|
||||
"$TRAVIS_SECURE_ENV_VARS" == "true" && \
|
||||
"$(python -c 'import sys; print(sys.version_info[0])')" == '2' ]]; then
|
||||
codecov
|
||||
fi
|
||||
|
||||
|
||||
@@ -12,10 +12,9 @@ install_requires = [
|
||||
"future>=0.16.0",
|
||||
"colorama>=0.3.9",
|
||||
"configobj>=5.0.6",
|
||||
"networkx==2.1",
|
||||
"networkx>=2.1",
|
||||
"pyyaml>=3.12",
|
||||
"gitpython>=2.1.8",
|
||||
"ntfsutils>=0.1.4",
|
||||
"setuptools>=34.0.0",
|
||||
"nanotime>=0.5.2",
|
||||
"pyasn1>=0.4.1",
|
||||
@@ -23,7 +22,7 @@ install_requires = [
|
||||
"jsonpath-rw==1.4.0",
|
||||
"requests>=2.18.4",
|
||||
"grandalf==0.6",
|
||||
"asciicanvas==0.0.3",
|
||||
"asciimatics>=1.10.0",
|
||||
]
|
||||
|
||||
# Extra dependencies for remote integrations
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
pytest>=3.3.1
|
||||
nose>=1.3.7
|
||||
mock>=2.0.0
|
||||
coverage
|
||||
codecov
|
||||
coverage>=4.5.1
|
||||
codecov>=2.0.15
|
||||
xmltodict>=0.11.0
|
||||
awscli>=1.14.18
|
||||
google-compute-engine
|
||||
|
||||
+2
-1
@@ -7,4 +7,5 @@ os.chdir(REPO_ROOT)
|
||||
os.putenv('PATH', '{}:{}'.format(os.path.join(REPO_ROOT, 'bin'), os.getenv('PATH')))
|
||||
os.putenv('DVC_HOME', REPO_ROOT)
|
||||
|
||||
check_call('nosetests -v --processes=-1 --process-timeout=200 --cover-inclusive --cover-erase --cover-package=dvc --with-coverage', shell=True)
|
||||
cmd = 'nosetests -v --processes=-1 --process-timeout=200 --cover-inclusive --cover-erase --cover-package=dvc --with-coverage'
|
||||
check_call(cmd, shell=True)
|
||||
|
||||
@@ -103,3 +103,12 @@ class TestSharedCacheDir(TestDir):
|
||||
self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[0]))), 1)
|
||||
self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[1]))), 1)
|
||||
self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[2]))), 1)
|
||||
|
||||
|
||||
class TestCacheLinkType(TestDvc):
|
||||
def test(self):
|
||||
ret = main(['config', 'cache.type', 'reflink,copy'])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
ret = main(['add', self.FOO])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
@@ -205,3 +205,15 @@ class TestCheckoutEmptyDir(TestDvc):
|
||||
|
||||
self.assertTrue(os.path.isdir(dname))
|
||||
self.assertEqual(len(os.listdir(dname)), 0)
|
||||
|
||||
|
||||
class TestCheckoutNotCachedFile(TestDvc):
|
||||
def test(self):
|
||||
cmd = 'python {} {} {}'.format(self.CODE, self.FOO, 'out')
|
||||
|
||||
self.dvc.add(self.FOO)
|
||||
stage = self.dvc.run(cmd=cmd,
|
||||
deps=[self.FOO, self.CODE],
|
||||
outs_no_cache=['out'])
|
||||
|
||||
self.dvc.checkout()
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
from subprocess import CalledProcessError
|
||||
from subprocess import check_output
|
||||
from subprocess import check_output, Popen
|
||||
from unittest import SkipTest
|
||||
import os
|
||||
import time
|
||||
@@ -94,6 +94,13 @@ def _should_test_hdfs():
|
||||
except (CalledProcessError, IOError):
|
||||
return False
|
||||
|
||||
p = Popen('hadoop fs -ls hdfs://127.0.0.1/',
|
||||
shell=True,
|
||||
executable=os.getenv('SHELL'))
|
||||
p.communicate()
|
||||
if p.returncode != 0:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
|
||||
+2
-1
@@ -29,7 +29,8 @@ class TestGC(TestDvc):
|
||||
self._test_gc()
|
||||
|
||||
def test_cli(self):
|
||||
ret = main(['gc'])
|
||||
ret = main(['gc', '-f'])
|
||||
self.assertEqual(ret, 0)
|
||||
self._test_gc()
|
||||
|
||||
def _test_gc(self):
|
||||
|
||||
+14
-1
@@ -1,9 +1,10 @@
|
||||
import os
|
||||
|
||||
from dvc.main import main
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.exceptions import DvcException, MoveNotDataSourceError
|
||||
|
||||
from tests.basic_env import TestDvc
|
||||
from tests.test_repro import TestRepro
|
||||
|
||||
|
||||
class TestMove(TestDvc):
|
||||
@@ -39,3 +40,15 @@ class TestCmdMove(TestDvc):
|
||||
|
||||
ret = main(['move', 'non-existing-file', 'dst'])
|
||||
self.assertNotEqual(ret, 0)
|
||||
|
||||
|
||||
class TestMoveNotDataSource(TestRepro):
|
||||
def test(self):
|
||||
from dvc.project import Project
|
||||
|
||||
self.dvc = Project(self._root_dir)
|
||||
with self.assertRaises(MoveNotDataSourceError):
|
||||
self.dvc.move(self.file1, 'dst')
|
||||
|
||||
ret = main(['move', self.file1, 'dst'])
|
||||
self.assertNotEqual(ret, 0)
|
||||
|
||||
@@ -72,3 +72,16 @@ class TestRemoteRemoveDefault(TestDvc):
|
||||
core = local_config.get(Config.SECTION_CORE, None)
|
||||
if core is not None:
|
||||
self.assertTrue(Config.SECTION_CORE_REMOTE not in core.keys())
|
||||
|
||||
|
||||
class TestRemoteRemove(TestDvc):
|
||||
def test(self):
|
||||
ret = main(['config', 'core.jobs', '1'])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
remote = 'mys3'
|
||||
ret = main(['remote', 'add', remote, 's3://bucket/name'])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
ret = main(['remote', 'remove', remote])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
@@ -146,6 +146,12 @@ class TestReproDry(TestReproChangedData):
|
||||
self.assertFalse(filecmp.cmp(self.file1, self.BAR, shallow=False))
|
||||
|
||||
|
||||
class TestReproUpToDate(TestRepro):
|
||||
def test(self):
|
||||
ret = main(['repro', self.file1_stage])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
|
||||
class TestReproDryNoExec(TestDvc):
|
||||
def test(self):
|
||||
deps = []
|
||||
@@ -217,6 +223,49 @@ class TestReproPipeline(TestReproChangedDeepData):
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
|
||||
class TestReproPipelines(TestDvc):
|
||||
def setUp(self):
|
||||
super(TestReproPipelines, self).setUp()
|
||||
|
||||
stages = self.dvc.add(self.FOO)
|
||||
self.assertEqual(len(stages), 1)
|
||||
self.foo_stage = stages[0]
|
||||
self.assertTrue(self.foo_stage is not None)
|
||||
|
||||
stages = self.dvc.add(self.BAR)
|
||||
self.assertEqual(len(stages), 1)
|
||||
self.bar_stage = stages[0]
|
||||
self.assertTrue(self.bar_stage is not None)
|
||||
|
||||
self.file1 = 'file1'
|
||||
self.file1_stage = self.file1 + '.dvc'
|
||||
self.dvc.run(fname=self.file1_stage,
|
||||
outs=[self.file1],
|
||||
deps=[self.FOO, self.CODE],
|
||||
cmd='python {} {} {}'.format(self.CODE, self.FOO, self.file1))
|
||||
|
||||
self.file2 = 'file2'
|
||||
self.file2_stage = self.file2 + '.dvc'
|
||||
self.dvc.run(fname=self.file2_stage,
|
||||
outs=[self.file2],
|
||||
deps=[self.BAR, self.CODE],
|
||||
cmd='python {} {} {}'.format(self.CODE, self.BAR, self.file2))
|
||||
|
||||
|
||||
def test(self):
|
||||
stages = self.dvc.reproduce(all_pipelines=True, force=True)
|
||||
self.assertEqual(len(stages), 4)
|
||||
names = [stage.relpath for stage in stages]
|
||||
self.assertTrue(self.foo_stage.relpath in names)
|
||||
self.assertTrue(self.bar_stage.relpath in names)
|
||||
self.assertTrue(self.file1_stage in names)
|
||||
self.assertTrue(self.file2_stage in names)
|
||||
|
||||
def test_cli(self):
|
||||
ret = main(['repro', '-f', '-P'])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
|
||||
class TestReproLocked(TestReproChangedData):
|
||||
def test(self):
|
||||
file2 = 'file2'
|
||||
|
||||
Reference in New Issue
Block a user