mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-18 23:10:14 +01:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c6811ae005 | ||
|
|
6ad674ca88 | ||
|
|
45afd36d76 | ||
|
|
0aac01ab88 | ||
|
|
ce3bcc4af9 | ||
|
|
06064061d6 | ||
|
|
cae6e712c6 | ||
|
|
4b525fe1fc | ||
|
|
13a2e4692d | ||
|
|
1de87bc91c | ||
|
|
82b10a8256 | ||
|
|
a736082595 | ||
|
|
4888ab82a7 | ||
|
|
bc93cc0bf6 | ||
|
|
971f3259f1 | ||
|
|
a4f9789aa4 | ||
|
|
584972ba0a | ||
|
|
b2b76b84d6 | ||
|
|
a6ea2f3db9 | ||
|
|
881f188016 | ||
|
|
c3f145591b | ||
|
|
1a8842d8c6 | ||
|
|
143a7f79a1 | ||
|
|
7bff8c6f89 | ||
|
|
2daa694bf2 | ||
|
|
27a96ef814 | ||
|
|
315bbfde3b | ||
|
|
53b37bb266 | ||
|
|
005db9bb3a | ||
|
|
7b36fd007c | ||
|
|
2c9b08a89b | ||
|
|
66a962976e | ||
|
|
4eff31dd5e | ||
|
|
377be75dd0 | ||
|
|
e015b9a167 | ||
|
|
2a7ec2d2df | ||
|
|
3d34f4e8c7 | ||
|
|
c10bd344ea | ||
|
|
6a8cd31ac9 | ||
|
|
2c49d7d004 | ||
|
|
990bbe32c1 | ||
|
|
f3668b0a4e | ||
|
|
73eaa2a611 | ||
|
|
e63ae3d676 | ||
|
|
9528ad6a1d | ||
|
|
aea92e0bdf | ||
|
|
d1077c4457 | ||
|
|
edf73dc52f | ||
|
|
b587f7833a | ||
|
|
998918c690 | ||
|
|
8aa53a5e71 | ||
|
|
cc2ebb2099 | ||
|
|
b9439f2e5b | ||
|
|
d112d2a829 | ||
|
|
f4fc3f47e4 | ||
|
|
417ad8706e | ||
|
|
2e09dbb592 | ||
|
|
67d63a4a5b | ||
|
|
8eb6fd5285 | ||
|
|
669b2ec303 | ||
|
|
04b3cd40e0 | ||
|
|
12d6292ed6 | ||
|
|
27edc4f027 | ||
|
|
5b0abe61ab | ||
|
|
1e1daeaae1 | ||
|
|
ece2b8ae0b | ||
|
|
fcea9dccc5 | ||
|
|
51b390146a | ||
|
|
1a6dedfe60 | ||
|
|
88f0913f91 | ||
|
|
b38112795a | ||
|
|
363db9125f | ||
|
|
89e03c8606 | ||
|
|
7bed7213dc | ||
|
|
76c2f812f0 | ||
|
|
8d151273c8 | ||
|
|
10e10f9535 | ||
|
|
3fd5603921 | ||
|
|
ff63376d49 | ||
|
|
5071b5026d | ||
|
|
c08804eddd | ||
|
|
cbc8cf23f9 | ||
|
|
5279b1ae8f | ||
|
|
057663696a | ||
|
|
8b513fb336 | ||
|
|
04373e81e8 | ||
|
|
2601766323 | ||
|
|
ad216acce4 | ||
|
|
5753944161 | ||
|
|
b7ac347c06 | ||
|
|
beedab77f1 | ||
|
|
b1b0e3413d | ||
|
|
c4593cd369 | ||
|
|
6a55d1502f | ||
|
|
e93a87f451 | ||
|
|
3fff8d771d | ||
|
|
69bb753980 | ||
|
|
1d654d0444 | ||
|
|
0618d9d0dc | ||
|
|
dea130cc61 | ||
|
|
bbd5096de7 | ||
|
|
545ea1efa5 | ||
|
|
e1841f07cb | ||
|
|
04efd1d6d3 | ||
|
|
89aa2a957e | ||
|
|
1da4546515 | ||
|
|
49c22cb139 | ||
|
|
4aa26a0a3a | ||
|
|
8d5675a92c | ||
|
|
2b1ea1e4b9 | ||
|
|
34faad3e96 | ||
|
|
9dab3f2e49 | ||
|
|
72e8b6983b | ||
|
|
682737e4a2 | ||
|
|
8932254c12 | ||
|
|
0c97e19d5c | ||
|
|
e698703969 | ||
|
|
571713484b | ||
|
|
65a9370281 | ||
|
|
0043bd5de3 | ||
|
|
33d04d0d59 | ||
|
|
5e3d7c6c2b | ||
|
|
59dcdb02ea | ||
|
|
23d00a15b9 | ||
|
|
2c3c0b08bd | ||
|
|
75d7fc2206 | ||
|
|
d704c41682 | ||
|
|
428811bb71 | ||
|
|
470825aec5 | ||
|
|
5fbfc135da | ||
|
|
b4e62772b8 | ||
|
|
912f0f72e5 | ||
|
|
5730b1d088 | ||
|
|
5c762f316e | ||
|
|
7791dc8cfb | ||
|
|
dc344056c5 | ||
|
|
87e82daad2 | ||
|
|
f044a6090f | ||
|
|
c68823e8d0 | ||
|
|
a17dc2504d | ||
|
|
537e7c75a0 | ||
|
|
76d6b37b1e | ||
|
|
8d13ba164e | ||
|
|
559c097394 | ||
|
|
c97bfea90f | ||
|
|
abf395da99 | ||
|
|
0687c4e199 | ||
|
|
d1931e3dc0 | ||
|
|
78c9d1edab | ||
|
|
7ef8645465 | ||
|
|
208540c66d | ||
|
|
216a6e5c1c | ||
|
|
8d8a94bcb7 | ||
|
|
1a18c4d3f1 | ||
|
|
211b957d17 | ||
|
|
4f6967a017 | ||
|
|
c1ac1b4c6e | ||
|
|
022eea8214 | ||
|
|
f1f1a97fbf | ||
|
|
a3aefe5f16 | ||
|
|
de3d8b528f | ||
|
|
9c041a375f | ||
|
|
086155daa6 | ||
|
|
d47e6c39ab | ||
|
|
249b60c030 | ||
|
|
392cd006e3 | ||
|
|
2b0391629b | ||
|
|
3b66176f6b | ||
|
|
baa6bcd035 | ||
|
|
b159ace6d7 | ||
|
|
ffde4a6a7a | ||
|
|
9a95698cd9 | ||
|
|
221dbf8245 | ||
|
|
3d15391c7a | ||
|
|
f6f2ebb83e | ||
|
|
ac80d59704 | ||
|
|
3574c8954b | ||
|
|
0d78705227 | ||
|
|
bf6ed76678 | ||
|
|
076a2145db | ||
|
|
bb3cca6530 | ||
|
|
165422a066 | ||
|
|
49baf38871 | ||
|
|
3461df1d8f | ||
|
|
03a3cf2f80 | ||
|
|
94911965c4 | ||
|
|
55b3c47bbf | ||
|
|
825da210cf | ||
|
|
ac97c4021d | ||
|
|
213e757b52 | ||
|
|
6f0e293c9a | ||
|
|
7783bb900b | ||
|
|
bdca1ce8cc | ||
|
|
80912b4a2f | ||
|
|
c46bba6889 | ||
|
|
1283689ed4 | ||
|
|
9c5bbecb2e | ||
|
|
316ea1d7cb | ||
|
|
2139978646 | ||
|
|
fa89ec8789 | ||
|
|
93ef937fc2 | ||
|
|
253315e4a8 | ||
|
|
9579cb7928 | ||
|
|
18f0ca0d92 | ||
|
|
d555bae6e1 | ||
|
|
4a43a29f11 | ||
|
|
2ff7d9b571 | ||
|
|
9d303b6b1b | ||
|
|
f682e556ec | ||
|
|
26e007f407 | ||
|
|
a7f14c2d3c | ||
|
|
79f9cedd37 | ||
|
|
623d3e2fed | ||
|
|
cb7eed71de | ||
|
|
9135944d7c | ||
|
|
ac4fff2c45 | ||
|
|
912538f165 | ||
|
|
35c5d4c294 | ||
|
|
3887e973c4 | ||
|
|
173a9fc7aa | ||
|
|
0281c2714c | ||
|
|
6d10bff23b | ||
|
|
ff5152ee5f | ||
|
|
c6f642fd6b | ||
|
|
ee0dcc384d | ||
|
|
8a484df228 | ||
|
|
c378e4b3d2 | ||
|
|
00d53f50b1 | ||
|
|
65ffc7d248 | ||
|
|
95479e6d6b | ||
|
|
79b9ae75e9 | ||
|
|
a46b1be925 | ||
|
|
a566e92ce0 | ||
|
|
adbfbfd732 | ||
|
|
d2b562e25a | ||
|
|
bcea7a132a | ||
|
|
d1aa1f3491 | ||
|
|
246b80d6a4 | ||
|
|
8f2d5c4195 | ||
|
|
bd9da20e20 | ||
|
|
c6138729f9 | ||
|
|
6743b36894 | ||
|
|
0dd995453f | ||
|
|
42a15d91f7 | ||
|
|
a6cb238982 | ||
|
|
1b4d7abce3 | ||
|
|
e6def1287d | ||
|
|
ce5a46d032 | ||
|
|
fcadd18069 | ||
|
|
a3588c4504 | ||
|
|
86322fbe35 | ||
|
|
c635adbfc7 | ||
|
|
6e95508e9e | ||
|
|
aaca0e7a38 | ||
|
|
487993995a | ||
|
|
e84f231c34 | ||
|
|
2b19b6bbfa | ||
|
|
db017543d5 | ||
|
|
bc2c84370d | ||
|
|
d34119d6f0 | ||
|
|
b940380e12 | ||
|
|
e3d39063ba | ||
|
|
11db27f333 | ||
|
|
15210f73b2 | ||
|
|
9be3f4f122 | ||
|
|
ddf1839d38 | ||
|
|
3ec0a5c4ae | ||
|
|
3814e0da4e | ||
|
|
292702473b | ||
|
|
6776f6e21c | ||
|
|
b261554818 | ||
|
|
10dd535176 | ||
|
|
e8949c1a77 | ||
|
|
6a8703e54e | ||
|
|
6cc0c30937 | ||
|
|
b15f5ffa3b | ||
|
|
d343f8562c | ||
|
|
321c1d5f8b | ||
|
|
f6e20b162c | ||
|
|
a249df3f5d | ||
|
|
fb4b979183 | ||
|
|
98a7150c65 | ||
|
|
dd99a09135 | ||
|
|
f854f5a97a | ||
|
|
9e068234ba | ||
|
|
6600ccf478 | ||
|
|
84abef7ac0 | ||
|
|
9ff2503a72 | ||
|
|
4ff4a1042e | ||
|
|
2195ff2b13 | ||
|
|
e539e74fde | ||
|
|
c345552460 | ||
|
|
cee8e1ec9f | ||
|
|
e4b8cf2e0e | ||
|
|
e2e7649afe | ||
|
|
77b2e45a64 | ||
|
|
e403568afe | ||
|
|
79eac34163 | ||
|
|
d9b7f7b97a | ||
|
|
cbe9fde7a6 | ||
|
|
24be60bf39 | ||
|
|
fe38b2768f | ||
|
|
105d8fc9b5 | ||
|
|
9d7581c6bd | ||
|
|
a14946d0bb | ||
|
|
6ad361b741 | ||
|
|
6fe6c04f6c | ||
|
|
23b167f5d7 | ||
|
|
2dde6e848b | ||
|
|
5e5705ae7e | ||
|
|
a2d3ced7f1 | ||
|
|
0804f84679 | ||
|
|
02da5eabdc | ||
|
|
85ab6a141b | ||
|
|
8d1e18d543 | ||
|
|
d42809abd3 | ||
|
|
f5bb3e4501 | ||
|
|
aab04a22ad | ||
|
|
ed8052c229 | ||
|
|
7baf368367 | ||
|
|
b5b8b8d06c | ||
|
|
75a6352d3d | ||
|
|
baac7a16bd | ||
|
|
e4137542df | ||
|
|
b9a5bde22f | ||
|
|
891ec0bad4 | ||
|
|
a0a138c69f | ||
|
|
1e4f6e9f09 | ||
|
|
9f5ba8c38f | ||
|
|
ef7aa024f9 | ||
|
|
48c9916937 | ||
|
|
04f0320126 | ||
|
|
536f369f7e | ||
|
|
23a4b296e9 | ||
|
|
a4482bac3b | ||
|
|
e5fd24f530 | ||
|
|
4dc61beab0 | ||
|
|
f02bf73587 | ||
|
|
cb13e45632 | ||
|
|
d0ce57adac | ||
|
|
eaa994527d | ||
|
|
89617501a4 | ||
|
|
8be51b6056 | ||
|
|
f7a17a9336 | ||
|
|
2788288e8e | ||
|
|
80e7040a21 | ||
|
|
bd4bbf1438 | ||
|
|
935cbc9513 | ||
|
|
98c5875668 | ||
|
|
42111b6348 | ||
|
|
3363b85367 | ||
|
|
83994d6ea7 | ||
|
|
568a8fb904 | ||
|
|
47d00e80d2 | ||
|
|
a9114d57a5 | ||
|
|
1bfcc12853 | ||
|
|
be817c5b84 | ||
|
|
1f15cf23e7 | ||
|
|
28e2f7d06f | ||
|
|
6bf853d635 | ||
|
|
8120104f2e | ||
|
|
1cb41f0a8a | ||
|
|
ad204cf915 | ||
|
|
ac3963878e | ||
|
|
dd891a8cd3 | ||
|
|
6c54810034 | ||
|
|
f23671a26a | ||
|
|
229983c403 | ||
|
|
6b042fb15a | ||
|
|
1879d49bc2 | ||
|
|
627606d6f2 | ||
|
|
7e4f39d8eb | ||
|
|
c408fa7530 | ||
|
|
0b2728c7ea | ||
|
|
00c5242558 | ||
|
|
84da47e2fd | ||
|
|
66f1777d9c | ||
|
|
0e9774a3f8 | ||
|
|
60f1deec47 | ||
|
|
0d7a2d3c83 | ||
|
|
2e543e49c7 | ||
|
|
1c371d036c | ||
|
|
4f89f3d19d | ||
|
|
6fd1e03364 | ||
|
|
df7dc6315f | ||
|
|
2f7a7638e2 | ||
|
|
0cea36a2dd | ||
|
|
537678def3 | ||
|
|
ee7f9666d3 | ||
|
|
48cc465696 | ||
|
|
e308d5697f | ||
|
|
7c2ef0c49c | ||
|
|
c8958d737f | ||
|
|
b6e721b486 | ||
|
|
f7a2377779 | ||
|
|
2e3ce3b3db | ||
|
|
2363057165 | ||
|
|
5a0e16a473 | ||
|
|
9e576cd55b | ||
|
|
4bbbdb334c | ||
|
|
2881cd204c | ||
|
|
2ee9525ba2 | ||
|
|
7cde8cbbca | ||
|
|
627412ff89 | ||
|
|
bf78d58793 | ||
|
|
6aa6a0c208 | ||
|
|
f49eeaaa3c | ||
|
|
4e49345ba2 | ||
|
|
6e6ed2d4fa | ||
|
|
c7e788874f | ||
|
|
6a6a4fe0b4 | ||
|
|
07fa6442b8 | ||
|
|
587cc08515 | ||
|
|
2dabcd7d2f | ||
|
|
adad65cc00 | ||
|
|
95f53a1b97 | ||
|
|
bb0903304e | ||
|
|
e598b0b4c7 | ||
|
|
3be898797a | ||
|
|
41ed1bca94 | ||
|
|
90781900e2 | ||
|
|
94ae130b56 | ||
|
|
e734ad1712 | ||
|
|
bc4a5d78e3 | ||
|
|
637fc29b79 | ||
|
|
0a3a124756 | ||
|
|
ff51b1ae04 | ||
|
|
9557c56a7e | ||
|
|
ebd8468b1c | ||
|
|
2cab8e03be | ||
|
|
94551b3554 | ||
|
|
b03dd85522 | ||
|
|
3820eb359b | ||
|
|
3f312a83c5 | ||
|
|
9bf0fc89df | ||
|
|
6c7bf390c6 | ||
|
|
5565ece1f8 | ||
|
|
ebecc9996b | ||
|
|
f9ba365a25 | ||
|
|
3410f8048a | ||
|
|
f54428d5ed | ||
|
|
6e1839df90 | ||
|
|
2d9b062fc4 | ||
|
|
169db1fe29 | ||
|
|
870d98bc18 | ||
|
|
dc5fec6a75 | ||
|
|
2885c00f80 | ||
|
|
d424fc3825 | ||
|
|
5cf8c8b598 | ||
|
|
805eb908d0 | ||
|
|
39fc1d2cc6 | ||
|
|
88afad7202 | ||
|
|
dd06f53bc7 | ||
|
|
be187fad6e | ||
|
|
1418ab81c9 | ||
|
|
6a01578dfb | ||
|
|
d98a166f87 | ||
|
|
f1a28c3812 | ||
|
|
99bd586151 | ||
|
|
6d120f474e | ||
|
|
6d6653840e | ||
|
|
7fea5d579c | ||
|
|
fef8ea4602 | ||
|
|
586fde5f95 | ||
|
|
918bb53b75 | ||
|
|
64da1b126f | ||
|
|
f7cb4afd33 | ||
|
|
71d9fb44fc | ||
|
|
794dcc2a58 | ||
|
|
c636581699 | ||
|
|
4668c10e6e | ||
|
|
9be161de54 | ||
|
|
79826b21a4 | ||
|
|
e6e11ef015 | ||
|
|
f476143c52 | ||
|
|
3946463551 | ||
|
|
774e635e60 | ||
|
|
9fedecdf55 | ||
|
|
69f5ea3f3a | ||
|
|
d63d1db5b6 | ||
|
|
d71df35f63 | ||
|
|
716c554b93 | ||
|
|
d24457c14f | ||
|
|
5f8b2f3b91 | ||
|
|
bb5f1de79f | ||
|
|
8f36587805 | ||
|
|
5b674c910d | ||
|
|
7897dd8c45 | ||
|
|
8a32292f46 | ||
|
|
97dfc88647 | ||
|
|
fcd0e34269 | ||
|
|
a438f77db2 | ||
|
|
fcbc6d115c | ||
|
|
1b1ca68b64 | ||
|
|
9724bd94aa | ||
|
|
3e08cd503c | ||
|
|
e9377dd65d | ||
|
|
581132bbe0 | ||
|
|
e051964a1b | ||
|
|
2aa9288f55 | ||
|
|
6996d0a9e6 | ||
|
|
1f30d29b57 | ||
|
|
abee5f07bb | ||
|
|
afd46e1a8a | ||
|
|
1bcd7408ec | ||
|
|
6882a781a4 | ||
|
|
50ea36e740 | ||
|
|
c7ed95f5ba | ||
|
|
5d3a009886 | ||
|
|
6683f13b4e | ||
|
|
3f73ef057a | ||
|
|
961bf8b34c | ||
|
|
11de88fcd4 | ||
|
|
e83dd1c972 | ||
|
|
62cd68af77 | ||
|
|
715cc717aa | ||
|
|
d5c015ff63 | ||
|
|
db374b6553 | ||
|
|
01efbac5a3 | ||
|
|
0f940f9167 | ||
|
|
e85368c8b8 | ||
|
|
fba1d8c181 | ||
|
|
6e82c6c748 | ||
|
|
468f4beee7 | ||
|
|
08b3273fcc | ||
|
|
b7b07d393a | ||
|
|
2542ceca0d | ||
|
|
9b032b72cb | ||
|
|
0a173c4731 | ||
|
|
65b5dcb601 | ||
|
|
425e44bac8 | ||
|
|
b952e599bf | ||
|
|
adc34eb312 | ||
|
|
71d03b5a43 | ||
|
|
db5cb125fb | ||
|
|
45fc17a074 | ||
|
|
e6bcdd3979 | ||
|
|
f471f7714b | ||
|
|
d37828c4e3 | ||
|
|
d9b1308447 | ||
|
|
ecf8045887 | ||
|
|
8d6813a112 | ||
|
|
35d318f0f0 | ||
|
|
0f8288fd9a | ||
|
|
5100e860fc | ||
|
|
8e06d9d8ca | ||
|
|
52083c2bc5 | ||
|
|
67e0c24ced | ||
|
|
c6c6a0521c | ||
|
|
459b21e0cf | ||
|
|
f634bc1916 | ||
|
|
3405a1ec47 | ||
|
|
aca3abff26 | ||
|
|
f274a745a0 | ||
|
|
e8818ef67a | ||
|
|
594b23bd94 | ||
|
|
449d4880d1 | ||
|
|
7c90cbab79 | ||
|
|
bd254c15ab | ||
|
|
8d662e4402 | ||
|
|
40cbaae2bd | ||
|
|
6219b28be6 | ||
|
|
95f3b49db8 | ||
|
|
bf1f9595f6 | ||
|
|
cafe703264 | ||
|
|
5afe59df72 | ||
|
|
a392289cbf | ||
|
|
9d943f7b21 | ||
|
|
591cd90583 | ||
|
|
d4161be595 | ||
|
|
82dfc989ad | ||
|
|
83b91328d2 | ||
|
|
fa9a79008b | ||
|
|
432fa56315 | ||
|
|
24d44695f7 | ||
|
|
f447437200 | ||
|
|
fde11a3b7f | ||
|
|
298b473a63 | ||
|
|
2f0c409668 | ||
|
|
88b6bb0c31 | ||
|
|
fb4fc72642 | ||
|
|
77a4145c7f | ||
|
|
12edcbaa04 | ||
|
|
5a3543b36f | ||
|
|
82e8ce46e8 | ||
|
|
f71c907a0e | ||
|
|
4d22842ccf | ||
|
|
b6199f836e | ||
|
|
5d85b19e77 | ||
|
|
0b90ec816f | ||
|
|
c51d8d8e93 | ||
|
|
555e802ed2 | ||
|
|
7ba47f7c5b | ||
|
|
15d7debb7e | ||
|
|
a4c63335d1 | ||
|
|
ce5376b6af | ||
|
|
d6c7c90db9 | ||
|
|
da832399aa | ||
|
|
a62ffb5518 | ||
|
|
61ee8b2c72 | ||
|
|
50578a5923 | ||
|
|
add9498e82 | ||
|
|
67634649bd | ||
|
|
5f8f199958 | ||
|
|
3a1a5f84a8 | ||
|
|
216897494c | ||
|
|
a28a1788a7 | ||
|
|
3f63592c82 | ||
|
|
44e65083af | ||
|
|
c0ede960ea | ||
|
|
d2352530fa | ||
|
|
318aa48839 | ||
|
|
f5e4e3b955 | ||
|
|
8da7bf6638 | ||
|
|
97a1f0ba99 | ||
|
|
1319b73fc8 | ||
|
|
16e70a718c | ||
|
|
077fee0705 | ||
|
|
ebee3ceae1 | ||
|
|
0831414d1c | ||
|
|
f3c8602125 | ||
|
|
b8371d1c76 | ||
|
|
b876a446d8 | ||
|
|
38aed70995 | ||
|
|
01f700a8be | ||
|
|
0b29f8748f | ||
|
|
820bd89a6c | ||
|
|
71752f68a8 | ||
|
|
b1558a9404 | ||
|
|
db7251b3ea | ||
|
|
dd7ff68f6c | ||
|
|
c274bfee3e | ||
|
|
16dda2f666 | ||
|
|
d0b730f62f | ||
|
|
c6c6405f27 | ||
|
|
a581373856 | ||
|
|
470160f851 | ||
|
|
387562b433 | ||
|
|
fb53760130 | ||
|
|
a87972c354 | ||
|
|
2613ab4738 | ||
|
|
bd3bcf7503 | ||
|
|
98d6ed6c26 | ||
|
|
9ec95022e2 | ||
|
|
a2767cc54f | ||
|
|
f23232238b | ||
|
|
6c773bea48 | ||
|
|
0a951f5ab3 | ||
|
|
25ff511edd | ||
|
|
d9fcadd417 | ||
|
|
d4982b360e | ||
|
|
11f7eb0c4e | ||
|
|
bca9376a9c | ||
|
|
4beb71c613 | ||
|
|
9680e040a3 | ||
|
|
48474e1d67 | ||
|
|
834f9502dd | ||
|
|
3a0c43e382 | ||
|
|
6886fc29d5 | ||
|
|
436a9fc817 | ||
|
|
56ed60f1cd | ||
|
|
3a914ceff2 | ||
|
|
eadc5033b7 | ||
|
|
8866d5b89a | ||
|
|
20ba323d18 | ||
|
|
83d831735c | ||
|
|
c733b8130b | ||
|
|
50c50a54e3 | ||
|
|
05cf6c26e3 | ||
|
|
4271ddf11c | ||
|
|
f65ea82ddf | ||
|
|
b511193113 | ||
|
|
048ae43e34 | ||
|
|
b074c49318 | ||
|
|
5ea2fd8367 | ||
|
|
b57f829b1b | ||
|
|
8148d51a97 | ||
|
|
594a99ba9d | ||
|
|
e49ccf26f1 | ||
|
|
78e5b166fe | ||
|
|
0ffcdd669a | ||
|
|
569e40eded | ||
|
|
5902e6066c | ||
|
|
582cb9e064 | ||
|
|
e380a4a858 | ||
|
|
35f5945720 | ||
|
|
ff0d91737b | ||
|
|
77b814f039 | ||
|
|
a734e07776 | ||
|
|
fecc38e602 | ||
|
|
6fe26a0bdc | ||
|
|
2cd4312423 | ||
|
|
e68137181e | ||
|
|
2e347847ec | ||
|
|
c136297b82 | ||
|
|
6218124764 | ||
|
|
22718faf9c | ||
|
|
a2bc64e85f | ||
|
|
f2a5058a74 | ||
|
|
38c585a5f9 | ||
|
|
408f96b31c | ||
|
|
9389422f0b | ||
|
|
5d169916de | ||
|
|
dd610c04fb | ||
|
|
585c27f354 | ||
|
|
346ef139dd | ||
|
|
331e3b2bf9 | ||
|
|
7a7c9e7713 | ||
|
|
1ef21039c4 | ||
|
|
42265339a8 | ||
|
|
fc53971f2b | ||
|
|
484b04cb68 | ||
|
|
88397ef430 | ||
|
|
6f4d0cd74a | ||
|
|
f03ea7a915 | ||
|
|
08dcd8490e | ||
|
|
5c1748c15c | ||
|
|
fbfdecd1de | ||
|
|
f3b2497fb9 | ||
|
|
e0a42f1265 | ||
|
|
2e78bb2a69 | ||
|
|
456ba271b3 | ||
|
|
95870ce769 | ||
|
|
92cbda749f | ||
|
|
33b26278fe | ||
|
|
ac3d37e58c | ||
|
|
a35b3aa67a | ||
|
|
7a3aa54293 | ||
|
|
b85c41c734 | ||
|
|
2bbefc91ef | ||
|
|
69f9c5d837 | ||
|
|
56e76d0d8b | ||
|
|
21bf52ee9d | ||
|
|
4ab9b68fbd | ||
|
|
82890ec2a4 | ||
|
|
962ee4ab77 | ||
|
|
c819ac9292 | ||
|
|
1cc2d103f4 | ||
|
|
d15143bb6f |
+12
-1
@@ -14,7 +14,9 @@ environment:
|
||||
GCP_CREDS:
|
||||
secure: 96fJ3r2i2GohbXHwnSs5N4EplQ7q8YmLpPWM0AC+f4s=
|
||||
CODECOV_TOKEN:
|
||||
secure: Hp3+vwZzsh1i2qjitm/hujpXA475YNhUSgrdpQ7dliCVQC+1lVcQMiDWQ0K4DEH4
|
||||
secure: XN4jRtmGE5Bqg8pPZkwNs7kn3UEI73Rkldqc0MGsQISZBm5TNJZOPofDMc1QnUsf
|
||||
AZURE_STORAGE_CONTAINER_NAME: appveyor-tests
|
||||
AZURE_STORAGE_CONNECTION_STRING: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;
|
||||
|
||||
matrix:
|
||||
|
||||
@@ -44,8 +46,13 @@ environment:
|
||||
# PYTHON_ARCH: "64"
|
||||
|
||||
install:
|
||||
- ps: Install-Product node
|
||||
- npm install -g azurite
|
||||
- ps: $AzuriteProcess = Start-Process azurite-blob -PassThru
|
||||
- cinst wget
|
||||
- cinst awscli
|
||||
- cinst gcloudsdk
|
||||
- refreshenv
|
||||
- cinst gsutil
|
||||
- cinst openssl.light
|
||||
- python -m pip install -U pip setuptools wheel
|
||||
@@ -66,8 +73,12 @@ test_script:
|
||||
|
||||
after_test:
|
||||
- if "%APPVEYOR_PULL_REQUEST_NUMBER%"=="" codecov
|
||||
- if "%APPVEYOR_REPO_TAG%"=="true" python -m dvc pull
|
||||
- if "%APPVEYOR_REPO_TAG%"=="true" .\scripts\build_windows.cmd
|
||||
|
||||
on_finish:
|
||||
- ps: Stop-Process $AzuriteProcess
|
||||
|
||||
artifacts:
|
||||
- path: dvc*.exe
|
||||
name: DVC Installer
|
||||
|
||||
@@ -5,3 +5,7 @@ omit =
|
||||
[report]
|
||||
exclude_lines =
|
||||
if __name__ == .__main__.:
|
||||
pass
|
||||
raise NotImplementedError
|
||||
# pragma: no cover
|
||||
# pragma: no cover
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
link.state
|
||||
link.state.lock
|
||||
lock
|
||||
config.local
|
||||
updater
|
||||
cache
|
||||
state
|
||||
state.lock
|
||||
@@ -0,0 +1,4 @@
|
||||
['remote "dvc"']
|
||||
url = s3://dvc-share/dvc/
|
||||
[core]
|
||||
remote = dvc
|
||||
@@ -0,0 +1,10 @@
|
||||
---
|
||||
name: Bug report
|
||||
about: Create a report to help us improve
|
||||
|
||||
---
|
||||
|
||||
|
||||
**Please provide information about your setup**
|
||||
DVC version(i.e. `dvc --version`), Platform and method of installation (pip, homebrew, pkg Mac, exe (Windows), DEB(Linux), RPM(Linux))
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
---
|
||||
name: Feature request
|
||||
about: Suggest an idea for this project
|
||||
|
||||
---
|
||||
|
||||
@@ -24,3 +24,5 @@ innosetup/config.ini
|
||||
.coverage
|
||||
|
||||
dvc/version.py
|
||||
|
||||
*.swp
|
||||
|
||||
+37
-5
@@ -1,4 +1,5 @@
|
||||
language: python
|
||||
cache: pip
|
||||
sudo: required
|
||||
dist: trusty
|
||||
group: deprecated-2017Q4
|
||||
@@ -6,7 +7,7 @@ python:
|
||||
- "2.7"
|
||||
- "3.4"
|
||||
- "3.5"
|
||||
# - "3.6" Not supported by pyinstaller yet
|
||||
- "3.6"
|
||||
# Travis doesn't support testing python on Mac yet, so we need
|
||||
# to workaround it by installing it directly with brew.
|
||||
#
|
||||
@@ -15,18 +16,24 @@ python:
|
||||
# version, that causes fpm to crash.
|
||||
matrix:
|
||||
include:
|
||||
# 3.7 needs Xenial image because of https://github.com/travis-ci/travis-ci/issues/9069
|
||||
- python: "3.7"
|
||||
dist: xenial
|
||||
sudo: true
|
||||
- if: type = cron OR tag is present
|
||||
os: osx
|
||||
osx_image: xcode9.3
|
||||
osx_image: xcode7.3
|
||||
language: ruby
|
||||
rvm: 2.4.0
|
||||
env:
|
||||
- PYTHON_VER=2.7.10
|
||||
- PYENV_ROOT="$HOME/.pyenv"
|
||||
- PATH="$PYENV_ROOT/bin:$PATH"
|
||||
- CPPFLAGS="-I/usr/local/opt/openssl/include $CPPFLAGS"
|
||||
- LDFLAGS="-L/usr/local/opt/openssl/lib $LDFLAGS"
|
||||
- if: type = cron OR tag is present
|
||||
os: osx
|
||||
osx_image: xcode9.3
|
||||
osx_image: xcode7.3
|
||||
language: ruby
|
||||
rvm: 2.4.0
|
||||
env:
|
||||
@@ -35,7 +42,7 @@ matrix:
|
||||
- PATH="$PYENV_ROOT/bin:$PATH"
|
||||
- if: type = cron OR tag is present
|
||||
os: osx
|
||||
osx_image: xcode9.3
|
||||
osx_image: xcode7.3
|
||||
language: ruby
|
||||
rvm: 2.4.0
|
||||
env:
|
||||
@@ -43,16 +50,41 @@ matrix:
|
||||
- PYENV_ROOT="$HOME/.pyenv"
|
||||
- PATH="$PYENV_ROOT/bin:$PATH"
|
||||
before_install:
|
||||
- bash ./scripts/ci/before_install.sh
|
||||
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
|
||||
eval "$(pyenv init -)";
|
||||
PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing --keep --verbose "$PYTHON_VER";
|
||||
pyenv shell "$PYTHON_VER";
|
||||
brew install openssl;
|
||||
brew cask install google-cloud-sdk;
|
||||
fi
|
||||
- which python
|
||||
- which pip
|
||||
- python --version
|
||||
- if [[ "$TRAVIS_OS_NAME" != "osx" ]]; then
|
||||
bash ./scripts/ci/install_hadoop.sh;
|
||||
source ~/.bashrc;
|
||||
hadoop fs -help;
|
||||
pushd ~;
|
||||
sudo mkdir -p /home/hadoop/hadoopinfra/hdfs/namenode/;
|
||||
sudo chmod 777 /home/hadoop/hadoopinfra/hdfs/namenode/;
|
||||
sudo chmod 777 -R /usr/local/hadoop;
|
||||
hdfs namenode -format;
|
||||
start-dfs.sh;
|
||||
jps;
|
||||
hadoop fs -ls /;
|
||||
hadoop fs -ls hdfs://127.0.0.1/;
|
||||
hadoop fs -mkdir hdfs://127.0.0.1/ololo;
|
||||
popd;
|
||||
fi
|
||||
install:
|
||||
- bash ./scripts/ci/install.sh
|
||||
- pip install flake8
|
||||
before_script:
|
||||
# stop the build if there are any readme formatting errors
|
||||
- python setup.py checkdocs
|
||||
# stop the build if there are any flake8 errors
|
||||
- flake8 dvc
|
||||
script:
|
||||
- bash ./scripts/ci/script.sh
|
||||
deploy:
|
||||
@@ -68,4 +100,4 @@ deploy:
|
||||
skip_cleanup: true
|
||||
on:
|
||||
tags: true
|
||||
repo: dataversioncontrol/dvc
|
||||
repo: iterative/dvc
|
||||
|
||||
+129
-51
@@ -1,20 +1,68 @@
|
||||
.. image:: https://img.shields.io/travis/dataversioncontrol/dvc/master.svg?label=Linux%20%26%20Mac%20OS
|
||||
:target: https://travis-ci.org/dataversioncontrol/dvc
|
||||
.. image:: https://dvc.org/static/img/logo-owl-readme.png
|
||||
:target: https://dvc.org
|
||||
:alt: DVC logo
|
||||
|
||||
.. image:: https://img.shields.io/appveyor/ci/dataversioncontrol/dvc/master.svg?label=Windows
|
||||
:target: https://ci.appveyor.com/project/dataversioncontrol/dvc/branch/master
|
||||
`Website <https://dvc.org>`_
|
||||
• `Docs <https://dvc.org/doc>`_
|
||||
• `Twitter <https://twitter.com/iterativeai>`_
|
||||
• `Chat (Community & Support) <https://dvc.org/chat>`_
|
||||
• `Tutorial <https://dvc.org/doc/tutorial>`_
|
||||
• `Mailing List <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_
|
||||
|
||||
.. image:: https://codeclimate.com/github/dataversioncontrol/dvc/badges/gpa.svg
|
||||
:target: https://codeclimate.com/github/dataversioncontrol/dvc
|
||||
.. image:: https://travis-ci.org/iterative/dvc.svg?branch=master
|
||||
:target: https://travis-ci.org/iterative/dvc
|
||||
:alt: Travis
|
||||
|
||||
.. image:: https://codecov.io/gh/dataversioncontrol/dvc/branch/master/graph/badge.svg
|
||||
:target: https://codecov.io/gh/dataversioncontrol/dvc
|
||||
.. image:: https://ci.appveyor.com/api/projects/status/github/iterative/dvc?branch=master&svg=true
|
||||
:target: https://ci.appveyor.com/project/iterative/dvc/branch/master
|
||||
:alt: Windows Build
|
||||
|
||||
Data Version Control or DVC is an open source tool for data science projects.
|
||||
It helps data scientists manage their code and data together in a simple form of Git-like commands.
|
||||
.. image:: https://codeclimate.com/github/iterative/dvc/badges/gpa.svg
|
||||
:target: https://codeclimate.com/github/iterative/dvc
|
||||
:alt: Code Climate
|
||||
|
||||
Get started
|
||||
.. image:: https://codecov.io/gh/iterative/dvc/branch/master/graph/badge.svg
|
||||
:target: https://codecov.io/gh/iterative/dvc
|
||||
:alt: Codecov
|
||||
|
||||
|
|
||||
|
||||
**Data Science Version Control** or **DVC** is an **open-source** tool for data science and
|
||||
machine learning projects. With a simple and flexible Git-like architecture and interface it
|
||||
helps data scientists:
|
||||
|
||||
#. manage **machine learning models** - versioning, including data sets and transformations (scripts) that were
|
||||
used to generate models;
|
||||
|
||||
#. make projects **reproducible**;
|
||||
|
||||
#. make projects **shareable**;
|
||||
|
||||
#. manage experiments with branching and **metrics** tracking;
|
||||
|
||||
It aims to replace tools like Excel and Docs that are being commonly used as a knowledge repo and
|
||||
a ledger for the team, ad-hoc scripts to track and move deploy different model versions, ad-hoc
|
||||
data file suffixes and prefixes.
|
||||
|
||||
.. contents:: **Contents**
|
||||
:backlinks: none
|
||||
|
||||
How DVC works
|
||||
=============
|
||||
|
||||
DVC is compatible with Git for storing code and the dependency graph (DAG), but not data files cache.
|
||||
To store and share data files cache DVC supports remotes - any cloud (S3, Azure, Google Cloud, etc) or any on-premise
|
||||
network storage (via SSH, for example).
|
||||
|
||||
.. image:: https://dvc.org/static/img/flow.gif
|
||||
:target: https://dvc.org/static/img/flow.gif
|
||||
:alt: how_dvc_works
|
||||
|
||||
Quick start
|
||||
===========
|
||||
|
||||
Please read `Get Started <https://dvc.org/doc/get-started>`_ for the full version. Common workflow commands include:
|
||||
|
||||
+-----------------------------------+-------------------------------------------------------------------+
|
||||
| Step | Command |
|
||||
+===================================+===================================================================+
|
||||
@@ -31,82 +79,112 @@ Get started
|
||||
| | | ``$ git commit -m 'The baseline model'`` |
|
||||
| | | ``$ git push`` |
|
||||
+-----------------------------------+-------------------------------------------------------------------+
|
||||
| Share data and ML models | | ``$ dvc config AWS.StoragePath mybucket/image_cnn`` |
|
||||
| Share data and ML models | | ``$ dvc remote add myremote s3://mybucket/image_cnn`` |
|
||||
| | | ``$ dvc core.remote myremote`` |
|
||||
| | | ``$ dvc push`` |
|
||||
+-----------------------------------+-------------------------------------------------------------------+
|
||||
|
||||
See more in `tutorial <https://blog.dataversioncontrol.com/data-version-control-tutorial-9146715eda46>`_.
|
||||
|
||||
Installation
|
||||
============
|
||||
|
||||
Packages
|
||||
--------
|
||||
|
||||
Operating system dependent packages are the recommended way to install DVC.
|
||||
The latest version of the packages can be found at GitHub releases page: https://github.com/dataversioncontrol/dvc/releases
|
||||
|
||||
Python Pip
|
||||
----------
|
||||
|
||||
DVC could be installed via the Python Package Index (PyPI).
|
||||
There are three options to install DVC: ``pip``, Homebrew, or an OS-specific package:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
pip install dvc
|
||||
|
||||
Homebrew (Mac OS)
|
||||
-----------------
|
||||
Package
|
||||
-------
|
||||
|
||||
Formula:
|
||||
^^^^^^^^
|
||||
Self-contained packages for Windows, Linux, Mac are available. The latest version of the packages can be found at
|
||||
GitHub `releases page <https://github.com/iterative/dvc/releases>`_.
|
||||
|
||||
Homebrew
|
||||
--------
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
brew install dataversioncontrol/homebrew-dvc/dvc
|
||||
$ brew install iterative/homebrew-dvc/dvc
|
||||
|
||||
Cask:
|
||||
^^^^^
|
||||
or:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
brew cask install dataversioncontrol/homebrew-dvc/dvc
|
||||
$ brew cask install iterative/homebrew-dvc/dvc
|
||||
|
||||
Links
|
||||
=====
|
||||
AUR
|
||||
---
|
||||
|
||||
Website: https://dataversioncontrol.com
|
||||
*Unofficial package*, any inquiries regarding the AUR package,
|
||||
`refer to the maintainer <https://github.com/mroutis/pkgbuilds>`_.
|
||||
|
||||
Tutorial: https://blog.dataversioncontrol.com/data-version-control-tutorial-9146715eda46
|
||||
.. code-block:: bash
|
||||
|
||||
Documentation: http://dataversioncontrol.com/docs/
|
||||
|
||||
Discussion: https://discuss.dataversioncontrol.com/
|
||||
# https://aur.archlinux.org/packages/dvc
|
||||
$ yay -S dvc
|
||||
|
||||
Related technologies
|
||||
====================
|
||||
|
||||
#. `Git-annex <https://git-annex.branchable.com/>`_ - DVC uses the idea of storing the content of large files (that you
|
||||
don't want to see in your Git repository) in a local key-value store and uses file hardlinks/symlinks instead of the
|
||||
copying actual files.
|
||||
|
||||
#. `Git-annex <https://git-annex.branchable.com/>`_ - DVC uses the idea of storing the content of large files (that you don't want to see in your Git repository) in a local key-value store and uses file hardlinks/symlinks instead of the copying actual files.
|
||||
#. `Git-LFS <https://git-lfs.github.com/>`_.
|
||||
#. Makefile (and it's analogues). DVC tracks dependencies (DAG).
|
||||
#. `Workflow Management Systems <https://en.wikipedia.org/wiki/Workflow_management_system>`_. DVC is workflow management system designed specificaly to manage machine learning experiments. DVC was built on top of Git.
|
||||
#. `Git-LFS <https://git-lfs.github.com/>`_ - DVC is compatible with any remote storage (S3, Google Cloud, Azure, SSH,
|
||||
etc). DVC utilizes reflinks or hardlinks to avoid copy operation on checkouts which makes much more efficient for
|
||||
large data files.
|
||||
|
||||
DVC is compatible with Git for storing code and the dependency graph (DAG), but not data files cache.
|
||||
Data files caches can be transferred separately - now data cache transfer throught AWS S3 and GCP storge are supported.
|
||||
#. *Makefile* (and its analogues). DVC tracks dependencies (DAG).
|
||||
|
||||
How DVC works
|
||||
=============
|
||||
#. `Workflow Management Systems <https://en.wikipedia.org/wiki/Workflow_management_system>`_. DVC is a workflow
|
||||
management system designed specifically to manage machine learning experiments. DVC is built on top of Git.
|
||||
|
||||
Contributing
|
||||
============
|
||||
Contributions are welcome! Please see our `Contributing Guide <https://dvc.org/doc/user-guide/contributing/>`_ for more
|
||||
details.
|
||||
|
||||
.. image:: https://s3-us-west-2.amazonaws.com/dvc-share/images/0.9/how_dvc_works.png
|
||||
:target: https://s3-us-west-2.amazonaws.com/dvc-share/images/0.9/how_dvc_works.png
|
||||
:alt: how_dvc_works
|
||||
.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/0
|
||||
:target: https://sourcerer.io/fame/efiop/iterative/dvc/links/0
|
||||
:alt: 0
|
||||
|
||||
.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/1
|
||||
:target: https://sourcerer.io/fame/efiop/iterative/dvc/links/1
|
||||
:alt: 1
|
||||
|
||||
.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/2
|
||||
:target: https://sourcerer.io/fame/efiop/iterative/dvc/links/2
|
||||
:alt: 2
|
||||
|
||||
.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/3
|
||||
:target: https://sourcerer.io/fame/efiop/iterative/dvc/links/3
|
||||
:alt: 3
|
||||
|
||||
.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/4
|
||||
:target: https://sourcerer.io/fame/efiop/iterative/dvc/links/4
|
||||
:alt: 4
|
||||
|
||||
.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/5
|
||||
:target: https://sourcerer.io/fame/efiop/iterative/dvc/links/5
|
||||
:alt: 5
|
||||
|
||||
.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/6
|
||||
:target: https://sourcerer.io/fame/efiop/iterative/dvc/links/6
|
||||
:alt: 6
|
||||
|
||||
.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/7
|
||||
:target: https://sourcerer.io/fame/efiop/iterative/dvc/links/7
|
||||
:alt: 7
|
||||
|
||||
Mailing List
|
||||
============
|
||||
|
||||
Want to stay up to date? Want to help improve DVC by participating in our ocassional polls? Subscribe to our `mailing list <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_. No spam, really low traffic.
|
||||
|
||||
Copyright
|
||||
=========
|
||||
|
||||
This project is distributed under the Apache license version 2.0 (see the LICENSE file in the project root).
|
||||
|
||||
By submitting a pull request for this project, you agree to license your contribution under the Apache license version 2.0 to this project.
|
||||
By submitting a pull request for this project, you agree to license your contribution under the Apache license version
|
||||
2.0 to this project.
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
REM pandoc -s doc.rst -o doc_simple.html
|
||||
pandoc -s --toc -H header.html -B top.html -A footer.html doc.rst -o doc.html
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
pandoc -s doc.rst -o doc_simple.html
|
||||
pandoc -s --toc -H header.html -B top.html -A footer.html doc.rst -o doc.html
|
||||
|
||||
File diff suppressed because one or more lines are too long
-997
@@ -1,997 +0,0 @@
|
||||
====================================
|
||||
Collaboration issues in data science
|
||||
====================================
|
||||
|
||||
Even with all the successes today in machine learning (ML), specifically deep learning and its applications in business, the data science community is still lacking good practices for organizing their projects and effectively collaborating across their varied ML projects.
|
||||
This is a massive challenge for the community and the industry now, when ML algorithms and methods are no longer simply "tribal knowledge" but are still difficult to implement, reuse, and manage.
|
||||
|
||||
To make progress in this challenge, many areas of the ML experimentation process need to be formalized. Many common questions need to be answered in an unified, principled way:
|
||||
|
||||
1. Source code and data versioning.
|
||||
|
||||
- How do you avoid any discrepancies between versions of the source code and versions of the data files when the data cannot fit into a repository?
|
||||
|
||||
2. Experiment time log.
|
||||
|
||||
- How do you track which of the hyperparameter changes contributed the most to producing your target metric? How do you monitor the extent of each change?
|
||||
|
||||
3. Navigating through experiments.
|
||||
|
||||
- How do you recover a model from last week without wasting time waiting for the model to re-train?
|
||||
|
||||
- How do you quickly switch between the large data source and a small data subset without modifying source code?
|
||||
|
||||
4. Reproducibility.
|
||||
|
||||
- How do you rerun a model's evaluation without re-training the model and preprocessing a raw dataset?
|
||||
|
||||
5. Managing and sharing large data files.
|
||||
|
||||
- How do you share models trained in a GPU environment with colleagues who do not have access to a GPU?
|
||||
|
||||
- How do you share the entire 147Gb of your project, with all of its data sources, intermediate data files, and models?
|
||||
|
||||
|
||||
Some of these questions are easy to answer individually.
|
||||
Any data scientist, engineer, or manager knows or could easily find answers to some of them.
|
||||
However, the variety of answers and approaches makes data science collaboration a nightmare.
|
||||
**A systematic approach is required.**
|
||||
|
||||
=========================
|
||||
Tools for data scientists
|
||||
=========================
|
||||
|
||||
Existing engineering tools
|
||||
__________________________
|
||||
|
||||
There is one common opinion regarding data science tooling.
|
||||
Data scientists as engineers are supposed to use the best practices and collaboration software from software engineering.
|
||||
Source code version control system (Git), continuous integration services (CI), and unit test frameworks are all expected to be utilized in the data science pipeline.
|
||||
|
||||
But a comprehensive look at data science processes shows that the software engineering toolset does not cover data science needs. Try to answer all the questions from the above using only engineering tools, and you are likely to be left wanting for more.
|
||||
|
||||
Experiment management software
|
||||
______________________________
|
||||
|
||||
To solve data scientists collaboration issues a new type of software was created - **experiment management software**.
|
||||
This software aims to cover the gap between data scientist needs and the existing toolset.
|
||||
|
||||
The experimentation software is usually **user interface (UI) based in contrast to the existing command line engineering tools**.
|
||||
The UI is a bridge to a **separate cloud based environment**.
|
||||
The cloud environment is usually not so fixible as local data scientists environement.
|
||||
And the cloud environment is not fully integrated with the local environment.
|
||||
|
||||
The separation of the local data scientist environment and the experimentation cloud environment creates another discrepancy issue and the environment synchronization requires addition work.
|
||||
Also, this style of software usually require external services, typically accompanied with a monthly bill.
|
||||
This might be a good solution for a particular companies or groups of data scientists.
|
||||
However a more accessible, free tool is needed for a wider audience.
|
||||
|
||||
============
|
||||
What is DVC?
|
||||
============
|
||||
|
||||
Data Version Control, or DVC, is **a new type of experiment management software** that has been built **on top of the existing engineering toolset** and particularly on a source code version control system (currently - Git).
|
||||
DVC reduces the gap between the existing tools and the data scientist needs.
|
||||
This gives an ability to **use the advantages of the experimentation software while reusing existing skills and intuition**.
|
||||
|
||||
The underlying source code control system **eliminates the need to use external services**.
|
||||
Data science experiment sharing and data scientist collaboration can be done through regular Git tools (commit messages, merges, pull requests, code comments), the same way it works for software engineers.
|
||||
|
||||
DVC implements a **Git experimentation methodology** where each experiment exists with its code as well as data, and can be represented as a separate Git branch or commit.
|
||||
|
||||
DVC uses a few core concepts:
|
||||
|
||||
- **Experiment** is equivalent to a Git branch. Each experiment (extract new features, change model hyperparameters, data cleaning, add a new data source) should be performed in a separate branch and then merged into the master branch only if the experiment is successful. DVC allows experiments to be integrated into a project's history and NEVER needs to recompute the results after a successful merge.
|
||||
|
||||
- **Experiment state** or state is equivalent to a Git snapshot (all committed files). Git checksum, branch name, or tag can be used as a reference to a experiment state.
|
||||
|
||||
- **Reproducibility** - an action to reproduce an experiment state. This action generates output files based on a set of input files and source code. This action usually changes experiment state.
|
||||
|
||||
- **Pipeline** - directed acyclic graph (DAG) of commands to reproduce an experiment state. The commands are connected by input and output files. Pipeline is defined by special **dvc-files** (which act like Makefiles).
|
||||
|
||||
- **Workflow** - set of experiments and relationships among them. Workflow corresponds to the entire Git repository.
|
||||
|
||||
- **Data files** - cached files (for large files). For data files the file content is stored outside of the Git repository on a local hard drive, but data file metadata is stored in Git for DVC needs (to maintain pipelines and reproducibility).
|
||||
|
||||
- **Data cache** - directory with all data files on a local hard drive or in cloud storage, but not in the Git repository.
|
||||
|
||||
- **Cloud storage** support is a compliment to the core DVC features. This is how a data scientist transfers large data files or shares a trained on GPU model to whom who does not have GPU.
|
||||
|
||||
|
||||
=============
|
||||
Core features
|
||||
=============
|
||||
|
||||
1. DVC works **on top of Git repositories** and has a similar command line interface and Git workflow.
|
||||
|
||||
2. It makes data science projects **reproducible** by creating lightweight pipelines of DAGs.
|
||||
|
||||
3. **Large data file versioning** works by creating pointers in your Git repository to the data cache on a local hard drive.
|
||||
|
||||
4. **Programming language agnostic**: Python, R, Julia, shell scripts, etc. ML library agnostic: Keras, Tensorflow, PyTorch, scipy, etc.
|
||||
|
||||
5. **Open-sourced** and **Self-served**. DVC is free and does not require any additional services.
|
||||
|
||||
6. DVC supports cloud storage (AWS S3 and GCP storage) for **data sources and pre-trained models sharing**.
|
||||
|
||||
|
||||
====================
|
||||
Related technologies
|
||||
====================
|
||||
|
||||
Due to the the novelty of this approach, DVC can be better understood in comparison to existing technologies and ideas.
|
||||
|
||||
DVC combines a number of existing technologies and ideas into a single product with the goal of bringing the best engineering practices into the data science process.
|
||||
|
||||
1. **Git**. The difference is:
|
||||
|
||||
- DVC extends Git by introducing the concept of *data files* - large files that should NOT be stored in a Git repository but still need to be tracked and versioned.
|
||||
|
||||
2. **Workflow management tools** (pipelines and DAGs): Apache Airflow, Luigi and etc. The differences are:
|
||||
|
||||
- DVC is focused on data science and modeling. As a result, DVC pipelines are lightweight, easy to create and modify. However, DVC lacks pipeline execution features like execution monitoring, execution error handling, and recovering.
|
||||
|
||||
- DVC is purely a command line tool that does not have a user interface and does not run any servers. Nevertheless, DVC can generate images with pipeline and experiment workflow visualization.
|
||||
|
||||
3. **Experiment management** software today is mostly designed for enterprise usage. An open-sourced experimentation tool example: http://studio.ml/. The differences are:
|
||||
|
||||
- DVC uses Git as the underlying platform for experiment tracking instead of a web application.
|
||||
|
||||
- DVC does not need to run any services. No user interface as a result, but we expect some UI services will be created on top of DVC.
|
||||
|
||||
- DVC has transparent design: DVC-files, meta files, state file, cache dirs have a simple format and can be easily reused by external tools.
|
||||
|
||||
4. **Git workflows** and Git usage methodologies such as Gitflow. The differences are:
|
||||
|
||||
- DVC supports a new experimentation methodology that integrates easily with a Git workflow. A separate branch should be created for each experiment, with a subsequent merge of this branch if it was successful.
|
||||
|
||||
- DVC innovates by allowing experimenters the ability to easily navigate through past experiments without recomputing them.
|
||||
|
||||
|
||||
5. **Makefile** (and it's analogues). The differences are:
|
||||
|
||||
- DVC utilizes a DAG:
|
||||
|
||||
- The DAG is defined by dvc-files with filenames *Dvcfile* or *<filename>.dvc*.
|
||||
|
||||
- One dvc-file defines one node in the DAG. All dvc-files in a repository make up a single pipeline (think a single Makefile). All dvc-files (and corresponding pipeline commands) are implicitly combined through their inputs and outputs, to simplify conflict resolving during merges.
|
||||
|
||||
- DVC provides a simple command *dvc run CMD* to generate a dvc-file automatically based on the provided command, dependencies, and outputs.
|
||||
|
||||
- File tracking:
|
||||
|
||||
- DVC tracks files based on checksum (md5) instead of file timestamps. This helps avoid running into heavy processes like model re-training when you checkout a previous, trained version of a modeling code (Makefile will retrain the model).
|
||||
|
||||
- DVC uses the files timestamps and inodes for optimization. This allows DVC to avoid recomputing all dependency files checksum, which would be highly problematic when working with large files (10Gb+).
|
||||
|
||||
|
||||
6. **Git-annex**. The differences are:
|
||||
|
||||
- DVC uses the idea of storing the content of large files (that you don't want to see in your Git repository) in a local key-value store and use file symlinks instead of the actual files.
|
||||
|
||||
- DVC uses hardlinks instead of symlinks to make user experience better.
|
||||
|
||||
- DVC optimizes checksum calculation.
|
||||
|
||||
- DVC stores data file metadata in Git repository *.dvc/*, not in the Git tree *.git/annex/*. As a result, all metadata can be shared through any Git server like Github (Git-annex loses all metadata when shared by Git server).
|
||||
|
||||
7. **Git-LFS** (Large File Storage). The differences are:
|
||||
|
||||
- DVC is fully compatible with Git. It does not require special Git servers like Git-LFS demands.
|
||||
|
||||
- DVC does not add any hooks to Git by default. To checkout data files, the *dvc checkout* command has to be run after each *git checkout* and *git clone* command.
|
||||
|
||||
- DVC creates hardlinks instead and changes data file permissions to read only. The *dvc checkout* command does not actually copy data files from cache to the working tree, as copying files is a heavy operation for large files (30Gb+).
|
||||
|
||||
|
||||
=================
|
||||
How does it work?
|
||||
=================
|
||||
|
||||
1. DVC is a command line tool that works on top of Git::
|
||||
|
||||
$ cd my_git_repo
|
||||
$ dvc init
|
||||
|
||||
2. DVC helps define pipelines of your commands, and keeps all the commands and dependencies in a Git repository::
|
||||
|
||||
$ dvc run -d input.csv -o results.csv python cnn_train.py --seed 20180227 --epoch 20 input.csv result.csv
|
||||
$ git add results.csv.dvc
|
||||
$ git commit -m 'Train CNN. 20 epochs.'
|
||||
|
||||
3. DVC is programming language agnostic. R command example::
|
||||
|
||||
$ dvc run -d result.csv -o plots.jpg Rscript plot.R result.csv plots.jpg
|
||||
$ git add plots.jpg.dvc
|
||||
$ git commit -m 'CNN plots'
|
||||
|
||||
4. DVC can reproduce a pipeline with respect to the pipeline's dependencies::
|
||||
|
||||
# The input dataset was changed
|
||||
$ dvc repro plots.jpg.dvc
|
||||
Reproducing 'output.p':
|
||||
python cnn_train.py --seed 20180227 --epoch 20 input.csv output.p
|
||||
Reproducing 'plots.jpg':
|
||||
Rscript plot.R result.csv plots.jpg
|
||||
|
||||
5. DVC introduces the concept of data files to Git repositories. DVC keeps data files outside of the repository but retains the metadata in Git::
|
||||
|
||||
$ git checkout a03_normbatch_vgg16 # checkout code and DVC meta data
|
||||
$ dvc checkout # checkout data files from the local cache (not Git)
|
||||
$ ls -l data/ # These LARGE files were copied from DVC cache, not from Git
|
||||
total 1017488
|
||||
-r-------- 2 501 staff 273M Jan 27 03:48 Posts-test.tsv
|
||||
-r-------- 2 501 staff 12G Jan 27 03:48 Posts-train.tsv
|
||||
|
||||
|
||||
6. DVC makes repositories reproducible. DVC metadata can be easily shared through any Git server, and allows for experiments to be easily reproduced::
|
||||
|
||||
$ git clone https://github.com/dataversioncontrol/myrepo.git
|
||||
$ cd myrepo
|
||||
# Reproduce data files
|
||||
$ dvc repro
|
||||
Reproducing 'output.p':
|
||||
python cnn_train.py --seed 20180227 --epoch 20 input.csv output.p
|
||||
Reproducing 'plots.jpg':
|
||||
Rscript plot.R result.csv plots.jpg
|
||||
|
||||
7. DVC's local cache can be transferred to your colleagues and partners through AWS S3 or GCP Storage::
|
||||
|
||||
$ git push
|
||||
$ dvc push # push the data cache to your cloud bucket
|
||||
|
||||
# On a colleague machine:
|
||||
$ git clone https://github.com/dataversioncontrol/myrepo.git
|
||||
$ cd myrepo
|
||||
$ git pull # get the data cache from cloud
|
||||
$ dvc checkout # checkout data files
|
||||
$ ls -l data/ # You just got gigabytes of data through Git and DVC:
|
||||
total 1017488
|
||||
-r-------- 2 501 staff 273M Jan 27 03:48 Posts-test.tsv
|
||||
|
||||
8. DVC works on Mac, Linux ,and Windows. A Windows example::
|
||||
|
||||
$ dir
|
||||
?????
|
||||
????
|
||||
|
||||
|
||||
==============================
|
||||
Getting Started with DVC
|
||||
==============================
|
||||
|
||||
To show DVC in action, let's play with an actual machine learning scenario.
|
||||
Let's explore the natural language processing (NLP) problem of predicting tags for a given
|
||||
StackOverflow question.
|
||||
For example, we want one classifier which can predict a post that is about the Java language by tagging it "Java".
|
||||
|
||||
First, let's download the model code and set up the Git repository::
|
||||
|
||||
$ mkdir myrepo
|
||||
$ cd myrepo
|
||||
$ mkdir code
|
||||
$ wget -nv -P code/ https://s3-us-west-2.amazonaws.com/dvc-share/so/code/featurization.py \
|
||||
https://s3-us-west-2.amazonaws.com/dvc-share/so/code/evaluate.py \
|
||||
https://s3-us-west-2.amazonaws.com/dvc-share/so/code/train_model.py \
|
||||
https://s3-us-west-2.amazonaws.com/dvc-share/so/code/split_train_test.py \
|
||||
https://s3-us-west-2.amazonaws.com/dvc-share/so/code/xml_to_tsv.py \
|
||||
https://s3-us-west-2.amazonaws.com/dvc-share/so/code/requirements.txt
|
||||
$ pip install -U -r code/requirements.txt
|
||||
$ git init
|
||||
$ git add code/
|
||||
$ git commit -m 'Download code'
|
||||
|
||||
The full pipeline can be built by running the code below::
|
||||
|
||||
$ # Initialize DVC repository (in your Git repository)
|
||||
$ dvc init
|
||||
|
||||
$ # Download a file and put it into the data/ directory.
|
||||
$ dvc import https://s3-us-west-2.amazonaws.com/dvc-share/so/25K/Posts.xml.tgz data/
|
||||
|
||||
$ # Extract XML from the archive.
|
||||
$ dvc run tar zxf data/Posts.xml.tgz -C data/
|
||||
|
||||
$ # Prepare the data.
|
||||
$ dvc run python code/xml_to_tsv.py data/Posts.xml data/Posts.tsv python
|
||||
|
||||
$ # Split training and testing dataset. Two output files.
|
||||
$ # 0.33 is the test dataset split ratio. 20170426 is a seed for randomization.
|
||||
$ dvc run python code/split_train_test.py data/Posts.tsv 0.33 20170426 data/Posts-train.tsv data/Posts-test.tsv
|
||||
|
||||
$ # Extract features from the data. Two TSV as inputs with two pickle matrices as outputs.
|
||||
$ dvc run python code/featurization.py data/Posts-train.tsv data/Posts-test.tsv data/matrix-train.p data/matrix-test.p
|
||||
|
||||
$ # Train ML model on the training dataset. 20170426 is another seed value.
|
||||
$ dvc run python code/train_model.py data/matrix-train.p 20170426 data/model.p
|
||||
|
||||
# Evaluate the model on the test dataset.
|
||||
$ dvc run python code/evaluate.py data/model.p data/matrix-test.p data/evaluation.txt
|
||||
|
||||
$ # The result.
|
||||
$ cat data/evaluation.txt
|
||||
AUC: 0.596182
|
||||
|
||||
Your code can be easily reproduced after some minor modification::
|
||||
|
||||
$ # Improve feature extraction step.
|
||||
$ vi code/featurization.py
|
||||
|
||||
$ # Commit all the changes.
|
||||
$ git commit -am "Add bigram features"
|
||||
[master 50b5a2a] Add bigram features
|
||||
1 file changed, 5 insertion(+), 2 deletion(-)
|
||||
|
||||
$ # Reproduce all required steps to get our target metrics file.
|
||||
$ dvc repro data/evaluation.txt
|
||||
Reproducing run command for data item data/matrix-train.p. Args: python code/featurization.py data/Posts-train.tsv data/Posts-test.tsv data/matrix-train.p data/matrix-test.p
|
||||
Reproducing run command for data item data/model.p. Args: python code/train_model.py data/matrix-train.p 20170426 data/model.p
|
||||
Reproducing run command for data item data/evaluation.txt. Args: python code/evaluate.py data/model.p data/matrix-test.p data/evaluation.txt
|
||||
Data item "data/evaluation.txt" was reproduced.
|
||||
|
||||
$ # Take a look at the target metric improvement.
|
||||
$ cat data/evaluation.txt
|
||||
AUC: 0.627196
|
||||
|
||||
It's easy to integrate DVC into your existing ML pipeline/processes without any significant effort to re-implement your code/application.
|
||||
|
||||
The key step to note is that DVC automatically derives the dependencies between the experiment steps and builds the dependency graph (DAG) transparently.
|
||||
|
||||
Not only can DVC streamline your work into a single, reproducible environment, it also makes it easy to share this environment by Git including the dependencies — an exciting collaboration feature which gives the ability to reproduce the research easily in a myriad of environments.
|
||||
|
||||
|
||||
============
|
||||
Installation
|
||||
============
|
||||
|
||||
Operating system dependent packages are the recommended way to install DVC.
|
||||
Some other methods of installation are available.
|
||||
|
||||
OS packages
|
||||
===========
|
||||
|
||||
DVC installation packages are available for Mac OS, Linux, and Windows.
|
||||
You can download the packages here at https://github.com/dataversioncontrol/dvc/releases/
|
||||
|
||||
Python pip
|
||||
==========
|
||||
|
||||
Another option is to use the standard Python pip package::
|
||||
|
||||
$ pip install dvc
|
||||
|
||||
**Note:** if you use *Anaconda*, it will work in *Anaconda’s* command prompt tool.
|
||||
At the moment, DVC does not provide a special installation package for the native *Anaconda* package manager *conda*.
|
||||
|
||||
Homebrew Cask
|
||||
=============
|
||||
|
||||
Mac OS users can install DVC by using the **brew** command::
|
||||
|
||||
$ brew cask install dataversioncontrol/homebrew-dvc/dvc
|
||||
|
||||
Development Version
|
||||
===================
|
||||
|
||||
If you would like to pull the latest version of DVC, you can do the following::
|
||||
|
||||
$ pip install git+git://github.com/dataversioncontrol/dvc
|
||||
|
||||
Note: this will automatically upgrade your DVC version to the latest version.
|
||||
|
||||
|
||||
=============
|
||||
Configuration
|
||||
=============
|
||||
|
||||
Once you install DVC, you will be able to start using it (in its local setup) immediately.
|
||||
|
||||
However, you can proceed to configure DVC (especially if you intend to use it in a *cloud-based* scenario).
|
||||
|
||||
DVC Files and Directories
|
||||
=========================
|
||||
|
||||
Once installed, DVC will populate its installation folder (hereafter referred to as .dvc)
|
||||
|
||||
* **.dvc/config** - This is a configuration file.
|
||||
The config file can be edited directly using command **dvc config NAME VALUE**.
|
||||
* **.dvc/cache** - the cache directory will contain your data files (the data directories of DVC repositories will only contain hardlinks to the data files in the global cache).
|
||||
**Note:** DVC includes the cache directory to **.gitignore** file during the initialization. And no data files (with actual content) will ever be pushed to Git repository,
|
||||
only dvc-files are needed to reproduce them.
|
||||
* **.dvc/state** - this file is created for optimization. The file contains data files checksum, timestamps, inodes, etc.
|
||||
|
||||
|
||||
Working with Cloud Data Storages
|
||||
======================================================
|
||||
|
||||
Using DVC with Cloud-based data storage is optional.
|
||||
By default, DVC is configured to use a local data storage only (.dvc/cache directory),
|
||||
which enables basic DVC usage scenarios out of the box.
|
||||
|
||||
DVC can use cloud storage as a common file storage.
|
||||
With cloud storage, you might use models and data files which were created by your team members
|
||||
without spending time and resources to re-build models and re-process data files.
|
||||
|
||||
As of this version, DVC supports two types of cloud-based storage providers:
|
||||
|
||||
* **AWS** - Amazon Web Services
|
||||
* **GCP** - Google Cloud Platform
|
||||
|
||||
The subsections below explain how to configure DVC to use each of them.
|
||||
|
||||
Using AWS
|
||||
---------------
|
||||
|
||||
To use AWS as cloud storage for your DVC repositories, you should update these **.dvc/config** options
|
||||
|
||||
* **Cloud = AWS** in *Global* section.
|
||||
* **StoragePath = /mybucket/dvc/tag_classifier** in **AWS** section - path to a cloud storage bucket and directory in the bucket.
|
||||
* **CredentialPath = ~/aws/credentials** in **AWS** section - path to AWS credentials in your local machine (AWS cli command line tools creates this directory).
|
||||
In Mac, default value is *~/.aws/credentials*, and it is *%USERPATH%/.aws/credentials* in Windows
|
||||
|
||||
|
||||
**Important:** do not forget to commit the config file change to Git: **git commit -am "Change cloud to AWS"**
|
||||
|
||||
Instead of manual file modification, we recommend you run the following commands::
|
||||
|
||||
$ dvc config Global.Cloud AWS # This step is not needed for new DVC repositories
|
||||
$ dvc config AWS.StoragePath /mybucket/dvc/tag_classifier
|
||||
$ dvc config AWS.CredentialPath ~/.aws/credentials # Not needed if AWS CLI is installed to default path
|
||||
$ dvc config AWS.CredentialSection default # Not needed if you have only one AWS account
|
||||
$ git commit -am "Change cloud to AWS"
|
||||
|
||||
|
||||
Using Google Cloud
|
||||
------------------
|
||||
|
||||
For using GCP (Google Cloud Platform) as cloud storage for your DVC repositories, you should update these **.dvc/config** options
|
||||
|
||||
* **Cloud = GCP** in *Global* section.
|
||||
* **StoragePath = /mybucket/dvc/tag_classifier** in GCP section - Run **dvc config GCP.StoragePath /my/path/to/a/bucket**
|
||||
* **ProjectName = MyCloud** - a GCP specific project name.
|
||||
|
||||
**Important:** do not forget to commit the config file change to Git: **git commit -am "Change cloud to GCP"**
|
||||
|
||||
Instead of manual file modification, we recommend you run the following commands::
|
||||
|
||||
$ dvc config Global.Cloud GCP
|
||||
$ dvc config GCP.StoragePath /mybucket/dvc/tag_classifier
|
||||
$ dvc config GCP.ProjectName MyCloud
|
||||
$ git commit -am "Change cloud to AWS"
|
||||
|
||||
|
||||
==================
|
||||
Using DVC Commands
|
||||
==================
|
||||
|
||||
DVC is a command-line tool.
|
||||
The typical use case for DVC goes as follows
|
||||
|
||||
* In an existing Git repository, initialize a DVC repository with **dvc init**.
|
||||
* Copy source files for modeling into the repository and convert the files into DVC data files with **dvc add** command.
|
||||
* Process source data files through your data processing and modeling code using the **dvc run** command.
|
||||
* Use **--outs** option to specify **dvc run** command outputs which will be converted to DVC data files after the code runs.
|
||||
* Clone a git repo with the code of your ML application pipeline. However, this will not copy your DVC cache. Use cloud storage settings and **dvc push** to share the cache (data).
|
||||
* Use **dvc repro** to quickly reproduce your pipeline on a new iteration, after your data item files or source code of your ML application are modified.
|
||||
|
||||
========================
|
||||
DVC Commands Cheat Sheet
|
||||
========================
|
||||
|
||||
Below is the quick summary of the most important commands
|
||||
|
||||
* **dvc -h** - Show how to use DVC and show the list of commands.
|
||||
* **dvc CMD -h** - Display help to use a specific DVC command (CMD).
|
||||
* **dvc init** - Initialize a new DVC repository.
|
||||
* **dvc add** - Add data file or data directory. The command converts regular files to DVC data files.
|
||||
* **dvc checkout** - Checkout data files and dirs into the working tree. The command should be executed after **git checkout** or cloning a repository.
|
||||
* **dvc run** - Generate a DVC file from a given command and execute the command. The command dependencies and outputs should be specified.
|
||||
* **dvc pull** - Pull data files from the cloud. Cloud settings for your DVC environment should be already configured prior to using this command.
|
||||
* **dvc push** - Push data files to the cloud. Cloud settings should be already configured.
|
||||
* **dvc status** - Show status of a data file in the DVC repository.
|
||||
* **dvc repro** - Reproduce a stage of pipeline. Default stage file is **Dvcfile**.
|
||||
* **dvc remove** - Remove data file (files or/and folders).
|
||||
* **dvc gc** - Collect garbage by cleaning DVC cache.
|
||||
* **dvc config** - Get or set configuration settings (as specified in dvc.conf).
|
||||
* **dvc fsck** - Data file consistency check.
|
||||
|
||||
=====================
|
||||
DVC Command Reference
|
||||
=====================
|
||||
|
||||
init
|
||||
====
|
||||
|
||||
This command initializes a DVC environment in a current Git repository.
|
||||
|
||||
.. code-block:: shell
|
||||
:linenos:
|
||||
|
||||
usage: dvc init [-h] [-q] [-v]
|
||||
optional arguments:
|
||||
-h, --help show this help message and exit
|
||||
-q, --quiet Be quiet.
|
||||
-v, --verbose Be verbose.
|
||||
|
||||
Example. Creating a new DVC repository::
|
||||
|
||||
$ mkdir tag_classifier
|
||||
$ cd tag_classifier
|
||||
|
||||
$ git init
|
||||
Initialized empty Git repository in /Users/dmitry/src/tag_classifier/.git/
|
||||
|
||||
$ dvc init
|
||||
$ git status
|
||||
On branch master
|
||||
|
||||
Initial commit
|
||||
|
||||
Changes to be committed:
|
||||
|
||||
(use "git rm --cached <file>..." to unstage)
|
||||
|
||||
new file: .dvc/.gitignore
|
||||
new file: .dvc/config
|
||||
|
||||
$ git commit -m 'Init DVC'
|
||||
[master (root-commit) 2db4618] Init DVC
|
||||
2 files changed, 41 insertions(+)
|
||||
create mode 100644 .dvc/.gitignore
|
||||
create mode 100644 .dvc/config
|
||||
|
||||
|
||||
add
|
||||
====
|
||||
|
||||
Converts files and directories to DVC data files.
|
||||
|
||||
The command does the conversion from a *regular file* to *DVC data file* in a few steps:
|
||||
|
||||
1. Calculate the file checksum.
|
||||
2. Create a cache file in the cache dir *.dvc/cache*.
|
||||
3. Create a corresponding DVC file.
|
||||
4. Replace the file with a hardlink to the cache file.
|
||||
|
||||
DVC stores the file's last modification timestamp, inode, and the checksum into a global state file *.dvc/state* to reduce time recomputing checksums later.
|
||||
|
||||
Note, this command does NOT copy any file contents and will run quickly even for a large files.
|
||||
Step (2) from the above is also made by hardlinks movement, not file content.
|
||||
The only heavy step is (1), which requires checksum calculation.
|
||||
|
||||
For directories, the command does the same steps for each file recursively.
|
||||
To retain information about the directory structure, a corresponding directory will be created in *.dvc/cache*.
|
||||
|
||||
.. code-block:: shell
|
||||
:linenos:
|
||||
|
||||
usage: dvc add [-h] [-q] [-v] targets [targets ...]
|
||||
|
||||
optional arguments:
|
||||
-h, --help show this help message and exit
|
||||
-q, --quiet Be quiet.
|
||||
-v, --verbose Be verbose.
|
||||
|
||||
Examples:
|
||||
|
||||
Convert files into data files::
|
||||
|
||||
$ mkdir raw
|
||||
$ cp ~/Downloads/dataset/* raw
|
||||
$ ls raw
|
||||
Badges.xml PostLinks.xml Votes.xml
|
||||
$ dvc add raw/Badges.tsv raw/PostLinks.tsv raw/Votes.tsv
|
||||
$ ls raw
|
||||
Badges.xml PostLinks.xml Votes.xml
|
||||
Badges.xml.dvc PostLinks.xml.dvc Votes.xml.dvc
|
||||
|
||||
Note, DVC files are created.
|
||||
|
||||
|
||||
checkout
|
||||
========
|
||||
|
||||
Checkout data files from cache.
|
||||
This command has to be called after *git checkout* since Git does not handle DVC data files.
|
||||
|
||||
The command restores data files from cache to the working tree and removes data files that are no longer on the working tree.
|
||||
|
||||
Note, this command does NOT copy any files - DVC uses hardlinks to perform data file restoration.
|
||||
This is crucial for large files where checking out as a 50Gb file might take a few minutes.
|
||||
For DVC, it will take less than a second to restore a 50Gb data file.
|
||||
|
||||
|
||||
.. code-block:: shell
|
||||
:linenos:
|
||||
|
||||
usage: dvc checkout [-h] [-q] [-v]
|
||||
|
||||
optional arguments:
|
||||
-h, --help show this help message and exit
|
||||
-q, --quiet Be quiet.
|
||||
-v, --verbose Be verbose.
|
||||
|
||||
Examples.
|
||||
|
||||
Checking out a branch example::
|
||||
|
||||
$ git checkout input_100K
|
||||
$ dvc checkout
|
||||
$ Remove 'data/model.p'
|
||||
$ Remove 'data/matrix-train.p'
|
||||
$ 'data/Posts-train.tsv': cache file not found
|
||||
|
||||
DVC does not report in the output which data files were restored.
|
||||
However, it reports removed files and files that DVC was unable to restore due to missing cache.
|
||||
To restore a file with a missing cache, the reproduction command should be called or the cache can be pulled from the cloud.
|
||||
|
||||
It might be convenient to assign Git hook to *git checkout*::
|
||||
|
||||
$ echo 'dvc checkout' > .git/hooks/post-checkout
|
||||
$ chmod +x .git/hooks/post-checkout
|
||||
$ git checkout input_100K # dvc checkout is not needed anymore
|
||||
$ Remove 'data/model.p'
|
||||
$ Remove 'data/matrix-train.p'
|
||||
$ 'data/Posts-train.tsv': cache file not found
|
||||
|
||||
run
|
||||
===
|
||||
|
||||
Generate a stage file from a given command and execute the command.
|
||||
The command dependencies and outputs should be specified.
|
||||
|
||||
By default, stage file name is **<file>.dvc** where **<file>** is file name of the first output.
|
||||
|
||||
For example, launch Python with a given python script and arguments. Or R script by Rscript command.
|
||||
|
||||
.. code-block:: shell
|
||||
:linenos:
|
||||
|
||||
usage: dvc run [-h] [-q] [-v] [-d DEPS] [-o OUTS] [-O OUTS_NO_CACHE] [-f FILE]
|
||||
[-c CWD] [--no-exec]
|
||||
...
|
||||
|
||||
positional arguments:
|
||||
command Command or command file to execute
|
||||
|
||||
optional arguments:
|
||||
-h, --help show this help message and exit
|
||||
-q, --quiet Be quiet.
|
||||
-v, --verbose Be verbose.
|
||||
-d DEPS, --deps DEPS Declare dependencies for reproducible cmd.
|
||||
-o OUTS, --outs OUTS Declare output data file or data directory.
|
||||
-O OUTS_NO_CACHE, --outs-no-cache OUTS_NO_CACHE
|
||||
Declare output regular file or directory (sync to Git,
|
||||
not DVC cache).
|
||||
-f FILE, --file FILE Specify name of the state file
|
||||
-c CWD, --cwd CWD Directory to run your command and place state file in
|
||||
--no-exec Only create stage file without actually running it
|
||||
|
||||
Examples:
|
||||
|
||||
Execute a Python script as the DVC pipeline step. Stage file was not specified, so a **model.p.dvc** stage file will be created::
|
||||
|
||||
$ # Train ML model on the training dataset. 20180226 is a seed value.
|
||||
$ dvc run -d matrix-train.p -d train_model.py -o model.p python train_model.py matrix-train.p 20180226 model.p
|
||||
|
||||
|
||||
Execute an R script as the DVC pipeline step::
|
||||
|
||||
$ dvc run -d parsingxml.R -d Posts.xml -o Posts.csv Rscript parsingxml.R Posts.xml Posts.csv
|
||||
|
||||
|
||||
Extract an XML file from an archive to the data/ subfolder::
|
||||
|
||||
$ mkdir data
|
||||
$ dvc run -d Posts.xml.tgz -o data/Posts.xml tar zxf Posts.xml.tgz -C data/
|
||||
|
||||
|
||||
push
|
||||
====
|
||||
|
||||
This command pushes all data file caches related to the current Git branch to cloud storage.
|
||||
Cloud storage settings need to be configured.
|
||||
See cloud storage configuration for more details on how to set up cloud storage.
|
||||
|
||||
.. code-block:: shell
|
||||
:linenos:
|
||||
|
||||
usage: dvc push [-h] [-q] [-v] [-j JOBS]
|
||||
|
||||
optional arguments:
|
||||
-h, --help show this help message and exit
|
||||
-q, --quiet Be quiet.
|
||||
-v, --verbose Be verbose.
|
||||
-j JOBS, --jobs JOBS Number of jobs to run simultaneously.
|
||||
|
||||
Examples:
|
||||
|
||||
Push all data file caches from the current Git branch to cloud::
|
||||
|
||||
$ dvc push
|
||||
(1/8): [########################################] 100% 72271bebdf053178a5cce48b4
|
||||
(2/8): [########################################] 100% d7208b910d1a40fedc2da5a44
|
||||
(3/8): [########################################] 100% 7f6ed2919af9c9e94c32ea13d
|
||||
(4/8): [########################################] 100% 5988519f8465218abb23ce0e0
|
||||
(5/8): [########################################] 100% 11de13709a78379d253a3d0f5
|
||||
(6/8): [########################################] 100% 3f9c7c3ae51db2eed7ba99e6e
|
||||
(7/8): [########################################] 100% cfdaa4bba57fa07d81ff96685
|
||||
(8/8): [####################### ] 57% 1de6178a9dd844e249ba05414
|
||||
|
||||
|
||||
pull
|
||||
====
|
||||
|
||||
This command pulls all data file caches from cloud storage.
|
||||
Cloud storage settings need to be configured.
|
||||
|
||||
.. code-block:: shell
|
||||
:linenos:
|
||||
|
||||
usage: dvc pull [-h] [-q] [-v] [-j JOBS]
|
||||
|
||||
optional arguments:
|
||||
-h, --help show this help message and exit
|
||||
-q, --quiet Be quiet.
|
||||
-v, --verbose Be verbose.
|
||||
-j JOBS, --jobs JOBS Number of jobs to run simultaneously.
|
||||
|
||||
Examples:
|
||||
|
||||
Pull all files from the current Git branch::
|
||||
|
||||
$ dvc pull
|
||||
(1/8): [########################################] 100% 54a6f1787490ba13fb811a46b
|
||||
(2/8): [########################################] 100% 5806dc797c08fb6ddd5d97d46
|
||||
(3/8): [########################################] 100% 5988519f8465218abb23ce0e0
|
||||
(4/8): [########################################] 100% 7f6ed2919af9c9e94c32ea13d
|
||||
(5/8): [########################################] 100% 11de13709a78379d253a3d0f5
|
||||
(6/8): [########################################] 100% c6f5a256d628e144db4181de8
|
||||
(7/8): [########################################] 100% 3f9c7c3ae51db2eed7ba99e6e
|
||||
(8/8): [########################################] 100% cfdaa4bba57fa07d81ff96685
|
||||
|
||||
status
|
||||
======
|
||||
|
||||
Show mismatches between local cache and cloud cache.
|
||||
|
||||
.. code-block:: shell
|
||||
:linenos:
|
||||
|
||||
usage: dvc status [-h] [-q] [-v] [-j JOBS]
|
||||
|
||||
optional arguments:
|
||||
-h, --help show this help message and exit
|
||||
-q, --quiet Be quiet.
|
||||
-v, --verbose Be verbose.
|
||||
-j JOBS, --jobs JOBS Number of jobs to run simultaneously.
|
||||
|
||||
Examples:
|
||||
|
||||
Show statuses::
|
||||
|
||||
$ dvc status
|
||||
new file: /Users/dmitry/src/myrepo_1/.dvc/cache/62f8c2ba93cfe5a6501136078f0336f9
|
||||
|
||||
repro
|
||||
=====
|
||||
|
||||
Reproduce DVC file and all stages the file depends on (recursively).
|
||||
Default file name is **Dvcfile**.
|
||||
However, DVC files can have any name followed by the **.dvc** suffix.
|
||||
|
||||
.. code-block:: shell
|
||||
:linenos:
|
||||
|
||||
usage: dvc repro [-h] [-q] [-v] [-f] [-s] [targets [targets ...]]
|
||||
|
||||
positional arguments:
|
||||
target DVC file to reproduce.
|
||||
|
||||
optional arguments:
|
||||
-h, --help show this help message and exit
|
||||
-q, --quiet Be quiet.
|
||||
-v, --verbose Be verbose.
|
||||
-f, --force Reproduce even if dependencies were not changed.
|
||||
-s, --single-item Reproduce only single data item without recursive dependencies check.
|
||||
|
||||
Examples:
|
||||
|
||||
Reproduce default stage file::
|
||||
|
||||
$ dvc repro
|
||||
Verifying data sources in 'data/Posts.xml.tgz.dvc'
|
||||
Reproducing 'Posts.xml.dvc':
|
||||
tar zxf data/Posts.xml.tgz -C data/
|
||||
Reproducing 'Posts.tsv.dvc':
|
||||
python code/xml_to_tsv.py data/Posts.xml data/Posts.tsv python
|
||||
Reproducing 'Posts-train.tsv.dvc':
|
||||
python code/split_train_test.py data/Posts.tsv 0.33 20170426 data/Posts-train.tsv data/Posts-test.tsv
|
||||
Reproducing 'matrix-train.p.dvc':
|
||||
python code/featurization.py data/Posts-train.tsv data/Posts-test.tsv data/matrix-train.p data/matrix-test.p
|
||||
Reproducing 'model.p.dvc':
|
||||
python code/train_model.py data/matrix-train.p 20170426 data/model.p
|
||||
|
||||
Reproduce the part of the pipeline where *Posts.tsv.dvc* is the target DVC file::
|
||||
|
||||
$ dvc repro Posts.tsv.dvc
|
||||
Reproducing 'Posts.xml.dvc':
|
||||
tar zxf data/Posts.xml.tgz -C data/
|
||||
Reproducing 'Posts.tsv.dvc':
|
||||
python code/xml_to_tsv.py data/Posts.xml data/Posts.tsv python
|
||||
|
||||
|
||||
remove
|
||||
======
|
||||
|
||||
Remove data file or data directory.
|
||||
|
||||
.. code-block:: shell
|
||||
:linenos:
|
||||
|
||||
usage: dvc remove [-h] [-q] [-v] targets [targets ...]
|
||||
|
||||
positional arguments:
|
||||
targets Target to remove - file or directory.
|
||||
|
||||
optional arguments:
|
||||
-h, --help show this help message and exit
|
||||
-q, --quiet Be quiet.
|
||||
-v, --verbose Be verbose.
|
||||
|
||||
Examples:
|
||||
|
||||
|
||||
Remove *matrix-train.p* data file::
|
||||
|
||||
$ dvc remove matrix-train.p
|
||||
|
||||
|
||||
|
||||
gc
|
||||
===
|
||||
|
||||
This command collects the garbage, removing unused cache files based on the current Git branch.
|
||||
If a data file was created in a different branch, then it will be removed by gc.
|
||||
If a data file has a few versions (and, of course. corresponding caches) - all caches except the current one will be removed.
|
||||
|
||||
.. code-block:: shell
|
||||
:linenos:
|
||||
|
||||
age: dvc gc [-h] [-q] [-v]
|
||||
|
||||
optional arguments:
|
||||
-h, --help show this help message and exit
|
||||
-q, --quiet Be quiet.
|
||||
-v, --verbose Be verbose.
|
||||
|
||||
Clean up example::
|
||||
|
||||
$ du -sh .dvc/cache/
|
||||
7.4G .dvc/cache/
|
||||
$ dvc gc
|
||||
'.dvc/cache/27e30965256ed4d3e71c2bf0c4caad2e' was removed
|
||||
'.dvc/cache/2e006be822767e8ba5d73ebad49ef082' was removed
|
||||
'.dvc/cache/2f412200dc53fb97dcac0353b609d199' was removed
|
||||
'.dvc/cache/541025db4da02fcab715ca2c2c8f4c19' was removed
|
||||
'.dvc/cache/62f8c2ba93cfe5a6501136078f0336f9' was removed
|
||||
'.dvc/cache/7c4521365288d69a03fa22ad3d399f32' was removed
|
||||
'.dvc/cache/9ff7365a8256766be8c363fac47fc0d4' was removed
|
||||
'.dvc/cache/a86ca87250ed8e54a9e2e8d6d34c252e' was removed
|
||||
'.dvc/cache/f64d65d4ccef9ff9d37ea4cf70b18700' was removed
|
||||
$ du -sh .dvc/cache/
|
||||
3.1G .dvc/cache/
|
||||
|
||||
|
||||
config
|
||||
======
|
||||
|
||||
Get or set config options. This command reads and overwrites the DVC config file *.dvc/config*.
|
||||
|
||||
|
||||
.. code-block:: shell
|
||||
:linenos:
|
||||
|
||||
usage: dvc config [-h] [-q] [-v] [-u] name [value]
|
||||
|
||||
positional arguments:
|
||||
name Option name
|
||||
value Option value
|
||||
|
||||
optional arguments:
|
||||
-h, --help show this help message and exit
|
||||
-q, --quiet Be quiet.
|
||||
-v, --verbose Be verbose.
|
||||
-u, --unset Unset option
|
||||
|
||||
Examples:
|
||||
|
||||
Specify an option name to get the option's value from config file::
|
||||
$ dvc config config Global.Cloud
|
||||
AWS
|
||||
|
||||
Overwrite the value::
|
||||
|
||||
$ dvc config Global.Cloud GCP
|
||||
$ git add .dvc/config
|
||||
$ git commit -m 'Change cloud to GCP'
|
||||
[input_100K a4c985f] Change cloud to GCP
|
||||
1 file changed, 1 insertion(+), 1 deletion(-)
|
||||
|
||||
fsck
|
||||
====
|
||||
|
||||
Data file consistency check.
|
||||
By default the commands outputs statuses of all corrupted data files (if any).
|
||||
Use *--all* option to see statuses of all data files.
|
||||
|
||||
The command checks:
|
||||
1. Cache file name which is equal to the file content checksum when DVC created the file.
|
||||
2. Checksum from local state file.
|
||||
3. Checksum regarding DVC files.
|
||||
4. The actual recomputed checksum. This is a computation heavy command for large data files. Enabled only by *--physical* option.
|
||||
|
||||
Data file is considered to be corrupted if one of the checksums does not match all others.
|
||||
|
||||
|
||||
.. code-block:: shell
|
||||
:linenos:
|
||||
|
||||
dvc fsck [-h] [-q] [-v] [-p] [-a] [targets [targets ...]]
|
||||
|
||||
positional arguments:
|
||||
targets Data files to check
|
||||
|
||||
optional arguments:
|
||||
-h, --help show this help message and exit
|
||||
-q, --quiet Be quiet.
|
||||
-v, --verbose Be verbose.
|
||||
-p, --physical Compute actual md5
|
||||
-a, --all Show all data files including correct ones
|
||||
|
||||
Examples.
|
||||
|
||||
|
||||
Check list of corrupted data files::
|
||||
|
||||
$ dvc fsck --physical
|
||||
File data/matrix-test.p:
|
||||
Error status: Checksum missmatch!!!
|
||||
Actual checksum: 7c4521365288d69a03fa22ad3d399f32
|
||||
Cache file name: 7c4521365288d69a03fa22ad3d399f32
|
||||
Local state checksum: 7c4521365288d69a03fa22ad3d399f32
|
||||
Local state mtime: 1517048086.0
|
||||
Actual mtime: 1517048086.0
|
||||
Stage file: eval_auc.txt.dvc
|
||||
Checksum: 7c4521365288d69a03fa22ad3d399f32
|
||||
Type: Dependency
|
||||
Stage file: matrix-train.p.dvc
|
||||
Checksum: 7c4521365288d69a03fa22ad3d399f32
|
||||
Type: Output
|
||||
Use cache: true
|
||||
|
||||
Common Arguments
|
||||
===========================================
|
||||
|
||||
Common Options
|
||||
--------------
|
||||
|
||||
As you can see, there are four optional arguments that are applicable to any DVC command. They are
|
||||
|
||||
.. code-block:: shell
|
||||
:linenos:
|
||||
|
||||
-h, --help show this help message and exit
|
||||
-q, --quiet Be quiet.
|
||||
-v, --verbose Be verbose.
|
||||
|
||||
Although these optional arguments are pretty self-explanatory, there is a note for DVC and Git commands that are used together.
|
||||
|
||||
* To see Git commands in DVC, you can set logging level to *Debug* (in **dvc.conf**) or run dvc with option *--verbose*
|
||||
|
||||
Number of DVC Jobs
|
||||
------------------
|
||||
|
||||
DVC can benefit from parallel processing and multiple processors/cores when performing cache push/pull operations.
|
||||
|
||||
By default, the number of DVC jobs is set to the number of available CPU cores. If you would like to change it to any other reasonable value, you could use *-j (--jobs)* option in DVC commands where applicable.
|
||||
@@ -1,13 +0,0 @@
|
||||
</div> <!-- end of content-containder -->
|
||||
<footer>
|
||||
<div class="footer">
|
||||
<div class="inner">
|
||||
<ul>
|
||||
<li><span>© 2017 dataversioncontrol.com</span></li>
|
||||
<li><a href="https://blog.dataversioncontrol.com/">Blog</a></li>
|
||||
<li class="last"><a href="https://twitter.com/FullStackML" class="twitter">
|
||||
<span class="ico"><svg width="16" height="16" viewBox="0 0 25 25"><path d="M21.725 5.338c-.744.47-1.605.804-2.513 1.006a3.978 3.978 0 0 0-2.942-1.293c-2.22 0-4.02 1.81-4.02 4.02 0 .32.034.63.07.94-3.31-.18-6.27-1.78-8.255-4.23a4.544 4.544 0 0 0-.574 2.01c.04 1.43.74 2.66 1.8 3.38-.63-.01-1.25-.19-1.79-.5v.08c0 1.93 1.38 3.56 3.23 3.95-.34.07-.7.12-1.07.14-.25-.02-.5-.04-.72-.07.49 1.58 1.97 2.74 3.74 2.8a8.49 8.49 0 0 1-5.02 1.72c-.3-.03-.62-.04-.93-.07A11.447 11.447 0 0 0 8.88 21c7.386 0 11.43-6.13 11.414-11.414.015-.21.01-.38 0-.578a7.604 7.604 0 0 0 2.01-2.08 7.27 7.27 0 0 1-2.297.645 3.856 3.856 0 0 0 1.72-2.23"></path></svg></span>@FullStackML</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
@@ -1,9 +0,0 @@
|
||||
<meta charset="utf-8">
|
||||
<meta name="description" content="DVC product documnetation">
|
||||
<meta name="HandheldFriendly" content="True">
|
||||
<meta name="MobileOptimized" content="320">
|
||||
<meta name="viewport" content="initial-scale=1.0, minimum-scale=1.0, maximum-scale=1.0, user-scalable=no">
|
||||
<title>Data Version Control (DVC) - Product Documentation</title>
|
||||
<link href="http://fonts.googleapis.com/css?family=Lora" rel="stylesheet" type="text/css">
|
||||
<link rel="stylesheet" href="css/style.css">
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 269 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 60 KiB |
@@ -1,133 +0,0 @@
|
||||
/**
|
||||
* jQuery.LocalScroll
|
||||
* Copyright (c) 2007-2009 Ariel Flesler - aflesler(at)gmail(dot)com | http://flesler.blogspot.com
|
||||
* Dual licensed under MIT and GPL.
|
||||
* Date: 3/11/2009
|
||||
*
|
||||
* @projectDescription Animated scrolling navigation, using anchors.
|
||||
* http://flesler.blogspot.com/2007/10/jquerylocalscroll-10.html
|
||||
* @author Ariel Flesler
|
||||
* @version 1.2.7
|
||||
*
|
||||
* @id jQuery.fn.localScroll
|
||||
* @param {Object} settings Hash of settings, it is passed in to jQuery.ScrollTo, none is required.
|
||||
* @return {jQuery} Returns the same jQuery object, for chaining.
|
||||
*
|
||||
* @example $('ul.links').localScroll();
|
||||
*
|
||||
* @example $('ul.links').localScroll({ filter:'.animated', duration:400, axis:'x' });
|
||||
*
|
||||
* @example $.localScroll({ target:'#pane', axis:'xy', queue:true, event:'mouseover' });
|
||||
*
|
||||
* Notes:
|
||||
* - The plugin requires jQuery.ScrollTo.
|
||||
* - The hash of settings, is passed to jQuery.ScrollTo, so the settings are valid for that plugin as well.
|
||||
* - jQuery.localScroll can be used if the desired links, are all over the document, it accepts the same settings.
|
||||
* - If the setting 'lazy' is set to true, then the binding will still work for later added anchors.
|
||||
* - If onBefore returns false, the event is ignored.
|
||||
**/
|
||||
;(function( $ ){
|
||||
var URI = location.href.replace(/#.*/,''); // local url without hash
|
||||
|
||||
var $localScroll = $.localScroll = function( settings ){
|
||||
$('body').localScroll( settings );
|
||||
};
|
||||
|
||||
// Many of these defaults, belong to jQuery.ScrollTo, check it's demo for an example of each option.
|
||||
// @see http://flesler.demos.com/jquery/scrollTo/
|
||||
// The defaults are public and can be overriden.
|
||||
$localScroll.defaults = {
|
||||
duration:1000, // How long to animate.
|
||||
axis:'y', // Which of top and left should be modified.
|
||||
event:'click', // On which event to react.
|
||||
stop:true, // Avoid queuing animations
|
||||
target: window, // What to scroll (selector or element). The whole window by default.
|
||||
reset: true // Used by $.localScroll.hash. If true, elements' scroll is resetted before actual scrolling
|
||||
/*
|
||||
lock:false, // ignore events if already animating
|
||||
lazy:false, // if true, links can be added later, and will still work.
|
||||
filter:null, // filter some anchors out of the matched elements.
|
||||
hash: false // if true, the hash of the selected link, will appear on the address bar.
|
||||
*/
|
||||
};
|
||||
|
||||
// If the URL contains a hash, it will scroll to the pointed element
|
||||
$localScroll.hash = function( settings ){
|
||||
if( location.hash ){
|
||||
settings = $.extend( {}, $localScroll.defaults, settings );
|
||||
settings.hash = false; // can't be true
|
||||
|
||||
if( settings.reset ){
|
||||
var d = settings.duration;
|
||||
delete settings.duration;
|
||||
$(settings.target).scrollTo( 0, settings );
|
||||
settings.duration = d;
|
||||
}
|
||||
scroll( 0, location, settings );
|
||||
}
|
||||
};
|
||||
|
||||
$.fn.localScroll = function( settings ){
|
||||
settings = $.extend( {}, $localScroll.defaults, settings );
|
||||
|
||||
return settings.lazy ?
|
||||
// use event delegation, more links can be added later.
|
||||
this.bind( settings.event, function( e ){
|
||||
// Could use closest(), but that would leave out jQuery -1.3.x
|
||||
var a = $([e.target, e.target.parentNode]).filter(filter)[0];
|
||||
// if a valid link was clicked
|
||||
if( a )
|
||||
scroll( e, a, settings ); // do scroll.
|
||||
}) :
|
||||
// bind concretely, to each matching link
|
||||
this.find('a,area')
|
||||
.filter( filter ).bind( settings.event, function(e){
|
||||
scroll( e, this, settings );
|
||||
}).end()
|
||||
.end();
|
||||
|
||||
function filter(){// is this a link that points to an anchor and passes a possible filter ? href is checked to avoid a bug in FF.
|
||||
return !!this.href && !!this.hash && this.href.replace(this.hash,'') == URI && (!settings.filter || $(this).is( settings.filter ));
|
||||
};
|
||||
};
|
||||
|
||||
function scroll( e, link, settings ){
|
||||
var id = link.hash.slice(1),
|
||||
elem = document.getElementById(id) || document.getElementsByName(id)[0];
|
||||
|
||||
if ( !elem )
|
||||
return;
|
||||
|
||||
if( e )
|
||||
e.preventDefault();
|
||||
|
||||
var $target = $( settings.target );
|
||||
|
||||
if( settings.lock && $target.is(':animated') ||
|
||||
settings.onBefore && settings.onBefore.call(settings, e, elem, $target) === false )
|
||||
return;
|
||||
|
||||
if( settings.stop )
|
||||
$target.stop(true); // remove all its animations
|
||||
|
||||
if( settings.hash ){
|
||||
var attr = elem.id == id ? 'id' : 'name',
|
||||
$a = $('<a> </a>').attr(attr, id).css({
|
||||
position:'absolute',
|
||||
top: $(window).scrollTop(),
|
||||
left: $(window).scrollLeft()
|
||||
});
|
||||
|
||||
elem[attr] = '';
|
||||
$('body').prepend($a);
|
||||
location = link.hash;
|
||||
$a.remove();
|
||||
elem[attr] = id;
|
||||
}
|
||||
|
||||
$target
|
||||
.scrollTo( elem, settings ) // do scroll
|
||||
.trigger('notify.serialScroll',[elem]); // notify serialScroll about this change
|
||||
};
|
||||
|
||||
})( jQuery );
|
||||
Vendored
-6
File diff suppressed because one or more lines are too long
@@ -1,218 +0,0 @@
|
||||
/*!
|
||||
* jQuery.ScrollTo
|
||||
* Copyright (c) 2007-2012 Ariel Flesler - aflesler(at)gmail(dot)com | http://flesler.blogspot.com
|
||||
* Dual licensed under MIT and GPL.
|
||||
* Date: 4/09/2012
|
||||
*
|
||||
* @projectDescription Easy element scrolling using jQuery.
|
||||
* http://flesler.blogspot.com/2007/10/jqueryscrollto.html
|
||||
* @author Ariel Flesler
|
||||
* @version 1.4.3.1
|
||||
*
|
||||
* @id jQuery.scrollTo
|
||||
* @id jQuery.fn.scrollTo
|
||||
* @param {String, Number, DOMElement, jQuery, Object} target Where to scroll the matched elements.
|
||||
* The different options for target are:
|
||||
* - A number position (will be applied to all axes).
|
||||
* - A string position ('44', '100px', '+=90', etc ) will be applied to all axes
|
||||
* - A jQuery/DOM element ( logically, child of the element to scroll )
|
||||
* - A string selector, that will be relative to the element to scroll ( 'li:eq(2)', etc )
|
||||
* - A hash { top:x, left:y }, x and y can be any kind of number/string like above.
|
||||
* - A percentage of the container's dimension/s, for example: 50% to go to the middle.
|
||||
* - The string 'max' for go-to-end.
|
||||
* @param {Number, Function} duration The OVERALL length of the animation, this argument can be the settings object instead.
|
||||
* @param {Object,Function} settings Optional set of settings or the onAfter callback.
|
||||
* @option {String} axis Which axis must be scrolled, use 'x', 'y', 'xy' or 'yx'.
|
||||
* @option {Number, Function} duration The OVERALL length of the animation.
|
||||
* @option {String} easing The easing method for the animation.
|
||||
* @option {Boolean} margin If true, the margin of the target element will be deducted from the final position.
|
||||
* @option {Object, Number} offset Add/deduct from the end position. One number for both axes or { top:x, left:y }.
|
||||
* @option {Object, Number} over Add/deduct the height/width multiplied by 'over', can be { top:x, left:y } when using both axes.
|
||||
* @option {Boolean} queue If true, and both axis are given, the 2nd axis will only be animated after the first one ends.
|
||||
* @option {Function} onAfter Function to be called after the scrolling ends.
|
||||
* @option {Function} onAfterFirst If queuing is activated, this function will be called after the first scrolling ends.
|
||||
* @return {jQuery} Returns the same jQuery object, for chaining.
|
||||
*
|
||||
* @desc Scroll to a fixed position
|
||||
* @example $('div').scrollTo( 340 );
|
||||
*
|
||||
* @desc Scroll relatively to the actual position
|
||||
* @example $('div').scrollTo( '+=340px', { axis:'y' } );
|
||||
*
|
||||
* @desc Scroll using a selector (relative to the scrolled element)
|
||||
* @example $('div').scrollTo( 'p.paragraph:eq(2)', 500, { easing:'swing', queue:true, axis:'xy' } );
|
||||
*
|
||||
* @desc Scroll to a DOM element (same for jQuery object)
|
||||
* @example var second_child = document.getElementById('container').firstChild.nextSibling;
|
||||
* $('#container').scrollTo( second_child, { duration:500, axis:'x', onAfter:function(){
|
||||
* alert('scrolled!!');
|
||||
* }});
|
||||
*
|
||||
* @desc Scroll on both axes, to different values
|
||||
* @example $('div').scrollTo( { top: 300, left:'+=200' }, { axis:'xy', offset:-20 } );
|
||||
*/
|
||||
|
||||
;(function( $ ){
|
||||
|
||||
var $scrollTo = $.scrollTo = function( target, duration, settings ){
|
||||
$(window).scrollTo( target, duration, settings );
|
||||
};
|
||||
|
||||
$scrollTo.defaults = {
|
||||
axis:'xy',
|
||||
duration: parseFloat($.fn.jquery) >= 1.3 ? 0 : 1,
|
||||
limit:true
|
||||
};
|
||||
|
||||
// Returns the element that needs to be animated to scroll the window.
|
||||
// Kept for backwards compatibility (specially for localScroll & serialScroll)
|
||||
$scrollTo.window = function( scope ){
|
||||
return $(window)._scrollable();
|
||||
};
|
||||
|
||||
// Hack, hack, hack :)
|
||||
// Returns the real elements to scroll (supports window/iframes, documents and regular nodes)
|
||||
$.fn._scrollable = function(){
|
||||
return this.map(function(){
|
||||
var elem = this,
|
||||
isWin = !elem.nodeName || $.inArray( elem.nodeName.toLowerCase(), ['iframe','#document','html','body'] ) != -1;
|
||||
|
||||
if( !isWin )
|
||||
return elem;
|
||||
|
||||
var doc = (elem.contentWindow || elem).document || elem.ownerDocument || elem;
|
||||
|
||||
return /webkit/i.test(navigator.userAgent) || doc.compatMode == 'BackCompat' ?
|
||||
doc.body :
|
||||
doc.documentElement;
|
||||
});
|
||||
};
|
||||
|
||||
$.fn.scrollTo = function( target, duration, settings ){
|
||||
if( typeof duration == 'object' ){
|
||||
settings = duration;
|
||||
duration = 0;
|
||||
}
|
||||
if( typeof settings == 'function' )
|
||||
settings = { onAfter:settings };
|
||||
|
||||
if( target == 'max' )
|
||||
target = 9e9;
|
||||
|
||||
settings = $.extend( {}, $scrollTo.defaults, settings );
|
||||
// Speed is still recognized for backwards compatibility
|
||||
duration = duration || settings.duration;
|
||||
// Make sure the settings are given right
|
||||
settings.queue = settings.queue && settings.axis.length > 1;
|
||||
|
||||
if( settings.queue )
|
||||
// Let's keep the overall duration
|
||||
duration /= 2;
|
||||
settings.offset = both( settings.offset );
|
||||
settings.over = both( settings.over );
|
||||
|
||||
return this._scrollable().each(function(){
|
||||
// Null target yields nothing, just like jQuery does
|
||||
if (target == null) return;
|
||||
|
||||
var elem = this,
|
||||
$elem = $(elem),
|
||||
targ = target, toff, attr = {},
|
||||
win = $elem.is('html,body');
|
||||
|
||||
switch( typeof targ ){
|
||||
// A number will pass the regex
|
||||
case 'number':
|
||||
case 'string':
|
||||
if( /^([+-]=)?\d+(\.\d+)?(px|%)?$/.test(targ) ){
|
||||
targ = both( targ );
|
||||
// We are done
|
||||
break;
|
||||
}
|
||||
// Relative selector, no break!
|
||||
targ = $(targ,this);
|
||||
if (!targ.length) return;
|
||||
case 'object':
|
||||
// DOMElement / jQuery
|
||||
if( targ.is || targ.style )
|
||||
// Get the real position of the target
|
||||
toff = (targ = $(targ)).offset();
|
||||
}
|
||||
$.each( settings.axis.split(''), function( i, axis ){
|
||||
var Pos = axis == 'x' ? 'Left' : 'Top',
|
||||
pos = Pos.toLowerCase(),
|
||||
key = 'scroll' + Pos,
|
||||
old = elem[key],
|
||||
max = $scrollTo.max(elem, axis);
|
||||
|
||||
if( toff ){// jQuery / DOMElement
|
||||
attr[key] = toff[pos] + ( win ? 0 : old - $elem.offset()[pos] );
|
||||
|
||||
// If it's a dom element, reduce the margin
|
||||
if( settings.margin ){
|
||||
attr[key] -= parseInt(targ.css('margin'+Pos)) || 0;
|
||||
attr[key] -= parseInt(targ.css('border'+Pos+'Width')) || 0;
|
||||
}
|
||||
|
||||
attr[key] += settings.offset[pos] || 0;
|
||||
|
||||
if( settings.over[pos] )
|
||||
// Scroll to a fraction of its width/height
|
||||
attr[key] += targ[axis=='x'?'width':'height']() * settings.over[pos];
|
||||
}else{
|
||||
var val = targ[pos];
|
||||
// Handle percentage values
|
||||
attr[key] = val.slice && val.slice(-1) == '%' ?
|
||||
parseFloat(val) / 100 * max
|
||||
: val;
|
||||
}
|
||||
|
||||
// Number or 'number'
|
||||
if( settings.limit && /^\d+$/.test(attr[key]) )
|
||||
// Check the limits
|
||||
attr[key] = attr[key] <= 0 ? 0 : Math.min( attr[key], max );
|
||||
|
||||
// Queueing axes
|
||||
if( !i && settings.queue ){
|
||||
// Don't waste time animating, if there's no need.
|
||||
if( old != attr[key] )
|
||||
// Intermediate animation
|
||||
animate( settings.onAfterFirst );
|
||||
// Don't animate this axis again in the next iteration.
|
||||
delete attr[key];
|
||||
}
|
||||
});
|
||||
|
||||
animate( settings.onAfter );
|
||||
|
||||
function animate( callback ){
|
||||
$elem.animate( attr, duration, settings.easing, callback && function(){
|
||||
callback.call(this, target, settings);
|
||||
});
|
||||
};
|
||||
|
||||
}).end();
|
||||
};
|
||||
|
||||
// Max scrolling position, works on quirks mode
|
||||
// It only fails (not too badly) on IE, quirks mode.
|
||||
$scrollTo.max = function( elem, axis ){
|
||||
var Dim = axis == 'x' ? 'Width' : 'Height',
|
||||
scroll = 'scroll'+Dim;
|
||||
|
||||
if( !$(elem).is('html,body') )
|
||||
return elem[scroll] - $(elem)[Dim.toLowerCase()]();
|
||||
|
||||
var size = 'client' + Dim,
|
||||
html = elem.ownerDocument.documentElement,
|
||||
body = elem.ownerDocument.body;
|
||||
|
||||
return Math.max( html[scroll], body[scroll] )
|
||||
- Math.min( html[size] , body[size] );
|
||||
};
|
||||
|
||||
function both( val ){
|
||||
return typeof val == 'object' ? val : { top:val, left:val };
|
||||
};
|
||||
|
||||
})( jQuery );
|
||||
@@ -1,83 +0,0 @@
|
||||
$(document).ready(function () {
|
||||
var AFFIX_TOP_LIMIT = 300;
|
||||
var AFFIX_OFFSET = 49;
|
||||
$('#menu-left').localScroll({hash:true, onAfterFirst:function(){$('html, body').scrollTo( {top:'-=25px'}, 'fast' );}});
|
||||
var $menu = $("#menu"),
|
||||
$btn = $("#menu-toggle");
|
||||
|
||||
$("#menu-toggle").on("click", function () {
|
||||
$menu.toggleClass("open");
|
||||
return false;
|
||||
});
|
||||
|
||||
|
||||
$(".docs-nav").each(function () {
|
||||
var $affixNav = $(this),
|
||||
$container = $affixNav.parent(),
|
||||
affixNavfixed = false,
|
||||
originalClassName = this.className,
|
||||
current = null,
|
||||
$links = $affixNav.find("a");
|
||||
|
||||
function getClosestHeader(top) {
|
||||
var last = $links.first();
|
||||
|
||||
if (top < AFFIX_TOP_LIMIT) {
|
||||
return last;
|
||||
}
|
||||
|
||||
for (var i = 0; i < $links.length; i++) {
|
||||
var $link = $links.eq(i),
|
||||
href = $link.attr("href");
|
||||
|
||||
if (href.charAt(0) === "#" && href.length > 1) {
|
||||
var $anchor = $(href).first();
|
||||
|
||||
if ($anchor.length > 0) {
|
||||
var offset = $anchor.offset();
|
||||
|
||||
if (top < offset.top - AFFIX_OFFSET) {
|
||||
return last;
|
||||
}
|
||||
|
||||
last = $link;
|
||||
}
|
||||
}
|
||||
}
|
||||
return last;
|
||||
}
|
||||
|
||||
|
||||
$(window).on("scroll", function (evt) {
|
||||
var top = window.scrollY,
|
||||
height = $affixNav.outerHeight(),
|
||||
max_bottom = $container.offset().top + $container.outerHeight(),
|
||||
bottom = top + height + AFFIX_OFFSET;
|
||||
|
||||
if (affixNavfixed) {
|
||||
if (top <= AFFIX_TOP_LIMIT) {
|
||||
$affixNav.removeClass("fixed");
|
||||
$affixNav.css("top", 0);
|
||||
affixNavfixed = false;
|
||||
} else if (bottom > max_bottom) {
|
||||
$affixNav.css("top", (max_bottom - height) - top);
|
||||
} else {
|
||||
$affixNav.css("top", AFFIX_OFFSET);
|
||||
}
|
||||
} else if (top > AFFIX_TOP_LIMIT) {
|
||||
$affixNav.addClass("fixed");
|
||||
affixNavfixed = true;
|
||||
}
|
||||
|
||||
var $current = getClosestHeader(top);
|
||||
|
||||
if (current !== $current) {
|
||||
$affixNav.find(".active").removeClass("active");
|
||||
$current.addClass("active");
|
||||
current = $current;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
prettyPrint();
|
||||
});
|
||||
File diff suppressed because one or more lines are too long
@@ -1,117 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
mkdir myrepo
|
||||
cd myrepo
|
||||
|
||||
# 1. Set up environment and get code
|
||||
git init
|
||||
echo "This is an empty readme" > README.md
|
||||
git add README.md
|
||||
git commit -m 'add readme'
|
||||
|
||||
mkdir code
|
||||
wget -nv -P code/ https://s3-us-west-2.amazonaws.com/dvc-share/so/code/featurization.py \
|
||||
https://s3-us-west-2.amazonaws.com/dvc-share/so/code/evaluate.py \
|
||||
https://s3-us-west-2.amazonaws.com/dvc-share/so/code/train_model.py \
|
||||
https://s3-us-west-2.amazonaws.com/dvc-share/so/code/split_train_test.py \
|
||||
https://s3-us-west-2.amazonaws.com/dvc-share/so/code/xml_to_tsv.py \
|
||||
https://s3-us-west-2.amazonaws.com/dvc-share/so/code/requirements.txt
|
||||
pip install -U -r code/requirements.txt
|
||||
git add code/
|
||||
git commit -m 'Download code'
|
||||
dvc init
|
||||
git commit -m 'Init DVC'
|
||||
|
||||
# 2. First steps in 100K dataset
|
||||
git checkout -b input_100K
|
||||
mkdir data
|
||||
wget https://s3-us-west-2.amazonaws.com/dvc-share/so/100K/Posts.xml.tgz -O data/Posts.xml.tgz
|
||||
#cp ../stackoverflow_small_xml/100K/Posts.xml.tgz data/
|
||||
dvc add data/Posts.xml.tgz
|
||||
dvc run -o data/Posts.xml -d data/Posts.xml.tgz tar zxf data/Posts.xml.tgz -C data/
|
||||
dvc run -d code/xml_to_tsv.py -d data/Posts.xml -o data/Posts.tsv python code/xml_to_tsv.py data/Posts.xml data/Posts.tsv python
|
||||
git add .
|
||||
git commit -m 'Extract TSV for 100K dataset'
|
||||
|
||||
# 3. Define full pipeline for 25K dataset
|
||||
git checkout master
|
||||
dvc checkout
|
||||
git checkout -b input_25K
|
||||
dvc checkout
|
||||
mkdir data
|
||||
wget https://s3-us-west-2.amazonaws.com/dvc-share/so/25K/Posts.xml.tgz -O data/Posts.xml.tgz
|
||||
#cp ../stackoverflow_small_xml/25K/Posts.xml.tgz data/
|
||||
dvc add data/Posts.xml.tgz
|
||||
dvc run -o data/Posts.xml -d data/Posts.xml.tgz tar zxf data/Posts.xml.tgz -C data/
|
||||
dvc run -d code/xml_to_tsv.py -d data/Posts.xml -o data/Posts.tsv python code/xml_to_tsv.py data/Posts.xml data/Posts.tsv python
|
||||
|
||||
dvc run -d code/split_train_test.py -d data/Posts.tsv -o data/Posts-train.tsv -o data/Posts-test.tsv python code/split_train_test.py data/Posts.tsv 0.33 20170426 data/Posts-train.tsv data/Posts-test.tsv
|
||||
dvc run -d code/featurization.py -d data/Posts-train.tsv -d data/Posts-test.tsv -o data/matrix-train.p -o data/matrix-test.p python code/featurization.py data/Posts-train.tsv data/Posts-test.tsv data/matrix-train.p data/matrix-test.p
|
||||
dvc run -d code/train_model.py -d data/matrix-train.p -o data/model.p python code/train_model.py data/matrix-train.p 20170426 data/model.p
|
||||
dvc run -d code/evaluate.py -d data/model.p -d data/matrix-test.p -O data/eval_auc.txt python code/evaluate.py data/model.p data/matrix-test.p data/eval_auc.txt
|
||||
dvc run -d data/eval_auc.txt
|
||||
cat data/eval_auc.txt
|
||||
# AUC: 0.596182
|
||||
|
||||
git status
|
||||
git add .
|
||||
git commit -m 'Full pipeline'
|
||||
|
||||
# 4. Integrate the 25K dataset pipeline into 100K branch
|
||||
git checkout input_100K
|
||||
dvc checkout
|
||||
git merge -X ours input_25K # You can also merge and resolve conflicts by hands!!!
|
||||
|
||||
dvc repro
|
||||
git add .
|
||||
git commit -m 'Repro'
|
||||
|
||||
# 5. Improve the 25K model
|
||||
git checkout input_25K
|
||||
dvc checkout
|
||||
vi code/train_model.py # Change: estimators=500 and n_jobs=6
|
||||
dvc repro
|
||||
git commit -am 'estimators=500'
|
||||
cat data/eval_auc.txt
|
||||
# 0.619262
|
||||
|
||||
vi code/featurization.py # Change: add ngram_range=(1, 2) to CountVectorizer
|
||||
dvc repro
|
||||
git commit -am 'Add bigrams'
|
||||
cat data/eval_auc.txt
|
||||
# 0.628989
|
||||
|
||||
vi code/featurization.py # Change: ngram_range=(1, 3)
|
||||
dvc repro
|
||||
git commit -am 'Add three-grams'
|
||||
cat data/eval_auc.txt
|
||||
# 0.630682
|
||||
|
||||
vi code/featurization.py # Change: ngram_range=(1, 4)
|
||||
dvc repro
|
||||
git commit -am 'Add 4-grams'
|
||||
cat data/eval_auc.txt
|
||||
# 0.621002
|
||||
|
||||
# 6. Revert the last step back
|
||||
# Create a branch for unsuccessful commit
|
||||
git checkout -b four_grams
|
||||
dvc checkout
|
||||
# The last successful commit which is the second last humman-created commit
|
||||
git log --pretty=oneline | grep -v "DVC repro-run: " | head -n 2 | tail -n 1
|
||||
# Move the current HEAD to the last successful commit
|
||||
git branch -f input_25K `git log --pretty=oneline | grep -v "DVC repro-run: " | head -n 2 | tail -n 1 | cut -d" " -f1`
|
||||
git checkout input_25K
|
||||
dvc checkout
|
||||
|
||||
# 7. Integrate the 25K model improvements into the 100K model
|
||||
git checkout input_100K
|
||||
dvc checkout
|
||||
git merge -X theirs input_25K
|
||||
dvc repro
|
||||
git commit -am 'Final model for 100K dataset'
|
||||
cat data/eval_auc.txt
|
||||
# 0.652154
|
||||
|
||||
@@ -1,227 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -x
|
||||
|
||||
GIT_REMOTE=https://github.com/dmpetrov/cnn_classify.git
|
||||
|
||||
mkdir cnn_classify
|
||||
cd cnn_classify
|
||||
git init
|
||||
dvc init
|
||||
dvc config AWS.StoragePath dvc-share/dvc_tutorial
|
||||
git add .
|
||||
git commit -m 'Init DVC'
|
||||
|
||||
mkdir data
|
||||
cp ../train.zip data/
|
||||
dvc add data/train.zip
|
||||
git add data/
|
||||
git status
|
||||
#On branch master
|
||||
#Changes to be committed:
|
||||
# (use "git reset HEAD <file>..." to unstage)
|
||||
#
|
||||
# new file: data/.gitignore
|
||||
# new file: data/train.zip.dvc
|
||||
git commit -m 'Raw data'
|
||||
|
||||
dvc run -d data/train.zip -o data/train unzip -d data/ data/train.zip
|
||||
git status
|
||||
#On branch master
|
||||
#Changes to be committed:
|
||||
# (use "git reset HEAD <file>..." to unstage)
|
||||
# (use "git checkout -- <file>..." to discard changes in working directory)
|
||||
#
|
||||
# modified: data/.gitignore
|
||||
#
|
||||
#Untracked files:
|
||||
# (use "git add <file>..." to include in what will be committed)
|
||||
# new file: train.dvc
|
||||
#
|
||||
#no changes added to commit (use "git add" and/or "git commit -a")
|
||||
|
||||
git add data/.gitignore train.dvc
|
||||
git commit -m 'Unzip files'
|
||||
|
||||
mkdir code
|
||||
wget -P code https://s3-us-west-2.amazonaws.com/dvc-share/tutorial_cnn_code/process_files.py
|
||||
wget -P code https://s3-us-west-2.amazonaws.com/dvc-share/tutorial_cnn_code/conf.py
|
||||
dvc run -d code/process_files.py -d data/train -o data/cats_and_dogs_small -f process.dvc python code/process_files.py
|
||||
git status
|
||||
#On branch master
|
||||
#Changes not staged for commit:
|
||||
# (use "git add <file>..." to update what will be committed)
|
||||
# (use "git checkout -- <file>..." to discard changes in working directory)
|
||||
#
|
||||
# modified: data/.gitignore
|
||||
#
|
||||
#Untracked files:
|
||||
# (use "git add <file>..." to include in what will be committed)
|
||||
#
|
||||
# code/
|
||||
# process.dvc
|
||||
#
|
||||
#no changes added to commit (use "git add" and/or "git commit -a")
|
||||
|
||||
echo '__pycache__/' >> code/.gitignore
|
||||
echo '*.pyc' >> code/.gitignore
|
||||
git add .
|
||||
git commit -m 'Process raw data'
|
||||
|
||||
wget -O code/model.py https://s3-us-west-2.amazonaws.com/dvc-share/tutorial_cnn_code/model_first_cnn.py
|
||||
|
||||
dvc run -d code/model.py -d code/conf.py -d data/cats_and_dogs_small -o data/model.h5 -o data/history.p python code/model.py
|
||||
git status
|
||||
#On branch master
|
||||
#Changes not staged for commit:
|
||||
# (use "git add <file>..." to update what will be committed)
|
||||
# (use "git checkout -- <file>..." to discard changes in working directory)
|
||||
#
|
||||
# modified: data/.gitignore
|
||||
#
|
||||
#Untracked files:
|
||||
# (use "git add <file>..." to include in what will be committed)
|
||||
#
|
||||
# code/model.py
|
||||
# model.h5.dvc
|
||||
#
|
||||
#no changes added to commit (use "git add" and/or "git commit -a")
|
||||
|
||||
git add .
|
||||
git commit -m 'First model'
|
||||
|
||||
wget -P code https://s3-us-west-2.amazonaws.com/dvc-share/tutorial_cnn_code/plot.py
|
||||
dvc run -d code/conf.py -d code/plot.py -d data/history.p -o data/plot_loss.jpeg -o data/plot_acc.jpeg python code/plot.py
|
||||
|
||||
dvc run -d data/plot_loss.jpeg
|
||||
|
||||
git status
|
||||
#On branch master
|
||||
#Changes not staged for commit:
|
||||
# (use "git add <file>..." to update what will be committed)
|
||||
# (use "git checkout -- <file>..." to discard changes in working directory)
|
||||
#
|
||||
# modified: data/.gitignore
|
||||
#
|
||||
#Untracked files:
|
||||
# (use "git add <file>..." to include in what will be committed)
|
||||
#
|
||||
# Dvcfile
|
||||
# code/plot.py
|
||||
# plot_loss.jpeg.dvc
|
||||
#
|
||||
#no changes added to commit (use "git add" and/or "git commit -a")
|
||||
|
||||
git add .
|
||||
git commit -m 'Accuracy and loss plots'
|
||||
|
||||
git tag -a v0.1-first-cnn -m 'First CNN model'
|
||||
|
||||
|
||||
############################################ v2
|
||||
rm code/model.py
|
||||
wget -O code/model.py https://s3-us-west-2.amazonaws.com/dvc-share/tutorial_cnn_code/model_augm.py
|
||||
# Actual changes: augmentation, Dropout=0.5, batch_size=20-->32, epochs=30-->100
|
||||
dvc repro
|
||||
|
||||
git status
|
||||
#On branch master
|
||||
#Changes not staged for commit:
|
||||
# (use "git add <file>..." to update what will be committed)
|
||||
# (use "git checkout -- <file>..." to discard changes in working directory)
|
||||
#
|
||||
# modified: Dvcfile
|
||||
# modified: code/model.py
|
||||
# modified: model.h5.dvc
|
||||
# modified: plot_loss.jpeg.dvc
|
||||
#
|
||||
#no changes added to commit (use "git add" and/or "git commit -a")
|
||||
|
||||
git add .
|
||||
git commit -m 'Retrain augmentation model'
|
||||
git tag -a v0.2-augm-cnn -m 'CNN model with augmentation'
|
||||
|
||||
########################################### v3
|
||||
|
||||
git checkout v0.1-first-cnn -b pre_trained
|
||||
dvc checkout
|
||||
rm code/model.py
|
||||
wget -O code/model.py https://s3-us-west-2.amazonaws.com/dvc-share/tutorial_cnn_code/model_vgg16_base.py
|
||||
|
||||
dvc repro
|
||||
git status
|
||||
#On branch pre_trained
|
||||
#Changes not staged for commit:
|
||||
# (use "git add <file>..." to update what will be committed)
|
||||
# (use "git checkout -- <file>..." to discard changes in working directory)
|
||||
#
|
||||
# modified: Dvcfile
|
||||
# modified: code/model.py
|
||||
# modified: model.h5.dvc
|
||||
# modified: plot_loss.jpeg.dvc
|
||||
#
|
||||
#no changes added to commit (use "git add" and/or "git commit -a")
|
||||
|
||||
git add .
|
||||
git commit -m 'Pretrained VGG16'
|
||||
git tag -a v0.3-vgg16_base -m 'Pretrained VGG16 tag'
|
||||
|
||||
|
||||
############################################### v4
|
||||
|
||||
git checkout v0.2-augm-cnn -b vgg16_augm
|
||||
dvc checkout
|
||||
rm code/model.py
|
||||
wget -O code/model.py https://s3-us-west-2.amazonaws.com/dvc-share/tutorial_cnn_code/model_vgg16_augm.py
|
||||
|
||||
dvc repro
|
||||
git status
|
||||
#On branch vgg16_augm
|
||||
#Changes not staged for commit:
|
||||
# (use "git add <file>..." to update what will be committed)
|
||||
# (use "git checkout -- <file>..." to discard changes in working directory)
|
||||
#
|
||||
# modified: Dvcfile
|
||||
# modified: code/model.py
|
||||
# modified: model.h5.dvc
|
||||
# modified: plot_loss.jpeg.dvc
|
||||
#
|
||||
#no changes added to commit (use "git add" and/or "git commit -a")
|
||||
|
||||
git add .
|
||||
git commit -m 'VGG16 with augm'
|
||||
git tag -a v0.4-vgg16_augm -m 'VGG16 with augm tag'
|
||||
|
||||
################################################# v5
|
||||
|
||||
rm code/model.py
|
||||
wget -O code/model.py https://s3-us-west-2.amazonaws.com/dvc-share/tutorial_cnn_code/model_vgg16_augm_fine.py
|
||||
dvc repro
|
||||
git status
|
||||
#On branch vgg16_augm
|
||||
#Changes not staged for commit:
|
||||
# (use "git add <file>..." to update what will be committed)
|
||||
# (use "git checkout -- <file>..." to discard changes in working directory)
|
||||
#
|
||||
# modified: Dvcfile
|
||||
# modified: code/model.py
|
||||
# modified: model.h5.dvc
|
||||
# modified: plot_loss.jpeg.dvc
|
||||
#
|
||||
#no changes added to commit (use "git add" and/or "git commit -a")
|
||||
|
||||
git add .
|
||||
git commit -m 'VGG, augmentation and fine-tuning'
|
||||
git tag -a v0.5-vgg_augm_fine -m 'VGG, augmentation and fine-tuning tag'
|
||||
|
||||
git checkout master
|
||||
git merge vgg16_augm
|
||||
|
||||
git remote add origin ${GIT_REMOTE}
|
||||
git push -u origin master --tags
|
||||
|
||||
git push -u origin pre_trained
|
||||
git push -u origin vgg16_augm
|
||||
|
||||
dvc push
|
||||
|
||||
+18
-7
@@ -4,8 +4,10 @@ DVC
|
||||
Make your data science projects reproducible and shareable.
|
||||
"""
|
||||
import os
|
||||
import warnings
|
||||
|
||||
VERSION_BASE = '0.9.7'
|
||||
|
||||
VERSION_BASE = '0.19.10'
|
||||
__version__ = VERSION_BASE
|
||||
|
||||
PACKAGEPATH = os.path.abspath(os.path.dirname(__file__))
|
||||
@@ -15,7 +17,8 @@ VERSIONPATH = os.path.join(PACKAGEPATH, 'version.py')
|
||||
if os.path.exists(os.path.join(HOMEPATH, 'setup.py')):
|
||||
# dvc is run directly from source without installation or
|
||||
# __version__ is called from setup.py
|
||||
if os.getenv('APPVEYOR_REPO_TAG', '').lower() != 'true' and os.getenv('TRAVIS_TAG', '') == '':
|
||||
if os.getenv('APPVEYOR_REPO_TAG', '').lower() != 'true' \
|
||||
and os.getenv('TRAVIS_TAG', '') == '':
|
||||
# Dynamically update version
|
||||
try:
|
||||
import git
|
||||
@@ -29,20 +32,28 @@ if os.path.exists(os.path.join(HOMEPATH, 'setup.py')):
|
||||
# and will provide a true version of the installed dvc
|
||||
with open(VERSIONPATH, 'w+') as fd:
|
||||
fd.write('# AUTOGENERATED by dvc/__init__.py\n')
|
||||
fd.write('version = "{}"'.format(__version__))
|
||||
except:
|
||||
fd.write('version = "{}"\n'.format(__version__))
|
||||
except Exception: # pragma: no cover
|
||||
pass
|
||||
else:
|
||||
else: # pragma: no cover
|
||||
# Remove version.py so that it doesn't get into the release
|
||||
if os.path.exists(VERSIONPATH):
|
||||
os.unlink(VERSIONPATH)
|
||||
else:
|
||||
else: # pragma: no cover
|
||||
# dvc was installed with pip or something. Hopefully we have our
|
||||
# auto-generated version.py to help us provide a true version
|
||||
try:
|
||||
from dvc.version import version
|
||||
__version__ = version
|
||||
except:
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
VERSION = __version__
|
||||
|
||||
|
||||
# Ignore numpy's runtime warnings: https://github.com/numpy/numpy/pull/432.
|
||||
# We don't directly import numpy, but our dependency networkx does, causing
|
||||
# these warnings in some environments. Luckily these warnings are benign and
|
||||
# we can simply ignore them so that they don't show up when you are using dvc.
|
||||
warnings.filterwarnings("ignore", message="numpy.dtype size changed")
|
||||
warnings.filterwarnings("ignore", message="numpy.ufunc size changed")
|
||||
|
||||
+33
-195
@@ -1,210 +1,48 @@
|
||||
import os
|
||||
import json
|
||||
import shutil
|
||||
|
||||
from dvc.state import State, LinkState
|
||||
from dvc.system import System
|
||||
from dvc.logger import Logger
|
||||
from dvc.utils import move, remove
|
||||
from dvc.lock import Lock
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.config import Config
|
||||
|
||||
|
||||
class Cache(object):
|
||||
CACHE_DIR = 'cache'
|
||||
CACHE_DIR_LOCK = 'cache.lock'
|
||||
CACHE_TYPES = ['reflink', 'hardlink', 'symlink', 'copy']
|
||||
CACHE_TYPE_MAP = {
|
||||
'copy': shutil.copyfile,
|
||||
'symlink': System.symlink,
|
||||
'hardlink': System.hardlink,
|
||||
'reflink': System.reflink,
|
||||
}
|
||||
|
||||
def __init__(self, root_dir, dvc_dir, cache_dir=None, cache_type=None):
|
||||
self.cache_type = cache_type
|
||||
def __init__(self, project):
|
||||
from dvc.remote import Remote
|
||||
|
||||
cache_dir = cache_dir if cache_dir else self.CACHE_DIR
|
||||
if os.path.isabs(cache_dir):
|
||||
self.cache_dir = cache_dir
|
||||
self.project = project
|
||||
|
||||
config = project.config._config[Config.SECTION_CACHE]
|
||||
|
||||
local = config.get(Config.SECTION_CACHE_LOCAL, None)
|
||||
if local:
|
||||
name = Config.SECTION_REMOTE_FMT.format(local)
|
||||
sect = project.config._config[name]
|
||||
else:
|
||||
self.cache_dir = os.path.abspath(os.path.realpath(os.path.join(dvc_dir, cache_dir)))
|
||||
sect = {}
|
||||
cache_dir = config.get(Config.SECTION_CACHE_DIR, self.CACHE_DIR)
|
||||
if not os.path.isabs(cache_dir):
|
||||
cache_dir = os.path.join(project.dvc_dir, cache_dir)
|
||||
cache_dir = os.path.abspath(os.path.realpath(cache_dir))
|
||||
sect[Config.SECTION_REMOTE_URL] = cache_dir
|
||||
t = config.get(Config.SECTION_CACHE_TYPE, None)
|
||||
if t:
|
||||
sect[Config.SECTION_CACHE_TYPE] = t
|
||||
|
||||
if not os.path.exists(self.cache_dir):
|
||||
os.mkdir(self.cache_dir)
|
||||
self.local = Remote(project, sect)
|
||||
|
||||
self.state = State(self.cache_dir)
|
||||
self.link_state = LinkState(root_dir, dvc_dir)
|
||||
self.lock = Lock(self.cache_dir, name=self.CACHE_DIR_LOCK)
|
||||
self.s3 = self._get_remote(config, Config.SECTION_CACHE_S3)
|
||||
self.gs = self._get_remote(config, Config.SECTION_CACHE_GS)
|
||||
self.ssh = self._get_remote(config, Config.SECTION_CACHE_SSH)
|
||||
self.hdfs = self._get_remote(config, Config.SECTION_CACHE_HDFS)
|
||||
self.azure = self._get_remote(config, Config.SECTION_CACHE_AZURE)
|
||||
|
||||
@staticmethod
|
||||
def init(root_dir, dvc_dir, cache_dir=None):
|
||||
return Cache(root_dir, dvc_dir, cache_dir=None)
|
||||
def _get_remote(self, config, name):
|
||||
from dvc.remote import Remote
|
||||
|
||||
def all(self):
|
||||
with self.lock:
|
||||
clist = []
|
||||
for entry in os.listdir(self.cache_dir):
|
||||
subdir = os.path.join(self.cache_dir, entry)
|
||||
if not os.path.isdir(subdir):
|
||||
continue
|
||||
|
||||
for cache in os.listdir(subdir):
|
||||
path = os.path.join(subdir, cache)
|
||||
clist.append(path)
|
||||
|
||||
return clist
|
||||
|
||||
def get(self, md5):
|
||||
if not md5:
|
||||
remote = config.get(name, None)
|
||||
if not remote:
|
||||
return None
|
||||
|
||||
return os.path.join(self.cache_dir, md5[0:2], md5[2:])
|
||||
|
||||
def path_to_md5(self, path):
|
||||
relpath = os.path.relpath(path, self.cache_dir)
|
||||
return os.path.dirname(relpath) + os.path.basename(relpath)
|
||||
|
||||
def _changed(self, md5):
|
||||
cache = self.get(md5)
|
||||
if self.state.changed(cache, md5=md5):
|
||||
if os.path.exists(cache):
|
||||
Logger.warn('Corrupted cache file {}'.format(os.path.relpath(cache)))
|
||||
remove(cache)
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def changed(self, md5):
|
||||
with self.lock:
|
||||
return self._changed(md5)
|
||||
|
||||
def link(self, src, link):
|
||||
dname = os.path.dirname(link)
|
||||
if not os.path.exists(dname):
|
||||
os.makedirs(dname)
|
||||
|
||||
if self.cache_type != None:
|
||||
types = [self.cache_type]
|
||||
else:
|
||||
types = self.CACHE_TYPES
|
||||
|
||||
for typ in types:
|
||||
try:
|
||||
self.CACHE_TYPE_MAP[typ](src, link)
|
||||
self.link_state.update(link)
|
||||
return
|
||||
except Exception as exc:
|
||||
msg = 'Cache type \'{}\' is not supported'.format(typ)
|
||||
Logger.debug(msg)
|
||||
if typ == types[-1]:
|
||||
raise DvcException(msg, cause=exc)
|
||||
|
||||
@staticmethod
|
||||
def load_dir_cache(path):
|
||||
if os.path.isabs(path):
|
||||
relpath = os.path.relpath(path)
|
||||
else:
|
||||
relpath = path
|
||||
|
||||
try:
|
||||
with open(path, 'r') as fd:
|
||||
d = json.load(fd)
|
||||
except Exception as exc:
|
||||
msg = u'Failed to load dir cache \'{}\''
|
||||
Logger.error(msg.format(relpath), exc)
|
||||
return []
|
||||
|
||||
if not isinstance(d, list):
|
||||
msg = u'Dir cache file format error \'{}\': skipping the file'
|
||||
Logger.error(msg.format(relpath))
|
||||
return []
|
||||
|
||||
return d
|
||||
|
||||
@staticmethod
|
||||
def get_dir_cache(path):
|
||||
res = {}
|
||||
d = Cache.load_dir_cache(path)
|
||||
|
||||
for entry in d:
|
||||
res[entry[State.PARAM_RELPATH]] = entry[State.PARAM_MD5]
|
||||
|
||||
return res
|
||||
|
||||
def dir_cache(self, cache):
|
||||
res = {}
|
||||
dir_cache = self.get_dir_cache(cache)
|
||||
|
||||
for relpath, md5 in dir_cache.items():
|
||||
res[relpath] = self.get(md5)
|
||||
|
||||
return res
|
||||
|
||||
@staticmethod
|
||||
def is_dir_cache(cache):
|
||||
return cache.endswith(State.MD5_DIR_SUFFIX)
|
||||
|
||||
def _checkout(self, path, md5):
|
||||
cache = self.get(md5)
|
||||
|
||||
if not cache or not os.path.exists(cache) or self._changed(md5):
|
||||
if cache:
|
||||
Logger.warn(u'\'{}({})\': cache file not found'.format(os.path.relpath(cache),
|
||||
os.path.relpath(path)))
|
||||
remove(path)
|
||||
return
|
||||
|
||||
if os.path.exists(path):
|
||||
msg = u'Data \'{}\' exists. Removing before checkout'
|
||||
Logger.debug(msg.format(os.path.relpath(path)))
|
||||
remove(path)
|
||||
|
||||
msg = u'Checking out \'{}\' with cache \'{}\''
|
||||
Logger.debug(msg.format(os.path.relpath(path), os.path.relpath(cache)))
|
||||
|
||||
if not self.is_dir_cache(cache):
|
||||
self.link(cache, path)
|
||||
return
|
||||
|
||||
dir_cache = self.dir_cache(cache)
|
||||
for relpath, c in dir_cache.items():
|
||||
p = os.path.join(path, relpath)
|
||||
self.link(c, p)
|
||||
|
||||
def checkout(self, path, md5):
|
||||
with self.lock:
|
||||
return self._checkout(path, md5)
|
||||
|
||||
def _save_file(self, path):
|
||||
md5 = self.state.update(path)
|
||||
cache = self.get(md5)
|
||||
if self._changed(md5):
|
||||
move(path, cache)
|
||||
self.state.update(cache)
|
||||
self._checkout(path, md5)
|
||||
|
||||
def _save_dir(self, path):
|
||||
md5 = self.state.update(path)
|
||||
cache = self.get(md5)
|
||||
dname = os.path.dirname(cache)
|
||||
dir_info = self.state.collect_dir(path)
|
||||
|
||||
for entry in dir_info:
|
||||
relpath = entry[State.PARAM_RELPATH]
|
||||
p = os.path.join(path, relpath)
|
||||
|
||||
self._save_file(p)
|
||||
|
||||
if not os.path.isdir(dname):
|
||||
os.makedirs(dname)
|
||||
|
||||
with open(cache, 'w+') as fd:
|
||||
json.dump(dir_info, fd, sort_keys=True)
|
||||
|
||||
def save(self, path):
|
||||
with self.lock:
|
||||
if os.path.isdir(path):
|
||||
self._save_dir(path)
|
||||
else:
|
||||
self._save_file(path)
|
||||
name = Config.SECTION_REMOTE_FMT.format(remote)
|
||||
sect = self.project.config._config[name]
|
||||
return Remote(self.project, sect)
|
||||
|
||||
+654
-103
File diff suppressed because it is too large
Load Diff
@@ -1,139 +0,0 @@
|
||||
import os
|
||||
import math
|
||||
import threading
|
||||
|
||||
import boto3
|
||||
|
||||
from dvc.config import Config
|
||||
from dvc.logger import Logger
|
||||
from dvc.progress import progress
|
||||
from dvc.cloud.base import DataCloudError, DataCloudBase
|
||||
|
||||
|
||||
def sizeof_fmt(num, suffix='B'):
|
||||
""" Convert number of bytes to human-readable string """
|
||||
for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
|
||||
if abs(num) < 1024.0:
|
||||
return "%3.1f%s%s" % (num, unit, suffix)
|
||||
num /= 1024.0
|
||||
return "%.1f%s%s" % (num, 'Y', suffix)
|
||||
|
||||
|
||||
def percent_cb(name, complete, total):
|
||||
""" Callback for updating target progress """
|
||||
Logger.debug('{}: {} transferred out of {}'.format(name,
|
||||
sizeof_fmt(complete),
|
||||
sizeof_fmt(total)))
|
||||
progress.update_target(name, complete, total)
|
||||
|
||||
|
||||
def create_cb(name):
|
||||
""" Create callback function for multipart object """
|
||||
return (lambda cur, tot: percent_cb(name, cur, tot))
|
||||
|
||||
|
||||
class Callback(object):
|
||||
def __init__(self, name, total):
|
||||
self.name = name
|
||||
self.total = total
|
||||
self.current = 0
|
||||
self.lock = threading.Lock()
|
||||
|
||||
def __call__(self, byts):
|
||||
with self.lock:
|
||||
self.current += byts
|
||||
progress.update_target(self.name, self.current, self.total)
|
||||
|
||||
|
||||
class AWSKey(object):
|
||||
def __init__(self, bucket, name):
|
||||
self.name = name
|
||||
self.bucket = bucket
|
||||
|
||||
|
||||
class DataCloudAWS(DataCloudBase):
|
||||
""" DataCloud class for Amazon Web Services """
|
||||
REGEX = r'^s3://(?P<path>.*)$'
|
||||
|
||||
@property
|
||||
def profile(self):
|
||||
return self._cloud_settings.cloud_config.get(Config.SECTION_AWS_PROFILE, None)
|
||||
|
||||
def connect(self):
|
||||
session = boto3.Session(profile_name=self.profile)
|
||||
self.s3 = session.resource('s3')
|
||||
bucket = self.s3.Bucket(self.storage_bucket)
|
||||
if bucket is None:
|
||||
raise DataCloudError('Storage path {} is not setup correctly'.format(self.storage_bucket))
|
||||
|
||||
def create_cb_pull(self, name, key):
|
||||
total = self.s3.Object(bucket_name=key.bucket, key=key.name).content_length
|
||||
return Callback(name, total)
|
||||
|
||||
def create_cb_push(self, name, fname):
|
||||
total = os.path.getsize(fname)
|
||||
return Callback(name, total)
|
||||
|
||||
def _pull_key(self, key, fname, no_progress_bar=False):
|
||||
Logger.debug("Pulling key '{}' from bucket '{}' to file '{}'".format(key.name,
|
||||
key.bucket,
|
||||
fname))
|
||||
self._makedirs(fname)
|
||||
|
||||
tmp_file = self.tmp_file(fname)
|
||||
name = os.path.relpath(fname, self._cloud_settings.cache.cache_dir)
|
||||
|
||||
if self._cmp_checksum(key, fname):
|
||||
Logger.debug('File "{}" matches with "{}".'.format(fname, key.name))
|
||||
return fname
|
||||
|
||||
Logger.debug('Downloading cache file from S3 "{}/{}" to "{}"'.format(key.bucket,
|
||||
key.name,
|
||||
fname))
|
||||
|
||||
if no_progress_bar:
|
||||
cb = None
|
||||
else:
|
||||
cb = self.create_cb_pull(name, key)
|
||||
|
||||
|
||||
try:
|
||||
self.s3.Object(key.bucket, key.name).download_file(tmp_file, Callback=cb)
|
||||
except Exception as exc:
|
||||
Logger.error('Failed to download "{}": {}'.format(key.name, exc))
|
||||
return None
|
||||
|
||||
os.rename(tmp_file, fname)
|
||||
|
||||
if not no_progress_bar:
|
||||
progress.finish_target(name)
|
||||
|
||||
Logger.debug('Downloading completed')
|
||||
|
||||
return fname
|
||||
|
||||
def _get_key(self, path):
|
||||
key_name = self.cache_file_key(path)
|
||||
try:
|
||||
self.s3.Object(self.storage_bucket, key_name).get()
|
||||
return AWSKey(self.storage_bucket, key_name)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _new_key(self, path):
|
||||
key_name = self.cache_file_key(path)
|
||||
return AWSKey(self.storage_bucket, key_name)
|
||||
|
||||
def _push_key(self, key, path):
|
||||
""" push, aws version """
|
||||
name = os.path.relpath(path, self._cloud_settings.cache.cache_dir)
|
||||
cb = self.create_cb_push(name, path)
|
||||
try:
|
||||
self.s3.Object(key.bucket, key.name).upload_file(path, Callback=cb)
|
||||
except Exception as exc:
|
||||
Logger.error('Failed to upload "{}": {}'.format(path, exc))
|
||||
return None
|
||||
|
||||
progress.finish_target(name)
|
||||
|
||||
return path
|
||||
@@ -1,229 +0,0 @@
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
|
||||
from dvc.logger import Logger
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.config import Config, ConfigError
|
||||
from dvc.cache import Cache
|
||||
|
||||
|
||||
STATUS_UNKNOWN = 0
|
||||
STATUS_OK = 1
|
||||
STATUS_MODIFIED = 2
|
||||
STATUS_NEW = 3
|
||||
STATUS_DELETED = 4
|
||||
|
||||
|
||||
STATUS_MAP = {
|
||||
# (local_exists, remote_exists, cmp)
|
||||
(True, True, True) : STATUS_OK,
|
||||
(True, True, False) : STATUS_MODIFIED,
|
||||
(True, False, None) : STATUS_NEW,
|
||||
(False, True, None) : STATUS_DELETED,
|
||||
}
|
||||
|
||||
|
||||
class DataCloudError(DvcException):
|
||||
""" Data Cloud exception """
|
||||
def __init__(self, msg):
|
||||
super(DataCloudError, self).__init__('Data sync error: {}'.format(msg))
|
||||
|
||||
|
||||
class CloudSettings(object):
|
||||
def __init__(self, cache=None, global_storage_path=None, cloud_config=None):
|
||||
self.cache = cache
|
||||
self.cloud_config = cloud_config
|
||||
self.global_storage_path = global_storage_path
|
||||
|
||||
|
||||
class DataCloudBase(object):
|
||||
""" Base class for DataCloud """
|
||||
|
||||
REGEX = r''
|
||||
|
||||
def __init__(self, cloud_settings):
|
||||
self._cloud_settings = cloud_settings
|
||||
|
||||
@property
|
||||
def url(self):
|
||||
config = self._cloud_settings.cloud_config
|
||||
return config.get(Config.SECTION_REMOTE_URL, None)
|
||||
|
||||
@classmethod
|
||||
def match(cls, url):
|
||||
return re.match(cls.REGEX, url)
|
||||
|
||||
def group(self, group):
|
||||
return self.match(self.url).group(group)
|
||||
|
||||
@property
|
||||
def path(self):
|
||||
return self.group('path')
|
||||
|
||||
@classmethod
|
||||
def supported(cls, url):
|
||||
return cls.match(url) != None
|
||||
|
||||
# backward compatibility
|
||||
@property
|
||||
def storage_path(self):
|
||||
""" get storage path
|
||||
|
||||
Precedence: Storage, then cloud specific
|
||||
"""
|
||||
|
||||
if self._cloud_settings.global_storage_path:
|
||||
return self._cloud_settings.global_storage_path
|
||||
|
||||
if not self.url:
|
||||
path = self._cloud_settings.cloud_config.get(Config.SECTION_CORE_STORAGEPATH, None)
|
||||
if path:
|
||||
Logger.warn('Using obsoleted config format. Consider updating.')
|
||||
else:
|
||||
path = self.path
|
||||
|
||||
return path
|
||||
|
||||
def _storage_path_parts(self):
|
||||
"""
|
||||
Split storage path into parts. I.e. 'dvc-test/myrepo' -> ['dvc', 'myrepo']
|
||||
"""
|
||||
return self.storage_path.strip('/').split('/', 1)
|
||||
|
||||
@property
|
||||
def storage_bucket(self):
|
||||
""" Data -> StoragePath takes precedence; if doesn't exist, use cloud-specific """
|
||||
return self._storage_path_parts()[0]
|
||||
|
||||
@property
|
||||
def storage_prefix(self):
|
||||
"""
|
||||
Prefix within the bucket. I.e. 'myrepo' in 'dvc-test/myrepo'.
|
||||
"""
|
||||
parts = self._storage_path_parts()
|
||||
if len(parts) > 1:
|
||||
return parts[1]
|
||||
return ''
|
||||
|
||||
def cache_file_key(self, fname):
|
||||
""" Key of a file within the bucket """
|
||||
relpath = os.path.relpath(fname, self._cloud_settings.cache.cache_dir)
|
||||
relpath = relpath.replace('\\', '/')
|
||||
return '{}/{}'.format(self.storage_prefix, relpath).strip('/')
|
||||
|
||||
@staticmethod
|
||||
def tmp_file(fname):
|
||||
""" Temporary name for a partial download """
|
||||
return fname + '.part'
|
||||
|
||||
def sanity_check(self):
|
||||
"""
|
||||
Cloud-specific method to check config for basic requirements.
|
||||
"""
|
||||
pass
|
||||
|
||||
def _push_key(self, key, path):
|
||||
pass
|
||||
|
||||
def collect(self, arg):
|
||||
path, local = arg
|
||||
ret = [path]
|
||||
|
||||
if not Cache.is_dir_cache(path):
|
||||
return ret
|
||||
|
||||
if local:
|
||||
if not os.path.isfile(path):
|
||||
return ret
|
||||
dir_path = path
|
||||
else:
|
||||
key = self._get_key(path)
|
||||
if not key:
|
||||
Logger.debug("File '{}' does not exist in the cloud".format(path))
|
||||
return ret
|
||||
tmp = os.path.join(tempfile.mkdtemp(), os.path.basename(path))
|
||||
self._pull_key(key, tmp, no_progress_bar=True)
|
||||
dir_path = tmp
|
||||
|
||||
for relpath, md5 in Cache.get_dir_cache(dir_path).items():
|
||||
cache = self._cloud_settings.cache.get(md5)
|
||||
ret.append(cache)
|
||||
|
||||
return ret
|
||||
|
||||
def _cmp_checksum(self, blob, fname):
|
||||
md5 = self._cloud_settings.cache.path_to_md5(fname)
|
||||
if self._cloud_settings.cache.state.changed(fname, md5=md5):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def push(self, path):
|
||||
key = self._get_key(path)
|
||||
if key:
|
||||
Logger.debug("File '{}' already uploaded to the cloud. Validating checksum...".format(path))
|
||||
if self._cmp_checksum(key, path):
|
||||
Logger.debug('File checksum matches. No uploading is needed.')
|
||||
return []
|
||||
Logger.debug('Checksum mismatch. Reuploading is required.')
|
||||
|
||||
key = self._new_key(path)
|
||||
return self._push_key(key, path)
|
||||
|
||||
def _makedirs(self, fname):
|
||||
dname = os.path.dirname(fname)
|
||||
try:
|
||||
os.makedirs(dname)
|
||||
except OSError as e:
|
||||
if e.errno != os.errno.EEXIST:
|
||||
raise
|
||||
|
||||
def _pull_key(self, key, path, no_progress_bar=False):
|
||||
""" Cloud-specific method of pulling keys """
|
||||
pass
|
||||
|
||||
def _get_key(self, path):
|
||||
""" Cloud-specific method of getting keys """
|
||||
pass
|
||||
|
||||
def pull(self, path):
|
||||
""" Generic method for pulling data from the cloud """
|
||||
key = self._get_key(path)
|
||||
if not key:
|
||||
Logger.error("File '{}' does not exist in the cloud".format(path))
|
||||
return None
|
||||
|
||||
return self._pull_key(key, path)
|
||||
|
||||
def _status(self, key, path):
|
||||
remote_exists = key != None
|
||||
local_exists = os.path.exists(path)
|
||||
|
||||
diff = None
|
||||
if remote_exists and local_exists:
|
||||
diff = self._cmp_checksum(key, path)
|
||||
|
||||
return STATUS_MAP.get((local_exists, remote_exists, diff), STATUS_UNKNOWN)
|
||||
|
||||
def status(self, path):
|
||||
"""
|
||||
Generic method for checking data item status.
|
||||
"""
|
||||
key = self._get_key(path)
|
||||
if not key:
|
||||
return STATUS_NEW
|
||||
|
||||
return self._status(key, path)
|
||||
|
||||
def connect(self):
|
||||
pass
|
||||
|
||||
def disconnect(self):
|
||||
pass
|
||||
|
||||
def __enter__(self):
|
||||
self.connect()
|
||||
|
||||
def __exit__(self, type, value, tb):
|
||||
self.disconnect()
|
||||
@@ -1,153 +0,0 @@
|
||||
import re
|
||||
from multiprocessing.pool import ThreadPool
|
||||
|
||||
from dvc.logger import Logger
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.config import Config, ConfigError
|
||||
from dvc.utils import map_progress
|
||||
|
||||
from dvc.cloud.aws import DataCloudAWS
|
||||
from dvc.cloud.gcp import DataCloudGCP
|
||||
from dvc.cloud.ssh import DataCloudSSH
|
||||
from dvc.cloud.local import DataCloudLOCAL
|
||||
from dvc.cloud.base import DataCloudBase, CloudSettings
|
||||
from dvc.cloud.base import STATUS_MODIFIED, STATUS_NEW, STATUS_DELETED
|
||||
|
||||
|
||||
class DataCloud(object):
|
||||
""" Generic class to do initial config parsing and redirect to proper DataCloud methods """
|
||||
|
||||
CLOUD_MAP = {
|
||||
'aws' : DataCloudAWS,
|
||||
'gcp' : DataCloudGCP,
|
||||
'ssh' : DataCloudSSH,
|
||||
'local' : DataCloudLOCAL,
|
||||
}
|
||||
|
||||
def __init__(self, cache=None, config=None):
|
||||
self._cache = cache
|
||||
self._config = config
|
||||
|
||||
remote = self._config[Config.SECTION_CORE].get(Config.SECTION_CORE_REMOTE, '')
|
||||
if remote == '':
|
||||
if config[Config.SECTION_CORE].get(Config.SECTION_CORE_CLOUD, None):
|
||||
# backward compatibility
|
||||
Logger.warn('Using obsoleted config format. Consider updating.')
|
||||
self._cloud = self.__init__compat()
|
||||
else:
|
||||
self._cloud = None
|
||||
return
|
||||
|
||||
self._cloud = self._init_remote(remote)
|
||||
|
||||
@staticmethod
|
||||
def supported(url):
|
||||
for cloud in DataCloud.CLOUD_MAP.values():
|
||||
if cloud.supported(url):
|
||||
return cloud
|
||||
return None
|
||||
|
||||
def _init_remote(self, remote):
|
||||
section = Config.SECTION_REMOTE_FMT.format(remote)
|
||||
cloud_config = self._config.get(section, None)
|
||||
if not cloud_config:
|
||||
raise ConfigError("Can't find remote section '{}' in config".format(section))
|
||||
|
||||
url = cloud_config[Config.SECTION_REMOTE_URL]
|
||||
cloud_type = self.supported(url)
|
||||
if not cloud_type:
|
||||
raise ConfigError("Unsupported url '{}'".format(url))
|
||||
|
||||
return self._init_cloud(cloud_config, cloud_type)
|
||||
|
||||
def __init__compat(self):
|
||||
cloud_name = self._config[Config.SECTION_CORE].get(Config.SECTION_CORE_CLOUD, '').strip().lower()
|
||||
if cloud_name == '':
|
||||
self._cloud = None
|
||||
return
|
||||
|
||||
cloud_type = self.CLOUD_MAP.get(cloud_name, None)
|
||||
if not cloud_type:
|
||||
raise ConfigError('Wrong cloud type %s specified' % cloud_name)
|
||||
|
||||
cloud_config = self._config.get(cloud_name, None)
|
||||
if not cloud_config:
|
||||
raise ConfigError('Can\'t find cloud section \'[%s]\' in config' % cloud_name)
|
||||
|
||||
return self._init_cloud(cloud_config, cloud_type)
|
||||
|
||||
def _init_cloud(self, cloud_config, cloud_type):
|
||||
global_storage_path = self._config[Config.SECTION_CORE].get(Config.SECTION_CORE_STORAGEPATH, None)
|
||||
if global_storage_path:
|
||||
Logger.warn('Using obsoleted config format. Consider updating.')
|
||||
|
||||
cloud_settings = CloudSettings(cache=self._cache,
|
||||
global_storage_path=global_storage_path,
|
||||
cloud_config=cloud_config)
|
||||
|
||||
cloud = cloud_type(cloud_settings)
|
||||
cloud.sanity_check()
|
||||
return cloud
|
||||
|
||||
def _collect(self, cloud, targets, jobs, local):
|
||||
collected = set()
|
||||
pool = ThreadPool(processes=jobs)
|
||||
args = zip(targets, [local]*len(targets))
|
||||
ret = pool.map(cloud.collect, args)
|
||||
|
||||
for r in ret:
|
||||
collected |= set(r)
|
||||
|
||||
return collected
|
||||
|
||||
def _get_cloud(self, remote):
|
||||
if remote:
|
||||
return self._init_remote(remote)
|
||||
|
||||
return self._cloud
|
||||
|
||||
def _filter(self, func, status, targets, jobs, remote):
|
||||
cloud = self._get_cloud(remote)
|
||||
if not cloud:
|
||||
return []
|
||||
|
||||
with cloud:
|
||||
filtered = []
|
||||
for t, s in self._status(cloud, targets, jobs):
|
||||
if s == STATUS_MODIFIED or s == status:
|
||||
filtered.append(t)
|
||||
|
||||
return map_progress(getattr(cloud, func), filtered, jobs)
|
||||
|
||||
def push(self, targets, jobs=1, remote=None):
|
||||
"""
|
||||
Push data items in a cloud-agnostic way.
|
||||
"""
|
||||
return self._filter('push', STATUS_NEW, targets, jobs, remote)
|
||||
|
||||
def pull(self, targets, jobs=1, remote=None):
|
||||
"""
|
||||
Pull data items in a cloud-agnostic way.
|
||||
"""
|
||||
return self._filter('pull', STATUS_DELETED, targets, jobs, remote)
|
||||
|
||||
def _collect_targets(self, cloud, targets, jobs=1):
|
||||
collected = set()
|
||||
collected |= self._collect(cloud, targets, jobs, True)
|
||||
collected |= self._collect(cloud, targets, jobs, False)
|
||||
return list(collected)
|
||||
|
||||
def _status(self, cloud, targets, jobs=1):
|
||||
collected = self._collect_targets(cloud, targets, jobs)
|
||||
return map_progress(cloud.status, collected, jobs)
|
||||
|
||||
def status(self, targets, jobs=1, remote=None):
|
||||
"""
|
||||
Check status of data items in a cloud-agnostic way.
|
||||
"""
|
||||
cloud = self._get_cloud(remote)
|
||||
if not cloud:
|
||||
return []
|
||||
|
||||
with cloud:
|
||||
return self._status(cloud, targets, jobs)
|
||||
@@ -1,81 +0,0 @@
|
||||
import base64
|
||||
import os
|
||||
|
||||
from google.cloud import storage as gc
|
||||
|
||||
from dvc.logger import Logger
|
||||
from dvc.config import Config, ConfigError
|
||||
from dvc.cloud.base import DataCloudError, DataCloudBase
|
||||
from dvc.progress import progress
|
||||
|
||||
|
||||
class DataCloudGCP(DataCloudBase):
|
||||
""" DataCloud class for Google Cloud Platform """
|
||||
REGEX = r'^gs://(?P<path>.*)$'
|
||||
|
||||
@property
|
||||
def gc_project_name(self):
|
||||
"""
|
||||
Get project name from config.
|
||||
"""
|
||||
return self._cloud_settings.cloud_config.get(Config.SECTION_GCP_PROJECTNAME, None)
|
||||
|
||||
def connect(self):
|
||||
client = gc.Client(project=self.gc_project_name)
|
||||
self.bucket = client.bucket(self.storage_bucket)
|
||||
if not self.bucket.exists():
|
||||
raise DataCloudError('sync up: google cloud bucket {} '
|
||||
'doesn\'t exist'.format(self.storage_bucket))
|
||||
|
||||
def _pull_key(self, key, path, no_progress_bar=False):
|
||||
self._makedirs(path)
|
||||
|
||||
name = os.path.relpath(path, self._cloud_settings.cache.cache_dir)
|
||||
tmp_file = self.tmp_file(path)
|
||||
|
||||
if self._cmp_checksum(key, path):
|
||||
Logger.debug('File "{}" matches with "{}".'.format(path, key.name))
|
||||
return path
|
||||
|
||||
Logger.debug('Downloading cache file from gc "{}/{}"'.format(key.bucket.name, key.name))
|
||||
|
||||
if not no_progress_bar:
|
||||
# percent_cb is not available for download_to_filename, so
|
||||
# lets at least update progress at keypoints(start, finish)
|
||||
progress.update_target(name, 0, None)
|
||||
|
||||
try:
|
||||
key.download_to_filename(tmp_file)
|
||||
except Exception as exc:
|
||||
Logger.error('Failed to download "{}": {}'.format(key.name, exc))
|
||||
return None
|
||||
|
||||
os.rename(tmp_file, path)
|
||||
|
||||
if not no_progress_bar:
|
||||
progress.finish_target(name)
|
||||
|
||||
Logger.debug('Downloading completed')
|
||||
|
||||
return path
|
||||
|
||||
def _get_key(self, path):
|
||||
key_name = self.cache_file_key(path)
|
||||
return self.bucket.get_blob(key_name)
|
||||
|
||||
def _new_key(self, path):
|
||||
key_name = self.cache_file_key(path)
|
||||
return self.bucket.blob(key_name)
|
||||
|
||||
def _push_key(self, key, path):
|
||||
""" push, gcp version """
|
||||
name = os.path.relpath(path, self._cloud_settings.cache.cache_dir)
|
||||
|
||||
progress.update_target(name, 0, None)
|
||||
|
||||
key.upload_from_filename(path)
|
||||
|
||||
progress.finish_target(name)
|
||||
Logger.debug('uploading %s completed' % path)
|
||||
|
||||
return path
|
||||
@@ -1,58 +0,0 @@
|
||||
import os
|
||||
import filecmp
|
||||
|
||||
from dvc.logger import Logger
|
||||
from dvc.utils import copyfile, file_md5
|
||||
from dvc.cloud.base import DataCloudError, DataCloudBase
|
||||
|
||||
|
||||
class LocalKey(object):
|
||||
def __init__(self, bucket, name):
|
||||
self.name = name
|
||||
self.bucket = bucket
|
||||
|
||||
@property
|
||||
def path(self):
|
||||
return os.path.join(self.bucket, self.name)
|
||||
|
||||
|
||||
class DataCloudLOCAL(DataCloudBase):
|
||||
"""
|
||||
Driver for local storage.
|
||||
"""
|
||||
REGEX = r'^(?P<path>(/+|.:\\+).*)$'
|
||||
|
||||
def cache_file_key(self, path):
|
||||
return os.path.relpath(os.path.abspath(path), self._cloud_settings.cache.cache_dir)
|
||||
|
||||
def _get_key(self, path):
|
||||
key_name = self.cache_file_key(path)
|
||||
key = LocalKey(self.storage_path, key_name)
|
||||
if os.path.exists(key.path) and os.path.isfile(key.path):
|
||||
return key
|
||||
return None
|
||||
|
||||
def _new_key(self, path):
|
||||
key_name = self.cache_file_key(path)
|
||||
key = LocalKey(self.storage_path, key_name)
|
||||
self._makedirs(key.path)
|
||||
return key
|
||||
|
||||
def _push_key(self, key, path):
|
||||
self._makedirs(key.path)
|
||||
copyfile(path, key.path)
|
||||
return path
|
||||
|
||||
def _pull_key(self, key, path, no_progress_bar=False):
|
||||
self._makedirs(path)
|
||||
|
||||
tmp_file = self.tmp_file(path)
|
||||
try:
|
||||
copyfile(key.path, tmp_file, no_progress_bar=no_progress_bar)
|
||||
except Exception as exc:
|
||||
Logger.error('Failed to copy "{}": {}'.format(key.path, exc))
|
||||
return None
|
||||
|
||||
os.rename(tmp_file, path)
|
||||
|
||||
return path
|
||||
@@ -1,110 +0,0 @@
|
||||
import os
|
||||
import errno
|
||||
import filecmp
|
||||
import paramiko
|
||||
import posixpath
|
||||
|
||||
from dvc.logger import Logger
|
||||
from dvc.utils import copyfile, file_md5
|
||||
from dvc.cloud.base import DataCloudError, DataCloudBase
|
||||
from dvc.cloud.aws import create_cb
|
||||
from dvc.progress import progress
|
||||
|
||||
|
||||
class SSHKey(object):
|
||||
def __init__(self, bucket, name):
|
||||
self.name = name
|
||||
self.bucket = bucket
|
||||
|
||||
@property
|
||||
def path(self):
|
||||
return os.path.join(self.bucket, self.name)
|
||||
|
||||
|
||||
class DataCloudSSH(DataCloudBase):
|
||||
"""
|
||||
Driver for remote storage over ssh.
|
||||
"""
|
||||
#NOTE: ~/ paths are temporarily forbidden
|
||||
REGEX = r'^(?P<user>.*)@(?P<host>.*):(?P<path>/+.*)$'
|
||||
|
||||
@property
|
||||
def hostname(self):
|
||||
return self.group('host')
|
||||
|
||||
@property
|
||||
def username(self):
|
||||
return self.group('user')
|
||||
|
||||
def connect(self):
|
||||
self.ssh = paramiko.SSHClient()
|
||||
|
||||
self.ssh.load_system_host_keys()
|
||||
self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
|
||||
self.ssh.connect(self.hostname, username=self.username)
|
||||
|
||||
def disconnect(self):
|
||||
self.ssh.close()
|
||||
|
||||
def get_sftp(self):
|
||||
#NOTE: SFTP doesn't seem to be thread safe
|
||||
return self.ssh.open_sftp()
|
||||
|
||||
def cache_file_key(self, path):
|
||||
relpath = os.path.relpath(os.path.abspath(path), self._cloud_settings.cache.cache_dir)
|
||||
return relpath.replace('\\', '/')
|
||||
|
||||
def _isfile_remote(self, path):
|
||||
try:
|
||||
self.get_sftp().open(path, 'r')
|
||||
return True
|
||||
except IOError as exc:
|
||||
if exc.errno != errno.ENOENT:
|
||||
raise
|
||||
return False
|
||||
|
||||
def _get_key(self, path):
|
||||
key_name = self.cache_file_key(path)
|
||||
key = SSHKey(self.path, key_name)
|
||||
if self._isfile_remote(key.path):
|
||||
return key
|
||||
return None
|
||||
|
||||
def _makedirs_remote(self, dname):
|
||||
try:
|
||||
self.get_sftp().chdir(dname)
|
||||
except IOError as exc:
|
||||
if exc.errno != errno.ENOENT:
|
||||
raise
|
||||
|
||||
parent = posixpath.dirname(dname)
|
||||
if len(parent):
|
||||
self._makedirs_remote(parent)
|
||||
self.get_sftp().mkdir(dname)
|
||||
|
||||
def _new_key(self, path):
|
||||
key_name = self.cache_file_key(path)
|
||||
key = SSHKey(self.path, key_name)
|
||||
self._makedirs_remote(posixpath.dirname(key.path))
|
||||
return key
|
||||
|
||||
def _push_key(self, key, path):
|
||||
self.get_sftp().put(path, key.path, callback=create_cb(key.name))
|
||||
progress.finish_target(key.name)
|
||||
return path
|
||||
|
||||
def _pull_key(self, key, path, no_progress_bar=False):
|
||||
self._makedirs(path)
|
||||
|
||||
tmp_file = self.tmp_file(path)
|
||||
try:
|
||||
self.get_sftp().get(key.path, tmp_file, callback=create_cb(key.name))
|
||||
except Exception as exc:
|
||||
Logger.error('Failed to copy "{}": {}'.format(key.path, exc))
|
||||
return None
|
||||
|
||||
os.rename(tmp_file, path)
|
||||
progress.finish_target(key.name)
|
||||
|
||||
return path
|
||||
+4
-3
@@ -1,13 +1,14 @@
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.command.common.base import CmdBase
|
||||
from dvc.command.base import CmdBase
|
||||
from dvc.logger import Logger
|
||||
|
||||
|
||||
class CmdAdd(CmdBase):
|
||||
def run(self):
|
||||
for target in self.args.targets:
|
||||
try:
|
||||
self.project.add(target)
|
||||
self.project.add(target, recursive=self.args.recursive)
|
||||
except DvcException as ex:
|
||||
self.project.logger.error('Failed to add {}', ex)
|
||||
Logger.error('Failed to add \'{}\''.format(target), ex)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import os
|
||||
|
||||
from dvc.project import Project
|
||||
from dvc.logger import Logger
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.lock import LockError
|
||||
@@ -8,6 +7,8 @@ from dvc.lock import LockError
|
||||
|
||||
class CmdBase(object):
|
||||
def __init__(self, args):
|
||||
from dvc.project import Project
|
||||
|
||||
self.project = Project(self._find_root())
|
||||
self.args = args
|
||||
self._set_loglevel(args)
|
||||
@@ -20,11 +21,15 @@ class CmdBase(object):
|
||||
Logger.be_verbose()
|
||||
|
||||
def _find_root(self):
|
||||
from dvc.project import Project
|
||||
|
||||
root = os.getcwd()
|
||||
while not os.path.ismount(root):
|
||||
while True:
|
||||
dvc_dir = os.path.join(root, Project.DVC_DIR)
|
||||
if os.path.isdir(dvc_dir):
|
||||
return root
|
||||
if os.path.ismount(root):
|
||||
break
|
||||
root = os.path.dirname(root)
|
||||
msg = "Not a dvc repository (checked up to mount point {})"
|
||||
raise DvcException(msg.format(root))
|
||||
@@ -34,7 +39,7 @@ class CmdBase(object):
|
||||
with self.project.lock:
|
||||
return self.run()
|
||||
except LockError as ex:
|
||||
self.project.logger.error('Failed to lock before running a command', ex)
|
||||
Logger.error('Failed to lock before running a command', ex)
|
||||
return 1
|
||||
|
||||
# Abstract methods that have to be implemented by any inheritance class
|
||||
@@ -1,5 +1,4 @@
|
||||
import os
|
||||
from dvc.command.common.base import CmdBase
|
||||
from dvc.command.base import CmdBase
|
||||
|
||||
|
||||
class CmdCheckout(CmdBase):
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
|
||||
class CmdCommonError(DvcException):
|
||||
def __init__(self, msg):
|
||||
super(CmdCommonError, self).__init__('{}'.format(msg))
|
||||
@@ -1,14 +1,15 @@
|
||||
import os
|
||||
import configobj
|
||||
|
||||
from dvc.command.common.base import CmdBase
|
||||
from dvc.command.base import CmdBase
|
||||
from dvc.logger import Logger
|
||||
from dvc.config import Config
|
||||
from dvc.project import Project
|
||||
|
||||
|
||||
class CmdConfig(CmdBase):
|
||||
def __init__(self, args):
|
||||
from dvc.project import Project
|
||||
|
||||
self.args = args
|
||||
root_dir = self._find_root()
|
||||
if args.local:
|
||||
@@ -38,7 +39,8 @@ class CmdConfig(CmdBase):
|
||||
try:
|
||||
self.configobj.write()
|
||||
except Exception as exc:
|
||||
Logger.error("Failed to write config '{}'".format(self.configobj.filename), exc)
|
||||
msg = "Failed to write config '{}'".format(self.configobj.filename)
|
||||
Logger.error(msg, exc)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
@@ -50,7 +52,7 @@ class CmdConfig(CmdBase):
|
||||
if opt in self.configobj[section].keys():
|
||||
del self.configobj[section][opt]
|
||||
|
||||
if len(self.configobj[section]) == 0 or opt == None:
|
||||
if len(self.configobj[section]) == 0 or opt is None:
|
||||
del self.configobj[section]
|
||||
|
||||
return self.save()
|
||||
@@ -69,7 +71,7 @@ class CmdConfig(CmdBase):
|
||||
|
||||
def check_opt(self):
|
||||
_section, _opt = self.args.name.strip().split('.', 1)
|
||||
add = (self.args.value != None and self.args.unset == False)
|
||||
add = (self.args.value is not None and self.args.unset is False)
|
||||
|
||||
section = self._get_key(self.configobj, _section, add)
|
||||
|
||||
|
||||
+26
-10
@@ -1,8 +1,4 @@
|
||||
import os
|
||||
|
||||
import dvc.cloud.base as cloud
|
||||
|
||||
from dvc.command.common.base import CmdBase
|
||||
from dvc.command.base import CmdBase
|
||||
|
||||
|
||||
class CmdDataBase(CmdBase):
|
||||
@@ -23,9 +19,16 @@ class CmdDataBase(CmdBase):
|
||||
class CmdDataPull(CmdDataBase):
|
||||
def do_run(self, target=None):
|
||||
try:
|
||||
self.project.pull(target=target, jobs=self.args.jobs, remote=self.args.remote)
|
||||
self.project.pull(target=target,
|
||||
jobs=self.args.jobs,
|
||||
remote=self.args.remote,
|
||||
show_checksums=self.args.show_checksums,
|
||||
all_branches=self.args.all_branches,
|
||||
all_tags=self.args.all_tags,
|
||||
with_deps=self.args.with_deps)
|
||||
except Exception as exc:
|
||||
self.project.logger.error('Failed to pull data from the cloud', exc)
|
||||
self.project.logger.error('Failed to pull data from the cloud',
|
||||
exc)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
@@ -33,7 +36,13 @@ class CmdDataPull(CmdDataBase):
|
||||
class CmdDataPush(CmdDataBase):
|
||||
def do_run(self, target=None):
|
||||
try:
|
||||
self.project.push(target=target, jobs=self.args.jobs, remote=self.args.remote)
|
||||
self.project.push(target=target,
|
||||
jobs=self.args.jobs,
|
||||
remote=self.args.remote,
|
||||
show_checksums=self.args.show_checksums,
|
||||
all_branches=self.args.all_branches,
|
||||
all_tags=self.args.all_tags,
|
||||
with_deps=self.args.with_deps)
|
||||
except Exception as exc:
|
||||
self.project.logger.error('Failed to push data to the cloud', exc)
|
||||
return 1
|
||||
@@ -43,8 +52,15 @@ class CmdDataPush(CmdDataBase):
|
||||
class CmdDataFetch(CmdDataBase):
|
||||
def do_run(self, target=None):
|
||||
try:
|
||||
self.project.fetch(target=target, jobs=self.args.jobs, remote=self.args.remote)
|
||||
self.project.fetch(target=target,
|
||||
jobs=self.args.jobs,
|
||||
remote=self.args.remote,
|
||||
show_checksums=self.args.show_checksums,
|
||||
all_branches=self.args.all_branches,
|
||||
all_tags=self.args.all_tags,
|
||||
with_deps=self.args.with_deps)
|
||||
except Exception as exc:
|
||||
self.project.logger.error('Failed to fetch data from the cloud', exc)
|
||||
self.project.logger.error('Failed to fetch data from the cloud',
|
||||
exc)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
from dvc.command.base import CmdBase
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
|
||||
class CmdDestroy(CmdBase):
|
||||
def run_cmd(self):
|
||||
try:
|
||||
msg = u'This will destroy all information about your pipelines, ' \
|
||||
u'all data files, as well as cache in .dvc/cache.\n' \
|
||||
u'Are you sure you want to continue?'
|
||||
|
||||
if not self.args.force \
|
||||
and not self.project.prompt.prompt(msg, False):
|
||||
msg = u'Cannot destroy without a confirmation from the ' \
|
||||
u'user. Use \'-f\' to force.'
|
||||
raise DvcException(msg)
|
||||
|
||||
self.project.destroy()
|
||||
except Exception as exc:
|
||||
self.project.logger.error('Failed to destroy DVC', exc)
|
||||
return 1
|
||||
return 0
|
||||
+21
-2
@@ -1,7 +1,26 @@
|
||||
from dvc.command.common.base import CmdBase
|
||||
from dvc.command.base import CmdBase
|
||||
|
||||
|
||||
class CmdGC(CmdBase):
|
||||
def run(self):
|
||||
self.project.gc()
|
||||
msg = 'This will remove all cache except the cache that is used in '
|
||||
if not self.args.all_branches and not self.args.all_tags:
|
||||
msg += 'the current git branch.'
|
||||
elif self.args.all_branches and not self.args.all_tags:
|
||||
msg += 'all git branches.'
|
||||
elif not self.args.all_branches and self.args.all_tags:
|
||||
msg += 'all git tags.'
|
||||
else:
|
||||
msg += 'all git branches and all git tags.'
|
||||
|
||||
self.project.logger.warn(msg)
|
||||
|
||||
msg = 'Are you sure you want to proceed?'
|
||||
if not self.args.force and not self.project.prompt.prompt(msg):
|
||||
return 1
|
||||
|
||||
self.project.gc(all_branches=self.args.all_branches,
|
||||
all_tags=self.args.all_tags,
|
||||
cloud=self.args.cloud,
|
||||
remote=self.args.remote)
|
||||
return 0
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.command.base import CmdBase
|
||||
from dvc.logger import Logger
|
||||
|
||||
|
||||
class CmdImport(CmdBase):
|
||||
def run(self):
|
||||
try:
|
||||
self.project.imp(self.args.url, self.args.out)
|
||||
except DvcException as ex:
|
||||
Logger.error('Failed to import {}'.format(self.args.url), ex)
|
||||
return 1
|
||||
return 0
|
||||
+3
-2
@@ -1,4 +1,3 @@
|
||||
from dvc.project import Project, InitError
|
||||
from dvc.logger import Logger
|
||||
|
||||
|
||||
@@ -7,8 +6,10 @@ class CmdInit(object):
|
||||
self.args = args
|
||||
|
||||
def run_cmd(self):
|
||||
from dvc.project import Project, InitError
|
||||
|
||||
try:
|
||||
Project.init('.', no_scm=self.args.no_scm)
|
||||
Project.init('.', no_scm=self.args.no_scm, force=self.args.force)
|
||||
except InitError as e:
|
||||
Logger.error('Failed to initiate dvc', e)
|
||||
return 1
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
from dvc.project import Project
|
||||
from dvc.logger import Logger
|
||||
from dvc.command.common.base import CmdBase
|
||||
from dvc.command.base import CmdBase
|
||||
|
||||
|
||||
class CmdInstall(CmdBase):
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.command.base import CmdBase
|
||||
|
||||
|
||||
class CmdLockBase(CmdBase):
|
||||
def run(self, unlock=False):
|
||||
for target in self.args.targets:
|
||||
try:
|
||||
self.project.lock_stage(target, unlock=unlock)
|
||||
except DvcException as ex:
|
||||
msg = 'Failed to {}lock \'{}\''
|
||||
msg.format('un' if unlock else '', target)
|
||||
self.project.logger.error(msg, ex)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
class CmdLock(CmdLockBase):
|
||||
def run(self):
|
||||
return super(CmdLock, self).run(False)
|
||||
|
||||
|
||||
class CmdUnlock(CmdLockBase):
|
||||
def run(self):
|
||||
return super(CmdUnlock, self).run(True)
|
||||
+72
-7
@@ -1,11 +1,76 @@
|
||||
from dvc.command.common.base import CmdBase
|
||||
from dvc.command.base import CmdBase
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
|
||||
class CmdMetrics(CmdBase):
|
||||
class CmdMetricsShow(CmdBase):
|
||||
def run(self):
|
||||
for branch, metric in self.project.metrics(self.args.path,
|
||||
json_path=self.args.json_path,
|
||||
tsv_path=self.args.tsv_path,
|
||||
htsv_path=self.args.htsv_path).items():
|
||||
self.project.logger.info("{}: {}".format(branch, metric))
|
||||
try:
|
||||
# backward compatibility
|
||||
if self.args.json_path:
|
||||
typ = 'json'
|
||||
xpath = self.args.json_path
|
||||
elif self.args.tsv_path:
|
||||
typ = 'tsv'
|
||||
xpath = self.args.tsv_path
|
||||
elif self.args.htsv_path:
|
||||
typ = 'htsv'
|
||||
xpath = self.args.htsv_path
|
||||
elif self.args.csv_path:
|
||||
typ = 'csv'
|
||||
xpath = self.args.csv_path
|
||||
elif self.args.hcsv_path:
|
||||
typ = 'hcsv'
|
||||
xpath = self.args.hcsv_path
|
||||
else:
|
||||
typ = self.args.type
|
||||
xpath = self.args.xpath
|
||||
|
||||
self.project.metrics_show(self.args.path,
|
||||
typ=typ,
|
||||
xpath=xpath,
|
||||
all_branches=self.args.all_branches,
|
||||
all_tags=self.args.all_tags)
|
||||
except DvcException as exc:
|
||||
self.project.logger.error('Failed to show metrics', exc)
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
class CmdMetricsModify(CmdBase):
|
||||
def run(self):
|
||||
try:
|
||||
self.project.metrics_modify(self.args.path,
|
||||
typ=self.args.type,
|
||||
xpath=self.args.xpath)
|
||||
except DvcException as exc:
|
||||
self.project.logger.error('Failed to modify metrics', exc)
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
class CmdMetricsAdd(CmdBase):
|
||||
def run(self):
|
||||
try:
|
||||
self.project.metrics_add(self.args.path,
|
||||
self.args.type,
|
||||
self.args.xpath)
|
||||
except DvcException as exc:
|
||||
msg = 'Failed to add metric file \'{}\''.format(self.args.path)
|
||||
self.project.logger.error(msg, exc)
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
class CmdMetricsRemove(CmdBase):
|
||||
def run(self):
|
||||
try:
|
||||
self.project.metrics_remove(self.args.path)
|
||||
except DvcException as exc:
|
||||
msg = 'Failed to remove metric file \'{}\''.format(self.args.path)
|
||||
self.project.logger.error(msg, exc)
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.command.base import CmdBase
|
||||
|
||||
|
||||
class CmdMove(CmdBase):
|
||||
def run(self):
|
||||
try:
|
||||
self.project.move(self.args.src, self.args.dst)
|
||||
except DvcException as ex:
|
||||
msg = 'Failed to move \'{}\' -> \'{}\''.format(self.args.src,
|
||||
self.args.dst)
|
||||
self.project.logger.error(msg, ex)
|
||||
return 1
|
||||
return 0
|
||||
@@ -0,0 +1,112 @@
|
||||
import os
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.command.base import CmdBase
|
||||
from dvc.dagascii import Dagascii
|
||||
|
||||
|
||||
class CmdPipelineShow(CmdBase):
|
||||
def _show(self, target, commands, outs):
|
||||
import networkx
|
||||
from dvc.stage import Stage
|
||||
|
||||
stage = Stage.load(self.project, target)
|
||||
G = self.project.graph()[0]
|
||||
stages = networkx.get_node_attributes(G, 'stage')
|
||||
node = os.path.relpath(stage.path, self.project.root_dir)
|
||||
|
||||
for n in networkx.dfs_postorder_nodes(G, node):
|
||||
if commands:
|
||||
self.project.logger.info(stages[n].cmd)
|
||||
elif outs:
|
||||
for out in stages[n].outs:
|
||||
self.project.logger.info(str(out))
|
||||
else:
|
||||
self.project.logger.info(n)
|
||||
|
||||
def __build_graph(self, target, commands, outs):
|
||||
import networkx
|
||||
from dvc.stage import Stage
|
||||
|
||||
stage = Stage.load(self.project, target)
|
||||
node = os.path.relpath(stage.path, self.project.root_dir)
|
||||
|
||||
pipelines = list(filter(lambda g: node in g.nodes(),
|
||||
self.project.pipelines()))
|
||||
|
||||
assert len(pipelines) == 1
|
||||
G = pipelines[0]
|
||||
stages = networkx.get_node_attributes(G, 'stage')
|
||||
|
||||
nodes = []
|
||||
for n in G.nodes():
|
||||
stage = stages[n]
|
||||
if commands:
|
||||
if stage.cmd is None:
|
||||
continue
|
||||
nodes.append(stage.cmd)
|
||||
elif outs:
|
||||
for out in stage.outs:
|
||||
nodes.append(str(out))
|
||||
else:
|
||||
nodes.append(stage.relpath)
|
||||
|
||||
edges = []
|
||||
for e in G.edges():
|
||||
from_stage = stages[e[0]]
|
||||
to_stage = stages[e[1]]
|
||||
if commands:
|
||||
if to_stage.cmd is None:
|
||||
continue
|
||||
edges.append((from_stage.cmd, to_stage.cmd))
|
||||
elif outs:
|
||||
for from_out in from_stage.outs:
|
||||
for to_out in to_stage.outs:
|
||||
edges.append((str(from_out),
|
||||
str(to_out)))
|
||||
else:
|
||||
edges.append((from_stage.relpath, to_stage.relpath))
|
||||
|
||||
return nodes, edges
|
||||
|
||||
def _show_ascii(self, target, commands, outs):
|
||||
nodes, edges = self.__build_graph(target, commands, outs)
|
||||
|
||||
if not nodes:
|
||||
return
|
||||
|
||||
d = Dagascii(nodes, edges)
|
||||
d.draw()
|
||||
|
||||
def __write_dot(self, target, commands, outs, filename):
|
||||
import networkx
|
||||
from networkx.drawing.nx_pydot import write_dot
|
||||
|
||||
_, edges = self.__build_graph(target, commands, outs)
|
||||
edges = [edge[::-1] for edge in edges]
|
||||
|
||||
simple_g = networkx.DiGraph()
|
||||
simple_g.add_edges_from(edges)
|
||||
write_dot(simple_g, filename)
|
||||
|
||||
def run(self, unlock=False):
|
||||
for target in self.args.targets:
|
||||
try:
|
||||
if self.args.ascii:
|
||||
self._show_ascii(target,
|
||||
self.args.commands,
|
||||
self.args.outs)
|
||||
elif self.args.dot:
|
||||
self.__write_dot(target,
|
||||
self.args.commands,
|
||||
self.args.outs,
|
||||
self.args.dot)
|
||||
else:
|
||||
self._show(target,
|
||||
self.args.commands,
|
||||
self.args.outs)
|
||||
except DvcException as ex:
|
||||
msg = 'Failed to show pipeline for \'{}\''.format(target)
|
||||
self.project.logger.error(msg, ex)
|
||||
return 1
|
||||
return 0
|
||||
+43
-3
@@ -1,4 +1,6 @@
|
||||
import os
|
||||
import re
|
||||
import configobj
|
||||
|
||||
from dvc.config import Config
|
||||
from dvc.command.config import CmdConfig
|
||||
@@ -8,13 +10,50 @@ from dvc.logger import Logger
|
||||
class CmdRemoteAdd(CmdConfig):
|
||||
def run(self):
|
||||
section = Config.SECTION_REMOTE_FMT.format(self.args.name)
|
||||
return self.set(section, Config.SECTION_REMOTE_URL, self.args.url)
|
||||
ret = self.set(section, Config.SECTION_REMOTE_URL, self.args.url)
|
||||
if ret != 0:
|
||||
return ret
|
||||
|
||||
if self.args.default:
|
||||
msg = 'Setting \'{}\' as a default remote.'.format(self.args.name)
|
||||
Logger.info(msg)
|
||||
ret = self.set(Config.SECTION_CORE,
|
||||
Config.SECTION_CORE_REMOTE,
|
||||
self.args.name)
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
class CmdRemoteRemove(CmdConfig):
|
||||
def _remove_default(self, config_file, remote):
|
||||
path = os.path.join(os.path.dirname(self.config_file),
|
||||
config_file)
|
||||
config = configobj.ConfigObj(path)
|
||||
|
||||
core = config.get(Config.SECTION_CORE, None)
|
||||
if core is None:
|
||||
return
|
||||
|
||||
default = core.get(Config.SECTION_CORE_REMOTE, None)
|
||||
if default is None:
|
||||
return
|
||||
|
||||
if default == remote:
|
||||
del config[Config.SECTION_CORE][Config.SECTION_CORE_REMOTE]
|
||||
if len(config[Config.SECTION_CORE]) == 0:
|
||||
del config[Config.SECTION_CORE]
|
||||
|
||||
config.write()
|
||||
|
||||
def run(self):
|
||||
section = Config.SECTION_REMOTE_FMT.format(self.args.name)
|
||||
return self.unset(section, Config.SECTION_REMOTE_URL)
|
||||
ret = self.unset(section)
|
||||
if ret != 0:
|
||||
return ret
|
||||
|
||||
self._remove_default(Config.CONFIG, self.args.name)
|
||||
self._remove_default(Config.CONFIG_LOCAL, self.args.name)
|
||||
return 0
|
||||
|
||||
|
||||
class CmdRemoteModify(CmdConfig):
|
||||
@@ -30,6 +69,7 @@ class CmdRemoteList(CmdConfig):
|
||||
r = re.match(Config.SECTION_REMOTE_REGEX, section)
|
||||
if r:
|
||||
name = r.group('name')
|
||||
url = self.configobj[section].get(Config.SECTION_REMOTE_URL, '')
|
||||
url = self.configobj[section].get(Config.SECTION_REMOTE_URL,
|
||||
'')
|
||||
Logger.info('{}\t{}'.format(name, url))
|
||||
return 0
|
||||
|
||||
@@ -1,13 +1,15 @@
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.command.common.base import CmdBase
|
||||
from dvc.command.base import CmdBase
|
||||
from dvc.logger import Logger
|
||||
|
||||
|
||||
class CmdRemove(CmdBase):
|
||||
def run(self):
|
||||
outs_only = not self.args.purge
|
||||
for target in self.args.targets:
|
||||
try:
|
||||
self.project.remove(target)
|
||||
self.project.remove(target, outs_only=outs_only)
|
||||
except DvcException as ex:
|
||||
self.project.logger.error('Failed to remove {}', ex)
|
||||
Logger.error('Failed to remove {}'.format(target), ex)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
+27
-22
@@ -1,34 +1,39 @@
|
||||
import os
|
||||
|
||||
from dvc.command.common.base import CmdBase
|
||||
from dvc.project import ReproductionError, StageNotFoundError
|
||||
from dvc.stage import Stage
|
||||
from dvc.command.base import CmdBase
|
||||
from dvc.command.status import CmdDataStatus
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
|
||||
class CmdRepro(CmdBase):
|
||||
def run(self):
|
||||
recursive = not self.args.single_item
|
||||
saved_dir = os.path.realpath(os.curdir)
|
||||
if self.args.cwd:
|
||||
os.chdir(self.args.cwd)
|
||||
|
||||
ret = 0
|
||||
for target in self.args.targets:
|
||||
try:
|
||||
self.project.reproduce(target,
|
||||
stages = self.project.reproduce(
|
||||
target,
|
||||
recursive=recursive,
|
||||
force=self.args.force)
|
||||
except ReproductionError as ex:
|
||||
force=self.args.force,
|
||||
dry=self.args.dry,
|
||||
interactive=self.args.interactive,
|
||||
pipeline=self.args.pipeline,
|
||||
all_pipelines=self.args.all_pipelines)
|
||||
|
||||
if len(stages) == 0:
|
||||
self.project.logger.info(CmdDataStatus.UP_TO_DATE_MSG)
|
||||
|
||||
if self.args.metrics:
|
||||
self.project.metrics_show()
|
||||
except DvcException as ex:
|
||||
msg = 'Failed to reproduce \'{}\''.format(target)
|
||||
self.project.logger.error(msg, ex)
|
||||
return 1
|
||||
except StageNotFoundError as ex:
|
||||
if os.path.exists(target):
|
||||
msg = '\'{}\' is not a dvc file.'.format(target)
|
||||
dvcfile = target + Stage.STAGE_FILE_SUFFIX
|
||||
if os.path.exists(target) and os.path.isfile(dvcfile):
|
||||
msg += ' Maybe you meant \'{}\'?'.format(dvcfile)
|
||||
else:
|
||||
msg = '\'{}\' does not exist.'.format(target)
|
||||
self.project.logger.error(msg)
|
||||
return 1
|
||||
except Exception as ex:
|
||||
msg = 'Failed to reproduce \'{}\' - unexpected error'.format(target)
|
||||
self.project.logger.error(msg, ex)
|
||||
return 1
|
||||
return 0
|
||||
ret = 1
|
||||
break
|
||||
|
||||
os.chdir(saved_dir)
|
||||
return ret
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
import os
|
||||
|
||||
from dvc.command.base import CmdBase
|
||||
|
||||
|
||||
class CmdRoot(CmdBase):
|
||||
def run(self):
|
||||
self.project.logger.info(os.path.relpath(self.project.root_dir))
|
||||
return 0
|
||||
+6
-31
@@ -1,21 +1,19 @@
|
||||
import os
|
||||
|
||||
from dvc.command.common.base import CmdBase
|
||||
from dvc.logger import Logger
|
||||
from dvc.stage import Stage
|
||||
from dvc.command.base import CmdBase
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
|
||||
class CmdRun(CmdBase):
|
||||
def run(self):
|
||||
fname = self.stage_file_name(self.args.file, self.args.outs, self.args.outs_no_cache)
|
||||
|
||||
try:
|
||||
if self.args.yes:
|
||||
self.project.prompt.default = True
|
||||
|
||||
self.project.run(cmd=' '.join(self.args.command),
|
||||
outs=self.args.outs,
|
||||
outs_no_cache=self.args.outs_no_cache,
|
||||
metrics_no_cache=self.args.metrics_no_cache,
|
||||
deps=self.args.deps,
|
||||
fname=fname,
|
||||
fname=self.args.file,
|
||||
cwd=self.args.cwd,
|
||||
no_exec=self.args.no_exec)
|
||||
except DvcException as ex:
|
||||
@@ -23,26 +21,3 @@ class CmdRun(CmdBase):
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def stage_file_name(args_file, args_outs, args_outs_no_cache):
|
||||
if args_file:
|
||||
return args_file
|
||||
|
||||
if args_outs:
|
||||
result = CmdRun._stage_file_basename(args_outs[0])
|
||||
elif args_outs_no_cache:
|
||||
result = CmdRun._stage_file_basename(args_outs_no_cache[0])
|
||||
else:
|
||||
result = Stage.STAGE_FILE
|
||||
|
||||
Logger.info(u'Using \'{}\' as a stage file'.format(result))
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _stage_file_basename(fname):
|
||||
result = os.path.basename(fname)
|
||||
if len(result) == 0:
|
||||
result = os.path.dirname(fname)
|
||||
result += Stage.STAGE_FILE_SUFFIX
|
||||
return result
|
||||
|
||||
+18
-8
@@ -1,16 +1,14 @@
|
||||
import os
|
||||
|
||||
from dvc.command.data_sync import CmdDataBase
|
||||
|
||||
|
||||
class CmdDataStatus(CmdDataBase):
|
||||
STATUS_LEN = 10
|
||||
STATUS_INDENT = '\t'
|
||||
UP_TO_DATE_MSG = "Pipeline is up to date. Nothing to reproduce."
|
||||
|
||||
def _normalize(self, s):
|
||||
s+=':'
|
||||
if len(s) >= self.STATUS_LEN:
|
||||
return s
|
||||
s += ':'
|
||||
assert len(s) < self.STATUS_LEN
|
||||
return s + (self.STATUS_LEN - len(s))*' '
|
||||
|
||||
def _show(self, status, indent=0):
|
||||
@@ -21,13 +19,25 @@ class CmdDataStatus(CmdDataBase):
|
||||
self.project.logger.info('{}{}'.format(ind, key))
|
||||
self._show(value, indent+1)
|
||||
else:
|
||||
self.project.logger.info('{}{}{}'.format(ind, self._normalize(value), key))
|
||||
msg = '{}{}{}'.format(ind, self._normalize(value), key)
|
||||
self.project.logger.info(msg)
|
||||
|
||||
def do_run(self, target=None):
|
||||
indent = 1 if self.args.cloud else 0
|
||||
try:
|
||||
status = self.project.status(target=target, jobs=self.args.jobs, cloud=self.args.cloud, remote=self.args.remote)
|
||||
self._show(status, indent)
|
||||
st = self.project.status(target=target,
|
||||
jobs=self.args.jobs,
|
||||
cloud=self.args.cloud,
|
||||
show_checksums=self.args.show_checksums,
|
||||
remote=self.args.remote,
|
||||
all_branches=self.args.all_branches,
|
||||
all_tags=self.args.all_tags,
|
||||
with_deps=self.args.with_deps)
|
||||
if st:
|
||||
self._show(st, indent)
|
||||
else:
|
||||
self.project.logger.info(self.UP_TO_DATE_MSG)
|
||||
|
||||
except Exception as exc:
|
||||
self.project.logger.error('Failed to obtain data status', exc)
|
||||
return 1
|
||||
|
||||
+124
-35
@@ -2,22 +2,55 @@
|
||||
DVC config objects.
|
||||
"""
|
||||
import os
|
||||
import schema
|
||||
import configobj
|
||||
from schema import Schema, Optional, And, Use, Regex
|
||||
|
||||
from dvc.cache import Cache
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
|
||||
class ConfigError(DvcException):
|
||||
""" DVC config exception """
|
||||
def __init__(self, ex=None):
|
||||
super(ConfigError, self).__init__('Config file error', ex)
|
||||
def __init__(self, msg, ex=None):
|
||||
super(ConfigError, self).__init__('Config file error: {}'.format(msg),
|
||||
ex)
|
||||
|
||||
|
||||
def supported_url(url):
|
||||
from dvc.cloud.data_cloud import DataCloud
|
||||
return DataCloud.supported(url) != None
|
||||
from dvc.remote import supported_url as supported
|
||||
return supported(url)
|
||||
|
||||
|
||||
def supported_cache_type(types):
|
||||
if isinstance(types, str):
|
||||
types = [t.strip() for t in types.split(',')]
|
||||
for t in types:
|
||||
if t not in ['reflink', 'hardlink', 'symlink', 'copy']:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def supported_loglevel(level):
|
||||
return level in ['info', 'debug', 'warning', 'error']
|
||||
|
||||
|
||||
def supported_cloud(cloud):
|
||||
return cloud in ['aws', 'gcp', 'local', '']
|
||||
|
||||
|
||||
def is_bool(val):
|
||||
return val.lower() in ['true', 'false']
|
||||
|
||||
|
||||
def to_bool(val):
|
||||
return val.lower() == 'true'
|
||||
|
||||
|
||||
def is_whole(val):
|
||||
return int(val) >= 0
|
||||
|
||||
|
||||
def is_percent(val):
|
||||
return int(val) >= 0 and int(val) <= 100
|
||||
|
||||
|
||||
class Config(object):
|
||||
@@ -26,43 +59,65 @@ class Config(object):
|
||||
|
||||
SECTION_CORE = 'core'
|
||||
SECTION_CORE_LOGLEVEL = 'loglevel'
|
||||
SECTION_CORE_LOGLEVEL_SCHEMA = schema.And(schema.Use(str.lower), lambda l: l in ('info', 'debug', 'warning', 'error'))
|
||||
SECTION_CORE_LOGLEVEL_SCHEMA = And(Use(str.lower), supported_loglevel)
|
||||
SECTION_CORE_REMOTE = 'remote'
|
||||
SECTION_CORE_INTERACTIVE_SCHEMA = And(str, is_bool, Use(to_bool))
|
||||
SECTION_CORE_INTERACTIVE = 'interactive'
|
||||
|
||||
SECTION_CACHE = 'cache'
|
||||
SECTION_CACHE_DIR = 'dir'
|
||||
SECTION_CACHE_TYPE = 'type'
|
||||
SECTION_CACHE_TYPE_SCHEMA = schema.And(schema.Use(str.lower), lambda t: t in Cache.CACHE_TYPE_MAP.keys())
|
||||
SECTION_CACHE_TYPE_SCHEMA = supported_cache_type
|
||||
SECTION_CACHE_LOCAL = 'local'
|
||||
SECTION_CACHE_S3 = 's3'
|
||||
SECTION_CACHE_GS = 'gs'
|
||||
SECTION_CACHE_SSH = 'ssh'
|
||||
SECTION_CACHE_HDFS = 'hdfs'
|
||||
SECTION_CACHE_AZURE = 'azure'
|
||||
SECTION_CACHE_SCHEMA = {
|
||||
schema.Optional(SECTION_CACHE_DIR, default=Cache.CACHE_DIR): str,
|
||||
schema.Optional(SECTION_CACHE_TYPE, default=None): SECTION_CACHE_TYPE_SCHEMA,
|
||||
Optional(SECTION_CACHE_LOCAL): str,
|
||||
Optional(SECTION_CACHE_S3): str,
|
||||
Optional(SECTION_CACHE_GS): str,
|
||||
Optional(SECTION_CACHE_HDFS): str,
|
||||
Optional(SECTION_CACHE_SSH): str,
|
||||
Optional(SECTION_CACHE_AZURE): str,
|
||||
|
||||
# backward compatibility
|
||||
Optional(SECTION_CACHE_DIR, default='cache'): str,
|
||||
Optional(SECTION_CACHE_TYPE, default=None): SECTION_CACHE_TYPE_SCHEMA,
|
||||
}
|
||||
|
||||
# backward compatibility
|
||||
SECTION_CORE_CLOUD = 'cloud'
|
||||
SECTION_CORE_CLOUD_SCHEMA = schema.And(schema.Use(str.lower), lambda c: c in ('aws', 'gcp', 'local', ''))
|
||||
SECTION_CORE_CLOUD_SCHEMA = And(Use(str.lower), supported_cloud)
|
||||
SECTION_CORE_STORAGEPATH = 'storagepath'
|
||||
|
||||
SECTION_CORE_SCHEMA = {
|
||||
schema.Optional(SECTION_CORE_LOGLEVEL, default='info'): schema.And(str, schema.Use(str.lower), SECTION_CORE_LOGLEVEL_SCHEMA),
|
||||
schema.Optional(SECTION_CORE_REMOTE, default=''): schema.And(str, schema.Use(str.lower)),
|
||||
Optional(SECTION_CORE_LOGLEVEL,
|
||||
default='info'): And(str, Use(str.lower),
|
||||
SECTION_CORE_LOGLEVEL_SCHEMA),
|
||||
Optional(SECTION_CORE_REMOTE, default=''): And(str, Use(str.lower)),
|
||||
Optional(SECTION_CORE_INTERACTIVE,
|
||||
default=False): SECTION_CORE_INTERACTIVE_SCHEMA,
|
||||
|
||||
# backward compatibility
|
||||
schema.Optional(SECTION_CORE_CLOUD, default=''): SECTION_CORE_CLOUD_SCHEMA,
|
||||
schema.Optional(SECTION_CORE_STORAGEPATH, default=''): str,
|
||||
Optional(SECTION_CORE_CLOUD, default=''): SECTION_CORE_CLOUD_SCHEMA,
|
||||
Optional(SECTION_CORE_STORAGEPATH, default=''): str,
|
||||
}
|
||||
|
||||
# backward compatibility
|
||||
SECTION_AWS = 'aws'
|
||||
SECTION_AWS_STORAGEPATH = 'storagepath'
|
||||
SECTION_AWS_CREDENTIALPATH = 'credentialpath'
|
||||
SECTION_AWS_ENDPOINT_URL = 'endpointurl'
|
||||
SECTION_AWS_REGION = 'region'
|
||||
SECTION_AWS_PROFILE = 'profile'
|
||||
SECTION_AWS_SCHEMA = {
|
||||
SECTION_AWS_STORAGEPATH: str,
|
||||
schema.Optional(SECTION_AWS_REGION): str,
|
||||
schema.Optional(SECTION_AWS_PROFILE, default='default'): str,
|
||||
schema.Optional(SECTION_AWS_CREDENTIALPATH, default = ''): str,
|
||||
Optional(SECTION_AWS_REGION): str,
|
||||
Optional(SECTION_AWS_PROFILE, default='default'): str,
|
||||
Optional(SECTION_AWS_CREDENTIALPATH, default=''): str,
|
||||
Optional(SECTION_AWS_ENDPOINT_URL, default=None): str,
|
||||
}
|
||||
|
||||
# backward compatibility
|
||||
@@ -71,7 +126,7 @@ class Config(object):
|
||||
SECTION_GCP_PROJECTNAME = 'projectname'
|
||||
SECTION_GCP_SCHEMA = {
|
||||
SECTION_GCP_STORAGEPATH: str,
|
||||
schema.Optional(SECTION_GCP_PROJECTNAME): str,
|
||||
Optional(SECTION_GCP_PROJECTNAME): str,
|
||||
}
|
||||
|
||||
# backward compatibility
|
||||
@@ -81,27 +136,49 @@ class Config(object):
|
||||
SECTION_LOCAL_STORAGEPATH: str,
|
||||
}
|
||||
|
||||
|
||||
SECTION_REMOTE_REGEX = r'^\s*remote\s*"(?P<name>.*)"\s*$'
|
||||
SECTION_REMOTE_FMT = 'remote "{}"'
|
||||
SECTION_REMOTE_URL = 'url'
|
||||
SECTION_REMOTE_USER = 'user'
|
||||
SECTION_REMOTE_PORT = 'port'
|
||||
SECTION_REMOTE_KEY_FILE = 'keyfile'
|
||||
SECTION_REMOTE_TIMEOUT = 'timeout'
|
||||
SECTION_REMOTE_PASSWORD = 'password'
|
||||
SECTION_REMOTE_ASK_PASSWORD = 'ask_password'
|
||||
SECTION_REMOTE_SCHEMA = {
|
||||
SECTION_REMOTE_URL: supported_url,
|
||||
schema.Optional(SECTION_AWS_REGION): str,
|
||||
schema.Optional(SECTION_AWS_PROFILE, default='default'): str,
|
||||
schema.Optional(SECTION_AWS_CREDENTIALPATH, default = ''): str,
|
||||
schema.Optional(SECTION_GCP_PROJECTNAME): str,
|
||||
SECTION_REMOTE_URL: And(supported_url, error="Unsupported URL"),
|
||||
Optional(SECTION_AWS_REGION): str,
|
||||
Optional(SECTION_AWS_PROFILE, default='default'): str,
|
||||
Optional(SECTION_AWS_CREDENTIALPATH, default=''): str,
|
||||
Optional(SECTION_AWS_ENDPOINT_URL, default=None): str,
|
||||
Optional(SECTION_GCP_PROJECTNAME): str,
|
||||
Optional(SECTION_CACHE_TYPE): SECTION_CACHE_TYPE_SCHEMA,
|
||||
Optional(SECTION_REMOTE_USER): str,
|
||||
Optional(SECTION_REMOTE_PORT): Use(int),
|
||||
Optional(SECTION_REMOTE_KEY_FILE): str,
|
||||
Optional(SECTION_REMOTE_TIMEOUT): Use(int),
|
||||
Optional(SECTION_REMOTE_PASSWORD): str,
|
||||
Optional(SECTION_REMOTE_ASK_PASSWORD): And(str, is_bool, Use(to_bool)),
|
||||
}
|
||||
|
||||
SECTION_STATE = 'state'
|
||||
SECTION_STATE_ROW_LIMIT = 'row_limit'
|
||||
SECTION_STATE_ROW_CLEANUP_QUOTA = 'row_cleanup_quota'
|
||||
SECTION_STATE_SCHEMA = {
|
||||
Optional(SECTION_STATE_ROW_LIMIT): And(Use(int), is_whole),
|
||||
Optional(SECTION_STATE_ROW_CLEANUP_QUOTA): And(Use(int), is_percent),
|
||||
}
|
||||
|
||||
SCHEMA = {
|
||||
schema.Optional(SECTION_CORE, default={}): SECTION_CORE_SCHEMA,
|
||||
schema.Optional(schema.Regex(SECTION_REMOTE_REGEX)): SECTION_REMOTE_SCHEMA,
|
||||
schema.Optional(SECTION_CACHE, default={}): SECTION_CACHE_SCHEMA,
|
||||
Optional(SECTION_CORE, default={}): SECTION_CORE_SCHEMA,
|
||||
Optional(Regex(SECTION_REMOTE_REGEX)): SECTION_REMOTE_SCHEMA,
|
||||
Optional(SECTION_CACHE, default={}): SECTION_CACHE_SCHEMA,
|
||||
Optional(SECTION_STATE, default={}): SECTION_STATE_SCHEMA,
|
||||
|
||||
# backward compatibility
|
||||
schema.Optional(SECTION_AWS, default={}): SECTION_AWS_SCHEMA,
|
||||
schema.Optional(SECTION_GCP, default={}): SECTION_GCP_SCHEMA,
|
||||
schema.Optional(SECTION_LOCAL, default={}): SECTION_LOCAL_SCHEMA,
|
||||
Optional(SECTION_AWS, default={}): SECTION_AWS_SCHEMA,
|
||||
Optional(SECTION_GCP, default={}): SECTION_GCP_SCHEMA,
|
||||
Optional(SECTION_LOCAL, default={}): SECTION_LOCAL_SCHEMA,
|
||||
}
|
||||
|
||||
def __init__(self, dvc_dir):
|
||||
@@ -114,19 +191,31 @@ class Config(object):
|
||||
local = configobj.ConfigObj(self.config_local_file)
|
||||
|
||||
# NOTE: schema doesn't support ConfigObj.Section validation, so we
|
||||
# need to convert our config to dict before passing it to schema.
|
||||
# need to convert our config to dict before passing it to
|
||||
self._config = self._lower(self._config)
|
||||
local = self._lower(local)
|
||||
self._config.update(local)
|
||||
self._config = self._merge(self._config, local)
|
||||
|
||||
self._config = schema.Schema(self.SCHEMA).validate(self._config)
|
||||
self._config = Schema(self.SCHEMA).validate(self._config)
|
||||
|
||||
# NOTE: now converting back to ConfigObj
|
||||
self._config = configobj.ConfigObj(self._config, write_empty_values=True)
|
||||
self._config = configobj.ConfigObj(self._config,
|
||||
write_empty_values=True)
|
||||
self._config.filename = self.config_file
|
||||
except Exception as ex:
|
||||
raise ConfigError(ex)
|
||||
|
||||
@staticmethod
|
||||
def _merge(first, second):
|
||||
res = {}
|
||||
sections = list(first.keys()) + list(second.keys())
|
||||
for section in sections:
|
||||
f = first.get(section, {}).copy()
|
||||
s = second.get(section, {}).copy()
|
||||
f.update(s)
|
||||
res[section] = f
|
||||
return res
|
||||
|
||||
@staticmethod
|
||||
def _lower(config):
|
||||
new_config = {}
|
||||
|
||||
+278
@@ -0,0 +1,278 @@
|
||||
import sys
|
||||
import math
|
||||
import select
|
||||
|
||||
from grandalf.graphs import Vertex, Edge, Graph
|
||||
from grandalf.layouts import SugiyamaLayout
|
||||
from grandalf.routing import route_with_lines, EdgeViewer
|
||||
|
||||
|
||||
class AsciiCanvas(object):
|
||||
def __init__(self, cols, lines):
|
||||
assert cols > 1
|
||||
assert lines > 1
|
||||
|
||||
self.cols = cols
|
||||
self.lines = lines
|
||||
|
||||
self.canvas = [[' '] * cols for l in range(lines)]
|
||||
|
||||
def draw(self):
|
||||
if sys.stdout.isatty(): # pragma: no cover
|
||||
from asciimatics.screen import Screen
|
||||
Screen.wrapper(self._do_draw)
|
||||
else:
|
||||
for line in self.canvas:
|
||||
print(''.join(line))
|
||||
|
||||
def _do_draw(self, screen): # pragma: no cover
|
||||
from asciimatics.event import KeyboardEvent
|
||||
|
||||
offset_x = 0
|
||||
offset_y = 0
|
||||
smaxrow, smaxcol = screen.dimensions
|
||||
assert smaxrow > 1
|
||||
assert smaxcol > 1
|
||||
smaxrow -= 1
|
||||
smaxcol -= 1
|
||||
|
||||
if self.lines + 1 > smaxrow:
|
||||
max_y = self.lines + 1 - smaxrow
|
||||
else:
|
||||
max_y = 0
|
||||
|
||||
if self.cols + 1 > smaxcol:
|
||||
max_x = self.cols + 1 - smaxcol
|
||||
else:
|
||||
max_x = 0
|
||||
|
||||
while True:
|
||||
for y in range(smaxrow + 1):
|
||||
y_index = offset_y + y
|
||||
line = []
|
||||
for x in range(smaxcol + 1):
|
||||
x_index = offset_x + x
|
||||
if len(self.canvas) > y_index \
|
||||
and len(self.canvas[y_index]) > x_index:
|
||||
line.append(self.canvas[y_index][x_index])
|
||||
else:
|
||||
line.append(' ')
|
||||
assert len(line) == (smaxcol + 1)
|
||||
screen.print_at(''.join(line), 0, y)
|
||||
|
||||
screen.refresh()
|
||||
|
||||
# NOTE: get_event() doesn't block by itself,
|
||||
# so we have to do the blocking ourselves.
|
||||
select.select([sys.stdin], [], [], None)
|
||||
|
||||
event = screen.get_event()
|
||||
if not isinstance(event, KeyboardEvent):
|
||||
continue
|
||||
|
||||
k = event.key_code
|
||||
if k == screen.KEY_DOWN or k == ord('s'):
|
||||
offset_y += 1
|
||||
elif k == screen.KEY_PAGE_DOWN or k == ord('S'):
|
||||
offset_y += smaxrow
|
||||
elif k == screen.KEY_UP or k == ord('w'):
|
||||
offset_y -= 1
|
||||
elif k == screen.KEY_PAGE_UP or k == ord('W'):
|
||||
offset_y -= smaxrow
|
||||
elif k == screen.KEY_RIGHT or k == ord('d'):
|
||||
offset_x += 1
|
||||
elif k == ord('D'):
|
||||
offset_x += smaxcol
|
||||
elif k == screen.KEY_LEFT or k == ord('a'):
|
||||
offset_x -= 1
|
||||
elif k == ord('A'):
|
||||
offset_x -= smaxcol
|
||||
elif k == ord('q') or k == ord('Q'):
|
||||
break
|
||||
|
||||
if offset_y > max_y:
|
||||
offset_y = max_y
|
||||
elif offset_y < 0:
|
||||
offset_y = 0
|
||||
|
||||
if offset_x > max_x:
|
||||
offset_x = max_x
|
||||
elif offset_x < 0:
|
||||
offset_x = 0
|
||||
|
||||
def point(self, x, y, char):
|
||||
assert x >= 0
|
||||
assert x < self.cols
|
||||
assert y >= 0
|
||||
assert y < self.lines
|
||||
|
||||
self.canvas[y][x] = char
|
||||
|
||||
def line(self, x0, y0, x1, y1, char):
|
||||
if x0 > x1:
|
||||
x1, x0 = x0, x1
|
||||
y1, y0 = y0, y1
|
||||
|
||||
dx = x1 - x0
|
||||
dy = y1 - y0
|
||||
|
||||
if dx == 0 and dy == 0:
|
||||
self.point(x0, y0, char)
|
||||
elif abs(dx) >= abs(dy):
|
||||
for x in range(x0, x1 + 1):
|
||||
if dx == 0:
|
||||
y = y0
|
||||
else:
|
||||
y = y0 + int(round((x - x0) * dy / float((dx))))
|
||||
self.point(x, y, char)
|
||||
elif y0 < y1:
|
||||
for y in range(y0, y1 + 1):
|
||||
if dy == 0:
|
||||
x = x0
|
||||
else:
|
||||
x = x0 + int(round((y - y0) * dx / float((dy))))
|
||||
self.point(x, y, char)
|
||||
else:
|
||||
for y in range(y1, y0 + 1):
|
||||
if dy == 0:
|
||||
x = x0
|
||||
else:
|
||||
x = x1 + int(round((y - y1) * dx / float((dy))))
|
||||
self.point(x, y, char)
|
||||
|
||||
def text(self, x, y, text):
|
||||
for i, c in enumerate(text):
|
||||
self.point(x + i, y, c)
|
||||
|
||||
def box(self, x0, y0, w, h):
|
||||
assert w > 1
|
||||
assert h > 1
|
||||
|
||||
w -= 1
|
||||
h -= 1
|
||||
|
||||
for x in range(x0, x0 + w):
|
||||
self.point(x, y0, '-')
|
||||
self.point(x, y0 + h, '-')
|
||||
|
||||
for y in range(y0, y0 + h):
|
||||
self.point(x0, y, '|')
|
||||
self.point(x0 + w, y, '|')
|
||||
|
||||
self.point(x0, y0, '+')
|
||||
self.point(x0 + w, y0, '+')
|
||||
self.point(x0, y0 + h, '+')
|
||||
self.point(x0 + w, y0 + h, '+')
|
||||
|
||||
|
||||
class Dagascii(object):
|
||||
def __init__(self, vertexes, edges):
|
||||
#
|
||||
# Just a reminder about naming conventions:
|
||||
# +------------X
|
||||
# |
|
||||
# |
|
||||
# |
|
||||
# |
|
||||
# Y
|
||||
#
|
||||
|
||||
V = {v: Vertex(" {} ".format(v)) for v in vertexes}
|
||||
# NOTE: reverting edges to correctly orientate the graph
|
||||
E = [Edge(V[e], V[s]) for s, e in edges]
|
||||
V = V.values()
|
||||
g = Graph(V, E)
|
||||
|
||||
class VertexViewer(object):
|
||||
h = 3 # top and bottom box edges + text
|
||||
|
||||
def __init__(self, name):
|
||||
self.w = len(name) + 2 # right and left bottom edges + text
|
||||
|
||||
for v in V:
|
||||
v.view = VertexViewer(v.data)
|
||||
|
||||
# NOTE: determine min box length to create the best layout
|
||||
minw = min([v.view.w for v in V])
|
||||
|
||||
for e in E:
|
||||
e.view = EdgeViewer()
|
||||
|
||||
sug = SugiyamaLayout(g.C[0])
|
||||
gr = g.C[0]
|
||||
r = list(filter(lambda x: len(x.e_in()) == 0, gr.sV))
|
||||
|
||||
sug.init_all(roots=r, optimize=True)
|
||||
|
||||
sug.yspace = VertexViewer.h
|
||||
sug.xspace = minw
|
||||
sug.route_edge = route_with_lines
|
||||
|
||||
sug.draw()
|
||||
|
||||
self.sug = sug
|
||||
|
||||
def draw(self):
|
||||
# NOTE: coordinates might me negative, so we need to shift
|
||||
# everything to the positive plane before we actually draw it.
|
||||
Xs = []
|
||||
Ys = []
|
||||
|
||||
for v in self.sug.g.sV:
|
||||
# NOTE: moving boxes w/2 to the left
|
||||
Xs.append(v.view.xy[0] - v.view.w/2.0)
|
||||
Xs.append(v.view.xy[0] + v.view.w/2.0)
|
||||
Ys.append(v.view.xy[1])
|
||||
Ys.append(v.view.xy[1] + v.view.h)
|
||||
|
||||
for e in self.sug.g.sE:
|
||||
for xy in e.view._pts:
|
||||
Xs.append(xy[0])
|
||||
Ys.append(xy[1])
|
||||
|
||||
minx = min(Xs)
|
||||
miny = min(Ys)
|
||||
maxx = max(Xs)
|
||||
maxy = max(Ys)
|
||||
|
||||
canvas_cols = int(math.ceil(math.ceil(maxx) - math.floor(minx))) + 1
|
||||
canvas_lines = int(round(maxy - miny))
|
||||
|
||||
canvas = AsciiCanvas(canvas_cols, canvas_lines)
|
||||
|
||||
# NOTE: first draw edges so that node boxes could overwrite them
|
||||
for e in self.sug.g.sE:
|
||||
assert len(e.view._pts) > 1
|
||||
for index in range(1, len(e.view._pts)):
|
||||
start = e.view._pts[index - 1]
|
||||
end = e.view._pts[index]
|
||||
|
||||
start_x = int(round(start[0] - minx))
|
||||
start_y = int(round(start[1] - miny))
|
||||
end_x = int(round(end[0] - minx))
|
||||
end_y = int(round(end[1] - miny))
|
||||
|
||||
assert start_x >= 0
|
||||
assert start_y >= 0
|
||||
assert end_x >= 0
|
||||
assert end_y >= 0
|
||||
|
||||
canvas.line(start_x, start_y,
|
||||
end_x, end_y,
|
||||
'*')
|
||||
|
||||
for v in self.sug.g.sV:
|
||||
# NOTE: moving boxes w/2 to the left
|
||||
x = v.view.xy[0] - v.view.w/2.0
|
||||
y = v.view.xy[1]
|
||||
|
||||
canvas.box(int(round(x - minx)),
|
||||
int(round(y - miny)),
|
||||
v.view.w,
|
||||
v.view.h)
|
||||
|
||||
canvas.text(int(round(x - minx)) + 1,
|
||||
int(round(y - miny)) + 1,
|
||||
v.data)
|
||||
|
||||
canvas.draw()
|
||||
@@ -0,0 +1,128 @@
|
||||
from dvc.logger import Logger
|
||||
from dvc.config import Config, ConfigError
|
||||
|
||||
from dvc.remote.s3 import RemoteS3
|
||||
from dvc.remote.gs import RemoteGS
|
||||
from dvc.remote.azure import RemoteAzure
|
||||
from dvc.remote.ssh import RemoteSSH
|
||||
from dvc.remote.hdfs import RemoteHDFS
|
||||
from dvc.remote.local import RemoteLOCAL
|
||||
|
||||
|
||||
class DataCloud(object):
|
||||
CLOUD_MAP = {
|
||||
'aws': RemoteS3,
|
||||
'gcp': RemoteGS,
|
||||
'azure': RemoteAzure,
|
||||
'ssh': RemoteSSH,
|
||||
'hdfs': RemoteHDFS,
|
||||
'local': RemoteLOCAL,
|
||||
}
|
||||
|
||||
def __init__(self, project, config=None):
|
||||
self.project = project
|
||||
self._config = config
|
||||
self._core = self._config[Config.SECTION_CORE]
|
||||
|
||||
remote = self._core.get(Config.SECTION_CORE_REMOTE, '')
|
||||
if remote == '':
|
||||
if self._core.get(Config.SECTION_CORE_CLOUD, None):
|
||||
# backward compatibility
|
||||
msg = 'Using obsoleted config format. Consider updating.'
|
||||
Logger.warn(msg)
|
||||
self._cloud = self.__init__compat()
|
||||
else:
|
||||
self._cloud = None
|
||||
return
|
||||
|
||||
self._cloud = self._init_remote(remote)
|
||||
|
||||
@staticmethod
|
||||
def supported(config):
|
||||
for cloud in DataCloud.CLOUD_MAP.values():
|
||||
if cloud.supported(config):
|
||||
return cloud
|
||||
return None
|
||||
|
||||
def _init_remote(self, remote):
|
||||
section = Config.SECTION_REMOTE_FMT.format(remote)
|
||||
cloud_config = self._config.get(section, None)
|
||||
if not cloud_config:
|
||||
msg = "Can't find remote section '{}' in config"
|
||||
raise ConfigError(msg.format(section))
|
||||
|
||||
cloud_type = self.supported(cloud_config)
|
||||
if not cloud_type:
|
||||
raise ConfigError("Unsupported cloud '{}'".format(cloud_config))
|
||||
|
||||
return self._init_cloud(cloud_config, cloud_type)
|
||||
|
||||
def __init__compat(self):
|
||||
name = self._core.get(Config.SECTION_CORE_CLOUD, '').strip().lower()
|
||||
if name == '':
|
||||
self._cloud = None
|
||||
return
|
||||
|
||||
cloud_type = self.CLOUD_MAP.get(name, None)
|
||||
if not cloud_type:
|
||||
msg = "Wrong cloud type {} specified"
|
||||
raise ConfigError(msg.format(name))
|
||||
|
||||
cloud_config = self._config.get(name, None)
|
||||
if not cloud_config:
|
||||
msg = "Can't find cloud section '{}' in config"
|
||||
raise ConfigError(msg.format(name))
|
||||
|
||||
return self._init_cloud(cloud_config, cloud_type)
|
||||
|
||||
def _init_cloud(self, cloud_config, cloud_type):
|
||||
global_storage_path = self._core.get(Config.SECTION_CORE_STORAGEPATH)
|
||||
if global_storage_path:
|
||||
Logger.warn('Using obsoleted config format. Consider updating.')
|
||||
|
||||
cloud = cloud_type(self.project, cloud_config)
|
||||
return cloud
|
||||
|
||||
def _get_cloud(self, remote, cmd):
|
||||
if remote:
|
||||
return self._init_remote(remote)
|
||||
|
||||
if self._cloud:
|
||||
return self._cloud
|
||||
|
||||
msg = "No remote repository specified. Setup default " \
|
||||
"repository with\n"\
|
||||
" dvc config core.remote <name>\n" \
|
||||
"or use:\n" \
|
||||
" dvc {} -r <name>".format(cmd)
|
||||
raise ConfigError(msg)
|
||||
|
||||
def push(self, targets, jobs=1, remote=None, show_checksums=False):
|
||||
"""
|
||||
Push data items in a cloud-agnostic way.
|
||||
"""
|
||||
return self.project.cache.local.push(targets,
|
||||
jobs=jobs,
|
||||
remote=self._get_cloud(remote,
|
||||
'push'),
|
||||
show_checksums=show_checksums)
|
||||
|
||||
def pull(self, targets, jobs=1, remote=None, show_checksums=False):
|
||||
"""
|
||||
Pull data items in a cloud-agnostic way.
|
||||
"""
|
||||
return self.project.cache.local.pull(targets,
|
||||
jobs=jobs,
|
||||
remote=self._get_cloud(remote,
|
||||
'pull'),
|
||||
show_checksums=show_checksums)
|
||||
|
||||
def status(self, targets, jobs=1, remote=None, show_checksums=False):
|
||||
"""
|
||||
Check status of data items in a cloud-agnostic way.
|
||||
"""
|
||||
cloud = self._get_cloud(remote, 'status')
|
||||
return self.project.cache.local.status(targets,
|
||||
jobs=jobs,
|
||||
remote=cloud,
|
||||
show_checksums=show_checksums)
|
||||
@@ -0,0 +1,69 @@
|
||||
import schema
|
||||
|
||||
try:
|
||||
from urlparse import urlparse
|
||||
except ImportError:
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.config import Config
|
||||
|
||||
from dvc.dependency.base import DependencyBase
|
||||
from dvc.dependency.s3 import DependencyS3
|
||||
from dvc.dependency.gs import DependencyGS
|
||||
from dvc.dependency.local import DependencyLOCAL
|
||||
from dvc.dependency.hdfs import DependencyHDFS
|
||||
from dvc.dependency.ssh import DependencySSH
|
||||
|
||||
from dvc.remote import Remote
|
||||
from dvc.remote.local import RemoteLOCAL
|
||||
from dvc.remote.s3 import RemoteS3
|
||||
from dvc.remote.hdfs import RemoteHDFS
|
||||
|
||||
DEPS = [DependencyHDFS,
|
||||
DependencyS3,
|
||||
DependencyGS,
|
||||
DependencySSH,
|
||||
DependencyLOCAL]
|
||||
|
||||
DEP_MAP = {'': DependencyLOCAL,
|
||||
'ssh': DependencySSH,
|
||||
's3': DependencyS3,
|
||||
'gs': DependencyGS,
|
||||
'hdfs': DependencyHDFS}
|
||||
|
||||
SCHEMA = {
|
||||
DependencyBase.PARAM_PATH: str,
|
||||
schema.Optional(RemoteLOCAL.PARAM_MD5): schema.Or(str, None),
|
||||
schema.Optional(RemoteS3.PARAM_ETAG): schema.Or(str, None),
|
||||
schema.Optional(RemoteHDFS.PARAM_CHECKSUM): schema.Or(str, None),
|
||||
}
|
||||
|
||||
|
||||
def _get(stage, p, info):
|
||||
parsed = urlparse(p)
|
||||
if parsed.scheme == 'remote':
|
||||
name = Config.SECTION_REMOTE_FMT.format(parsed.netloc)
|
||||
sect = stage.project.config._config[name]
|
||||
remote = Remote(stage.project, sect)
|
||||
return DEP_MAP[remote.scheme](stage, p, info, remote=remote)
|
||||
|
||||
for d in DEPS:
|
||||
if d.supported(p):
|
||||
return d(stage, p, info)
|
||||
raise DvcException('Dependency \'{}\' is not supported'.format(p))
|
||||
|
||||
|
||||
def loadd_from(stage, d_list):
|
||||
ret = []
|
||||
for d in d_list:
|
||||
p = d.pop(DependencyBase.PARAM_PATH)
|
||||
ret.append(_get(stage, p, d))
|
||||
return ret
|
||||
|
||||
|
||||
def loads_from(stage, s_list):
|
||||
ret = []
|
||||
for s in s_list:
|
||||
ret.append(_get(stage, s, {}))
|
||||
return ret
|
||||
@@ -0,0 +1,79 @@
|
||||
import re
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
|
||||
class DependencyError(DvcException):
|
||||
def __init__(self, path, msg):
|
||||
msg = 'Dependency \'{}\' error: {}'
|
||||
super(DependencyError, self).__init__(msg.format(path, msg))
|
||||
|
||||
|
||||
class DependencyDoesNotExistError(DependencyError):
|
||||
def __init__(self, path):
|
||||
msg = 'does not exist'
|
||||
super(DependencyDoesNotExistError, self).__init__(path, msg)
|
||||
|
||||
|
||||
class DependencyIsNotFileOrDirError(DependencyError):
|
||||
def __init__(self, path):
|
||||
msg = 'not a file or directory'
|
||||
super(DependencyIsNotFileOrDirError, self).__init__(path, msg)
|
||||
|
||||
|
||||
class DependencyBase(object):
|
||||
REGEX = None
|
||||
|
||||
PARAM_PATH = 'path'
|
||||
|
||||
def __init__(self, stage, path):
|
||||
self.stage = stage
|
||||
self.project = stage.project
|
||||
self.path = path
|
||||
|
||||
def __str__(self):
|
||||
return self.path
|
||||
|
||||
@classmethod
|
||||
def match(cls, url):
|
||||
return re.match(cls.REGEX, url)
|
||||
|
||||
def group(self, name):
|
||||
match = self.match(self.path)
|
||||
if not match:
|
||||
return None
|
||||
return match.group(name)
|
||||
|
||||
@classmethod
|
||||
def supported(cls, url):
|
||||
return cls.match(url) is not None
|
||||
|
||||
@property
|
||||
def sep(self):
|
||||
return '/'
|
||||
|
||||
@property
|
||||
def exists(self):
|
||||
return self.remote.exists([self.path_info])[0]
|
||||
|
||||
def changed(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def status(self):
|
||||
if self.changed():
|
||||
# FIXME better msgs
|
||||
if self.path_info['scheme'] == 'local':
|
||||
p = self.rel_path
|
||||
else:
|
||||
p = self.path
|
||||
return {p: 'changed'}
|
||||
return {}
|
||||
|
||||
def save(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def dumpd(self):
|
||||
return {self.PARAM_PATH: self.path}
|
||||
|
||||
def download(self, to_info):
|
||||
self.remote.download([self.path_info], [to_info])
|
||||
@@ -0,0 +1,25 @@
|
||||
import posixpath
|
||||
|
||||
try:
|
||||
from urlparse import urlparse
|
||||
except ImportError:
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from dvc.dependency.s3 import DependencyS3
|
||||
from dvc.remote.gs import RemoteGS
|
||||
|
||||
|
||||
class DependencyGS(DependencyS3):
|
||||
REGEX = RemoteGS.REGEX
|
||||
|
||||
def __init__(self, stage, path, info=None, remote=None):
|
||||
super(DependencyGS, self).__init__(stage, path)
|
||||
self.info = info
|
||||
self.remote = remote if remote else RemoteGS(stage.project, {})
|
||||
bucket = remote.bucket if remote else urlparse(path).netloc
|
||||
key = urlparse(path).path.lstrip('/')
|
||||
if remote:
|
||||
key = posixpath.join(remote.prefix, key)
|
||||
self.path_info = {'scheme': 'gs',
|
||||
'bucket': bucket,
|
||||
'key': key}
|
||||
@@ -0,0 +1,35 @@
|
||||
import posixpath
|
||||
|
||||
try:
|
||||
from urlparse import urlparse
|
||||
except ImportError:
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from dvc.dependency.base import DependencyBase
|
||||
from dvc.remote.hdfs import RemoteHDFS
|
||||
|
||||
|
||||
class DependencyHDFS(DependencyBase):
|
||||
REGEX = RemoteHDFS.REGEX
|
||||
|
||||
def __init__(self, stage, path, info=None, remote=None):
|
||||
super(DependencyHDFS, self).__init__(stage, path)
|
||||
self.info = info
|
||||
self.remote = remote if remote else RemoteHDFS(stage.project, {})
|
||||
if remote:
|
||||
path = posixpath.join(remote.url, urlparse(path).path.lstrip('/'))
|
||||
user = remote.user if remote else self.group('user')
|
||||
self.path_info = {'scheme': 'hdfs',
|
||||
'user': user,
|
||||
'url': path}
|
||||
|
||||
def changed(self):
|
||||
return self.info != self.remote.save_info(self.path_info)
|
||||
|
||||
def save(self):
|
||||
self.info = self.remote.save_info(self.path_info)
|
||||
|
||||
def dumpd(self):
|
||||
ret = self.info
|
||||
ret[self.PARAM_PATH] = self.path
|
||||
return ret
|
||||
@@ -0,0 +1,89 @@
|
||||
import os
|
||||
|
||||
try:
|
||||
from urlparse import urlparse
|
||||
except ImportError:
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from dvc.dependency.base import DependencyBase
|
||||
from dvc.dependency.base import DependencyDoesNotExistError
|
||||
from dvc.dependency.base import DependencyIsNotFileOrDirError
|
||||
from dvc.logger import Logger
|
||||
from dvc.remote.local import RemoteLOCAL
|
||||
|
||||
|
||||
class DependencyLOCAL(DependencyBase):
|
||||
REGEX = r'^(?P<path>(/+|.:\\+)?[^:]*)$'
|
||||
|
||||
DoesNotExistError = DependencyDoesNotExistError
|
||||
IsNotFileOrDirError = DependencyIsNotFileOrDirError
|
||||
|
||||
def __init__(self, stage, path, info=None, remote=None):
|
||||
self.stage = stage
|
||||
self.project = stage.project
|
||||
self.info = info
|
||||
if remote is not None:
|
||||
self.remote = remote
|
||||
else:
|
||||
self.remote = RemoteLOCAL(stage.project, {})
|
||||
|
||||
if remote:
|
||||
path = os.path.join(remote.prefix, urlparse(path).path.lstrip('/'))
|
||||
|
||||
if not os.path.isabs(path):
|
||||
path = self.remote.ospath(path)
|
||||
path = os.path.join(stage.cwd, path)
|
||||
self.path = os.path.abspath(os.path.normpath(path))
|
||||
|
||||
self.path_info = {'scheme': 'local',
|
||||
'path': self.path}
|
||||
|
||||
def __str__(self):
|
||||
return self.rel_path
|
||||
|
||||
@property
|
||||
def is_local(self):
|
||||
assert os.path.isabs(self.path)
|
||||
assert os.path.isabs(self.project.root_dir)
|
||||
return self.path.startswith(self.project.root_dir)
|
||||
|
||||
@property
|
||||
def sep(self):
|
||||
return os.sep
|
||||
|
||||
@property
|
||||
def rel_path(self):
|
||||
return os.path.relpath(self.path)
|
||||
|
||||
def changed(self):
|
||||
if not self.exists:
|
||||
return True
|
||||
|
||||
info = self.remote.save_info(self.path_info)
|
||||
|
||||
return self.info != info
|
||||
|
||||
def save(self):
|
||||
if not self.exists:
|
||||
raise self.DoesNotExistError(self.rel_path)
|
||||
|
||||
if not os.path.isfile(self.path) \
|
||||
and not os.path.isdir(self.path): # pragma: no cover
|
||||
raise self.IsNotFileOrDirError(self.rel_path)
|
||||
|
||||
if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \
|
||||
(os.path.isdir(self.path) and len(os.listdir(self.path)) == 0):
|
||||
Logger.warn("File/directory '{}' is empty.".format(self.rel_path))
|
||||
|
||||
self.info = self.remote.save_info(self.path_info)
|
||||
|
||||
def dumpd(self):
|
||||
if self.is_local:
|
||||
path = self.remote.unixpath(os.path.relpath(self.path,
|
||||
self.stage.cwd))
|
||||
else:
|
||||
path = self.path
|
||||
|
||||
info = self.info.copy()
|
||||
info[self.PARAM_PATH] = path
|
||||
return info
|
||||
@@ -0,0 +1,37 @@
|
||||
import posixpath
|
||||
|
||||
try:
|
||||
from urlparse import urlparse
|
||||
except ImportError:
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from dvc.dependency.base import DependencyBase
|
||||
from dvc.remote.s3 import RemoteS3
|
||||
|
||||
|
||||
class DependencyS3(DependencyBase):
|
||||
REGEX = RemoteS3.REGEX
|
||||
|
||||
def __init__(self, stage, path, info=None, remote=None):
|
||||
super(DependencyS3, self).__init__(stage, path)
|
||||
self.info = info
|
||||
self.remote = remote if remote else RemoteS3(stage.project, {})
|
||||
|
||||
bucket = remote.bucket if remote else urlparse(path).netloc
|
||||
key = urlparse(path).path.lstrip('/')
|
||||
if remote:
|
||||
key = posixpath.join(remote.prefix, key)
|
||||
self.path_info = {'scheme': 's3',
|
||||
'bucket': bucket,
|
||||
'key': key}
|
||||
|
||||
def changed(self):
|
||||
return self.info != self.remote.save_info(self.path_info)
|
||||
|
||||
def save(self):
|
||||
self.info = self.remote.save_info(self.path_info)
|
||||
|
||||
def dumpd(self):
|
||||
ret = self.info
|
||||
ret[self.PARAM_PATH] = self.path
|
||||
return ret
|
||||
@@ -0,0 +1,44 @@
|
||||
import posixpath
|
||||
|
||||
try:
|
||||
from urlparse import urlparse
|
||||
except ImportError:
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from dvc.dependency.base import DependencyBase
|
||||
from dvc.remote.ssh import RemoteSSH
|
||||
|
||||
|
||||
class DependencySSH(DependencyBase):
|
||||
REGEX = RemoteSSH.REGEX
|
||||
|
||||
def __init__(self, stage, path, info=None, remote=None):
|
||||
super(DependencySSH, self).__init__(stage, path)
|
||||
self.info = info
|
||||
self.remote = remote if remote else RemoteSSH(stage.project, {})
|
||||
|
||||
host = remote.host if remote else self.group('host')
|
||||
port = remote.port if remote else RemoteSSH.DEFAULT_PORT
|
||||
user = remote.user if remote else self.group('user')
|
||||
if remote:
|
||||
path = posixpath.join(remote.prefix,
|
||||
urlparse(path).path.lstrip('/'))
|
||||
else:
|
||||
path = self.match(self.path).group('path')
|
||||
|
||||
self.path_info = {'scheme': 'ssh',
|
||||
'host': host,
|
||||
'port': port,
|
||||
'user': user,
|
||||
'path': path}
|
||||
|
||||
def changed(self):
|
||||
return self.info != self.remote.save_info(self.path_info)
|
||||
|
||||
def save(self):
|
||||
self.info = self.remote.save_info(self.path_info)
|
||||
|
||||
def dumpd(self):
|
||||
ret = self.info
|
||||
ret[self.PARAM_PATH] = self.path
|
||||
return ret
|
||||
+28
-1
@@ -8,6 +8,33 @@ class DvcException(Exception):
|
||||
if cause:
|
||||
try:
|
||||
self.cause_tb = traceback.format_exc()
|
||||
except Exception:
|
||||
except Exception: # pragma: no cover
|
||||
pass
|
||||
super(DvcException, self).__init__(msg)
|
||||
|
||||
|
||||
class UnsupportedRemoteError(DvcException):
|
||||
def __init__(self, config):
|
||||
msg = "Remote '{}' is not supported.".format(config)
|
||||
super(UnsupportedRemoteError, self).__init__(msg)
|
||||
|
||||
|
||||
class OutputDuplicationError(DvcException):
|
||||
def __init__(self, output, stages):
|
||||
assert isinstance(output, str)
|
||||
assert isinstance(stages, list)
|
||||
assert all(isinstance(stage, str) for stage in stages)
|
||||
msg = "File '{}' is specified as an output in more than one stage:{}"
|
||||
s = ""
|
||||
for stage in stages:
|
||||
s += "\n " + stage
|
||||
super(OutputDuplicationError, self).__init__(msg.format(output, s))
|
||||
|
||||
|
||||
class MoveNotDataSourceError(DvcException):
|
||||
def __init__(self, path):
|
||||
msg = "Move is not permitted for stages that are not data sources. " \
|
||||
"You need to either move '{path}' to a new location and edit " \
|
||||
"it by hand, or remove '{path}' and create a new one at the " \
|
||||
"desired location."
|
||||
super(MoveNotDataSourceError, self).__init__(msg.format(path=path))
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
# Based on https://eli.thegreenplace.net/2011/10/19/
|
||||
# perls-guess-if-file-is-text-or-binary-implemented-in-python
|
||||
|
||||
import sys
|
||||
PY3 = sys.version_info[0] == 3
|
||||
|
||||
# A function that takes an integer in the 8-bit range and returns
|
||||
# a single-character byte object in py3 / a single-character string
|
||||
# in py2.
|
||||
#
|
||||
int2byte = (lambda x: bytes((x,))) if PY3 else chr
|
||||
|
||||
|
||||
_text_characters = (
|
||||
b''.join(int2byte(i) for i in range(32, 127)) +
|
||||
b'\n\r\t\f\b')
|
||||
|
||||
|
||||
def istextfile(fname, blocksize=512):
|
||||
""" Uses heuristics to guess whether the given file is text or binary,
|
||||
by reading a single block of bytes from the file.
|
||||
If more than 30% of the chars in the block are non-text, or there
|
||||
are NUL ('\x00') bytes in the block, assume this is a binary file.
|
||||
"""
|
||||
with open(fname, 'rb') as fd:
|
||||
block = fd.read(blocksize)
|
||||
|
||||
if b'\x00' in block:
|
||||
# Files with null bytes are binary
|
||||
return False
|
||||
elif not block:
|
||||
# An empty file is considered a valid text file
|
||||
return True
|
||||
|
||||
# Use translate's 'deletechars' argument to efficiently remove all
|
||||
# occurrences of _text_characters from the block
|
||||
nontext = block.translate(None, _text_characters)
|
||||
return float(len(nontext)) / len(block) <= 0.30
|
||||
+2
-5
@@ -17,15 +17,12 @@ class Lock(object):
|
||||
self.lock_file = os.path.join(dvc_dir, name)
|
||||
self._lock = None
|
||||
|
||||
@staticmethod
|
||||
def init(dvc_dir):
|
||||
return Lock(dvc_dir)
|
||||
|
||||
def _do_lock(self):
|
||||
try:
|
||||
self._lock = zc.lockfile.LockFile(self.lock_file)
|
||||
except zc.lockfile.LockError:
|
||||
raise LockError('Cannot perform the cmd since DVC is busy and locked. Please retry the cmd later.')
|
||||
raise LockError('Cannot perform the cmd since DVC is busy and '
|
||||
'locked. Please retry the cmd later.')
|
||||
|
||||
def lock(self):
|
||||
try:
|
||||
|
||||
+38
-11
@@ -19,9 +19,16 @@ class Logger(object):
|
||||
}
|
||||
|
||||
COLOR_MAP = {
|
||||
'debug': colorama.Fore.BLUE,
|
||||
'warn': colorama.Fore.YELLOW,
|
||||
'error': colorama.Fore.RED
|
||||
'green': colorama.Fore.GREEN,
|
||||
'yellow': colorama.Fore.YELLOW,
|
||||
'blue': colorama.Fore.BLUE,
|
||||
'red': colorama.Fore.RED,
|
||||
}
|
||||
|
||||
LEVEL_COLOR_MAP = {
|
||||
'debug': 'blue',
|
||||
'warn': 'yellow',
|
||||
'error': 'red',
|
||||
}
|
||||
|
||||
def __init__(self, loglevel=None):
|
||||
@@ -58,12 +65,12 @@ class Logger(object):
|
||||
Logger.logger().setLevel(logging.DEBUG)
|
||||
|
||||
@staticmethod
|
||||
def colorize(msg, typ):
|
||||
def colorize(msg, color):
|
||||
header = ''
|
||||
footer = ''
|
||||
|
||||
if sys.stdout.isatty():
|
||||
header = Logger.COLOR_MAP.get(typ.lower(), '')
|
||||
if sys.stdout.isatty(): # pragma: no cover
|
||||
header = Logger.COLOR_MAP.get(color.lower(), '')
|
||||
footer = colorama.Style.RESET_ALL
|
||||
|
||||
return u'{}{}{}'.format(header, msg, footer)
|
||||
@@ -82,22 +89,42 @@ class Logger(object):
|
||||
|
||||
return (str_exc, str_tb)
|
||||
|
||||
@staticmethod
|
||||
def _prefix(msg, typ):
|
||||
color = Logger.LEVEL_COLOR_MAP.get(typ.lower(), '')
|
||||
return Logger.colorize('{}: '.format(msg), color)
|
||||
|
||||
@staticmethod
|
||||
def error_prefix():
|
||||
return Logger._prefix('Error', 'error')
|
||||
|
||||
@staticmethod
|
||||
def warning_prefix():
|
||||
return Logger._prefix('Warning', 'warn')
|
||||
|
||||
@staticmethod
|
||||
def debug_prefix():
|
||||
return Logger._prefix('Debug', 'debug')
|
||||
|
||||
@staticmethod
|
||||
def error(msg, exc=None):
|
||||
prefix = Logger.error_prefix()
|
||||
str_exc, str_tb = Logger.parse_exc(exc)
|
||||
if Logger.logger().getEffectiveLevel() == logging.DEBUG and exc:
|
||||
str_tb = str_tb if str_tb else traceback.format_exc()
|
||||
Logger.logger().error(str_tb)
|
||||
return Logger.logger().error(Logger.colorize(msg + str_exc, 'error'))
|
||||
Logger.logger().error(prefix + str_tb)
|
||||
chat = "\n\nHaving any troubles? Hit us up at dvc.org/support, " \
|
||||
"we are always happy to help!"
|
||||
return Logger.logger().error(prefix + msg + str_exc + chat)
|
||||
|
||||
@staticmethod
|
||||
def warn(msg):
|
||||
return Logger.logger().warn(Logger.colorize(msg, 'warn'))
|
||||
return Logger.logger().warn(Logger.warning_prefix() + msg)
|
||||
|
||||
@staticmethod
|
||||
def debug(msg):
|
||||
return Logger.logger().debug(Logger.colorize(msg, 'debug'))
|
||||
return Logger.logger().debug(Logger.debug_prefix() + msg)
|
||||
|
||||
@staticmethod
|
||||
def info(msg):
|
||||
return Logger.logger().info(Logger.colorize(msg, 'info'))
|
||||
return Logger.logger().info(msg)
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
from dvc.logger import Logger
|
||||
from dvc.cli import parse_args
|
||||
from dvc.command.common.base import CmdBase
|
||||
from dvc.command.base import CmdBase
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
|
||||
-217
@@ -1,217 +0,0 @@
|
||||
import os
|
||||
import schema
|
||||
import posixpath
|
||||
import ntpath
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.logger import Logger
|
||||
from dvc.utils import remove
|
||||
|
||||
|
||||
class OutputError(DvcException):
|
||||
pass
|
||||
|
||||
|
||||
class CmdOutputError(DvcException):
|
||||
def __init__(self, path, msg):
|
||||
super(CmdOutputError, self).__init__('Output file \'{}\' error: {}'.format(path, msg))
|
||||
|
||||
|
||||
class CmdOutputNoCacheError(CmdOutputError):
|
||||
def __init__(self, path):
|
||||
super(CmdOutputNoCacheError, self).__init__(path, 'no cache')
|
||||
|
||||
|
||||
class CmdOutputOutsideOfRepoError(CmdOutputError):
|
||||
def __init__(self, path):
|
||||
super(CmdOutputOutsideOfRepoError, self).__init__(path, 'outside of repository')
|
||||
|
||||
|
||||
class CmdOutputDoesNotExistError(CmdOutputError):
|
||||
def __init__(self, path):
|
||||
super(CmdOutputDoesNotExistError, self).__init__(path, 'does not exist')
|
||||
|
||||
|
||||
class CmdOutputIsNotFileOrDirError(CmdOutputError):
|
||||
def __init__(self, path):
|
||||
super(CmdOutputIsNotFileOrDirError, self).__init__(path, 'not a file or directory')
|
||||
|
||||
|
||||
class CmdOutputAlreadyTrackedError(CmdOutputError):
|
||||
def __init__(self, path):
|
||||
super(CmdOutputAlreadyTrackedError, self).__init__(path, 'already tracked by scm(e.g. git)')
|
||||
|
||||
|
||||
class Dependency(object):
|
||||
PARAM_PATH = 'path'
|
||||
PARAM_MD5 = 'md5'
|
||||
MD5_DIR_SUFFIX = '.dir'
|
||||
|
||||
SCHEMA = {
|
||||
PARAM_PATH: str,
|
||||
schema.Optional(PARAM_MD5): schema.Or(str, None),
|
||||
}
|
||||
|
||||
def __init__(self, project, path, md5=None):
|
||||
self.project = project
|
||||
self.path = os.path.abspath(os.path.normpath(path))
|
||||
|
||||
if not self.path.startswith(self.project.root_dir):
|
||||
raise CmdOutputOutsideOfRepoError(self.rel_path)
|
||||
|
||||
self.md5 = md5
|
||||
|
||||
@property
|
||||
def rel_path(self):
|
||||
return os.path.relpath(self.path)
|
||||
|
||||
def _changed_md5(self):
|
||||
if not os.path.exists(self.path):
|
||||
return True
|
||||
|
||||
return self.project.state.changed(self.path, self.md5)
|
||||
|
||||
@staticmethod
|
||||
def _changed_msg(changed):
|
||||
if changed:
|
||||
return 'changed'
|
||||
return "didn't change"
|
||||
|
||||
def changed(self):
|
||||
ret = self._changed_md5()
|
||||
|
||||
msg = u'Dependency \'{}\' {}'.format(self.rel_path, self._changed_msg(ret))
|
||||
self.project.logger.debug(msg)
|
||||
|
||||
return ret
|
||||
|
||||
def status(self):
|
||||
if self.changed():
|
||||
#FIXME better msgs
|
||||
return {self.rel_path: 'changed'}
|
||||
return {}
|
||||
|
||||
def save(self):
|
||||
if not os.path.exists(self.path):
|
||||
raise CmdOutputDoesNotExistError(self.rel_path)
|
||||
|
||||
if not os.path.isfile(self.path) and not os.path.isdir(self.path):
|
||||
raise CmdOutputIsNotFileOrDirError(self.rel_path)
|
||||
|
||||
if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \
|
||||
(os.path.isdir(self.path) and len(os.listdir(self.path)) == 0):
|
||||
self.project.logger.warn("File/directory '{}' is empty.".format(self.rel_path))
|
||||
|
||||
self.md5 = self.project.state.update(self.path)
|
||||
|
||||
@staticmethod
|
||||
def unixpath(path):
|
||||
assert not ntpath.isabs(path)
|
||||
assert not posixpath.isabs(path)
|
||||
return path.replace('\\', '/')
|
||||
|
||||
def dumpd(self, cwd):
|
||||
return {
|
||||
Output.PARAM_PATH: self.unixpath(os.path.relpath(self.path, cwd)),
|
||||
Output.PARAM_MD5: self.md5,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def loadd(cls, project, d, cwd=os.curdir):
|
||||
relpath = os.path.normpath(Output.unixpath(d[Output.PARAM_PATH]))
|
||||
path = os.path.join(cwd, relpath)
|
||||
md5 = d.get(Output.PARAM_MD5, None)
|
||||
return cls(project, path, md5=md5)
|
||||
|
||||
@classmethod
|
||||
def loadd_from(cls, project, d_list, cwd=os.curdir):
|
||||
return [cls.loadd(project, x, cwd=cwd) for x in d_list]
|
||||
|
||||
@classmethod
|
||||
def loads(cls, project, s, cwd=os.curdir):
|
||||
return cls(project, os.path.join(cwd, s), md5=None)
|
||||
|
||||
@classmethod
|
||||
def loads_from(cls, project, s_list, cwd=os.curdir):
|
||||
return [cls.loads(project, x, cwd=cwd) for x in s_list]
|
||||
|
||||
|
||||
class Output(Dependency):
|
||||
PARAM_CACHE = 'cache'
|
||||
|
||||
SCHEMA = Dependency.SCHEMA
|
||||
SCHEMA[schema.Optional(PARAM_CACHE)] = bool
|
||||
|
||||
def __init__(self, project, path, md5=None, use_cache=True):
|
||||
super(Output, self).__init__(project, path, md5=md5)
|
||||
self.use_cache = use_cache
|
||||
|
||||
@property
|
||||
def cache(self):
|
||||
if not self.use_cache:
|
||||
return None
|
||||
|
||||
return self.project.cache.get(self.md5)
|
||||
|
||||
def dumpd(self, cwd):
|
||||
ret = super(Output, self).dumpd(cwd)
|
||||
ret[Output.PARAM_CACHE] = self.use_cache
|
||||
return ret
|
||||
|
||||
@classmethod
|
||||
def loadd(cls, project, d, cwd=os.curdir):
|
||||
ret = super(Output, cls).loadd(project, d, cwd=cwd)
|
||||
ret.use_cache = d.get(Output.PARAM_CACHE, True)
|
||||
return ret
|
||||
|
||||
@classmethod
|
||||
def loads(cls, project, s, use_cache=True, cwd=os.curdir):
|
||||
ret = super(Output, cls).loads(project, s, cwd=cwd)
|
||||
ret.use_cache = use_cache
|
||||
return ret
|
||||
|
||||
@classmethod
|
||||
def loads_from(cls, project, s_list, use_cache=False, cwd=os.curdir):
|
||||
return [cls.loads(project, x, use_cache=use_cache, cwd=cwd) for x in s_list]
|
||||
|
||||
def changed(self):
|
||||
if super(Output, self).changed():
|
||||
return True
|
||||
|
||||
if self.use_cache and self.project.cache.changed(self.md5):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def checkout(self):
|
||||
if not self.use_cache:
|
||||
return
|
||||
|
||||
msg = u'Checking out \'{}\' with cache \'{}\''
|
||||
self.project.logger.debug(msg.format(self.rel_path, self.md5))
|
||||
|
||||
if not self.changed():
|
||||
msg = u'Data file \'{}\' with cache \'{}\' didn\'t change, skipping checkout.'
|
||||
self.project.logger.debug(msg.format(self.rel_path, self.md5))
|
||||
return
|
||||
|
||||
self.project.cache.checkout(self.path, self.md5)
|
||||
|
||||
def save(self):
|
||||
super(Output, self).save()
|
||||
|
||||
if not self.use_cache:
|
||||
return
|
||||
|
||||
self.project.logger.debug(u'Saving \'{}\' to \'{}\''.format(self.rel_path, self.md5))
|
||||
|
||||
if self.project.scm.is_tracked(self.path):
|
||||
raise CmdOutputAlreadyTrackedError(self.rel_path)
|
||||
|
||||
if not self.changed():
|
||||
return
|
||||
|
||||
self.project.cache.save(self.path)
|
||||
|
||||
def remove(self):
|
||||
remove(self.path)
|
||||
@@ -0,0 +1,62 @@
|
||||
import schema
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.config import Config
|
||||
|
||||
from dvc.dependency import SCHEMA, urlparse
|
||||
from dvc.dependency.base import DependencyBase
|
||||
from dvc.output.s3 import OutputS3
|
||||
from dvc.output.gs import OutputGS
|
||||
from dvc.output.local import OutputLOCAL
|
||||
from dvc.output.hdfs import OutputHDFS
|
||||
from dvc.output.ssh import OutputSSH
|
||||
|
||||
from dvc.remote import Remote
|
||||
|
||||
|
||||
OUTS = [OutputHDFS, OutputS3, OutputGS, OutputSSH, OutputLOCAL]
|
||||
|
||||
OUTS_MAP = {'hdfs': OutputHDFS,
|
||||
's3': OutputS3,
|
||||
'gs': OutputGS,
|
||||
'ssh': OutputSSH,
|
||||
'': OutputLOCAL}
|
||||
|
||||
SCHEMA[schema.Optional(OutputLOCAL.PARAM_CACHE)] = bool
|
||||
SCHEMA[schema.Optional(OutputLOCAL.PARAM_METRIC)] = OutputLOCAL.METRIC_SCHEMA
|
||||
|
||||
|
||||
def _get(stage, p, info, cache, metric):
|
||||
parsed = urlparse(p)
|
||||
if parsed.scheme == 'remote':
|
||||
name = Config.SECTION_REMOTE_FMT.format(parsed.netloc)
|
||||
sect = stage.project.config._config[name]
|
||||
remote = Remote(stage.project, sect)
|
||||
return OUTS_MAP[remote.scheme](stage,
|
||||
p,
|
||||
info,
|
||||
cache=cache,
|
||||
remote=remote,
|
||||
metric=metric)
|
||||
|
||||
for o in OUTS:
|
||||
if o.supported(p):
|
||||
return o(stage, p, info, cache=cache, remote=None, metric=metric)
|
||||
raise DvcException('Output \'{}\' is not supported'.format(p))
|
||||
|
||||
|
||||
def loadd_from(stage, d_list):
|
||||
ret = []
|
||||
for d in d_list:
|
||||
p = d.pop(DependencyBase.PARAM_PATH)
|
||||
cache = d.pop(OutputLOCAL.PARAM_CACHE, True)
|
||||
metric = d.pop(OutputLOCAL.PARAM_METRIC, False)
|
||||
ret.append(_get(stage, p, info=d, cache=cache, metric=metric))
|
||||
return ret
|
||||
|
||||
|
||||
def loads_from(stage, s_list, use_cache=True, metric=False):
|
||||
ret = []
|
||||
for s in s_list:
|
||||
ret.append(_get(stage, s, info={}, cache=use_cache, metric=metric))
|
||||
return ret
|
||||
@@ -0,0 +1,24 @@
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
|
||||
class OutputError(DvcException):
|
||||
def __init__(self, path, msg):
|
||||
msg = 'Output \'{}\' error: {}'.format(path, msg)
|
||||
super(OutputError, self).__init__(msg)
|
||||
|
||||
|
||||
class OutputDoesNotExistError(OutputError):
|
||||
def __init__(self, path):
|
||||
super(OutputDoesNotExistError, self).__init__(path, 'does not exist')
|
||||
|
||||
|
||||
class OutputIsNotFileOrDirError(OutputError):
|
||||
def __init__(self, path):
|
||||
msg = 'not a file or directory'
|
||||
super(OutputIsNotFileOrDirError, self).__init__(path, msg)
|
||||
|
||||
|
||||
class OutputAlreadyTrackedError(OutputError):
|
||||
def __init__(self, path):
|
||||
msg = 'already tracked by scm(e.g. git)'
|
||||
super(OutputAlreadyTrackedError, self).__init__(path, msg)
|
||||
@@ -0,0 +1,53 @@
|
||||
from dvc.dependency.gs import DependencyGS
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
|
||||
class OutputGS(DependencyGS):
|
||||
PARAM_CACHE = 'cache'
|
||||
PARAM_METRIC = 'metric'
|
||||
|
||||
def __init__(self,
|
||||
stage,
|
||||
path,
|
||||
info=None,
|
||||
remote=None,
|
||||
cache=True,
|
||||
metric=False):
|
||||
super(OutputGS, self).__init__(stage, path, info, remote=remote)
|
||||
self.use_cache = cache
|
||||
self.metric = metric
|
||||
if cache and self.project.cache.gs is None:
|
||||
raise DvcException("No cache location setup for \'gs\' outputs.")
|
||||
|
||||
def dumpd(self):
|
||||
ret = super(OutputGS, self).dumpd()
|
||||
ret[self.PARAM_CACHE] = self.use_cache
|
||||
ret[self.PARAM_METRIC] = self.metric
|
||||
return ret
|
||||
|
||||
def changed(self):
|
||||
if super(OutputGS, self).changed():
|
||||
return True
|
||||
|
||||
if self.use_cache \
|
||||
and self.project.cache.gs.changed(self.path_info, self.info):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def checkout(self):
|
||||
if not self.use_cache:
|
||||
return
|
||||
|
||||
self.project.cache.gs.checkout(self.path_info, self.info)
|
||||
|
||||
def save(self):
|
||||
super(OutputGS, self).save()
|
||||
|
||||
if not self.use_cache:
|
||||
return
|
||||
|
||||
self.info = self.project.cache.gs.save(self.path_info)
|
||||
|
||||
def remove(self, ignore_remove=False):
|
||||
self.remote.remove(self.path_info)
|
||||
@@ -0,0 +1,53 @@
|
||||
from dvc.dependency.hdfs import DependencyHDFS
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
|
||||
class OutputHDFS(DependencyHDFS):
|
||||
PARAM_CACHE = 'cache'
|
||||
PARAM_METRIC = 'metric'
|
||||
|
||||
def __init__(self,
|
||||
stage,
|
||||
path,
|
||||
info=None,
|
||||
remote=None,
|
||||
cache=True,
|
||||
metric=False):
|
||||
super(OutputHDFS, self).__init__(stage, path, info, remote=remote)
|
||||
self.use_cache = cache
|
||||
self.metric = metric
|
||||
if cache and self.project.cache.hdfs is None:
|
||||
raise DvcException("No cache location setup for \'hdfs\' outputs.")
|
||||
|
||||
def dumpd(self):
|
||||
ret = super(OutputHDFS, self).dumpd()
|
||||
ret[self.PARAM_CACHE] = self.use_cache
|
||||
ret[self.PARAM_METRIC] = self.metric
|
||||
return ret
|
||||
|
||||
def changed(self):
|
||||
if super(OutputHDFS, self).changed():
|
||||
return True
|
||||
|
||||
if self.use_cache \
|
||||
and self.project.cache.hdfs.changed(self.path_info, self.info):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def checkout(self):
|
||||
if not self.use_cache:
|
||||
return
|
||||
|
||||
self.project.cache.hdfs.checkout(self.path_info, self.info)
|
||||
|
||||
def save(self):
|
||||
super(OutputHDFS, self).save()
|
||||
|
||||
if not self.use_cache:
|
||||
return
|
||||
|
||||
self.info = self.project.cache.hdfs.save(self.path_info)
|
||||
|
||||
def remove(self, ignore_remove=False):
|
||||
self.remote.remove(self.path_info)
|
||||
@@ -0,0 +1,130 @@
|
||||
import os
|
||||
from schema import Optional, Or
|
||||
|
||||
from dvc.output.base import OutputDoesNotExistError, OutputIsNotFileOrDirError
|
||||
from dvc.output.base import OutputAlreadyTrackedError
|
||||
from dvc.dependency.local import DependencyLOCAL
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.istextfile import istextfile
|
||||
|
||||
|
||||
class OutputLOCAL(DependencyLOCAL):
|
||||
PARAM_CACHE = 'cache'
|
||||
PARAM_METRIC = 'metric'
|
||||
PARAM_METRIC_TYPE = 'type'
|
||||
PARAM_METRIC_XPATH = 'xpath'
|
||||
|
||||
METRIC_SCHEMA = Or(None, bool,
|
||||
{Optional(PARAM_METRIC_TYPE): Or(str, None),
|
||||
Optional(PARAM_METRIC_XPATH): Or(str, None)})
|
||||
|
||||
DoesNotExistError = OutputDoesNotExistError
|
||||
IsNotFileOrDirError = OutputIsNotFileOrDirError
|
||||
|
||||
def __init__(self,
|
||||
stage,
|
||||
path,
|
||||
info=None,
|
||||
remote=None,
|
||||
cache=True,
|
||||
metric=False):
|
||||
super(OutputLOCAL, self).__init__(stage, path, info, remote=remote)
|
||||
self.use_cache = cache
|
||||
self.metric = metric
|
||||
|
||||
@property
|
||||
def md5(self):
|
||||
return self.info.get(self.project.cache.local.PARAM_MD5, None)
|
||||
|
||||
@property
|
||||
def cache(self):
|
||||
return self.project.cache.local.get(self.md5)
|
||||
|
||||
def dumpd(self):
|
||||
ret = super(OutputLOCAL, self).dumpd()
|
||||
ret[self.PARAM_CACHE] = self.use_cache
|
||||
|
||||
if isinstance(self.metric, dict):
|
||||
if self.PARAM_METRIC_XPATH in self.metric and \
|
||||
not self.metric[self.PARAM_METRIC_XPATH]:
|
||||
del self.metric[self.PARAM_METRIC_XPATH]
|
||||
|
||||
if self.metric:
|
||||
ret[self.PARAM_METRIC] = self.metric
|
||||
|
||||
return ret
|
||||
|
||||
def changed(self):
|
||||
if not self.use_cache:
|
||||
return super(OutputLOCAL, self).changed()
|
||||
|
||||
return self.project.cache.local.changed(self.path_info, self.info)
|
||||
|
||||
def checkout(self):
|
||||
if not self.use_cache:
|
||||
return
|
||||
|
||||
self.project.cache.local.checkout(self.path_info, self.info)
|
||||
|
||||
def _verify_metric(self):
|
||||
if not self.metric:
|
||||
return
|
||||
|
||||
if os.path.isdir(self.path):
|
||||
msg = 'Directory \'{}\' cannot be used as metrics.'
|
||||
raise DvcException(msg.format(self.rel_path))
|
||||
|
||||
if not istextfile(self.path):
|
||||
msg = 'Binary file \'{}\' cannot be used as metrics.'
|
||||
raise DvcException(msg.format(self.rel_path))
|
||||
|
||||
def save(self):
|
||||
if not self.use_cache:
|
||||
super(OutputLOCAL, self).save()
|
||||
self._verify_metric()
|
||||
msg = 'Output \'{}\' doesn\'t use cache. Skipping saving.'
|
||||
self.project.logger.info(msg.format(self.rel_path))
|
||||
return
|
||||
|
||||
if not os.path.exists(self.path):
|
||||
raise self.DoesNotExistError(self.rel_path)
|
||||
|
||||
if not os.path.isfile(self.path) \
|
||||
and not os.path.isdir(self.path): # pragma: no cover
|
||||
raise self.IsNotFileOrDirError(self.rel_path)
|
||||
|
||||
if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \
|
||||
(os.path.isdir(self.path) and len(os.listdir(self.path)) == 0):
|
||||
msg = "File/directory '{}' is empty.".format(self.rel_path)
|
||||
self.project.logger.warn(msg)
|
||||
|
||||
if not self.changed():
|
||||
msg = 'Output \'{}\' didn\'t change. Skipping saving.'
|
||||
self.project.logger.info(msg.format(self.rel_path))
|
||||
return
|
||||
|
||||
if self.is_local:
|
||||
if self.project.scm.is_tracked(self.path):
|
||||
raise OutputAlreadyTrackedError(self.rel_path)
|
||||
|
||||
if self.use_cache:
|
||||
self.project.scm.ignore(self.path)
|
||||
|
||||
self.info = self.project.cache.local.save(self.path_info)
|
||||
|
||||
def remove(self, ignore_remove=False):
|
||||
self.remote.remove(self.path_info)
|
||||
if ignore_remove and self.use_cache and self.is_local:
|
||||
self.project.scm.ignore_remove(self.path)
|
||||
|
||||
def move(self, out):
|
||||
if self.use_cache and self.is_local:
|
||||
self.project.scm.ignore_remove(self.path)
|
||||
|
||||
self.remote.move(self.path_info, out.path_info)
|
||||
self.path = out.path
|
||||
self.path_info = out.path_info
|
||||
self.save()
|
||||
|
||||
if self.use_cache and self.is_local:
|
||||
self.project.scm.ignore(self.path)
|
||||
@@ -0,0 +1,53 @@
|
||||
from dvc.dependency.s3 import DependencyS3
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
|
||||
class OutputS3(DependencyS3):
|
||||
PARAM_CACHE = 'cache'
|
||||
PARAM_METRIC = 'metric'
|
||||
|
||||
def __init__(self,
|
||||
stage,
|
||||
path,
|
||||
info=None,
|
||||
remote=None,
|
||||
cache=True,
|
||||
metric=False):
|
||||
super(OutputS3, self).__init__(stage, path, info, remote=remote)
|
||||
self.use_cache = cache
|
||||
self.metric = metric
|
||||
if cache and self.project.cache.s3 is None:
|
||||
raise DvcException("No cache location setup for \'s3\' outputs.")
|
||||
|
||||
def dumpd(self):
|
||||
ret = super(OutputS3, self).dumpd()
|
||||
ret[self.PARAM_CACHE] = self.use_cache
|
||||
ret[self.PARAM_METRIC] = self.metric
|
||||
return ret
|
||||
|
||||
def changed(self):
|
||||
if super(OutputS3, self).changed():
|
||||
return True
|
||||
|
||||
if self.use_cache \
|
||||
and self.project.cache.s3.changed(self.path_info, self.info):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def checkout(self):
|
||||
if not self.use_cache:
|
||||
return
|
||||
|
||||
self.project.cache.s3.checkout(self.path_info, self.info)
|
||||
|
||||
def save(self):
|
||||
super(OutputS3, self).save()
|
||||
|
||||
if not self.use_cache:
|
||||
return
|
||||
|
||||
self.info = self.project.cache.s3.save(self.path_info)
|
||||
|
||||
def remove(self, ignore_remove=False):
|
||||
self.remote.remove(self.path_info)
|
||||
@@ -0,0 +1,53 @@
|
||||
from dvc.dependency.ssh import DependencySSH
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
|
||||
class OutputSSH(DependencySSH):
|
||||
PARAM_CACHE = 'cache'
|
||||
PARAM_METRIC = 'metric'
|
||||
|
||||
def __init__(self,
|
||||
stage,
|
||||
path,
|
||||
info=None,
|
||||
remote=None,
|
||||
cache=True,
|
||||
metric=False):
|
||||
super(OutputSSH, self).__init__(stage, path, info, remote=remote)
|
||||
self.use_cache = cache
|
||||
self.metric = metric
|
||||
if cache and self.project.cache.ssh is None:
|
||||
raise DvcException("No cache location setup for \'ssh\' outputs.")
|
||||
|
||||
def dumpd(self):
|
||||
ret = super(OutputSSH, self).dumpd()
|
||||
ret[self.PARAM_CACHE] = self.use_cache
|
||||
ret[self.PARAM_METRIC] = self.metric
|
||||
return ret
|
||||
|
||||
def changed(self):
|
||||
if super(OutputSSH, self).changed():
|
||||
return True
|
||||
|
||||
if self.use_cache \
|
||||
and self.project.cache.ssh.changed(self.path_info, self.info):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def checkout(self):
|
||||
if not self.use_cache:
|
||||
return
|
||||
|
||||
self.project.cache.ssh.checkout(self.path_info, self.info)
|
||||
|
||||
def save(self):
|
||||
super(OutputSSH, self).save()
|
||||
|
||||
if not self.use_cache:
|
||||
return
|
||||
|
||||
self.info = self.project.cache.ssh.save(self.path_info)
|
||||
|
||||
def remove(self, ignore_remove=False):
|
||||
self.remote.remove(self.path_info)
|
||||
+12
-7
@@ -1,6 +1,7 @@
|
||||
from __future__ import print_function
|
||||
import sys
|
||||
import threading
|
||||
import threading
|
||||
|
||||
|
||||
class Progress(object):
|
||||
"""
|
||||
@@ -15,6 +16,10 @@ class Progress(object):
|
||||
self._n_total = total
|
||||
self._n_finished = 0
|
||||
|
||||
@property
|
||||
def is_finished(self):
|
||||
return self._n_total == self._n_finished
|
||||
|
||||
def _clearln(self):
|
||||
print('\r\x1b[K', end='')
|
||||
|
||||
@@ -52,10 +57,9 @@ class Progress(object):
|
||||
Make a progress bar out of info, which looks like:
|
||||
(1/2): [########################################] 100% master.zip
|
||||
"""
|
||||
total_len = 100
|
||||
bar_len = 30
|
||||
|
||||
if total == None:
|
||||
if total is None:
|
||||
progress = 0
|
||||
percent = "?% "
|
||||
else:
|
||||
@@ -63,15 +67,16 @@ class Progress(object):
|
||||
progress = int((100 * current)/total) if current < total else 100
|
||||
percent = str(progress) + "% "
|
||||
|
||||
num = "({}/{}): ".format(self._n_finished + 1, self._n_total)
|
||||
if self._n_total > 1:
|
||||
num = "({}/{}): ".format(self._n_finished + 1, self._n_total)
|
||||
else:
|
||||
num = ""
|
||||
|
||||
n_sh = int((progress * bar_len)/100)
|
||||
n_sp = bar_len - n_sh
|
||||
bar = "[" + '#'*n_sh + ' '*n_sp + "] "
|
||||
|
||||
name_len = total_len - len(num + bar + percent)
|
||||
name = target_name[:name_len] if len(target_name) > name_len else target_name
|
||||
return num + bar + percent + target_name
|
||||
|
||||
return num + bar + percent + name
|
||||
|
||||
progress = Progress()
|
||||
|
||||
+754
-183
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,40 @@
|
||||
import sys
|
||||
|
||||
from dvc.progress import progress
|
||||
|
||||
try:
|
||||
# NOTE: in Python3 raw_input() was renamed to input()
|
||||
input = raw_input
|
||||
except NameError:
|
||||
pass
|
||||
|
||||
|
||||
class Prompt(object):
|
||||
def __init__(self):
|
||||
self.default = None
|
||||
|
||||
def prompt(self, msg, default=False): # pragma: no cover
|
||||
if self.default is not None:
|
||||
return self.default
|
||||
|
||||
if not sys.stdout.isatty():
|
||||
return default
|
||||
|
||||
answer = input(msg + u' (y/n)\n').lower()
|
||||
while answer not in ['yes', 'no', 'y', 'n']:
|
||||
answer = input('Enter \'yes\' or \'no\'.\n').lower()
|
||||
|
||||
return answer[0] == "y"
|
||||
|
||||
def prompt_password(self, msg): # pragma: no cover
|
||||
import getpass
|
||||
|
||||
if not sys.stdout.isatty():
|
||||
return None
|
||||
|
||||
msg = 'Enter password for {}:\n'.format(msg)
|
||||
|
||||
if not progress.is_finished:
|
||||
msg = u'\n' + msg
|
||||
|
||||
return getpass.getpass(msg)
|
||||
@@ -0,0 +1,27 @@
|
||||
from dvc.remote.local import RemoteLOCAL
|
||||
from dvc.remote.s3 import RemoteS3
|
||||
from dvc.remote.gs import RemoteGS
|
||||
from dvc.remote.hdfs import RemoteHDFS
|
||||
from dvc.remote.ssh import RemoteSSH
|
||||
from dvc.remote.azure import RemoteAzure
|
||||
|
||||
from dvc.config import Config
|
||||
from dvc.exceptions import UnsupportedRemoteError
|
||||
|
||||
|
||||
REMOTES = [RemoteHDFS, RemoteSSH, RemoteS3, RemoteGS, RemoteAzure, RemoteLOCAL]
|
||||
|
||||
|
||||
def supported_url(url):
|
||||
config = {Config.SECTION_REMOTE_URL: url}
|
||||
for r in REMOTES:
|
||||
if r.supported(config):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def Remote(project, config):
|
||||
for r in REMOTES:
|
||||
if r.supported(config):
|
||||
return r(project, config)
|
||||
raise UnsupportedRemoteError(str(config))
|
||||
@@ -0,0 +1,244 @@
|
||||
from __future__ import absolute_import
|
||||
import os
|
||||
import re
|
||||
|
||||
try:
|
||||
from azure.storage.blob import BlockBlobService
|
||||
except ImportError:
|
||||
BlockBlobService = None
|
||||
|
||||
from dvc.logger import Logger
|
||||
from dvc.progress import progress
|
||||
from dvc.config import Config
|
||||
from dvc.remote.base import RemoteBase
|
||||
|
||||
|
||||
class Callback(object):
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
|
||||
def __call__(self, current, total):
|
||||
progress.update_target(self.name, current, total)
|
||||
|
||||
|
||||
class RemoteAzure(RemoteBase):
|
||||
scheme = 'azure'
|
||||
REGEX = (r'^azure://'
|
||||
r'(ContainerName=(?P<container_name>[^;]+);?)?'
|
||||
r'(?P<connection_string>.+)?$')
|
||||
REQUIRES = {'azure-storage-blob': BlockBlobService}
|
||||
PARAM_ETAG = 'etag'
|
||||
COPY_POLL_SECONDS = 5
|
||||
|
||||
def __init__(self, project, config):
|
||||
super(RemoteAzure, self).__init__(project, config)
|
||||
self.project = project
|
||||
|
||||
self.url = config.get(Config.SECTION_REMOTE_URL)
|
||||
match = re.match(self.REGEX, self.url)
|
||||
|
||||
self.bucket = (
|
||||
match.group('container_name')
|
||||
or os.getenv('AZURE_STORAGE_CONTAINER_NAME'))
|
||||
|
||||
self.connection_string = (
|
||||
match.group('connection_string')
|
||||
or os.getenv('AZURE_STORAGE_CONNECTION_STRING'))
|
||||
|
||||
if not self.bucket:
|
||||
raise ValueError('Azure Storage container name missing')
|
||||
|
||||
if not self.connection_string:
|
||||
raise ValueError('Azure Storage connection string missing')
|
||||
|
||||
self.__blob_service = None
|
||||
|
||||
@property
|
||||
def blob_service(self):
|
||||
if self.__blob_service is None:
|
||||
self.__blob_service = BlockBlobService(
|
||||
connection_string=self.connection_string)
|
||||
self.__blob_service.create_container(self.bucket)
|
||||
return self.__blob_service
|
||||
|
||||
# FIXME: temporarily disabled because of the lack of test for external azure
|
||||
# dependencies/outputs/cache.
|
||||
#
|
||||
# def _get_etag(self, bucket, key):
|
||||
# try:
|
||||
# blob = self.blob_service.get_blob_properties(bucket, key)
|
||||
# return blob.properties.etag
|
||||
# except Exception:
|
||||
# return None
|
||||
#
|
||||
# def save_info(self, path_info):
|
||||
# if path_info['scheme'] != self.scheme:
|
||||
# raise NotImplementedError
|
||||
#
|
||||
# return {self.PARAM_ETAG: self._get_etag(
|
||||
# path_info['bucket'], path_info['key'])}
|
||||
#
|
||||
# def save(self, path_info):
|
||||
# if path_info['scheme'] != self.scheme:
|
||||
# raise NotImplementedError
|
||||
#
|
||||
# etag = self._get_etag(path_info['bucket'], path_info['key'])
|
||||
# dest_key = '{}/{}'.format(etag[0:2], etag[2:])
|
||||
#
|
||||
# self._copy_blob(to_bucket=self.bucket,
|
||||
# to_key=dest_key,
|
||||
# from_bucket=path_info['bucket'],
|
||||
# from_key=path_info['key'])
|
||||
#
|
||||
# return {self.PARAM_ETAG: etag}
|
||||
#
|
||||
# def changed(self, path_info, checksum_info):
|
||||
# return checksum_info != self.save_info(path_info)
|
||||
#
|
||||
# def _copy_blob(self, to_bucket, to_key, from_bucket, from_key):
|
||||
# source = self.blob_service.make_blob_url(from_bucket, from_key)
|
||||
#
|
||||
# copy = self.blob_service.copy_blob(to_bucket, to_key, source)
|
||||
#
|
||||
# if self.COPY_POLL_SECONDS <= 0:
|
||||
# return
|
||||
#
|
||||
# while copy.status != 'success':
|
||||
# time.sleep(self.COPY_POLL_SECONDS)
|
||||
# copy = self.blob_service.get_blob_properties(
|
||||
# to_bucket, to_key).properties.copy
|
||||
#
|
||||
# def checkout(self, path_info, checksum_info):
|
||||
# if path_info['scheme'] != self.scheme:
|
||||
# raise NotImplementedError
|
||||
#
|
||||
# etag = checksum_info.get(self.PARAM_ETAG, None)
|
||||
# if not etag:
|
||||
# return
|
||||
#
|
||||
# self._copy_blob(to_bucket=path_info['bucket'],
|
||||
# to_key=path_info['key'],
|
||||
# from_bucket=self.bucket,
|
||||
# from_key='{}/{}'.format(etag[0:2], etag[2:]))
|
||||
#
|
||||
# def remove(self, path_info):
|
||||
# if path_info['scheme'] != self.scheme:
|
||||
# raise NotImplementedError
|
||||
#
|
||||
# Logger.debug('Removing azure://{}/{}'.format(path_info['bucket'],
|
||||
# path_info['key']))
|
||||
#
|
||||
# self.blob_service.delete_blob(path_info['bucket'], path_info['key'])
|
||||
|
||||
def md5s_to_path_infos(self, md5s):
|
||||
return [{
|
||||
'scheme': self.scheme,
|
||||
'bucket': self.bucket,
|
||||
'key': '{}/{}'.format(md5[0:2], md5[2:])
|
||||
} for md5 in md5s]
|
||||
|
||||
def exists(self, path_infos):
|
||||
keys = {blob.name
|
||||
for blob in self.blob_service.list_blobs(self.bucket)}
|
||||
|
||||
ret = []
|
||||
for path_info in path_infos:
|
||||
if path_info['scheme'] != self.scheme:
|
||||
raise NotImplementedError
|
||||
|
||||
key = path_info['key']
|
||||
ret.append(key in keys)
|
||||
|
||||
return ret
|
||||
|
||||
def upload(self, from_infos, to_infos, names=None):
|
||||
names = self._verify_path_args(to_infos, from_infos, names)
|
||||
|
||||
for from_info, to_info, name in zip(from_infos, to_infos, names):
|
||||
if to_info['scheme'] != self.scheme:
|
||||
raise NotImplementedError
|
||||
|
||||
if from_info['scheme'] != 'local':
|
||||
raise NotImplementedError
|
||||
|
||||
bucket = to_info['bucket']
|
||||
key = to_info['key']
|
||||
|
||||
Logger.debug("Uploading '{}' to '{}/{}'".format(
|
||||
from_info['path'], bucket, key))
|
||||
|
||||
if not name:
|
||||
name = os.path.basename(from_info['path'])
|
||||
|
||||
cb = Callback(name)
|
||||
|
||||
try:
|
||||
self.blob_service.create_blob_from_path(
|
||||
bucket, key, from_info['path'], progress_callback=cb)
|
||||
except Exception as ex:
|
||||
Logger.error("Failed to upload '{}'".format(from_info['path']),
|
||||
ex)
|
||||
else:
|
||||
progress.finish_target(name)
|
||||
|
||||
def download(self,
|
||||
from_infos,
|
||||
to_infos,
|
||||
no_progress_bar=False,
|
||||
names=None):
|
||||
names = self._verify_path_args(from_infos, to_infos, names)
|
||||
|
||||
for to_info, from_info, name in zip(to_infos, from_infos, names):
|
||||
if from_info['scheme'] != self.scheme:
|
||||
raise NotImplementedError
|
||||
|
||||
if to_info['scheme'] != 'local':
|
||||
raise NotImplementedError
|
||||
|
||||
bucket = from_info['bucket']
|
||||
key = from_info['key']
|
||||
|
||||
Logger.debug("Downloading '{}/{}' to '{}'".format(
|
||||
bucket, key, to_info['path']))
|
||||
|
||||
tmp_file = self.tmp_file(to_info['path'])
|
||||
if not name:
|
||||
name = os.path.basename(to_info['path'])
|
||||
|
||||
cb = None if no_progress_bar else Callback(name)
|
||||
|
||||
self._makedirs(to_info['path'])
|
||||
|
||||
try:
|
||||
self.blob_service.get_blob_to_path(
|
||||
bucket, key, tmp_file, progress_callback=cb)
|
||||
except Exception as exc:
|
||||
Logger.error("Failed to download '{}/{}'".format(
|
||||
bucket, key), exc)
|
||||
else:
|
||||
os.rename(tmp_file, to_info['path'])
|
||||
|
||||
if not no_progress_bar:
|
||||
progress.finish_target(name)
|
||||
|
||||
# FIXME: temporarily disabled because of the lack of test for external azure
|
||||
# dependencies/outputs/cache.
|
||||
#
|
||||
# def gc(self, cinfos):
|
||||
# used = [info[self.PARAM_ETAG] for info in cinfos['azure']]
|
||||
# used += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
|
||||
#
|
||||
# all_blobs = self.blob_service.list_blobs(self.bucket)
|
||||
#
|
||||
# removed = False
|
||||
# for blob in all_blobs:
|
||||
# etag = blob.properties.etag
|
||||
# if etag in used:
|
||||
# continue
|
||||
# path_info = {'scheme': self.scheme,
|
||||
# 'key': blob.name,
|
||||
# 'bucket': self.bucket}
|
||||
# self.remove(path_info)
|
||||
# removed = True
|
||||
#
|
||||
# return removed
|
||||
@@ -0,0 +1,134 @@
|
||||
import os
|
||||
import re
|
||||
import errno
|
||||
from multiprocessing import cpu_count
|
||||
|
||||
from dvc.config import Config
|
||||
from dvc.logger import Logger
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
|
||||
STATUS_OK = 1
|
||||
STATUS_NEW = 3
|
||||
STATUS_DELETED = 4
|
||||
|
||||
|
||||
STATUS_MAP = {
|
||||
# (local_exists, remote_exists)
|
||||
(True, True): STATUS_OK,
|
||||
(False, False): STATUS_OK,
|
||||
(True, False): STATUS_NEW,
|
||||
(False, True): STATUS_DELETED,
|
||||
}
|
||||
|
||||
|
||||
class DataCloudError(DvcException):
|
||||
""" Data Cloud exception """
|
||||
def __init__(self, msg):
|
||||
super(DataCloudError, self).__init__('Data sync error: {}'.format(msg))
|
||||
|
||||
|
||||
class RemoteBase(object):
|
||||
REGEX = None
|
||||
REQUIRES = {}
|
||||
JOBS = 4 * cpu_count()
|
||||
|
||||
def __init__(self, project, config):
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def supported(cls, config):
|
||||
url = config[Config.SECTION_REMOTE_URL]
|
||||
url_ok = cls.match(url)
|
||||
deps_ok = all(cls.REQUIRES.values())
|
||||
if url_ok and not deps_ok:
|
||||
missing = [k for k, v in cls.REQUIRES.items() if v is None]
|
||||
msg = "URL \'{}\' is supported but requires these missing " \
|
||||
"dependencies: {}. If you have installed dvc using pip, " \
|
||||
"choose one of these options to proceed: \n" \
|
||||
"\n" \
|
||||
" 1) Install specific missing dependencies:\n" \
|
||||
" pip install {}\n" \
|
||||
" 2) Install dvc package that includes those missing " \
|
||||
"dependencies: \n" \
|
||||
" pip install dvc[{}]\n" \
|
||||
" 3) Install dvc package with all possible " \
|
||||
"dependencies included: \n" \
|
||||
" pip install dvc[all]\n" \
|
||||
"\n" \
|
||||
"If you have installed dvc from a binary package and you " \
|
||||
"are still seeing this message, please report it to us " \
|
||||
"using https://github.com/iterative/dvc/issues. Thank you!"
|
||||
msg = msg.format(url, missing, " ".join(missing), cls.scheme)
|
||||
Logger.warn(msg)
|
||||
|
||||
return url_ok and deps_ok
|
||||
|
||||
@classmethod
|
||||
def match(cls, url):
|
||||
return re.match(cls.REGEX, url)
|
||||
|
||||
def group(self, name):
|
||||
m = self.match(self.url)
|
||||
if not m:
|
||||
return None
|
||||
return m.group(name)
|
||||
|
||||
@staticmethod
|
||||
def tmp_file(fname):
|
||||
""" Temporary name for a partial download """
|
||||
# FIXME probably better use uuid()
|
||||
return fname + '.part'
|
||||
|
||||
def save_info(self, path_info):
|
||||
raise NotImplementedError
|
||||
|
||||
def changed(self, path_info, checksum_info):
|
||||
raise NotImplementedError
|
||||
|
||||
def save(self, path_info):
|
||||
raise NotImplementedError
|
||||
|
||||
def checkout(self, path_info, checksum_info):
|
||||
raise NotImplementedError
|
||||
|
||||
def download(self, from_infos, to_infos, no_progress_bar=False, name=None):
|
||||
raise NotImplementedError
|
||||
|
||||
def upload(self, from_infos, to_infos, path_info, name=None):
|
||||
raise NotImplementedError
|
||||
|
||||
def remove(self, path_info):
|
||||
raise NotImplementedError
|
||||
|
||||
def move(self, path_info):
|
||||
raise NotImplementedError
|
||||
|
||||
def _makedirs(self, fname):
|
||||
dname = os.path.dirname(fname)
|
||||
|
||||
try:
|
||||
os.makedirs(dname)
|
||||
except OSError as e:
|
||||
if e.errno != errno.EEXIST:
|
||||
raise
|
||||
|
||||
def md5s_to_path_infos(self, md5s):
|
||||
raise NotImplementedError
|
||||
|
||||
def exists(self, path_infos):
|
||||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
def _verify_path_args(cls, from_infos, to_infos, names=None):
|
||||
assert isinstance(from_infos, list)
|
||||
assert isinstance(to_infos, list)
|
||||
assert len(from_infos) == len(to_infos)
|
||||
|
||||
if not names:
|
||||
names = len(to_infos) * [None]
|
||||
else:
|
||||
assert isinstance(names, list)
|
||||
assert len(names) == len(to_infos)
|
||||
|
||||
return names
|
||||
@@ -0,0 +1,293 @@
|
||||
import os
|
||||
import posixpath
|
||||
|
||||
try:
|
||||
from google.cloud import storage
|
||||
except ImportError:
|
||||
storage = None
|
||||
|
||||
try:
|
||||
from urlparse import urlparse
|
||||
except ImportError:
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from dvc.logger import Logger
|
||||
from dvc.remote.base import RemoteBase
|
||||
from dvc.remote.local import RemoteLOCAL
|
||||
from dvc.config import Config
|
||||
from dvc.progress import progress
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
|
||||
class RemoteGS(RemoteBase):
|
||||
scheme = 'gs'
|
||||
REGEX = r'^gs://(?P<path>.*)$'
|
||||
REQUIRES = {'google.cloud.storage': storage}
|
||||
PARAM_ETAG = 'etag'
|
||||
|
||||
def __init__(self, project, config):
|
||||
self.project = project
|
||||
storagepath = 'gs://'
|
||||
storagepath += config.get(Config.SECTION_AWS_STORAGEPATH, '/')
|
||||
storagepath.lstrip('/')
|
||||
self.url = config.get(Config.SECTION_REMOTE_URL, storagepath)
|
||||
self.projectname = config.get(Config.SECTION_GCP_PROJECTNAME, None)
|
||||
|
||||
@property
|
||||
def bucket(self):
|
||||
return urlparse(self.url).netloc
|
||||
|
||||
@property
|
||||
def prefix(self):
|
||||
return urlparse(self.url).path.lstrip('/')
|
||||
|
||||
@property
|
||||
def gs(self):
|
||||
return storage.Client()
|
||||
|
||||
def get_etag(self, bucket, key):
|
||||
blob = self.gs.bucket(bucket).get_blob(key)
|
||||
if not blob:
|
||||
return None
|
||||
|
||||
return blob.etag
|
||||
|
||||
def save_info(self, path_info):
|
||||
if path_info['scheme'] != 'gs':
|
||||
raise NotImplementedError
|
||||
|
||||
return {self.PARAM_ETAG: self.get_etag(path_info['bucket'],
|
||||
path_info['key'])}
|
||||
|
||||
@staticmethod
|
||||
def to_string(path_info):
|
||||
return "{}://{}/{}".format(path_info['scheme'],
|
||||
path_info['bucket'],
|
||||
path_info['key'])
|
||||
|
||||
def changed_cache(self, etag):
|
||||
key = posixpath.join(self.prefix, etag[0:2], etag[2:])
|
||||
cache = {'scheme': 'gs', 'bucket': self.bucket, 'key': key}
|
||||
|
||||
if {self.PARAM_ETAG: etag} != self.save_info(cache):
|
||||
if self.exists([cache])[0]:
|
||||
msg = 'Corrupted cache file {}'
|
||||
Logger.warn(msg.format(self.to_string(cache)))
|
||||
self.remove(cache)
|
||||
return True
|
||||
return False
|
||||
|
||||
def changed(self, path_info, checksum_info):
|
||||
if not self.exists([path_info])[0]:
|
||||
return True
|
||||
|
||||
etag = checksum_info.get(self.PARAM_ETAG, None)
|
||||
if etag is None:
|
||||
return True
|
||||
|
||||
if self.changed_cache(etag):
|
||||
return True
|
||||
|
||||
return checksum_info != self.save_info(path_info)
|
||||
|
||||
def _copy(self, from_info, to_info, gs=None):
|
||||
gs = gs if gs else self.gs
|
||||
|
||||
blob = gs.bucket(from_info['bucket']).get_blob(from_info['key'])
|
||||
if not blob:
|
||||
msg = '{} doesn\'t exist in the cloud'
|
||||
raise DvcException(msg.format(from_info['key']))
|
||||
|
||||
bucket = self.gs.bucket(to_info['bucket'])
|
||||
bucket.copy_blob(blob,
|
||||
self.gs.bucket(to_info['bucket']),
|
||||
new_name=to_info['key'])
|
||||
|
||||
def save(self, path_info):
|
||||
if path_info['scheme'] != 'gs':
|
||||
raise NotImplementedError
|
||||
|
||||
etag = self.get_etag(path_info['bucket'], path_info['key'])
|
||||
key = posixpath.join(self.prefix, etag[0:2], etag[2:])
|
||||
to_info = {'scheme': 'gs', 'bucket': self.bucket, 'key': key}
|
||||
|
||||
self._copy(path_info, to_info)
|
||||
|
||||
return {self.PARAM_ETAG: etag}
|
||||
|
||||
def checkout(self, path_info, checksum_info):
|
||||
if path_info['scheme'] != 'gs':
|
||||
raise NotImplementedError
|
||||
|
||||
etag = checksum_info.get(self.PARAM_ETAG, None)
|
||||
if not etag:
|
||||
return
|
||||
|
||||
if not self.changed(path_info, checksum_info):
|
||||
msg = "Data '{}' didn't change."
|
||||
Logger.info(msg.format(self.to_string(path_info)))
|
||||
return
|
||||
|
||||
if self.changed_cache(etag):
|
||||
msg = "Cache '{}' not found. File '{}' won't be created."
|
||||
Logger.warn(msg.format(etag, self.to_string(path_info)))
|
||||
return
|
||||
|
||||
if self.exists([path_info])[0]:
|
||||
msg = "Data '{}' exists. Removing before checkout."
|
||||
Logger.warn(msg.format(self.to_string(path_info)))
|
||||
self.remove(path_info)
|
||||
return
|
||||
|
||||
msg = "Checking out '{}' with cache '{}'."
|
||||
Logger.info(msg.format(self.to_string(path_info), etag))
|
||||
|
||||
key = posixpath.join(self.prefix, etag[0:2], etag[2:])
|
||||
from_info = {'scheme': 'gs', 'bucket': self.bucket, 'key': key}
|
||||
|
||||
self._copy(from_info, path_info)
|
||||
|
||||
def remove(self, path_info):
|
||||
if path_info['scheme'] != 'gs':
|
||||
raise NotImplementedError
|
||||
|
||||
Logger.debug("Removing gs://{}/{}".format(path_info['bucket'],
|
||||
path_info['key']))
|
||||
|
||||
blob = self.gs.bucket(path_info['bucket']).get_blob(path_info['key'])
|
||||
if not blob:
|
||||
return
|
||||
|
||||
blob.delete()
|
||||
|
||||
def md5s_to_path_infos(self, md5s):
|
||||
return [{'scheme': 'gs',
|
||||
'bucket': self.bucket,
|
||||
'key': posixpath.join(self.prefix,
|
||||
md5[0:2], md5[2:])} for md5 in md5s]
|
||||
|
||||
def exists(self, path_infos):
|
||||
ret = []
|
||||
gs = self.gs
|
||||
|
||||
bucket = gs.bucket(self.bucket)
|
||||
keys = [blob.name for blob in bucket.list_blobs(prefix=self.prefix)]
|
||||
|
||||
for path_info in path_infos:
|
||||
exists = False
|
||||
if path_info['key'] in keys:
|
||||
exists = True
|
||||
ret.append(exists)
|
||||
return ret
|
||||
|
||||
def upload(self, from_infos, to_infos, names=None):
|
||||
names = self._verify_path_args(to_infos, from_infos, names)
|
||||
|
||||
gs = self.gs
|
||||
|
||||
for from_info, to_info, name in zip(from_infos, to_infos, names):
|
||||
if to_info['scheme'] != 'gs':
|
||||
raise NotImplementedError
|
||||
|
||||
if from_info['scheme'] != 'local':
|
||||
raise NotImplementedError
|
||||
|
||||
Logger.debug("Uploading '{}' to '{}/{}'".format(from_info['path'],
|
||||
to_info['bucket'],
|
||||
to_info['key']))
|
||||
|
||||
if not name:
|
||||
name = os.path.basename(from_info['path'])
|
||||
|
||||
progress.update_target(name, 0, None)
|
||||
|
||||
try:
|
||||
bucket = gs.bucket(to_info['bucket'])
|
||||
blob = bucket.blob(to_info['key'])
|
||||
blob.upload_from_filename(from_info['path'])
|
||||
except Exception as exc:
|
||||
msg = "Failed to upload '{}' to '{}/{}'"
|
||||
Logger.error(msg.format(from_info['path'],
|
||||
to_info['bucket'],
|
||||
to_info['key']), exc)
|
||||
continue
|
||||
|
||||
progress.finish_target(name)
|
||||
|
||||
def download(self,
|
||||
from_infos,
|
||||
to_infos,
|
||||
no_progress_bar=False,
|
||||
names=None):
|
||||
names = self._verify_path_args(from_infos, to_infos, names)
|
||||
|
||||
gs = self.gs
|
||||
|
||||
for to_info, from_info, name in zip(to_infos, from_infos, names):
|
||||
if from_info['scheme'] != 'gs':
|
||||
raise NotImplementedError
|
||||
|
||||
if to_info['scheme'] == 'gs':
|
||||
self._copy(from_info, to_info, gs=gs)
|
||||
continue
|
||||
|
||||
if to_info['scheme'] != 'local':
|
||||
raise NotImplementedError
|
||||
|
||||
msg = "Downloading '{}/{}' to '{}'".format(from_info['bucket'],
|
||||
from_info['key'],
|
||||
to_info['path'])
|
||||
Logger.debug(msg)
|
||||
|
||||
tmp_file = self.tmp_file(to_info['path'])
|
||||
if not name:
|
||||
name = os.path.basename(to_info['path'])
|
||||
|
||||
if not no_progress_bar:
|
||||
# percent_cb is not available for download_to_filename, so
|
||||
# lets at least update progress at keypoints(start, finish)
|
||||
progress.update_target(name, 0, None)
|
||||
|
||||
self._makedirs(to_info['path'])
|
||||
|
||||
try:
|
||||
bucket = gs.bucket(from_info['bucket'])
|
||||
blob = bucket.get_blob(from_info['key'])
|
||||
blob.download_to_filename(tmp_file)
|
||||
except Exception as exc:
|
||||
msg = "Failed to download '{}/{}' to '{}'"
|
||||
Logger.error(msg.format(from_info['bucket'],
|
||||
from_info['key'],
|
||||
to_info['path']), exc)
|
||||
continue
|
||||
|
||||
os.rename(tmp_file, to_info['path'])
|
||||
|
||||
if not no_progress_bar:
|
||||
progress.finish_target(name)
|
||||
|
||||
def _path_to_etag(self, path):
|
||||
relpath = posixpath.relpath(path, self.prefix)
|
||||
return posixpath.dirname(relpath) + posixpath.basename(relpath)
|
||||
|
||||
def _all_etags(self):
|
||||
blobs = self.gs.bucket(self.bucket).list_blobs(prefix=self.prefix)
|
||||
blobs = list(blobs)
|
||||
return [self._path_to_etag(blob.name) for blob in blobs]
|
||||
|
||||
def gc(self, cinfos):
|
||||
used = [info[self.PARAM_ETAG] for info in cinfos['gs']]
|
||||
used += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
|
||||
|
||||
removed = False
|
||||
for etag in self._all_etags():
|
||||
if etag in used:
|
||||
continue
|
||||
path_info = {'scheme': 'gs',
|
||||
'bucket': self.bucket,
|
||||
'key': posixpath.join(self.prefix,
|
||||
etag[0:2], etag[2:])}
|
||||
self.remove(path_info)
|
||||
removed = True
|
||||
|
||||
return removed
|
||||
@@ -0,0 +1,270 @@
|
||||
import os
|
||||
import re
|
||||
import getpass
|
||||
import posixpath
|
||||
from subprocess import Popen, PIPE
|
||||
|
||||
from dvc.config import Config
|
||||
from dvc.remote.base import RemoteBase
|
||||
from dvc.remote.local import RemoteLOCAL
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.logger import Logger
|
||||
from dvc.utils import fix_env
|
||||
|
||||
|
||||
class RemoteHDFS(RemoteBase):
|
||||
scheme = 'hdfs'
|
||||
REGEX = r'^hdfs://((?P<user>.*)@)?.*$'
|
||||
PARAM_CHECKSUM = 'checksum'
|
||||
|
||||
def __init__(self, project, config):
|
||||
self.project = project
|
||||
self.url = config.get(Config.SECTION_REMOTE_URL, '/')
|
||||
self.user = self.group('user')
|
||||
if not self.user:
|
||||
self.user = config.get(Config.SECTION_REMOTE_USER,
|
||||
getpass.getuser())
|
||||
|
||||
def hadoop_fs(self, cmd, user=None):
|
||||
cmd = 'hadoop fs -' + cmd
|
||||
if user:
|
||||
cmd = 'HADOOP_USER_NAME={} '.format(user) + cmd
|
||||
|
||||
# NOTE: close_fds doesn't work with redirected stdin/stdout/stderr.
|
||||
# See https://github.com/iterative/dvc/issues/1197.
|
||||
close_fds = (os.name != 'nt')
|
||||
|
||||
p = Popen(cmd,
|
||||
shell=True,
|
||||
close_fds=close_fds,
|
||||
executable=os.getenv('SHELL'),
|
||||
env=fix_env(os.environ),
|
||||
stdin=PIPE,
|
||||
stdout=PIPE,
|
||||
stderr=PIPE)
|
||||
out, err = p.communicate()
|
||||
if p.returncode != 0:
|
||||
raise DvcException('HDFS command failed: {}: {}'.format(cmd, err))
|
||||
return out.decode('utf-8')
|
||||
|
||||
@staticmethod
|
||||
def _group(regex, s, gname):
|
||||
match = re.match(regex, s)
|
||||
assert match is not None
|
||||
return match.group(gname)
|
||||
|
||||
def checksum(self, path_info):
|
||||
regex = r'.*\t.*\t(?P<checksum>.*)'
|
||||
stdout = self.hadoop_fs('checksum {}'.format(path_info['url']),
|
||||
user=path_info['user'])
|
||||
return self._group(regex, stdout, 'checksum')
|
||||
|
||||
def cp(self, from_info, to_info):
|
||||
self.hadoop_fs('mkdir -p {}'.format(posixpath.dirname(to_info['url'])),
|
||||
user=to_info['user'])
|
||||
self.hadoop_fs('cp -f {} {}'.format(from_info['url'], to_info['url']),
|
||||
user=to_info['user'])
|
||||
|
||||
def rm(self, path_info):
|
||||
self.hadoop_fs('rm {}'.format(path_info['url']),
|
||||
user=path_info['user'])
|
||||
|
||||
def save_info(self, path_info):
|
||||
if path_info['scheme'] != 'hdfs':
|
||||
raise NotImplementedError
|
||||
|
||||
assert path_info.get('url')
|
||||
|
||||
return {self.PARAM_CHECKSUM: self.checksum(path_info)}
|
||||
|
||||
@staticmethod
|
||||
def to_string(path_info):
|
||||
return "{}://{}".format(path_info['scheme'],
|
||||
path_info['url'])
|
||||
|
||||
def changed_cache(self, checksum):
|
||||
cache = {}
|
||||
cache['scheme'] = 'hdfs'
|
||||
cache['user'] = self.user
|
||||
cache['url'] = posixpath.join(self.url, checksum[0:2], checksum[2:])
|
||||
|
||||
if {self.PARAM_CHECKSUM: checksum} != self.save_info(cache):
|
||||
if self.exists([cache])[0]:
|
||||
msg = 'Corrupted cache file {}'
|
||||
Logger.warn(msg.format(self.to_string(cache)))
|
||||
self.remove(cache)
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def changed(self, path_info, checksum_info):
|
||||
if not self.exists([path_info])[0]:
|
||||
return True
|
||||
|
||||
checksum = checksum_info.get(self.PARAM_CHECKSUM, None)
|
||||
if checksum is None:
|
||||
return True
|
||||
|
||||
if self.changed_cache(self, checksum):
|
||||
return True
|
||||
|
||||
return checksum_info != self.save_info(path_info)
|
||||
|
||||
def save(self, path_info):
|
||||
if path_info['scheme'] != 'hdfs':
|
||||
raise NotImplementedError
|
||||
|
||||
assert path_info.get('url')
|
||||
|
||||
checksum = self.checksum(path_info)
|
||||
dest = path_info.copy()
|
||||
dest['url'] = posixpath.join(self.url, checksum[0:2], checksum[2:])
|
||||
|
||||
self.cp(path_info, dest)
|
||||
|
||||
return {self.PARAM_CHECKSUM: checksum}
|
||||
|
||||
def checkout(self, path_info, checksum_info):
|
||||
if path_info['scheme'] != 'hdfs':
|
||||
raise NotImplementedError
|
||||
|
||||
assert path_info.get('url')
|
||||
|
||||
checksum = checksum_info.get(self.PARAM_CHECKSUM, None)
|
||||
if not checksum:
|
||||
return
|
||||
|
||||
if not self.changed(path_info, checksum_info):
|
||||
msg = "Data '{}' didn't change."
|
||||
Logger.info(msg.format(self.to_string(path_info)))
|
||||
return
|
||||
|
||||
if self.changed_cache(checksum):
|
||||
msg = "Cache '{}' not found. File '{}' won't be created."
|
||||
Logger.warn(msg.format(checksum, self.to_string(path_info)))
|
||||
return
|
||||
|
||||
if self.exists([path_info])[0]:
|
||||
msg = "Data '{}' exists. Removing before checkout."
|
||||
Logger.warn(msg.format(self.to_string(path_info)))
|
||||
self.remove(path_info)
|
||||
return
|
||||
|
||||
msg = "Checking out '{}' with cache '{}'."
|
||||
Logger.info(msg.format(self.to_string(path_info), checksum))
|
||||
|
||||
src = path_info.copy()
|
||||
src['url'] = posixpath.join(self.url, checksum[0:2], checksum[2:])
|
||||
|
||||
self.cp(src, path_info)
|
||||
|
||||
def remove(self, path_info):
|
||||
if path_info['scheme'] != 'hdfs':
|
||||
raise NotImplementedError
|
||||
|
||||
assert path_info.get('url')
|
||||
|
||||
Logger.debug('Removing {}'.format(path_info['url']))
|
||||
|
||||
self.rm(path_info)
|
||||
|
||||
def md5s_to_path_infos(self, md5s):
|
||||
return [{'scheme': 'hdfs',
|
||||
'user': self.user,
|
||||
'url': posixpath.join(self.url,
|
||||
md5[0:2], md5[2:])} for md5 in md5s]
|
||||
|
||||
def exists(self, path_infos):
|
||||
try:
|
||||
stdout = self.hadoop_fs('ls -R {}'.format(self.url))
|
||||
except DvcException:
|
||||
return len(path_infos) * [False]
|
||||
|
||||
lines = stdout.split('\n')
|
||||
lurl = []
|
||||
for line in lines:
|
||||
if not line.startswith('-'):
|
||||
continue
|
||||
lurl.append(line.split()[-1])
|
||||
|
||||
ret = []
|
||||
for path_info in path_infos:
|
||||
exists = False
|
||||
if path_info['url'] in lurl:
|
||||
exists = True
|
||||
ret.append(exists)
|
||||
|
||||
return ret
|
||||
|
||||
def upload(self, from_infos, to_infos, names=None):
|
||||
names = self._verify_path_args(to_infos, from_infos, names)
|
||||
|
||||
for from_info, to_info, name in zip(from_infos, to_infos, names):
|
||||
if to_info['scheme'] != 'hdfs':
|
||||
raise NotImplementedError
|
||||
|
||||
if from_info['scheme'] != 'local':
|
||||
raise NotImplementedError
|
||||
|
||||
cmd = 'mkdir -p {}'.format(posixpath.dirname(to_info['url']))
|
||||
self.hadoop_fs(cmd, user=to_info['user'])
|
||||
|
||||
cmd = 'copyFromLocal {} {}'.format(from_info['path'],
|
||||
to_info['url'])
|
||||
self.hadoop_fs(cmd, user=to_info['user'])
|
||||
|
||||
def download(self,
|
||||
from_infos,
|
||||
to_infos,
|
||||
no_progress_bar=False,
|
||||
names=None):
|
||||
names = self._verify_path_args(from_infos, to_infos, names)
|
||||
|
||||
for to_info, from_info, name in zip(to_infos, from_infos, names):
|
||||
if from_info['scheme'] != 'hdfs':
|
||||
raise NotImplementedError
|
||||
|
||||
if to_info['scheme'] == 'hdfs':
|
||||
self.cp(from_info, to_info)
|
||||
continue
|
||||
|
||||
if to_info['scheme'] != 'local':
|
||||
raise NotImplementedError
|
||||
|
||||
dname = os.path.dirname(to_info['path'])
|
||||
if not os.path.exists(dname):
|
||||
os.makedirs(dname)
|
||||
|
||||
cmd = 'copyToLocal {} {}'.format(from_info['url'], to_info['path'])
|
||||
self.hadoop_fs(cmd, user=from_info['user'])
|
||||
|
||||
def _path_to_checksum(self, path):
|
||||
relpath = posixpath.relpath(path, self.url)
|
||||
return posixpath.dirname(relpath) + posixpath.basename(relpath)
|
||||
|
||||
def _all_checksums(self):
|
||||
stdout = self.hadoop_fs('ls -R {}'.format(self.url))
|
||||
lines = stdout.split('\n')
|
||||
flist = []
|
||||
for line in lines:
|
||||
if not line.startswith('-'):
|
||||
continue
|
||||
flist.append(line.split()[-1])
|
||||
return [self._path_to_checksum(path) for path in flist]
|
||||
|
||||
def gc(self, cinfos):
|
||||
used = [info[self.PARAM_CHECKSUM] for info in cinfos['hdfs']]
|
||||
used += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
|
||||
|
||||
removed = False
|
||||
for checksum in self._all_checksums():
|
||||
if checksum in used:
|
||||
continue
|
||||
path_info = {'scheme': 'hdfs',
|
||||
'user': self.user,
|
||||
'url': posixpath.join(self.url,
|
||||
checksum[0:2], checksum[2:])}
|
||||
self.remove(path_info)
|
||||
removed = True
|
||||
|
||||
return removed
|
||||
@@ -0,0 +1,676 @@
|
||||
import os
|
||||
import uuid
|
||||
import json
|
||||
import ntpath
|
||||
import shutil
|
||||
import posixpath
|
||||
from operator import itemgetter
|
||||
|
||||
from dvc.system import System
|
||||
from dvc.remote.base import RemoteBase, STATUS_MAP
|
||||
from dvc.logger import Logger
|
||||
from dvc.utils import remove, move, copyfile, dict_md5, to_chunks
|
||||
from dvc.utils import LARGE_DIR_SIZE
|
||||
from dvc.config import Config
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.progress import progress
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
|
||||
class RemoteLOCAL(RemoteBase):
|
||||
scheme = ''
|
||||
REGEX = r'^(?P<path>(/+|.:\\+).*)$'
|
||||
PARAM_MD5 = 'md5'
|
||||
PARAM_PATH = 'path'
|
||||
PARAM_RELPATH = 'relpath'
|
||||
MD5_DIR_SUFFIX = '.dir'
|
||||
|
||||
CACHE_TYPES = ['reflink', 'hardlink', 'symlink', 'copy']
|
||||
CACHE_TYPE_MAP = {
|
||||
'copy': shutil.copyfile,
|
||||
'symlink': System.symlink,
|
||||
'hardlink': System.hardlink,
|
||||
'reflink': System.reflink,
|
||||
}
|
||||
|
||||
def __init__(self, project, config):
|
||||
self.project = project
|
||||
self.state = self.project.state
|
||||
storagepath = config.get(Config.SECTION_AWS_STORAGEPATH, None)
|
||||
self.cache_dir = config.get(Config.SECTION_REMOTE_URL, storagepath)
|
||||
|
||||
types = config.get(Config.SECTION_CACHE_TYPE, None)
|
||||
if types:
|
||||
if isinstance(types, str):
|
||||
types = [t.strip() for t in types.split(',')]
|
||||
self.cache_types = types
|
||||
else:
|
||||
self.cache_types = self.CACHE_TYPES
|
||||
|
||||
if self.cache_dir is not None and not os.path.exists(self.cache_dir):
|
||||
os.mkdir(self.cache_dir)
|
||||
|
||||
@property
|
||||
def url(self):
|
||||
return self.cache_dir
|
||||
|
||||
@property
|
||||
def prefix(self):
|
||||
return self.cache_dir
|
||||
|
||||
def all(self):
|
||||
clist = []
|
||||
for entry in os.listdir(self.cache_dir):
|
||||
subdir = os.path.join(self.cache_dir, entry)
|
||||
if not os.path.isdir(subdir):
|
||||
continue
|
||||
|
||||
for cache in os.listdir(subdir):
|
||||
path = os.path.join(subdir, cache)
|
||||
clist.append(self.path_to_md5(path))
|
||||
|
||||
return clist
|
||||
|
||||
def get(self, md5):
|
||||
if not md5:
|
||||
return None
|
||||
|
||||
return os.path.join(self.cache_dir, md5[0:2], md5[2:])
|
||||
|
||||
def path_to_md5(self, path):
|
||||
relpath = os.path.relpath(path, self.cache_dir)
|
||||
return os.path.dirname(relpath) + os.path.basename(relpath)
|
||||
|
||||
def changed_cache(self, md5):
|
||||
cache = self.get(md5)
|
||||
if self.state.changed(cache, md5=md5):
|
||||
if os.path.exists(cache):
|
||||
msg = 'Corrupted cache file {}'
|
||||
Logger.warn(msg.format(os.path.relpath(cache)))
|
||||
remove(cache)
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def link(self, cache, path):
|
||||
assert os.path.isfile(cache)
|
||||
|
||||
dname = os.path.dirname(path)
|
||||
if not os.path.exists(dname):
|
||||
os.makedirs(dname)
|
||||
|
||||
i = len(self.cache_types)
|
||||
while i > 0:
|
||||
try:
|
||||
self.CACHE_TYPE_MAP[self.cache_types[0]](cache, path)
|
||||
msg = "Created '{}': {} -> {}"
|
||||
Logger.info(msg.format(self.cache_types[0],
|
||||
os.path.relpath(cache),
|
||||
os.path.relpath(path)))
|
||||
return
|
||||
except DvcException as exc:
|
||||
msg = 'Cache type \'{}\' is not supported: {}'
|
||||
Logger.debug(msg.format(self.cache_types[0], str(exc)))
|
||||
del self.cache_types[0]
|
||||
i -= 1
|
||||
|
||||
raise DvcException('No possible cache types left to try out.')
|
||||
|
||||
@classmethod
|
||||
def ospath(cls, path):
|
||||
if os.name == 'nt':
|
||||
return cls.ntpath(path)
|
||||
return cls.unixpath(path)
|
||||
|
||||
@staticmethod
|
||||
def unixpath(path):
|
||||
assert not ntpath.isabs(path)
|
||||
assert not posixpath.isabs(path)
|
||||
return path.replace('\\', '/')
|
||||
|
||||
@staticmethod
|
||||
def ntpath(path):
|
||||
assert not ntpath.isabs(path)
|
||||
assert not posixpath.isabs(path)
|
||||
return path.replace('/', '\\')
|
||||
|
||||
def collect_dir_cache(self, dname):
|
||||
dir_info = []
|
||||
|
||||
bar = False
|
||||
for root, dirs, files in os.walk(dname):
|
||||
if len(files) > LARGE_DIR_SIZE:
|
||||
msg = "Computing md5 for a large directory {}. " \
|
||||
"This is only done once."
|
||||
Logger.info(msg.format(os.path.relpath(dname)))
|
||||
bar = True
|
||||
title = os.path.relpath(dname)
|
||||
processed = 0
|
||||
total = len(files)
|
||||
progress.update_target(title, 0, total)
|
||||
|
||||
for fname in files:
|
||||
path = os.path.join(root, fname)
|
||||
relpath = self.unixpath(os.path.relpath(path, dname))
|
||||
|
||||
if bar:
|
||||
progress.update_target(title, processed, total)
|
||||
processed += 1
|
||||
|
||||
md5 = self.state.update(path)
|
||||
dir_info.append({self.PARAM_RELPATH: relpath,
|
||||
self.PARAM_MD5: md5})
|
||||
|
||||
if bar:
|
||||
progress.finish_target(title)
|
||||
|
||||
# NOTE: sorting the list by path to ensure reproducibility
|
||||
dir_info = sorted(dir_info, key=itemgetter(self.PARAM_RELPATH))
|
||||
|
||||
md5 = dict_md5(dir_info) + self.MD5_DIR_SUFFIX
|
||||
if self.changed_cache(md5):
|
||||
self.dump_dir_cache(md5, dir_info)
|
||||
|
||||
return (md5, dir_info)
|
||||
|
||||
def load_dir_cache(self, md5):
|
||||
path = self.get(md5)
|
||||
|
||||
assert self.is_dir_cache(path)
|
||||
|
||||
try:
|
||||
with open(path, 'r') as fd:
|
||||
d = json.load(fd)
|
||||
except Exception as exc:
|
||||
msg = u'Failed to load dir cache \'{}\''
|
||||
Logger.error(msg.format(os.path.relpath(path)), exc)
|
||||
return []
|
||||
|
||||
if not isinstance(d, list):
|
||||
msg = u'Dir cache file format error \'{}\': skipping the file'
|
||||
Logger.error(msg.format(os.path.relpath(path)))
|
||||
return []
|
||||
|
||||
for info in d:
|
||||
info['relpath'] = self.ospath(info['relpath'])
|
||||
|
||||
return d
|
||||
|
||||
def dump_dir_cache(self, md5, dir_info):
|
||||
path = self.get(md5)
|
||||
dname = os.path.dirname(path)
|
||||
|
||||
assert self.is_dir_cache(path)
|
||||
assert isinstance(dir_info, list)
|
||||
|
||||
if not os.path.isdir(dname):
|
||||
os.makedirs(dname)
|
||||
|
||||
# NOTE: Writing first and renaming after that
|
||||
# to make sure that the operation is atomic.
|
||||
tmp = '{}.{}'.format(path, str(uuid.uuid4()))
|
||||
with open(tmp, 'w+') as fd:
|
||||
json.dump(dir_info, fd, sort_keys=True)
|
||||
move(tmp, path)
|
||||
|
||||
@classmethod
|
||||
def is_dir_cache(cls, cache):
|
||||
return cache.endswith(cls.MD5_DIR_SUFFIX)
|
||||
|
||||
def checkout(self, path_info, checksum_info):
|
||||
path = path_info['path']
|
||||
md5 = checksum_info.get(self.PARAM_MD5, None)
|
||||
cache = self.get(md5)
|
||||
|
||||
if not cache:
|
||||
msg = 'No cache info for \'{}\'. Skipping checkout.'
|
||||
Logger.warn(msg.format(os.path.relpath(path)))
|
||||
return
|
||||
|
||||
if not self.changed(path_info, checksum_info):
|
||||
msg = "Data '{}' didn't change."
|
||||
Logger.info(msg.format(os.path.relpath(path)))
|
||||
return
|
||||
|
||||
if self.changed_cache(md5):
|
||||
msg = u'Cache \'{}\' not found. File \'{}\' won\'t be created.'
|
||||
Logger.warn(msg.format(md5, os.path.relpath(path)))
|
||||
remove(path)
|
||||
return
|
||||
|
||||
if os.path.exists(path):
|
||||
msg = u'Data \'{}\' exists. Removing before checkout.'
|
||||
Logger.warn(msg.format(os.path.relpath(path)))
|
||||
remove(path)
|
||||
|
||||
msg = u'Checking out \'{}\' with cache \'{}\'.'
|
||||
Logger.info(msg.format(os.path.relpath(path), md5))
|
||||
|
||||
if not self.is_dir_cache(cache):
|
||||
self.link(cache, path)
|
||||
self.state.update_link(path)
|
||||
return
|
||||
|
||||
# Create dir separately so that dir is created
|
||||
# even if there are no files in it
|
||||
if not os.path.exists(path):
|
||||
os.makedirs(path)
|
||||
|
||||
for entry in self.load_dir_cache(md5):
|
||||
md5 = entry[self.PARAM_MD5]
|
||||
c = self.get(md5)
|
||||
relpath = entry[self.PARAM_RELPATH]
|
||||
p = os.path.join(path, relpath)
|
||||
self.link(c, p)
|
||||
self.state.update_link(path)
|
||||
|
||||
def _move(self, inp, outp):
|
||||
# moving in two stages to make last the move atomic in
|
||||
# case inp and outp are in different filesystems
|
||||
tmp = '{}.{}'.format(outp, str(uuid.uuid4()))
|
||||
move(inp, tmp)
|
||||
move(tmp, outp)
|
||||
|
||||
def _save_file(self, path_info):
|
||||
path = path_info['path']
|
||||
md5 = self.state.update(path)
|
||||
assert md5 is not None
|
||||
|
||||
cache = self.get(md5)
|
||||
|
||||
if self.changed_cache(md5):
|
||||
self._move(path, cache)
|
||||
else:
|
||||
remove(path)
|
||||
|
||||
self.link(cache, path)
|
||||
self.state.update_link(path)
|
||||
|
||||
return {self.PARAM_MD5: md5}
|
||||
|
||||
def _save_dir(self, path_info):
|
||||
path = path_info['path']
|
||||
md5, dir_info = self.state.update_info(path)
|
||||
|
||||
for entry in dir_info:
|
||||
relpath = entry[self.PARAM_RELPATH]
|
||||
m = entry[self.PARAM_MD5]
|
||||
p = os.path.join(path, relpath)
|
||||
c = self.get(m)
|
||||
|
||||
if self.changed_cache(m):
|
||||
self._move(p, c)
|
||||
else:
|
||||
remove(p)
|
||||
|
||||
self.link(c, p)
|
||||
|
||||
self.state.update_link(path)
|
||||
|
||||
return {self.PARAM_MD5: md5}
|
||||
|
||||
def save(self, path_info):
|
||||
if path_info['scheme'] != 'local':
|
||||
raise NotImplementedError
|
||||
|
||||
path = path_info['path']
|
||||
|
||||
msg = "Saving '{}' to cache '{}'."
|
||||
Logger.info(msg.format(os.path.relpath(path),
|
||||
os.path.relpath(self.cache_dir)))
|
||||
|
||||
if os.path.isdir(path):
|
||||
return self._save_dir(path_info)
|
||||
else:
|
||||
return self._save_file(path_info)
|
||||
|
||||
def save_info(self, path_info):
|
||||
if path_info['scheme'] != 'local':
|
||||
raise NotImplementedError
|
||||
|
||||
return {self.PARAM_MD5: self.state.update(path_info['path'])}
|
||||
|
||||
def changed(self, path_info, checksum_info):
|
||||
if not self.exists([path_info])[0]:
|
||||
return True
|
||||
|
||||
md5 = checksum_info.get(self.PARAM_MD5, None)
|
||||
if md5 is None:
|
||||
return True
|
||||
|
||||
if self.changed_cache(md5):
|
||||
return True
|
||||
|
||||
return checksum_info != self.save_info(path_info)
|
||||
|
||||
def remove(self, path_info):
|
||||
if path_info['scheme'] != 'local':
|
||||
raise NotImplementedError
|
||||
|
||||
remove(path_info['path'])
|
||||
|
||||
def move(self, from_info, to_info):
|
||||
if from_info['scheme'] != 'local' or to_info['scheme'] != 'local':
|
||||
raise NotImplementedError
|
||||
|
||||
move(from_info['path'], to_info['path'])
|
||||
|
||||
def md5s_to_path_infos(self, md5s):
|
||||
return [{'scheme': 'local',
|
||||
'path': os.path.join(self.prefix,
|
||||
md5[0:2], md5[2:])} for md5 in md5s]
|
||||
|
||||
def exists(self, path_infos):
|
||||
ret = []
|
||||
for path_info in path_infos:
|
||||
ret.append(os.path.exists(path_info['path']))
|
||||
return ret
|
||||
|
||||
def upload(self, from_infos, to_infos, names=None):
|
||||
names = self._verify_path_args(to_infos, from_infos, names)
|
||||
|
||||
for from_info, to_info, name in zip(from_infos, to_infos, names):
|
||||
if to_info['scheme'] != 'local':
|
||||
raise NotImplementedError
|
||||
|
||||
if from_info['scheme'] != 'local':
|
||||
raise NotImplementedError
|
||||
|
||||
Logger.debug("Uploading '{}' to '{}'".format(from_info['path'],
|
||||
to_info['path']))
|
||||
|
||||
if not name:
|
||||
name = os.path.basename(from_info['path'])
|
||||
|
||||
self._makedirs(to_info['path'])
|
||||
|
||||
try:
|
||||
copyfile(from_info['path'], to_info['path'], name=name)
|
||||
except Exception as exc:
|
||||
msg = "Failed to upload '{}' tp '{}'"
|
||||
Logger.error(msg.format(from_info['path'],
|
||||
to_info['path']), exc)
|
||||
|
||||
def download(self,
|
||||
from_infos,
|
||||
to_infos,
|
||||
no_progress_bar=False,
|
||||
names=None):
|
||||
names = self._verify_path_args(from_infos, to_infos, names)
|
||||
|
||||
for to_info, from_info, name in zip(to_infos, from_infos, names):
|
||||
if from_info['scheme'] != 'local':
|
||||
raise NotImplementedError
|
||||
|
||||
if to_info['scheme'] != 'local':
|
||||
raise NotImplementedError
|
||||
|
||||
Logger.debug("Downloading '{}' to '{}'".format(from_info['path'],
|
||||
to_info['path']))
|
||||
|
||||
if not name:
|
||||
name = os.path.basename(to_info['path'])
|
||||
|
||||
self._makedirs(to_info['path'])
|
||||
tmp_file = self.tmp_file(to_info['path'])
|
||||
try:
|
||||
copyfile(from_info['path'],
|
||||
tmp_file,
|
||||
no_progress_bar=no_progress_bar,
|
||||
name=name)
|
||||
except Exception as exc:
|
||||
msg = "Failed to download '{}' to '{}'"
|
||||
Logger.error(msg.format(from_info['path'],
|
||||
to_info['path']), exc)
|
||||
continue
|
||||
|
||||
os.rename(tmp_file, to_info['path'])
|
||||
|
||||
def _collect(self, checksum_infos, push=False):
|
||||
missing = []
|
||||
collected = []
|
||||
for info in checksum_infos:
|
||||
md5 = info[self.PARAM_MD5]
|
||||
cache = self.get(md5)
|
||||
|
||||
if not self.is_dir_cache(info[self.PARAM_MD5]):
|
||||
continue
|
||||
|
||||
if not os.path.exists(cache):
|
||||
missing.append(info)
|
||||
continue
|
||||
|
||||
for i in self.load_dir_cache(md5):
|
||||
if info.get('branch'):
|
||||
i['branch'] = info['branch']
|
||||
i[self.PARAM_PATH] = os.path.join(info[self.PARAM_PATH],
|
||||
i[self.PARAM_RELPATH])
|
||||
collected.append(i.copy())
|
||||
|
||||
collected.extend(checksum_infos)
|
||||
return collected, missing
|
||||
|
||||
def _group(self, checksum_infos, show_checksums=False):
|
||||
by_md5 = {}
|
||||
|
||||
for info in checksum_infos:
|
||||
md5 = info[self.PARAM_MD5]
|
||||
|
||||
if show_checksums:
|
||||
by_md5[md5] = md5
|
||||
continue
|
||||
|
||||
name = info[self.PARAM_PATH]
|
||||
branch = info.get('branch')
|
||||
if branch:
|
||||
name += '({})'.format(branch)
|
||||
|
||||
if md5 not in by_md5.keys():
|
||||
by_md5[md5] = ''
|
||||
else:
|
||||
by_md5[md5] += ' '
|
||||
|
||||
by_md5[md5] += name
|
||||
|
||||
return list(by_md5.keys()), list(by_md5.values())
|
||||
|
||||
def gc(self, checksum_infos):
|
||||
checksum_infos = self._collect(checksum_infos['local'])[0]
|
||||
used_md5s = [info[self.PARAM_MD5] for info in checksum_infos]
|
||||
|
||||
removed = False
|
||||
for md5 in self.all():
|
||||
if md5 in used_md5s:
|
||||
continue
|
||||
remove(self.get(md5))
|
||||
removed = True
|
||||
|
||||
return removed
|
||||
|
||||
def status(self, checksum_infos, remote, jobs=None, show_checksums=False):
|
||||
Logger.info("Preparing to pull data from {}".format(remote.url))
|
||||
title = "Collecting information"
|
||||
|
||||
progress.set_n_total(1)
|
||||
progress.update_target(title, 0, 100)
|
||||
|
||||
checksum_infos, missing = self._collect(checksum_infos)
|
||||
checksum_infos += missing
|
||||
|
||||
progress.update_target(title, 10, 100)
|
||||
|
||||
md5s, names = self._group(checksum_infos,
|
||||
show_checksums=show_checksums)
|
||||
|
||||
progress.update_target(title, 20, 100)
|
||||
|
||||
path_infos = remote.md5s_to_path_infos(md5s)
|
||||
|
||||
progress.update_target(title, 30, 100)
|
||||
|
||||
remote_exists = remote.exists(path_infos)
|
||||
|
||||
progress.update_target(title, 90, 100)
|
||||
|
||||
local_exists = [not self.changed_cache(md5) for md5 in md5s]
|
||||
|
||||
progress.finish_target(title)
|
||||
|
||||
return [(name, STATUS_MAP[l, r]) for name, l, r in zip(names,
|
||||
local_exists,
|
||||
remote_exists)]
|
||||
|
||||
def _do_pull(self,
|
||||
checksum_infos,
|
||||
remote,
|
||||
jobs=None,
|
||||
show_checksums=False):
|
||||
title = "Collecting information"
|
||||
|
||||
progress.set_n_total(1)
|
||||
progress.update_target(title, 0, 100)
|
||||
|
||||
grouped = zip(*self._group(checksum_infos,
|
||||
show_checksums=show_checksums))
|
||||
|
||||
progress.update_target(title, 10, 100)
|
||||
|
||||
md5s = []
|
||||
names = []
|
||||
# NOTE: filter files that are not corrupted
|
||||
for md5, name in grouped:
|
||||
if self.changed_cache(md5):
|
||||
md5s.append(md5)
|
||||
names.append(name)
|
||||
|
||||
progress.update_target(title, 30, 100)
|
||||
|
||||
cache = [{'scheme': 'local', 'path': self.get(md5)} for md5 in md5s]
|
||||
|
||||
progress.update_target(title, 50, 100)
|
||||
|
||||
path_infos = remote.md5s_to_path_infos(md5s)
|
||||
|
||||
progress.update_target(title, 60, 100)
|
||||
|
||||
# NOTE: dummy call to try to establish a connection
|
||||
# to see if we need to ask user for a password.
|
||||
remote.exists(remote.md5s_to_path_infos(['000']))
|
||||
|
||||
progress.update_target(title, 70, 100)
|
||||
|
||||
assert len(path_infos) == len(cache) == len(md5s) == len(names)
|
||||
|
||||
if jobs is None:
|
||||
jobs = remote.JOBS
|
||||
|
||||
chunks = list(zip(to_chunks(path_infos, jobs),
|
||||
to_chunks(cache, jobs),
|
||||
to_chunks(names, jobs)))
|
||||
|
||||
progress.finish_target(title)
|
||||
|
||||
progress.set_n_total(len(names))
|
||||
|
||||
if len(chunks) == 0:
|
||||
return
|
||||
|
||||
futures = []
|
||||
with ThreadPoolExecutor(max_workers=len(chunks)) as executor:
|
||||
for from_infos, to_infos, names in chunks:
|
||||
res = executor.submit(remote.download,
|
||||
from_infos,
|
||||
to_infos,
|
||||
names=names)
|
||||
futures.append(res)
|
||||
|
||||
for f in futures:
|
||||
f.result()
|
||||
|
||||
def pull(self, checksum_infos, remote, jobs=None, show_checksums=False):
|
||||
Logger.info("Preparing to pull data from {}".format(remote.url))
|
||||
|
||||
# NOTE: try fetching missing dir info
|
||||
checksum_infos, missing = self._collect(checksum_infos)
|
||||
if len(missing) > 0:
|
||||
self._do_pull(missing,
|
||||
remote,
|
||||
jobs,
|
||||
show_checksums=show_checksums)
|
||||
checksum_infos += self._collect(missing)[0]
|
||||
|
||||
self._do_pull(checksum_infos,
|
||||
remote,
|
||||
jobs,
|
||||
show_checksums=show_checksums)
|
||||
|
||||
def push(self, checksum_infos, remote, jobs=None, show_checksums=False):
|
||||
Logger.info("Preparing to push data to {}".format(remote.url))
|
||||
title = "Collecting information"
|
||||
|
||||
progress.set_n_total(1)
|
||||
progress.update_target(title, 0, 100)
|
||||
|
||||
checksum_infos = self._collect(checksum_infos)[0]
|
||||
|
||||
progress.update_target(title, 10, 100)
|
||||
|
||||
# NOTE: verifying that our cache is not corrupted
|
||||
def func(info):
|
||||
return not self.changed_cache(info[self.PARAM_MD5])
|
||||
checksum_infos = list(filter(func, checksum_infos))
|
||||
|
||||
progress.update_target(title, 20, 100)
|
||||
|
||||
# NOTE: filter files that are already uploaded
|
||||
md5s = [i[self.PARAM_MD5] for i in checksum_infos]
|
||||
exists = remote.exists(remote.md5s_to_path_infos(md5s))
|
||||
|
||||
progress.update_target(title, 30, 100)
|
||||
|
||||
def func(entry):
|
||||
return not entry[0]
|
||||
|
||||
assert len(exists) == len(checksum_infos)
|
||||
infos_exist = list(filter(func, zip(exists, checksum_infos)))
|
||||
checksum_infos = [i for e, i in infos_exist]
|
||||
|
||||
progress.update_target(title, 70, 100)
|
||||
|
||||
md5s, names = self._group(checksum_infos,
|
||||
show_checksums=show_checksums)
|
||||
cache = [{'scheme': 'local', 'path': self.get(md5)} for md5 in md5s]
|
||||
|
||||
progress.update_target(title, 80, 100)
|
||||
|
||||
path_infos = remote.md5s_to_path_infos(md5s)
|
||||
|
||||
assert len(path_infos) == len(cache) == len(md5s) == len(names)
|
||||
|
||||
progress.update_target(title, 90, 100)
|
||||
|
||||
if jobs is None:
|
||||
jobs = remote.JOBS
|
||||
|
||||
chunks = list(zip(to_chunks(path_infos, jobs),
|
||||
to_chunks(cache, jobs),
|
||||
to_chunks(names, jobs)))
|
||||
|
||||
progress.finish_target(title)
|
||||
|
||||
progress.set_n_total(len(names))
|
||||
|
||||
if len(chunks) == 0:
|
||||
return
|
||||
|
||||
futures = []
|
||||
with ThreadPoolExecutor(max_workers=len(chunks)) as executor:
|
||||
for to_infos, from_infos, names in chunks:
|
||||
res = executor.submit(remote.upload,
|
||||
from_infos,
|
||||
to_infos,
|
||||
names=names)
|
||||
futures.append(res)
|
||||
|
||||
for f in futures:
|
||||
f.result()
|
||||
@@ -0,0 +1,357 @@
|
||||
import os
|
||||
import threading
|
||||
import posixpath
|
||||
|
||||
try:
|
||||
import boto3
|
||||
except ImportError:
|
||||
boto3 = None
|
||||
|
||||
try:
|
||||
from urlparse import urlparse
|
||||
except ImportError:
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from dvc.logger import Logger
|
||||
from dvc.progress import progress
|
||||
from dvc.config import Config
|
||||
from dvc.remote.base import RemoteBase
|
||||
from dvc.remote.local import RemoteLOCAL
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
|
||||
class Callback(object):
|
||||
def __init__(self, name, total):
|
||||
self.name = name
|
||||
self.total = total
|
||||
self.current = 0
|
||||
self.lock = threading.Lock()
|
||||
|
||||
def __call__(self, byts):
|
||||
with self.lock:
|
||||
self.current += byts
|
||||
progress.update_target(self.name, self.current, self.total)
|
||||
|
||||
|
||||
class RemoteS3(RemoteBase):
|
||||
scheme = 's3'
|
||||
REGEX = r'^s3://(?P<path>.*)$'
|
||||
REQUIRES = {'boto3': boto3}
|
||||
PARAM_ETAG = 'etag'
|
||||
|
||||
def __init__(self, project, config):
|
||||
import configobj
|
||||
|
||||
self.project = project
|
||||
storagepath = 's3://' + config.get(Config.SECTION_AWS_STORAGEPATH,
|
||||
'').lstrip('/')
|
||||
self.url = config.get(Config.SECTION_REMOTE_URL, storagepath)
|
||||
self.region = config.get(Config.SECTION_AWS_REGION, None)
|
||||
self.profile = os.getenv('AWS_PROFILE',
|
||||
config.get(Config.SECTION_AWS_PROFILE,
|
||||
'default'))
|
||||
self.endpoint_url = config.get(Config.SECTION_AWS_ENDPOINT_URL, None)
|
||||
|
||||
credentialpath = config.get(Config.SECTION_AWS_CREDENTIALPATH, None)
|
||||
if credentialpath:
|
||||
creds_conf = configobj.ConfigObj(credentialpath)
|
||||
creds = creds_conf.get(self.profile, {})
|
||||
else:
|
||||
creds = self._get_aws_config_from_env()
|
||||
|
||||
self.creds = creds
|
||||
self.region = creds.get('region', self.region)
|
||||
self.aws_access_key_id = creds.get('aws_access_key_id', None)
|
||||
self.aws_secret_access_key = creds.get('aws_secret_access_key', None)
|
||||
|
||||
@property
|
||||
def bucket(self):
|
||||
return urlparse(self.url).netloc
|
||||
|
||||
@property
|
||||
def prefix(self):
|
||||
return urlparse(self.url).path.lstrip('/')
|
||||
|
||||
@property
|
||||
def s3(self):
|
||||
if not self.creds:
|
||||
session = boto3.session.Session(profile_name=self.profile)
|
||||
else:
|
||||
session = boto3.session.Session(
|
||||
aws_access_key_id=self.aws_access_key_id,
|
||||
aws_secret_access_key=self.aws_secret_access_key,
|
||||
region_name=self.region)
|
||||
return session.client('s3', endpoint_url=self.endpoint_url)
|
||||
|
||||
def _get_aws_config_from_env(self):
|
||||
creds = {
|
||||
'region': os.getenv('AWS_DEFAULT_REGION'),
|
||||
'aws_access_key_id': os.getenv('AWS_ACCESS_KEY_ID'),
|
||||
'aws_secret_access_key': os.getenv('AWS_SECRET_ACCESS_KEY'),
|
||||
}
|
||||
return {k: v for k, v in creds.items() if v is not None}
|
||||
|
||||
def get_etag(self, bucket, key):
|
||||
try:
|
||||
obj = self.s3.head_object(Bucket=bucket, Key=key)
|
||||
except Exception:
|
||||
raise DvcException('s3://{}/{} does not exist'.format(bucket, key))
|
||||
|
||||
return obj['ETag'].strip('"')
|
||||
|
||||
def save_info(self, path_info):
|
||||
if path_info['scheme'] != 's3':
|
||||
raise NotImplementedError
|
||||
|
||||
return {self.PARAM_ETAG: self.get_etag(path_info['bucket'],
|
||||
path_info['key'])}
|
||||
|
||||
def changed(self, path_info, checksum_info):
|
||||
if not self.exists([path_info])[0]:
|
||||
return True
|
||||
|
||||
etag = checksum_info.get(self.PARAM_ETAG, None)
|
||||
if etag is None:
|
||||
return True
|
||||
|
||||
if self.changed_cache(etag):
|
||||
return True
|
||||
|
||||
return checksum_info != self.save_info(path_info)
|
||||
|
||||
def _copy(self, from_info, to_info, s3=None):
|
||||
s3 = s3 if s3 else self.s3
|
||||
|
||||
source = {'Bucket': from_info['bucket'],
|
||||
'Key': from_info['key']}
|
||||
self.s3.copy(source, to_info['bucket'], to_info['key'])
|
||||
|
||||
def save(self, path_info):
|
||||
if path_info['scheme'] != 's3':
|
||||
raise NotImplementedError
|
||||
|
||||
etag = self.get_etag(path_info['bucket'], path_info['key'])
|
||||
key = posixpath.join(self.prefix, etag[0:2], etag[2:])
|
||||
to_info = {'scheme': 's3', 'bucket': self.bucket, 'key': key}
|
||||
|
||||
self._copy(path_info, to_info)
|
||||
|
||||
return {self.PARAM_ETAG: etag}
|
||||
|
||||
@staticmethod
|
||||
def to_string(path_info):
|
||||
return "s3://{}/{}".format(path_info['bucket'], path_info['key'])
|
||||
|
||||
def changed_cache(self, etag):
|
||||
key = posixpath.join(self.prefix, etag[0:2], etag[2:])
|
||||
cache = {'scheme': 's3', 'bucket': self.bucket, 'key': key}
|
||||
|
||||
if {self.PARAM_ETAG: etag} != self.save_info(cache):
|
||||
if self.exists([cache])[0]:
|
||||
msg = 'Corrupted cache file {}'
|
||||
Logger.warn(msg.format(self.to_string(cache)))
|
||||
self.remove(cache)
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def checkout(self, path_info, checksum_info):
|
||||
if path_info['scheme'] != 's3':
|
||||
raise NotImplementedError
|
||||
|
||||
etag = checksum_info.get(self.PARAM_ETAG, None)
|
||||
if not etag:
|
||||
return
|
||||
|
||||
if not self.changed(path_info, checksum_info):
|
||||
msg = "Data '{}' didn't change."
|
||||
Logger.info(msg.format(self.to_string(path_info)))
|
||||
return
|
||||
|
||||
if self.changed_cache(etag):
|
||||
msg = "Cache '{}' not found. File '{}' won't be created."
|
||||
Logger.warn(msg.format(etag, self.to_string(path_info)))
|
||||
return
|
||||
|
||||
if self.exists([path_info])[0]:
|
||||
msg = "Data '{}' exists. Removing before checkout."
|
||||
Logger.warn(msg.format(self.to_string(path_info)))
|
||||
self.remove(path_info)
|
||||
return
|
||||
|
||||
msg = "Checking out '{}' with cache '{}'."
|
||||
Logger.info(msg.format(self.to_string(path_info), etag))
|
||||
|
||||
key = posixpath.join(self.prefix, etag[0:2], etag[2:])
|
||||
from_info = {'scheme': 's3', 'bucket': self.bucket, 'key': key}
|
||||
|
||||
self._copy(from_info, path_info)
|
||||
|
||||
def remove(self, path_info):
|
||||
if path_info['scheme'] != 's3':
|
||||
raise NotImplementedError
|
||||
|
||||
Logger.debug('Removing s3://{}/{}'.format(path_info['bucket'],
|
||||
path_info['key']))
|
||||
|
||||
self.s3.delete_object(Bucket=path_info['bucket'],
|
||||
Key=path_info['key'])
|
||||
|
||||
def md5s_to_path_infos(self, md5s):
|
||||
return [{'scheme': self.scheme,
|
||||
'bucket': self.bucket,
|
||||
'key': posixpath.join(self.prefix,
|
||||
md5[0:2], md5[2:])} for md5 in md5s]
|
||||
|
||||
def _all_keys(self):
|
||||
s3 = self.s3
|
||||
|
||||
keys = []
|
||||
kwargs = {'Bucket': self.bucket,
|
||||
'Prefix': self.prefix}
|
||||
while True:
|
||||
resp = s3.list_objects_v2(**kwargs)
|
||||
contents = resp.get('Contents', None)
|
||||
if not contents:
|
||||
break
|
||||
|
||||
for obj in contents:
|
||||
keys.append(obj['Key'])
|
||||
|
||||
token = resp.get('NextContinuationToken', None)
|
||||
if not token:
|
||||
break
|
||||
|
||||
kwargs['ContinuationToken'] = token
|
||||
|
||||
return keys
|
||||
|
||||
def exists(self, path_infos):
|
||||
# NOTE: We mostly use exists() method when filtering a bulk of cache
|
||||
# files to decide if we need to download/upload them and in s3
|
||||
# list_objects_v2() is much-much faster than trying to check keys
|
||||
# one-by-one.
|
||||
ret = []
|
||||
|
||||
keys = self._all_keys()
|
||||
|
||||
for path_info in path_infos:
|
||||
exists = False
|
||||
if path_info['key'] in keys:
|
||||
exists = True
|
||||
ret.append(exists)
|
||||
|
||||
return ret
|
||||
|
||||
def upload(self, from_infos, to_infos, names=None):
|
||||
names = self._verify_path_args(to_infos, from_infos, names)
|
||||
|
||||
s3 = self.s3
|
||||
|
||||
for from_info, to_info, name in zip(from_infos, to_infos, names):
|
||||
if to_info['scheme'] != 's3':
|
||||
raise NotImplementedError
|
||||
|
||||
if from_info['scheme'] != 'local':
|
||||
raise NotImplementedError
|
||||
|
||||
Logger.debug("Uploading '{}' to '{}/{}'".format(from_info['path'],
|
||||
to_info['bucket'],
|
||||
to_info['key']))
|
||||
|
||||
if not name:
|
||||
name = os.path.basename(from_info['path'])
|
||||
|
||||
total = os.path.getsize(from_info['path'])
|
||||
cb = Callback(name, total)
|
||||
|
||||
try:
|
||||
s3.upload_file(from_info['path'],
|
||||
to_info['bucket'],
|
||||
to_info['key'],
|
||||
Callback=cb)
|
||||
except Exception as exc:
|
||||
msg = "Failed to upload '{}'".format(from_info['path'])
|
||||
Logger.error(msg, exc)
|
||||
continue
|
||||
|
||||
progress.finish_target(name)
|
||||
|
||||
def download(self,
|
||||
from_infos,
|
||||
to_infos,
|
||||
no_progress_bar=False,
|
||||
names=None):
|
||||
names = self._verify_path_args(from_infos, to_infos, names)
|
||||
|
||||
s3 = self.s3
|
||||
|
||||
for to_info, from_info, name in zip(to_infos, from_infos, names):
|
||||
if from_info['scheme'] != 's3':
|
||||
raise NotImplementedError
|
||||
|
||||
if to_info['scheme'] == 's3':
|
||||
self._copy(from_info, to_info, s3=s3)
|
||||
continue
|
||||
|
||||
if to_info['scheme'] != 'local':
|
||||
raise NotImplementedError
|
||||
|
||||
msg = "Downloading '{}/{}' to '{}'".format(from_info['bucket'],
|
||||
from_info['key'],
|
||||
to_info['path'])
|
||||
Logger.debug(msg)
|
||||
|
||||
tmp_file = self.tmp_file(to_info['path'])
|
||||
if not name:
|
||||
name = os.path.basename(to_info['path'])
|
||||
|
||||
if no_progress_bar:
|
||||
cb = None
|
||||
else:
|
||||
total = s3.head_object(Bucket=from_info['bucket'],
|
||||
Key=from_info['key'])['ContentLength']
|
||||
cb = Callback(name, total)
|
||||
|
||||
self._makedirs(to_info['path'])
|
||||
|
||||
try:
|
||||
s3.download_file(from_info['bucket'],
|
||||
from_info['key'],
|
||||
tmp_file,
|
||||
Callback=cb)
|
||||
except Exception as exc:
|
||||
msg = "Failed to download '{}/{}'".format(from_info['bucket'],
|
||||
from_info['key'])
|
||||
Logger.error(msg, exc)
|
||||
return
|
||||
|
||||
os.rename(tmp_file, to_info['path'])
|
||||
|
||||
if not no_progress_bar:
|
||||
progress.finish_target(name)
|
||||
|
||||
def _path_to_etag(self, path):
|
||||
relpath = posixpath.relpath(path, self.prefix)
|
||||
return posixpath.dirname(relpath) + posixpath.basename(relpath)
|
||||
|
||||
def _all(self):
|
||||
keys = self._all_keys()
|
||||
return [self._path_to_etag(key) for key in keys]
|
||||
|
||||
def gc(self, cinfos):
|
||||
used_etags = [info[self.PARAM_ETAG] for info in cinfos['s3']]
|
||||
used_etags += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
|
||||
|
||||
removed = False
|
||||
for etag in self._all():
|
||||
if etag in used_etags:
|
||||
continue
|
||||
path_info = {'scheme': 's3',
|
||||
'key': posixpath.join(self.prefix,
|
||||
etag[0:2], etag[2:]),
|
||||
'bucket': self.bucket}
|
||||
self.remove(path_info)
|
||||
removed = True
|
||||
|
||||
return removed
|
||||
@@ -0,0 +1,435 @@
|
||||
import os
|
||||
import getpass
|
||||
import posixpath
|
||||
|
||||
try:
|
||||
import paramiko
|
||||
except ImportError:
|
||||
paramiko = None
|
||||
|
||||
from dvc.logger import Logger
|
||||
from dvc.progress import progress
|
||||
from dvc.remote.base import RemoteBase
|
||||
from dvc.remote.local import RemoteLOCAL
|
||||
from dvc.config import Config
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
|
||||
def sizeof_fmt(num, suffix='B'):
|
||||
""" Convert number of bytes to human-readable string """
|
||||
for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
|
||||
if abs(num) < 1024.0:
|
||||
return "%3.1f%s%s" % (num, unit, suffix)
|
||||
num /= 1024.0
|
||||
return "%.1f%s%s" % (num, 'Y', suffix)
|
||||
|
||||
|
||||
def percent_cb(name, complete, total):
|
||||
""" Callback for updating target progress """
|
||||
Logger.debug('{}: {} transferred out of {}'.format(name,
|
||||
sizeof_fmt(complete),
|
||||
sizeof_fmt(total)))
|
||||
progress.update_target(name, complete, total)
|
||||
|
||||
|
||||
def create_cb(name):
|
||||
""" Create callback function for multipart object """
|
||||
return (lambda cur, tot: percent_cb(name, cur, tot))
|
||||
|
||||
|
||||
class RemoteSSH(RemoteBase):
|
||||
scheme = 'ssh'
|
||||
|
||||
# NOTE: temporarily only absolute paths are allowed
|
||||
REGEX = r'^ssh://((?P<user>.*)@)?(?P<host>[^/]*):(?P<path>/.*)$'
|
||||
|
||||
REQUIRES = {'paramiko': paramiko}
|
||||
|
||||
JOBS = 4
|
||||
|
||||
PARAM_MD5 = 'md5'
|
||||
|
||||
DEFAULT_PORT = 22
|
||||
TIMEOUT = 1800
|
||||
|
||||
def __init__(self, project, config):
|
||||
self.project = project
|
||||
self.url = config.get(Config.SECTION_REMOTE_URL, '/')
|
||||
self.host = self.group('host')
|
||||
self.user = self.group('user')
|
||||
if not self.user:
|
||||
self.user = config.get(Config.SECTION_REMOTE_USER,
|
||||
getpass.getuser())
|
||||
self.prefix = self.group('path')
|
||||
self.port = config.get(Config.SECTION_REMOTE_PORT, self.DEFAULT_PORT)
|
||||
self.keyfile = config.get(Config.SECTION_REMOTE_KEY_FILE, None)
|
||||
self.timeout = config.get(Config.SECTION_REMOTE_TIMEOUT, self.TIMEOUT)
|
||||
self.password = config.get(Config.SECTION_REMOTE_PASSWORD, None)
|
||||
self.ask_password = config.get(Config.SECTION_REMOTE_ASK_PASSWORD,
|
||||
False)
|
||||
|
||||
def md5s_to_path_infos(self, md5s):
|
||||
return [{'scheme': 'ssh',
|
||||
'host': self.host,
|
||||
'user': self.user,
|
||||
'port': self.port,
|
||||
'path': posixpath.join(self.prefix,
|
||||
md5[0:2], md5[2:])} for md5 in md5s]
|
||||
|
||||
def ssh(self, host=None, user=None, port=None):
|
||||
msg = "Establishing ssh connection with '{}' " \
|
||||
"through port '{}' as user '{}'"
|
||||
Logger.debug(msg.format(host, port, user))
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
|
||||
ssh.load_system_host_keys()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
|
||||
if self.ask_password and self.password is None:
|
||||
msg = "host '{}' port '{}' user '{}'".format(host, port, user)
|
||||
self.password = self.project.prompt.prompt_password(msg)
|
||||
|
||||
ssh.connect(host,
|
||||
username=user,
|
||||
port=port,
|
||||
key_filename=self.keyfile,
|
||||
timeout=self.timeout,
|
||||
password=self.password)
|
||||
|
||||
return ssh
|
||||
|
||||
def exists(self, path_infos):
|
||||
ret = []
|
||||
ssh = self.ssh(host=self.host,
|
||||
user=self.user,
|
||||
port=self.port)
|
||||
cmd = 'find {} -type f -follow -print'.format(self.prefix)
|
||||
stdout = self._exec(ssh, cmd)
|
||||
plist = stdout.split()
|
||||
ssh.close()
|
||||
|
||||
for path_info in path_infos:
|
||||
exists = False
|
||||
if path_info['path'] in plist:
|
||||
exists = True
|
||||
ret.append(exists)
|
||||
|
||||
return ret
|
||||
|
||||
def _exec(self, ssh, cmd):
|
||||
stdin, stdout, stderr = ssh.exec_command(cmd)
|
||||
channel = stdout.channel
|
||||
|
||||
stdin.close()
|
||||
channel.shutdown_write()
|
||||
|
||||
stdout_chunks = []
|
||||
stderr_chunks = []
|
||||
while (not channel.closed
|
||||
or channel.recv_ready()
|
||||
or channel.recv_stderr_ready()):
|
||||
import select
|
||||
got_chunk = False
|
||||
readq, _, _ = select.select([stdout.channel], [], [], self.timeout)
|
||||
for c in readq:
|
||||
if c.recv_ready():
|
||||
stdout_chunks.append(stdout.channel.recv(len(c.in_buffer)))
|
||||
got_chunk = True
|
||||
|
||||
if c.recv_stderr_ready():
|
||||
stderr_len = len(c.in_stderr_buffer)
|
||||
s = stderr.channel.recv_stderr(stderr_len)
|
||||
stderr_chunks.append(s)
|
||||
got_chunk = True
|
||||
|
||||
if not got_chunk \
|
||||
and stdout.channel.exit_status_ready() \
|
||||
and not stderr.channel.recv_stderr_ready() \
|
||||
and not stdout.channel.recv_ready():
|
||||
stdout.channel.shutdown_read()
|
||||
stdout.channel.close()
|
||||
break
|
||||
|
||||
stdout.close()
|
||||
stderr.close()
|
||||
|
||||
if stdout.channel.recv_exit_status() != 0:
|
||||
err = b''.join(stderr_chunks).decode('utf-8')
|
||||
msg = 'SSH command \'{}\' failed: {}'.format(cmd, err)
|
||||
raise DvcException(msg)
|
||||
|
||||
return b''.join(stdout_chunks).decode('utf-8')
|
||||
|
||||
def md5(self, path_info):
|
||||
if path_info['scheme'] != 'ssh':
|
||||
raise NotImplementedError
|
||||
|
||||
ssh = self.ssh(host=path_info['host'],
|
||||
user=path_info['user'],
|
||||
port=path_info['port'])
|
||||
|
||||
# Use different md5 commands depending on os
|
||||
stdout = self._exec(ssh, 'uname').strip()
|
||||
if stdout == 'Darwin':
|
||||
md5cmd = 'md5'
|
||||
index = -1
|
||||
elif stdout == 'Linux':
|
||||
md5cmd = 'md5sum'
|
||||
index = 0
|
||||
else:
|
||||
msg = '\'{}\' is not supported as a remote'.format(stdout)
|
||||
raise DvcException(msg)
|
||||
|
||||
stdout = self._exec(ssh, '{} {}'.format(md5cmd, path_info['path']))
|
||||
md5 = stdout.split()[index]
|
||||
ssh.close()
|
||||
|
||||
assert len(md5) == 32
|
||||
|
||||
return md5
|
||||
|
||||
def cp(self, from_info, to_info, ssh=None):
|
||||
if from_info['scheme'] != 'ssh' or to_info['scheme'] != 'ssh':
|
||||
raise NotImplementedError
|
||||
|
||||
assert from_info['host'] == to_info['host']
|
||||
assert from_info['user'] == to_info['user']
|
||||
|
||||
s = ssh if ssh else self.ssh(host=from_info['host'],
|
||||
user=from_info['user'],
|
||||
port=from_info['port'])
|
||||
|
||||
dname = posixpath.dirname(to_info['path'])
|
||||
self._exec(s, 'mkdir -p {}'.format(dname))
|
||||
self._exec(s, 'cp {} {}'.format(from_info['path'], to_info['path']))
|
||||
|
||||
if not ssh:
|
||||
s.close()
|
||||
|
||||
def save_info(self, path_info):
|
||||
if path_info['scheme'] != 'ssh':
|
||||
raise NotImplementedError
|
||||
|
||||
return {self.PARAM_MD5: self.md5(path_info)}
|
||||
|
||||
@staticmethod
|
||||
def to_string(path_info):
|
||||
return "{}://{}@{}:{}".format(path_info['scheme'],
|
||||
path_info['user'],
|
||||
path_info['host'],
|
||||
path_info['path'])
|
||||
|
||||
def changed_cache(self, md5):
|
||||
cache = {}
|
||||
cache['scheme'] = 'ssh'
|
||||
cache['host'] = self.host
|
||||
cache['port'] = self.port
|
||||
cache['user'] = self.user
|
||||
cache['path'] = posixpath.join(self.prefix, md5[0:2], md5[2:])
|
||||
|
||||
if {self.PARAM_MD5: md5} != self.save_info(cache):
|
||||
if self.exists([cache])[0]:
|
||||
msg = 'Corrupted cache file {}'
|
||||
Logger.warn(msg.format(self.to_string(cache)))
|
||||
self.remove(cache)
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def changed(self, path_info, checksum_info):
|
||||
if not self.exists([path_info])[0]:
|
||||
return True
|
||||
|
||||
md5 = checksum_info.get(self.PARAM_MD5, None)
|
||||
if md5 is None:
|
||||
return True
|
||||
|
||||
if self.changed_code(md5):
|
||||
return True
|
||||
|
||||
return checksum_info != self.save_info(path_info)
|
||||
|
||||
def save(self, path_info):
|
||||
if path_info['scheme'] != 'ssh':
|
||||
raise NotImplementedError
|
||||
|
||||
md5 = self.md5(path_info)
|
||||
dest = path_info.copy()
|
||||
dest['path'] = posixpath.join(self.prefix, md5[0:2], md5[2:])
|
||||
|
||||
self.cp(path_info, dest)
|
||||
|
||||
return {self.PARAM_MD5: md5}
|
||||
|
||||
def checkout(self, path_info, checksum_info):
|
||||
if path_info['scheme'] != 'ssh':
|
||||
raise NotImplementedError
|
||||
|
||||
md5 = checksum_info.get(self.PARAM_MD5, None)
|
||||
if not md5:
|
||||
return
|
||||
|
||||
if not self.changed(path_info, checksum_info):
|
||||
msg = "Data '{}' didn't change."
|
||||
Logger.info(msg.format(self.to_string(path_info)))
|
||||
return
|
||||
|
||||
if self.changed_cache(md5):
|
||||
msg = "Cache '{}' not found. File '{}' won't be created."
|
||||
Logger.warn(msg.format(md5, self.to_string(path_info)))
|
||||
return
|
||||
|
||||
if self.exists([path_info])[0]:
|
||||
msg = "Data '{}' exists. Removing before checkout."
|
||||
Logger.warn(msg.format(self.to_string(path_info)))
|
||||
self.remove(path_info)
|
||||
return
|
||||
|
||||
msg = "Checking out '{}' with cache '{}'."
|
||||
Logger.info(msg.format(self.to_string(path_info), md5))
|
||||
|
||||
src = path_info.copy()
|
||||
src['path'] = posixpath.join(self.prefix, md5[0:2], md5[2:])
|
||||
|
||||
self.cp(src, path_info)
|
||||
|
||||
def remove(self, path_info):
|
||||
if path_info['scheme'] != 'ssh':
|
||||
raise NotImplementedError
|
||||
|
||||
Logger.debug('Removing ssh://{}@{}/{}'.format(path_info['user'],
|
||||
path_info['host'],
|
||||
path_info['path']))
|
||||
|
||||
ssh = self.ssh(host=path_info['host'],
|
||||
user=path_info['user'],
|
||||
port=path_info['port'])
|
||||
ssh.open_sftp().remove(path_info['path'])
|
||||
ssh.close()
|
||||
|
||||
def download(self,
|
||||
from_infos,
|
||||
to_infos,
|
||||
no_progress_bar=False,
|
||||
names=None):
|
||||
names = self._verify_path_args(from_infos, to_infos, names)
|
||||
|
||||
ssh = self.ssh(host=from_infos[0]['host'],
|
||||
user=from_infos[0]['user'],
|
||||
port=from_infos[0]['port'])
|
||||
|
||||
for to_info, from_info, name in zip(to_infos, from_infos, names):
|
||||
if from_info['scheme'] != 'ssh':
|
||||
raise NotImplementedError
|
||||
|
||||
if to_info['scheme'] == 'ssh':
|
||||
assert from_info['host'] == to_info['host']
|
||||
assert from_info['port'] == to_info['port']
|
||||
assert from_info['user'] == to_info['user']
|
||||
self.cp(from_info, to_info, ssh=ssh)
|
||||
continue
|
||||
|
||||
if to_info['scheme'] != 'local':
|
||||
raise NotImplementedError
|
||||
|
||||
msg = "Downloading '{}/{}' to '{}'".format(from_info['host'],
|
||||
from_info['path'],
|
||||
to_info['path'])
|
||||
Logger.debug(msg)
|
||||
|
||||
if not name:
|
||||
name = os.path.basename(to_info['path'])
|
||||
|
||||
self._makedirs(to_info['path'])
|
||||
tmp_file = self.tmp_file(to_info['path'])
|
||||
try:
|
||||
ssh.open_sftp().get(from_info['path'],
|
||||
tmp_file,
|
||||
callback=create_cb(name))
|
||||
except Exception as exc:
|
||||
msg = "Failed to download '{}/{}' to '{}'"
|
||||
Logger.error(msg.format(from_info['host'],
|
||||
from_info['path'],
|
||||
to_info['path']), exc)
|
||||
continue
|
||||
|
||||
os.rename(tmp_file, to_info['path'])
|
||||
progress.finish_target(name)
|
||||
|
||||
ssh.close()
|
||||
|
||||
def upload(self, from_infos, to_infos, names=None):
|
||||
names = self._verify_path_args(to_infos, from_infos, names)
|
||||
|
||||
ssh = self.ssh(host=to_infos[0]['host'],
|
||||
user=to_infos[0]['user'],
|
||||
port=to_infos[0]['port'])
|
||||
sftp = ssh.open_sftp()
|
||||
|
||||
for from_info, to_info, name in zip(from_infos, to_infos, names):
|
||||
if to_info['scheme'] != 'ssh':
|
||||
raise NotImplementedError
|
||||
|
||||
if from_info['scheme'] != 'local':
|
||||
raise NotImplementedError
|
||||
|
||||
Logger.debug("Uploading '{}' to '{}/{}'".format(from_info['path'],
|
||||
to_info['host'],
|
||||
to_info['path']))
|
||||
|
||||
if not name:
|
||||
name = os.path.basename(from_info['path'])
|
||||
|
||||
dname = posixpath.dirname(to_info['path'])
|
||||
self._exec(ssh, 'mkdir -p {}'.format(dname))
|
||||
|
||||
try:
|
||||
sftp.put(from_info['path'],
|
||||
to_info['path'],
|
||||
callback=create_cb(name))
|
||||
except Exception as exc:
|
||||
msg = "Failed to upload '{}' to '{}/{}'"
|
||||
Logger.error(msg.format(from_info['path'],
|
||||
to_info['host'],
|
||||
to_info['path'], exc))
|
||||
continue
|
||||
|
||||
progress.finish_target(name)
|
||||
|
||||
sftp.close()
|
||||
ssh.close()
|
||||
|
||||
def _path_to_md5(self, path):
|
||||
relpath = posixpath.relpath(path, self.prefix)
|
||||
return posixpath.dirname(relpath) + posixpath.basename(relpath)
|
||||
|
||||
def _all_md5s(self):
|
||||
ssh = self.ssh(host=self.host,
|
||||
user=self.user,
|
||||
port=self.port)
|
||||
cmd = 'find {} -type f -follow -print'.format(self.prefix)
|
||||
stdout = self._exec(ssh, cmd)
|
||||
flist = stdout.split()
|
||||
ssh.close()
|
||||
|
||||
return [self._path_to_md5(path) for path in flist]
|
||||
|
||||
def gc(self, cinfos):
|
||||
used = [info[self.PARAM_MD5] for info in cinfos['ssh']]
|
||||
used += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
|
||||
|
||||
removed = False
|
||||
for md5 in self._all_md5s():
|
||||
if md5 in used:
|
||||
continue
|
||||
path_info = {'scheme': 'ssh',
|
||||
'user': self.user,
|
||||
'host': self.host,
|
||||
'port': self.port,
|
||||
'path': posixpath.join(self.prefix,
|
||||
md5[0:2], md5[2:])}
|
||||
self.remove(path_info)
|
||||
removed = True
|
||||
|
||||
return removed
|
||||
+105
-19
@@ -1,6 +1,8 @@
|
||||
import os
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.logger import Logger
|
||||
from dvc.utils import fix_env
|
||||
|
||||
|
||||
class SCMError(DvcException):
|
||||
@@ -12,13 +14,18 @@ class FileNotInRepoError(DvcException):
|
||||
|
||||
|
||||
class Base(object):
|
||||
def __init__(self, root_dir=os.curdir):
|
||||
self.root_dir = os.path.abspath(os.path.realpath(root_dir))
|
||||
def __init__(self, root_dir=os.curdir, project=None):
|
||||
self.project = project
|
||||
self.root_dir = root_dir
|
||||
|
||||
@staticmethod
|
||||
def is_repo(root_dir):
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def is_submodule(root_dir):
|
||||
return True
|
||||
|
||||
def ignore(self, path):
|
||||
pass
|
||||
|
||||
@@ -43,6 +50,36 @@ class Base(object):
|
||||
def branch(self, branch):
|
||||
pass
|
||||
|
||||
def brancher(self,
|
||||
branches=None,
|
||||
all_branches=False,
|
||||
tags=None,
|
||||
all_tags=False):
|
||||
if not branches and not all_branches \
|
||||
and not tags and not all_tags:
|
||||
yield ''
|
||||
return
|
||||
|
||||
saved = self.active_branch()
|
||||
revs = []
|
||||
|
||||
if branches is not None:
|
||||
revs.extend(branches)
|
||||
elif all_branches:
|
||||
revs.extend(self.list_branches())
|
||||
elif tags is not None:
|
||||
revs.extend(tags)
|
||||
elif all_tags:
|
||||
revs.extend(self.list_tags())
|
||||
else:
|
||||
revs.extend([saved])
|
||||
|
||||
for rev in revs:
|
||||
self.checkout(rev)
|
||||
yield rev
|
||||
|
||||
self.checkout(saved)
|
||||
|
||||
def untracked_files(self):
|
||||
pass
|
||||
|
||||
@@ -55,6 +92,9 @@ class Base(object):
|
||||
def list_branches(self):
|
||||
pass
|
||||
|
||||
def list_tags(self):
|
||||
pass
|
||||
|
||||
def install(self):
|
||||
pass
|
||||
|
||||
@@ -63,28 +103,46 @@ class Git(Base):
|
||||
GITIGNORE = '.gitignore'
|
||||
GIT_DIR = '.git'
|
||||
|
||||
def __init__(self, root_dir=os.curdir):
|
||||
super(Git, self).__init__(root_dir)
|
||||
def __init__(self, root_dir=os.curdir, project=None):
|
||||
super(Git, self).__init__(root_dir, project=project)
|
||||
|
||||
import git
|
||||
from git.exc import InvalidGitRepositoryError
|
||||
try:
|
||||
self.repo = git.Repo(root_dir)
|
||||
except InvalidGitRepositoryError:
|
||||
raise SCMError('{} is not a git repository'.format(root_dir))
|
||||
msg = '{} is not a git repository'
|
||||
raise SCMError(msg.format(root_dir))
|
||||
|
||||
# NOTE: fixing LD_LIBRARY_PATH for binary built by PyInstaller.
|
||||
# http://pyinstaller.readthedocs.io/en/stable/runtime-information.html
|
||||
env = fix_env(None)
|
||||
lp = env.get('LD_LIBRARY_PATH', None)
|
||||
self.repo.git.update_environment(LD_LIBRARY_PATH=lp)
|
||||
|
||||
@staticmethod
|
||||
def is_repo(root_dir):
|
||||
git_dir = os.path.join(root_dir, Git.GIT_DIR)
|
||||
return os.path.isdir(git_dir)
|
||||
return os.path.isdir(Git._get_git_dir(root_dir))
|
||||
|
||||
@staticmethod
|
||||
def is_submodule(root_dir):
|
||||
return os.path.isfile(Git._get_git_dir(root_dir))
|
||||
|
||||
@staticmethod
|
||||
def _get_git_dir(root_dir):
|
||||
return os.path.join(root_dir, Git.GIT_DIR)
|
||||
|
||||
@property
|
||||
def dir(self):
|
||||
return self.repo.git_dir
|
||||
|
||||
def ignore_file(self):
|
||||
return self.GITIGNORE
|
||||
|
||||
def _get_gitignore(self, path):
|
||||
assert os.path.isabs(path)
|
||||
entry = os.path.basename(path)
|
||||
gitignore = os.path.join(os.path.abspath(os.path.dirname(path)),
|
||||
self.GITIGNORE)
|
||||
gitignore = os.path.abspath(os.path.realpath(gitignore))
|
||||
gitignore = os.path.join(os.path.dirname(path), self.GITIGNORE)
|
||||
|
||||
if not gitignore.startswith(self.root_dir):
|
||||
raise FileNotInRepoError(path)
|
||||
@@ -97,15 +155,24 @@ class Git(Base):
|
||||
ignore_list = []
|
||||
if os.path.exists(gitignore):
|
||||
ignore_list = open(gitignore, 'r').readlines()
|
||||
filtered = list(filter(lambda x: x.strip() == entry.strip(), ignore_list))
|
||||
filtered = list(filter(lambda x: x.strip() == entry.strip(),
|
||||
ignore_list))
|
||||
if len(filtered) != 0:
|
||||
return
|
||||
|
||||
msg = "Adding '{}' to '{}'.".format(os.path.relpath(path),
|
||||
os.path.relpath(gitignore))
|
||||
Logger.info(msg)
|
||||
|
||||
content = entry
|
||||
if len(ignore_list) > 0:
|
||||
content = '\n' + content
|
||||
|
||||
open(gitignore, 'a').write(content)
|
||||
with open(gitignore, 'a') as fd:
|
||||
fd.write(content)
|
||||
|
||||
if self.project is not None:
|
||||
self.project._files_to_git_add.append(os.path.relpath(gitignore))
|
||||
|
||||
def ignore_remove(self, path):
|
||||
entry, gitignore = self._get_gitignore(path)
|
||||
@@ -121,8 +188,20 @@ class Git(Base):
|
||||
with open(gitignore, 'w') as fd:
|
||||
fd.writelines(filtered)
|
||||
|
||||
if self.project is not None:
|
||||
self.project._files_to_git_add.append(os.path.relpath(gitignore))
|
||||
|
||||
def add(self, paths):
|
||||
self.repo.index.add(paths)
|
||||
# NOTE: GitPython is not currently able to handle index version >= 3.
|
||||
# See https://github.com/iterative/dvc/issues/610 for more details.
|
||||
try:
|
||||
self.repo.index.add(paths)
|
||||
except AssertionError as exc:
|
||||
msg = 'Failed to add \'{}\' to git. You can add those files '
|
||||
msg += 'manually using \'git add\'. '
|
||||
msg += 'See \'https://github.com/iterative/dvc/issues/610\' '
|
||||
msg += 'for more details.'
|
||||
Logger.error(msg.format(str(paths)), exc)
|
||||
|
||||
def commit(self, msg):
|
||||
self.repo.index.commit(msg)
|
||||
@@ -149,17 +228,24 @@ class Git(Base):
|
||||
def list_branches(self):
|
||||
return [h.name for h in self.repo.heads]
|
||||
|
||||
def list_tags(self):
|
||||
return [t.name for t in self.repo.tags]
|
||||
|
||||
def install(self):
|
||||
hook = os.path.join(self.root_dir, self.GIT_DIR, 'hooks', 'post-checkout')
|
||||
hook = os.path.join(self.root_dir,
|
||||
self.GIT_DIR,
|
||||
'hooks',
|
||||
'post-checkout')
|
||||
if os.path.isfile(hook):
|
||||
raise SCMError('Git hook \'{}\' already exists.'.format(os.path.relpath(hook)))
|
||||
msg = 'Git hook \'{}\' already exists.'
|
||||
raise SCMError(msg.format(os.path.relpath(hook)))
|
||||
with open(hook, 'w+') as fd:
|
||||
fd.write('#!/bin/sh\nexec dvc checkout\n')
|
||||
os.chmod(hook, 0o777)
|
||||
|
||||
|
||||
def SCM(root_dir=os.curdir, no_scm=False):
|
||||
if Git.is_repo(root_dir):
|
||||
return Git(root_dir)
|
||||
def SCM(root_dir, no_scm=False, project=None):
|
||||
if Git.is_repo(root_dir) or Git.is_submodule(root_dir):
|
||||
return Git(root_dir, project=project)
|
||||
|
||||
return Base(root_dir)
|
||||
return Base(root_dir, project=project)
|
||||
|
||||
+289
-83
@@ -1,13 +1,14 @@
|
||||
import os
|
||||
import yaml
|
||||
import itertools
|
||||
import posixpath
|
||||
import subprocess
|
||||
import schema
|
||||
from schema import Schema, SchemaError, Optional, Or, And
|
||||
|
||||
import dvc.dependency as dependency
|
||||
import dvc.output as output
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.output import Output, Dependency, OutputError
|
||||
from dvc.logger import Logger
|
||||
from dvc.utils import dict_md5
|
||||
from dvc.utils import dict_md5, fix_env
|
||||
|
||||
|
||||
class StageCmdFailedError(DvcException):
|
||||
@@ -17,11 +18,42 @@ class StageCmdFailedError(DvcException):
|
||||
|
||||
|
||||
class StageFileFormatError(DvcException):
|
||||
def __init__(self):
|
||||
super(StageFileFormatError, self).__init__('Stage file format error')
|
||||
def __init__(self, fname, e):
|
||||
msg = "Stage file '{}' format error: {}".format(fname, str(e))
|
||||
super(StageFileFormatError, self).__init__(msg)
|
||||
|
||||
|
||||
class MissingDataSource(OutputError):
|
||||
class StageFileDoesNotExistError(DvcException):
|
||||
def __init__(self, fname):
|
||||
msg = "'{}' does not exist.".format(fname)
|
||||
super(StageFileDoesNotExistError, self).__init__(msg)
|
||||
|
||||
|
||||
class StageFileIsNotDvcFileError(DvcException):
|
||||
def __init__(self, fname):
|
||||
msg = "'{}' is not a dvc file".format(fname)
|
||||
super(StageFileIsNotDvcFileError, self).__init__(msg)
|
||||
|
||||
|
||||
class StageFileBadNameError(DvcException):
|
||||
def __init__(self, msg):
|
||||
super(StageFileBadNameError, self).__init__(msg)
|
||||
|
||||
|
||||
class MissingDep(DvcException):
|
||||
def __init__(self, deps):
|
||||
assert len(deps) > 0
|
||||
|
||||
if len(deps) > 1:
|
||||
dep = 'dependencies'
|
||||
else:
|
||||
dep = 'dependency'
|
||||
|
||||
msg = u'missing {}: {}'.format(dep, ', '.join(map(str, deps)))
|
||||
super(MissingDep, self).__init__(msg)
|
||||
|
||||
|
||||
class MissingDataSource(DvcException):
|
||||
def __init__(self, missing_files):
|
||||
assert len(missing_files) > 0
|
||||
|
||||
@@ -41,15 +73,25 @@ class Stage(object):
|
||||
PARAM_CMD = 'cmd'
|
||||
PARAM_DEPS = 'deps'
|
||||
PARAM_OUTS = 'outs'
|
||||
PARAM_LOCKED = 'locked'
|
||||
|
||||
SCHEMA = {
|
||||
schema.Optional(PARAM_MD5): schema.Or(str, None),
|
||||
schema.Optional(PARAM_CMD): schema.Or(str, None),
|
||||
schema.Optional(PARAM_DEPS): schema.Or(schema.And(list, schema.Schema([Dependency.SCHEMA])), None),
|
||||
schema.Optional(PARAM_OUTS): schema.Or(schema.And(list, schema.Schema([Output.SCHEMA])), None),
|
||||
Optional(PARAM_MD5): Or(str, None),
|
||||
Optional(PARAM_CMD): Or(str, None),
|
||||
Optional(PARAM_DEPS): Or(And(list, Schema([dependency.SCHEMA])), None),
|
||||
Optional(PARAM_OUTS): Or(And(list, Schema([output.SCHEMA])), None),
|
||||
Optional(PARAM_LOCKED): bool,
|
||||
}
|
||||
|
||||
def __init__(self, project, path=None, cmd=None, cwd=None, deps=[], outs=[], md5=None):
|
||||
def __init__(self,
|
||||
project,
|
||||
path=None,
|
||||
cmd=None,
|
||||
cwd=os.curdir,
|
||||
deps=[],
|
||||
outs=[],
|
||||
md5=None,
|
||||
locked=False):
|
||||
self.project = project
|
||||
self.path = path
|
||||
self.cmd = cmd
|
||||
@@ -57,6 +99,7 @@ class Stage(object):
|
||||
self.outs = outs
|
||||
self.deps = deps
|
||||
self.md5 = md5
|
||||
self.locked = locked
|
||||
|
||||
@property
|
||||
def relpath(self):
|
||||
@@ -66,98 +109,171 @@ class Stage(object):
|
||||
def is_data_source(self):
|
||||
return self.cmd is None
|
||||
|
||||
@staticmethod
|
||||
def is_stage_filename(path):
|
||||
if not path.endswith(Stage.STAGE_FILE_SUFFIX) \
|
||||
and os.path.basename(path) != Stage.STAGE_FILE:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def is_stage_file(path):
|
||||
if not os.path.isfile(path):
|
||||
return False
|
||||
|
||||
if not path.endswith(Stage.STAGE_FILE_SUFFIX) and os.path.basename(path) != Stage.STAGE_FILE:
|
||||
return False
|
||||
|
||||
return True
|
||||
return Stage.is_stage_filename(path)
|
||||
|
||||
def changed_md5(self):
|
||||
md5 = self.dumpd().get(self.PARAM_MD5, None)
|
||||
|
||||
# backward compatibility
|
||||
if self.md5 == None:
|
||||
if self.md5 is None:
|
||||
return False
|
||||
|
||||
if self.md5 and md5 and self.md5 == md5:
|
||||
return False
|
||||
|
||||
msg = "Dvc file '{}' md5 changed(expected '{}', actual '{}')"
|
||||
self.project.logger.debug(msg.format(self.relpath, self.md5, md5))
|
||||
return True
|
||||
|
||||
def changed(self):
|
||||
@property
|
||||
def is_callback(self):
|
||||
return not self.is_data_source and len(self.deps) == 0
|
||||
|
||||
@property
|
||||
def is_import(self):
|
||||
return not self.cmd and \
|
||||
len(self.deps) == 1 and \
|
||||
len(self.outs) == 1
|
||||
|
||||
def changed(self, print_info=False):
|
||||
ret = False
|
||||
|
||||
if not self.is_data_source and len(self.deps) == 0:
|
||||
if print_info:
|
||||
log = self.project.logger.info
|
||||
else:
|
||||
log = self.project.logger.debug
|
||||
|
||||
if self.is_callback:
|
||||
msg = "Dvc file '{}' is a 'callback' stage (has a command and " \
|
||||
"no dependencies) and thus always considered as changed."
|
||||
self.project.logger.warn(msg.format(self.relpath))
|
||||
ret = True
|
||||
|
||||
for entry in itertools.chain(self.outs, self.deps):
|
||||
if entry.changed():
|
||||
if not self.locked:
|
||||
for dep in self.deps:
|
||||
if not dep.changed():
|
||||
continue
|
||||
if print_info:
|
||||
msg = "Dependency '{}' of '{}' changed."
|
||||
log(msg.format(dep, self.relpath))
|
||||
ret = True
|
||||
|
||||
for out in self.outs:
|
||||
if not out.changed():
|
||||
continue
|
||||
if print_info:
|
||||
msg = "Output '{}' of '{}' changed."
|
||||
log(msg.format(out, self.relpath))
|
||||
ret = True
|
||||
|
||||
if self.changed_md5():
|
||||
if print_info:
|
||||
msg = "Dvc file '{}' changed."
|
||||
log(msg.format(self.relpath))
|
||||
ret = True
|
||||
|
||||
if ret:
|
||||
self.project.logger.debug(u'Dvc file \'{}\' changed'.format(self.relpath))
|
||||
msg = "Stage '{}' changed.".format(self.relpath)
|
||||
color = 'yellow'
|
||||
else:
|
||||
self.project.logger.debug(u'Dvc file \'{}\' didn\'t change'.format(self.relpath))
|
||||
msg = "Stage '{}' didn't change.".format(self.relpath)
|
||||
color = 'green'
|
||||
|
||||
log(Logger.colorize(msg, color))
|
||||
|
||||
return ret
|
||||
|
||||
def remove_outs(self):
|
||||
def remove_outs(self, ignore_remove=False):
|
||||
for out in self.outs:
|
||||
out.remove()
|
||||
if out.use_cache:
|
||||
self.project.scm.ignore_remove(out.path)
|
||||
out.remove(ignore_remove=ignore_remove)
|
||||
|
||||
def remove(self):
|
||||
self.remove_outs()
|
||||
self.remove_outs(ignore_remove=True)
|
||||
os.unlink(self.path)
|
||||
|
||||
def reproduce(self, force=False):
|
||||
if not self.changed() and not force:
|
||||
def reproduce(self, force=False, dry=False, interactive=False):
|
||||
if not self.changed(print_info=True) and not force:
|
||||
return None
|
||||
|
||||
if self.cmd:
|
||||
if (self.cmd or self.is_import) and not self.locked and not dry:
|
||||
# Removing outputs only if we actually have command to reproduce
|
||||
self.remove_outs()
|
||||
self.remove_outs(ignore_remove=False)
|
||||
|
||||
self.run()
|
||||
msg = "Going to reproduce '{}'. Are you sure you want to continue?"
|
||||
msg = msg.format(self.relpath)
|
||||
if interactive \
|
||||
and not self.project.prompt.prompt(msg):
|
||||
raise DvcException('Reproduction aborted by the user')
|
||||
|
||||
self.project.logger.info(u'Reproducing \'{}\''.format(self.relpath))
|
||||
|
||||
self.run(dry=dry)
|
||||
|
||||
msg = u'\'{}\' was reproduced'.format(self.relpath)
|
||||
self.project.logger.debug(msg)
|
||||
|
||||
return self
|
||||
|
||||
@staticmethod
|
||||
def validate(d):
|
||||
def validate(d, fname=None):
|
||||
try:
|
||||
schema.Schema(Stage.SCHEMA).validate(d)
|
||||
except schema.SchemaError as exc:
|
||||
Logger.debug(str(exc))
|
||||
raise StageFileFormatError()
|
||||
Schema(Stage.SCHEMA).validate(d)
|
||||
except SchemaError as exc:
|
||||
raise StageFileFormatError(fname, exc)
|
||||
|
||||
@staticmethod
|
||||
def loadd(project, d, path):
|
||||
Stage.validate(d)
|
||||
|
||||
Stage.validate(d, fname=os.path.relpath(path))
|
||||
path = os.path.abspath(path)
|
||||
cwd = os.path.dirname(path)
|
||||
cmd = d.get(Stage.PARAM_CMD, None)
|
||||
deps = Dependency.loadd_from(project, d.get(Stage.PARAM_DEPS, []), cwd=cwd)
|
||||
outs = Output.loadd_from(project, d.get(Stage.PARAM_OUTS, []), cwd=cwd)
|
||||
md5 = d.get(Stage.PARAM_MD5, None)
|
||||
locked = d.get(Stage.PARAM_LOCKED, False)
|
||||
|
||||
return Stage(project=project,
|
||||
path=path,
|
||||
cmd=cmd,
|
||||
cwd=cwd,
|
||||
deps=deps,
|
||||
outs=outs,
|
||||
md5=md5)
|
||||
stage = Stage(project=project,
|
||||
path=path,
|
||||
cmd=cmd,
|
||||
cwd=cwd,
|
||||
md5=md5,
|
||||
locked=locked)
|
||||
|
||||
stage.deps = dependency.loadd_from(stage, d.get(Stage.PARAM_DEPS, []))
|
||||
stage.outs = output.loadd_from(stage, d.get(Stage.PARAM_OUTS, []))
|
||||
|
||||
return stage
|
||||
|
||||
@classmethod
|
||||
def _stage_fname_cwd(cls, fname, cwd, outs, add):
|
||||
if fname and cwd:
|
||||
return (fname, cwd)
|
||||
|
||||
if not outs:
|
||||
return (cls.STAGE_FILE, cwd if cwd else os.getcwd())
|
||||
|
||||
out = outs[0]
|
||||
if out.path_info['scheme'] == 'local':
|
||||
path = os.path
|
||||
else:
|
||||
path = posixpath
|
||||
|
||||
if not fname:
|
||||
fname = path.basename(out.path) + cls.STAGE_FILE_SUFFIX
|
||||
|
||||
if not cwd or (add and out.is_local):
|
||||
cwd = path.dirname(out.path)
|
||||
|
||||
return (fname, cwd)
|
||||
|
||||
@staticmethod
|
||||
def loads(project=None,
|
||||
@@ -165,32 +281,81 @@ class Stage(object):
|
||||
deps=[],
|
||||
outs=[],
|
||||
outs_no_cache=[],
|
||||
metrics_no_cache=[],
|
||||
fname=None,
|
||||
cwd=os.curdir):
|
||||
cwd=os.curdir,
|
||||
locked=False,
|
||||
add=False,
|
||||
overwrite=True):
|
||||
stage = Stage(project=project,
|
||||
cwd=cwd,
|
||||
cmd=cmd,
|
||||
locked=locked)
|
||||
|
||||
stage.outs = output.loads_from(stage, outs, use_cache=True)
|
||||
stage.outs += output.loads_from(stage, outs_no_cache, use_cache=False)
|
||||
stage.outs += output.loads_from(stage, metrics_no_cache,
|
||||
use_cache=False, metric=True)
|
||||
stage.deps = dependency.loads_from(stage, deps)
|
||||
|
||||
if fname is not None and os.path.basename(fname) != fname:
|
||||
msg = "Stage file name '{}' should not contain subdirectories. " \
|
||||
"Use '-c|--cwd' to change location of the stage file."
|
||||
raise StageFileBadNameError(msg.format(fname))
|
||||
|
||||
fname, cwd = Stage._stage_fname_cwd(fname, cwd, stage.outs, add=add)
|
||||
|
||||
cwd = os.path.abspath(cwd)
|
||||
path = os.path.join(cwd, fname)
|
||||
outputs = Output.loads_from(project, outs, use_cache=True, cwd=cwd)
|
||||
outputs += Output.loads_from(project, outs_no_cache, use_cache=False, cwd=cwd)
|
||||
dependencies = Dependency.loads_from(project, deps, cwd=cwd)
|
||||
|
||||
return Stage(project=project,
|
||||
path=path,
|
||||
cmd=cmd,
|
||||
cwd=cwd,
|
||||
outs=outputs,
|
||||
deps=dependencies)
|
||||
if os.path.exists(path):
|
||||
relpath = os.path.relpath(path)
|
||||
msg = "'{}' already exists. " \
|
||||
"Do you wish to run the command and overwrite it?"
|
||||
if not overwrite \
|
||||
and not project.prompt.prompt(msg.format(relpath), False):
|
||||
raise DvcException("'{}' already exists".format(relpath))
|
||||
|
||||
stage.cwd = cwd
|
||||
stage.path = path
|
||||
|
||||
return stage
|
||||
|
||||
@staticmethod
|
||||
def _check_dvc_filename(fname):
|
||||
if not Stage.is_stage_filename(fname):
|
||||
msg = "Bad stage filename '{}'. Stage files should be named " \
|
||||
"'Dvcfile' or have a '.dvc' suffix(e.g. '{}.dvc')."
|
||||
raise StageFileBadNameError(msg.format(os.path.relpath(fname),
|
||||
os.path.basename(fname)))
|
||||
|
||||
@staticmethod
|
||||
def _check_dvc_file(fname):
|
||||
sname = fname + Stage.STAGE_FILE_SUFFIX
|
||||
if Stage.is_stage_file(sname):
|
||||
Logger.info("Do you mean '{}'?".format(sname))
|
||||
|
||||
@staticmethod
|
||||
def load(project, fname):
|
||||
if not os.path.exists(fname):
|
||||
Stage._check_dvc_file(fname)
|
||||
raise StageFileDoesNotExistError(fname)
|
||||
|
||||
Stage._check_dvc_filename(fname)
|
||||
|
||||
if not Stage.is_stage_file(fname):
|
||||
Stage._check_dvc_file(fname)
|
||||
raise StageFileIsNotDvcFileError(fname)
|
||||
|
||||
with open(fname, 'r') as fd:
|
||||
return Stage.loadd(project, yaml.safe_load(fd), fname)
|
||||
|
||||
def dumpd(self):
|
||||
deps = [x.dumpd(self.cwd) for x in self.deps]
|
||||
outs = [x.dumpd(self.cwd) for x in self.outs]
|
||||
deps = [x.dumpd() for x in self.deps]
|
||||
outs = [x.dumpd() for x in self.outs]
|
||||
|
||||
ret = {}
|
||||
if self.cmd != None:
|
||||
if self.cmd is not None:
|
||||
ret[Stage.PARAM_CMD] = self.cmd
|
||||
|
||||
if len(deps):
|
||||
@@ -199,6 +364,9 @@ class Stage(object):
|
||||
if len(outs):
|
||||
ret[Stage.PARAM_OUTS] = outs
|
||||
|
||||
if self.locked:
|
||||
ret[Stage.PARAM_LOCKED] = self.locked
|
||||
|
||||
ret[Stage.PARAM_MD5] = dict_md5(ret)
|
||||
|
||||
return ret
|
||||
@@ -207,39 +375,74 @@ class Stage(object):
|
||||
if not fname:
|
||||
fname = self.path
|
||||
|
||||
self._check_dvc_filename(fname)
|
||||
|
||||
msg = "Saving information to '{}'.".format(os.path.relpath(fname))
|
||||
Logger.info(msg)
|
||||
|
||||
with open(fname, 'w') as fd:
|
||||
yaml.safe_dump(self.dumpd(), fd, default_flow_style=False)
|
||||
|
||||
self.project._files_to_git_add.append(os.path.relpath(fname))
|
||||
|
||||
def save(self):
|
||||
for dep in self.deps:
|
||||
dep.save()
|
||||
|
||||
for out in self.outs:
|
||||
out.save()
|
||||
if out.use_cache:
|
||||
self.project.scm.ignore(out.path)
|
||||
|
||||
def run(self):
|
||||
if not self.is_data_source:
|
||||
self.project.logger.info(u'Reproducing \'{}\':\n\t{}'.format(self.relpath, self.cmd))
|
||||
def _check_missing_deps(self):
|
||||
missing = []
|
||||
for dep in self.deps:
|
||||
if not dep.exists:
|
||||
missing.append(dep)
|
||||
|
||||
p = subprocess.Popen(self.cmd, cwd=self.cwd, shell=True)
|
||||
p.communicate()
|
||||
if p.returncode != 0:
|
||||
raise StageCmdFailedError(self)
|
||||
|
||||
self.save()
|
||||
if len(missing) > 0:
|
||||
raise MissingDep(missing)
|
||||
|
||||
self.project.logger.debug(u'\'{}\' was reproduced'.format(self.relpath))
|
||||
def run(self, dry=False):
|
||||
if self.locked:
|
||||
msg = u'Verifying outputs in locked stage \'{}\''
|
||||
self.project.logger.info(msg.format(self.relpath))
|
||||
if not dry:
|
||||
self.check_missing_outputs()
|
||||
elif self.is_import:
|
||||
msg = u'Importing \'{}\' -> \'{}\''
|
||||
self.project.logger.info(msg.format(self.deps[0].path,
|
||||
self.outs[0].path))
|
||||
|
||||
if not dry:
|
||||
self.deps[0].download(self.outs[0].path_info)
|
||||
elif self.is_data_source:
|
||||
msg = u'Verifying data sources in \'{}\''.format(self.relpath)
|
||||
self.project.logger.info(msg)
|
||||
if not dry:
|
||||
self.check_missing_outputs()
|
||||
else:
|
||||
self.project.logger.info(u'Verifying data sources in \'{}\''.format(self.relpath))
|
||||
self.check_missing_outputs()
|
||||
msg = u'Running command:\n\t{}'.format(self.cmd)
|
||||
self.project.logger.info(msg)
|
||||
|
||||
if not dry:
|
||||
self._check_missing_deps()
|
||||
p = subprocess.Popen(self.cmd,
|
||||
cwd=self.cwd,
|
||||
shell=True,
|
||||
env=fix_env(os.environ),
|
||||
executable=os.getenv('SHELL'))
|
||||
p.communicate()
|
||||
if p.returncode != 0:
|
||||
raise StageCmdFailedError(self)
|
||||
|
||||
if not dry:
|
||||
self.save()
|
||||
|
||||
def check_missing_outputs(self):
|
||||
missing_outs = [out.rel_path for out in self.outs if not os.path.exists(out.rel_path)]
|
||||
if missing_outs:
|
||||
raise MissingDataSource(missing_outs)
|
||||
outs = [out for out in self.outs if not out.exists]
|
||||
paths = [out.path if out.path_info['scheme'] != 'local' else
|
||||
out.rel_path for out in outs]
|
||||
if paths:
|
||||
raise MissingDataSource(paths)
|
||||
|
||||
def checkout(self):
|
||||
for out in self.outs:
|
||||
@@ -258,10 +461,13 @@ class Stage(object):
|
||||
|
||||
def status(self):
|
||||
ret = {}
|
||||
ret.update(self._status(self.deps, 'deps'))
|
||||
|
||||
if not self.locked:
|
||||
ret.update(self._status(self.deps, 'deps'))
|
||||
|
||||
ret.update(self._status(self.outs, 'outs'))
|
||||
|
||||
if ret or self.changed_md5():
|
||||
if ret or self.changed_md5() or self.is_callback:
|
||||
return {self.relpath: ret}
|
||||
|
||||
return {}
|
||||
|
||||
+246
-147
@@ -1,37 +1,13 @@
|
||||
import os
|
||||
import json
|
||||
import time
|
||||
import sqlite3
|
||||
import nanotime
|
||||
import threading
|
||||
|
||||
from dvc.lock import Lock
|
||||
from dvc.config import Config
|
||||
from dvc.system import System
|
||||
from dvc.utils import file_md5, dict_md5, remove
|
||||
from dvc.utils import file_md5, remove
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
|
||||
class StateEntry(object):
|
||||
PARAM_MTIME = 'mtime'
|
||||
PARAM_MD5 = 'md5'
|
||||
|
||||
def __init__(self, md5, mtime):
|
||||
self.mtime = mtime
|
||||
self.md5 = md5
|
||||
|
||||
def update(self, md5, mtime):
|
||||
self.mtime = mtime
|
||||
self.md5 = md5
|
||||
|
||||
@staticmethod
|
||||
def loadd(d):
|
||||
mtime = d[StateEntry.PARAM_MTIME]
|
||||
md5 = d[StateEntry.PARAM_MD5]
|
||||
return StateEntry(md5, mtime)
|
||||
|
||||
def dumpd(self):
|
||||
return {
|
||||
self.PARAM_MD5: self.md5,
|
||||
self.PARAM_MTIME: self.mtime,
|
||||
}
|
||||
from dvc.logger import Logger
|
||||
|
||||
|
||||
class StateDuplicateError(DvcException):
|
||||
@@ -39,45 +15,64 @@ class StateDuplicateError(DvcException):
|
||||
|
||||
|
||||
class State(object):
|
||||
VERSION = 1
|
||||
STATE_FILE = 'state'
|
||||
STATE_LOCK_FILE = 'state.lock'
|
||||
STATE_TABLE = 'state'
|
||||
STATE_TABLE_LAYOUT = "inode UNSIGNED INTEGER PRIMARY KEY, " \
|
||||
"mtime TEXT NOT NULL, " \
|
||||
"md5 TEXT NOT NULL, " \
|
||||
"timestamp TEXT NOT NULL"
|
||||
|
||||
PARAM_RELPATH = 'relpath'
|
||||
PARAM_MD5 = 'md5'
|
||||
MD5_DIR_SUFFIX = '.dir'
|
||||
STATE_INFO_TABLE = 'state_info'
|
||||
STATE_INFO_TABLE_LAYOUT = 'count INTEGER'
|
||||
STATE_INFO_ROW = 1
|
||||
|
||||
def __init__(self, dvc_dir):
|
||||
self.dvc_dir = dvc_dir
|
||||
self.state_file = os.path.join(dvc_dir, self.STATE_FILE)
|
||||
self._db = self.load()
|
||||
self._lock = threading.Lock()
|
||||
self._lock_file = Lock(dvc_dir, self.STATE_LOCK_FILE)
|
||||
LINK_STATE_TABLE = 'link_state'
|
||||
LINK_STATE_TABLE_LAYOUT = "path TEXT PRIMARY KEY, " \
|
||||
"inode UNSIGNED INTEGER NOT NULL, " \
|
||||
"mtime TEXT NOT NULL"
|
||||
|
||||
@staticmethod
|
||||
def init(dvc_dir):
|
||||
return State(dvc_dir)
|
||||
STATE_ROW_LIMIT = 10000000
|
||||
STATE_ROW_CLEANUP_QUOTA = 50
|
||||
|
||||
def collect_dir(self, dname):
|
||||
dir_info = []
|
||||
def __init__(self, project, config):
|
||||
self.project = project
|
||||
self.dvc_dir = project.dvc_dir
|
||||
self.root_dir = project.root_dir
|
||||
|
||||
for root, dirs, files in os.walk(dname):
|
||||
for fname in files:
|
||||
path = os.path.join(root, fname)
|
||||
relpath = os.path.relpath(path, dname)
|
||||
self.row_limit = 100
|
||||
self.row_cleanup_quota = 50
|
||||
|
||||
md5 = self.update(path, dump=False)
|
||||
dir_info.append({self.PARAM_RELPATH: relpath, self.PARAM_MD5: md5})
|
||||
c = config.get(Config.SECTION_STATE, {})
|
||||
self.row_limit = c.get(Config.SECTION_STATE_ROW_LIMIT,
|
||||
self.STATE_ROW_LIMIT)
|
||||
self.row_cleanup_quota = c.get(Config.SECTION_STATE_ROW_CLEANUP_QUOTA,
|
||||
self.STATE_ROW_CLEANUP_QUOTA)
|
||||
|
||||
if not self.dvc_dir:
|
||||
self.state_file = None
|
||||
return
|
||||
|
||||
self.state_file = os.path.join(self.dvc_dir, self.STATE_FILE)
|
||||
self.db = None
|
||||
self.c = None
|
||||
self.inserts = 0
|
||||
|
||||
def __enter__(self):
|
||||
self.load()
|
||||
|
||||
def __exit__(self, type, value, tb):
|
||||
self.dump()
|
||||
|
||||
return dir_info
|
||||
@staticmethod
|
||||
def init(project):
|
||||
return State(project)
|
||||
|
||||
def compute_md5(self, path):
|
||||
def _collect(self, path):
|
||||
if os.path.isdir(path):
|
||||
dir_info = self.collect_dir(path)
|
||||
return dict_md5(dir_info) + self.MD5_DIR_SUFFIX
|
||||
return self.project.cache.local.collect_dir_cache(path)
|
||||
else:
|
||||
return file_md5(path)[0]
|
||||
return (file_md5(path)[0], None)
|
||||
|
||||
def changed(self, path, md5):
|
||||
actual = self.update(path)
|
||||
@@ -87,117 +82,223 @@ class State(object):
|
||||
|
||||
return actual.split('.')[0] != md5.split('.')[0]
|
||||
|
||||
def load(self):
|
||||
if not os.path.isfile(self.state_file):
|
||||
return {}
|
||||
def _execute(self, cmd):
|
||||
Logger.debug(cmd)
|
||||
return self.c.execute(cmd)
|
||||
|
||||
with open(self.state_file, 'r') as fd:
|
||||
return json.load(fd)
|
||||
def _load(self):
|
||||
from dvc import VERSION
|
||||
|
||||
cmd = "PRAGMA user_version;"
|
||||
self._execute(cmd)
|
||||
ret = self.c.fetchall()
|
||||
assert len(ret) == 1
|
||||
assert len(ret[0]) == 1
|
||||
assert isinstance(ret[0][0], int)
|
||||
self.version = ret[0][0]
|
||||
|
||||
if self.version > self.VERSION:
|
||||
msg = "You are using an old version '{}' of dvc that is using " \
|
||||
"state file version '{}' which is not compatible with the " \
|
||||
"state file version '{}' that is used in this projet. " \
|
||||
"Please upgrade right now!"
|
||||
raise DvcException(msg.format(VERSION,
|
||||
self.VERSION,
|
||||
self.version))
|
||||
elif self.version < self.VERSION:
|
||||
msg = "State file version '{}' is too old. " \
|
||||
"Reformatting to the current version '{}'."
|
||||
self.project.logger.warn(msg.format(self.version, self.VERSION))
|
||||
cmd = "DROP TABLE IF EXISTS {};"
|
||||
self._execute(cmd.format(self.STATE_TABLE))
|
||||
self._execute(cmd.format(self.STATE_INFO_TABLE))
|
||||
self._execute(cmd.format(self.LINK_STATE_TABLE))
|
||||
|
||||
# Check that the state file is indeed a database
|
||||
cmd = "CREATE TABLE IF NOT EXISTS {} ({})"
|
||||
self._execute(cmd.format(self.STATE_TABLE,
|
||||
self.STATE_TABLE_LAYOUT))
|
||||
self._execute(cmd.format(self.STATE_INFO_TABLE,
|
||||
self.STATE_INFO_TABLE_LAYOUT))
|
||||
self._execute(cmd.format(self.LINK_STATE_TABLE,
|
||||
self.LINK_STATE_TABLE_LAYOUT))
|
||||
|
||||
cmd = "INSERT OR IGNORE INTO {} (count) SELECT 0 " \
|
||||
"WHERE NOT EXISTS (SELECT * FROM {})"
|
||||
self._execute(cmd.format(self.STATE_INFO_TABLE,
|
||||
self.STATE_INFO_TABLE))
|
||||
|
||||
cmd = "PRAGMA user_version = {};"
|
||||
self._execute(cmd.format(self.VERSION))
|
||||
|
||||
def load(self):
|
||||
retries = 1
|
||||
while True:
|
||||
assert self.db is None
|
||||
assert self.c is None
|
||||
assert self.inserts == 0
|
||||
self.db = sqlite3.connect(self.state_file)
|
||||
self.c = self.db.cursor()
|
||||
|
||||
# Try loading once to check that the file is indeed a database
|
||||
# and reformat it if it is not.
|
||||
try:
|
||||
self._load()
|
||||
return
|
||||
except sqlite3.DatabaseError:
|
||||
self.c.close()
|
||||
self.db.close()
|
||||
self.db = None
|
||||
self.c = None
|
||||
self.inserts = 0
|
||||
if retries > 0:
|
||||
os.unlink(self.state_file)
|
||||
retries -= 1
|
||||
else:
|
||||
raise
|
||||
|
||||
def dump(self):
|
||||
with open(self.state_file, 'w+') as fd:
|
||||
json.dump(self._db, fd)
|
||||
assert self.db is not None
|
||||
|
||||
cmd = "SELECT count from {} WHERE rowid={}"
|
||||
self._execute(cmd.format(self.STATE_INFO_TABLE,
|
||||
self.STATE_INFO_ROW))
|
||||
ret = self.c.fetchall()
|
||||
assert len(ret) == 1
|
||||
assert len(ret[0]) == 1
|
||||
count = ret[0][0] + self.inserts
|
||||
|
||||
if count > self.row_limit:
|
||||
msg = "Cleaning up state. This might take a while."
|
||||
self.project.logger.warn(msg)
|
||||
|
||||
delete = (count - self.row_limit)
|
||||
delete += int(self.row_limit * (self.row_cleanup_quota/100.))
|
||||
cmd = "DELETE FROM {} WHERE timestamp IN (" \
|
||||
"SELECT timestamp FROM {} ORDER BY timestamp ASC LIMIT {});"
|
||||
self._execute(cmd.format(self.STATE_TABLE,
|
||||
self.STATE_TABLE,
|
||||
delete))
|
||||
|
||||
self._execute("VACUUM")
|
||||
|
||||
cmd = "SELECT COUNT(*) FROM {}"
|
||||
|
||||
self._execute(cmd.format(self.STATE_TABLE))
|
||||
ret = self.c.fetchall()
|
||||
assert len(ret) == 1
|
||||
assert len(ret[0]) == 1
|
||||
count = ret[0][0]
|
||||
|
||||
cmd = "UPDATE {} SET count = {} WHERE rowid = {}"
|
||||
self._execute(cmd.format(self.STATE_INFO_TABLE,
|
||||
count,
|
||||
self.STATE_INFO_ROW))
|
||||
|
||||
self.db.commit()
|
||||
self.c.close()
|
||||
self.db.close()
|
||||
self.db = None
|
||||
self.c = None
|
||||
self.inserts = 0
|
||||
|
||||
@staticmethod
|
||||
def mtime(path):
|
||||
return str(int(nanotime.timestamp(os.path.getmtime(path))))
|
||||
mtime = os.path.getmtime(path)
|
||||
if os.path.isdir(path):
|
||||
for root, dirs, files in os.walk(path):
|
||||
for entry in list(dirs) + list(files):
|
||||
p = os.path.join(root, entry)
|
||||
m = os.path.getmtime(p)
|
||||
if m > mtime:
|
||||
mtime = m
|
||||
|
||||
return str(int(nanotime.timestamp(mtime)))
|
||||
|
||||
@staticmethod
|
||||
def inode(path):
|
||||
return str(System.inode(path))
|
||||
return System.inode(path)
|
||||
|
||||
def update(self, path, dump=True):
|
||||
def _do_update(self, path):
|
||||
if not os.path.exists(path):
|
||||
return None
|
||||
return (None, None)
|
||||
|
||||
mtime = self.mtime(path)
|
||||
inode = self.inode(path)
|
||||
|
||||
md5 = self._get(inode, mtime)
|
||||
if md5:
|
||||
return md5
|
||||
cmd = 'SELECT * from {} WHERE inode={}'.format(self.STATE_TABLE,
|
||||
inode)
|
||||
|
||||
md5 = self.compute_md5(path)
|
||||
state = StateEntry(md5, mtime)
|
||||
d = state.dumpd()
|
||||
self._execute(cmd)
|
||||
ret = self.c.fetchall()
|
||||
if len(ret) == 0:
|
||||
md5, info = self._collect(path)
|
||||
cmd = 'INSERT INTO {}(inode, mtime, md5, timestamp) ' \
|
||||
'VALUES ({}, "{}", "{}", "{}")'
|
||||
self._execute(cmd.format(self.STATE_TABLE,
|
||||
inode,
|
||||
mtime,
|
||||
md5,
|
||||
int(nanotime.timestamp(time.time()))))
|
||||
self.inserts += 1
|
||||
else:
|
||||
assert len(ret) == 1
|
||||
assert len(ret[0]) == 4
|
||||
i, m, md5, timestamp = ret[0]
|
||||
assert i == inode
|
||||
if mtime != m:
|
||||
md5, info = self._collect(path)
|
||||
cmd = 'UPDATE {} SET ' \
|
||||
'mtime = "{}", md5 = "{}", timestamp = "{}" ' \
|
||||
'WHERE inode = {}'
|
||||
self._execute(cmd.format(self.STATE_TABLE,
|
||||
mtime,
|
||||
md5,
|
||||
int(nanotime.timestamp(time.time())),
|
||||
inode))
|
||||
else:
|
||||
info = None
|
||||
cmd = 'UPDATE {} SET timestamp = "{}" WHERE inode = {}'
|
||||
self._execute(cmd.format(self.STATE_TABLE,
|
||||
int(nanotime.timestamp(time.time())),
|
||||
inode))
|
||||
|
||||
with self._lock:
|
||||
with self._lock_file:
|
||||
self._db[inode] = d
|
||||
|
||||
if dump:
|
||||
self.dump()
|
||||
|
||||
return md5
|
||||
|
||||
def _get(self, inode, mtime):
|
||||
with self._lock:
|
||||
with self._lock_file:
|
||||
d = self._db.get(inode, None)
|
||||
|
||||
if not d:
|
||||
return None
|
||||
|
||||
state = StateEntry.loadd(d)
|
||||
if mtime == state.mtime:
|
||||
return state.md5
|
||||
|
||||
return None
|
||||
|
||||
def get(self, path):
|
||||
mtime = self.mtime(path)
|
||||
inode = self.inode(path)
|
||||
|
||||
return self._get(inode, mtime)
|
||||
|
||||
|
||||
class LinkStateEntry(object):
|
||||
PARAM_MTIME = 'mtime'
|
||||
PARAM_INODE = 'inode'
|
||||
|
||||
def __init__(self, inode, mtime):
|
||||
self.mtime = mtime
|
||||
self.inode = inode
|
||||
|
||||
@staticmethod
|
||||
def loadd(d):
|
||||
mtime = d[LinkStateEntry.PARAM_MTIME]
|
||||
inode = d[LinkStateEntry.PARAM_INODE]
|
||||
return LinkStateEntry(inode, mtime)
|
||||
|
||||
def dumpd(self):
|
||||
return {
|
||||
self.PARAM_INODE: self.inode,
|
||||
self.PARAM_MTIME: self.mtime,
|
||||
}
|
||||
|
||||
|
||||
class LinkState(State):
|
||||
STATE_FILE = 'link.state'
|
||||
STATE_LOCK_FILE = STATE_FILE + '.lock'
|
||||
|
||||
def __init__(self, root_dir, dvc_dir):
|
||||
super(LinkState, self).__init__(dvc_dir)
|
||||
self.root_dir = root_dir
|
||||
return (md5, info)
|
||||
|
||||
def update(self, path):
|
||||
return self._do_update(path)[0]
|
||||
|
||||
def update_info(self, path):
|
||||
md5, info = self._do_update(path)
|
||||
if not info:
|
||||
info = self.project.cache.local.load_dir_cache(md5)
|
||||
return (md5, info)
|
||||
|
||||
def update_link(self, path):
|
||||
if not os.path.exists(path):
|
||||
return
|
||||
|
||||
mtime = self.mtime(path)
|
||||
inode = self.inode(path)
|
||||
relpath = os.path.relpath(path, self.root_dir)
|
||||
|
||||
with self._lock:
|
||||
with self._lock_file:
|
||||
state = LinkStateEntry(inode, mtime)
|
||||
d = state.dumpd()
|
||||
self._db[os.path.relpath(path, self.root_dir)] = d
|
||||
self.dump()
|
||||
cmd = 'REPLACE INTO {}(path, inode, mtime) ' \
|
||||
'VALUES ("{}", {}, "{}")'.format(self.LINK_STATE_TABLE,
|
||||
relpath,
|
||||
inode,
|
||||
mtime)
|
||||
self._execute(cmd)
|
||||
|
||||
def _do_remove_all(self):
|
||||
for p, s in self._db.items():
|
||||
def remove_unused_links(self, used):
|
||||
unused = []
|
||||
|
||||
self._execute('SELECT * FROM {}'.format(self.LINK_STATE_TABLE))
|
||||
for row in self.c:
|
||||
p, i, m = row
|
||||
path = os.path.join(self.root_dir, p)
|
||||
state = LinkStateEntry.loadd(s)
|
||||
|
||||
if path in used:
|
||||
continue
|
||||
|
||||
if not os.path.exists(path):
|
||||
continue
|
||||
@@ -205,13 +306,11 @@ class LinkState(State):
|
||||
inode = self.inode(path)
|
||||
mtime = self.mtime(path)
|
||||
|
||||
if inode == state.inode and mtime == state.mtime:
|
||||
if i == inode and m == mtime:
|
||||
Logger.debug('Removing \'{}\' as unused link.'.format(path))
|
||||
remove(path)
|
||||
unused.append(p)
|
||||
|
||||
self._db = {}
|
||||
|
||||
def remove_all(self):
|
||||
with self._lock:
|
||||
with self._lock_file:
|
||||
self._do_remove_all()
|
||||
self.dump()
|
||||
for p in unused:
|
||||
cmd = 'DELETE FROM {} WHERE path = "{}"'
|
||||
self._execute(cmd.format(self.LINK_STATE_TABLE, p))
|
||||
|
||||
+150
-12
@@ -1,10 +1,4 @@
|
||||
import os
|
||||
import ctypes
|
||||
import reflink
|
||||
|
||||
if os.name == 'nt':
|
||||
import ntfsutils.hardlink as winlink
|
||||
from ntfsutils.fs import getdirinfo
|
||||
|
||||
|
||||
class System(object):
|
||||
@@ -14,13 +8,24 @@ class System(object):
|
||||
|
||||
@staticmethod
|
||||
def hardlink(source, link_name):
|
||||
import ctypes
|
||||
if System.is_unix():
|
||||
return os.link(source, link_name)
|
||||
os.link(source, link_name)
|
||||
return
|
||||
|
||||
return winlink.create(source, link_name)
|
||||
CreateHardLink = ctypes.windll.kernel32.CreateHardLinkW
|
||||
CreateHardLink.argtypes = [ctypes.c_wchar_p,
|
||||
ctypes.c_wchar_p,
|
||||
ctypes.c_void_p]
|
||||
CreateHardLink.restype = ctypes.wintypes.BOOL
|
||||
|
||||
res = CreateHardLink(link_name, source, None)
|
||||
if res == 0:
|
||||
raise ctypes.WinError()
|
||||
|
||||
@staticmethod
|
||||
def symlink(source, link_name):
|
||||
import ctypes
|
||||
if System.is_unix():
|
||||
return os.symlink(source, link_name)
|
||||
|
||||
@@ -35,15 +40,148 @@ class System(object):
|
||||
if func(link_name, source, flags) == 0:
|
||||
raise ctypes.WinError()
|
||||
|
||||
@staticmethod
|
||||
def _reflink_darwin(src, dst):
|
||||
import ctypes
|
||||
|
||||
clib = ctypes.CDLL('libc.dylib')
|
||||
if not hasattr(clib, 'clonefile'):
|
||||
return -1
|
||||
|
||||
clonefile = clib.clonefile
|
||||
clonefile.argtypes = [ctypes.c_char_p,
|
||||
ctypes.c_char_p,
|
||||
ctypes.c_int]
|
||||
clonefile.restype = ctypes.c_int
|
||||
|
||||
return clonefile(ctypes.c_char_p(src.encode('utf-8')),
|
||||
ctypes.c_char_p(dst.encode('utf-8')),
|
||||
ctypes.c_int(0))
|
||||
|
||||
@staticmethod
|
||||
def _reflink_windows(src, dst):
|
||||
return -1
|
||||
|
||||
@staticmethod
|
||||
def _reflink_linux(src, dst):
|
||||
import os
|
||||
import fcntl
|
||||
|
||||
FICLONE = 0x40049409
|
||||
|
||||
s = open(src, 'r')
|
||||
d = open(dst, 'w+')
|
||||
|
||||
try:
|
||||
ret = fcntl.ioctl(d.fileno(), FICLONE, s.fileno())
|
||||
except IOError:
|
||||
s.close()
|
||||
d.close()
|
||||
os.unlink(dst)
|
||||
raise
|
||||
|
||||
s.close()
|
||||
d.close()
|
||||
|
||||
if ret != 0:
|
||||
os.unlink(dst)
|
||||
|
||||
return ret
|
||||
|
||||
@staticmethod
|
||||
def reflink(source, link_name):
|
||||
return reflink.reflink(source, link_name)
|
||||
import platform
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
system = platform.system()
|
||||
try:
|
||||
if system == 'Windows':
|
||||
ret = System._reflink_windows(source, link_name)
|
||||
elif system == 'Darwin':
|
||||
ret = System._reflink_darwin(source, link_name)
|
||||
elif system == 'Linux':
|
||||
ret = System._reflink_linux(source, link_name)
|
||||
else:
|
||||
ret = -1
|
||||
except IOError:
|
||||
ret = -1
|
||||
|
||||
if ret != 0:
|
||||
raise DvcException('Reflink is not supported')
|
||||
|
||||
@staticmethod
|
||||
def getdirinfo(path):
|
||||
import ctypes
|
||||
from ctypes import c_void_p, c_wchar_p, Structure, WinError
|
||||
from ctypes.wintypes import DWORD, HANDLE, POINTER, BOOL
|
||||
|
||||
FILE_FLAG_BACKUP_SEMANTICS = 0x02000000
|
||||
FILE_SHARE_READ = 0x00000001
|
||||
OPEN_EXISTING = 3
|
||||
|
||||
class FILETIME(Structure):
|
||||
_fields_ = [("dwLowDateTime", DWORD),
|
||||
("dwHighDateTime", DWORD)]
|
||||
|
||||
class BY_HANDLE_FILE_INFORMATION(Structure):
|
||||
_fields_ = [("dwFileAttributes", DWORD),
|
||||
("ftCreationTime", FILETIME),
|
||||
("ftLastAccessTime", FILETIME),
|
||||
("ftLastWriteTime", FILETIME),
|
||||
("dwVolumeSerialNumber", DWORD),
|
||||
("nFileSizeHigh", DWORD),
|
||||
("nFileSizeLow", DWORD),
|
||||
("nNumberOfLinks", DWORD),
|
||||
("nFileIndexHigh", DWORD),
|
||||
("nFileIndexLow", DWORD)]
|
||||
|
||||
flags = FILE_FLAG_BACKUP_SEMANTICS
|
||||
|
||||
func = ctypes.windll.kernel32.CreateFileW
|
||||
func.argtypes = [c_wchar_p,
|
||||
DWORD,
|
||||
DWORD,
|
||||
c_void_p,
|
||||
DWORD,
|
||||
DWORD,
|
||||
HANDLE]
|
||||
func.restype = HANDLE
|
||||
|
||||
hfile = func(path,
|
||||
0,
|
||||
FILE_SHARE_READ,
|
||||
None,
|
||||
OPEN_EXISTING,
|
||||
flags,
|
||||
None)
|
||||
if hfile is None:
|
||||
raise WinError()
|
||||
|
||||
func = ctypes.windll.kernel32.GetFileInformationByHandle
|
||||
func.argtypes = [HANDLE, POINTER(BY_HANDLE_FILE_INFORMATION)]
|
||||
func.restype = BOOL
|
||||
|
||||
info = BY_HANDLE_FILE_INFORMATION()
|
||||
rv = func(hfile, info)
|
||||
|
||||
func = ctypes.windll.kernel32.CloseHandle
|
||||
func.argtypes = [HANDLE]
|
||||
func.restype = BOOL
|
||||
|
||||
func(hfile)
|
||||
|
||||
if rv == 0:
|
||||
raise WinError()
|
||||
|
||||
return info
|
||||
|
||||
@staticmethod
|
||||
def inode(path):
|
||||
if System.is_unix():
|
||||
return os.stat(path).st_ino
|
||||
return os.lstat(path).st_ino
|
||||
|
||||
# getdirinfo from ntfsutils works on both files and dirs
|
||||
info = getdirinfo(path)
|
||||
return hash((info.dwVolumeSerialNumber, info.nFileIndexHigh, info.nFileIndexLow))
|
||||
info = System.getdirinfo(path)
|
||||
return hash((info.dwVolumeSerialNumber,
|
||||
info.nFileIndexHigh,
|
||||
info.nFileIndexLow))
|
||||
|
||||
+17
-11
@@ -1,15 +1,17 @@
|
||||
import os
|
||||
import time
|
||||
import json
|
||||
import requests
|
||||
|
||||
from dvc import VERSION_BASE
|
||||
from dvc.logger import Logger
|
||||
|
||||
class Updater(object):
|
||||
URL = 'https://4ki8820rsf.execute-api.us-east-2.amazonaws.com/prod/latest-version'
|
||||
|
||||
class Updater(object): # pragma: no cover
|
||||
URL = 'https://4ki8820rsf.execute-api.us-east-2.amazonaws.com/' \
|
||||
'prod/latest-version'
|
||||
UPDATER_FILE = 'updater'
|
||||
TIMEOUT = 7 * 24 * 60 * 60 #every week
|
||||
TIMEOUT = 24 * 60 * 60 # every day
|
||||
TIMEOUT_GET = 10
|
||||
|
||||
def __init__(self, dvc_dir):
|
||||
self.dvc_dir = dvc_dir
|
||||
@@ -34,22 +36,26 @@ class Updater(object):
|
||||
|
||||
os.unlink(self.updater_file)
|
||||
|
||||
Logger.info('Checking for updates...')
|
||||
|
||||
try:
|
||||
r = requests.get(self.URL)
|
||||
j = json.loads(r.content)
|
||||
r = requests.get(self.URL, timeout=self.TIMEOUT_GET)
|
||||
j = r.json()
|
||||
latest = j['version']
|
||||
open(self.updater_file, 'w+').close()
|
||||
except Exception as exc:
|
||||
Logger.debug('Failed to obtain latest version: {}'.format(str(exc)))
|
||||
msg = 'Failed to obtain latest version: {}'.format(str(exc))
|
||||
Logger.debug(msg)
|
||||
return
|
||||
|
||||
l_major, l_minor, l_patch = latest.split('.')
|
||||
c_major, c_minor, c_patch = current.split('.')
|
||||
l_major, l_minor, l_patch = [int(x) for x in latest.split('.')]
|
||||
c_major, c_minor, c_patch = [int(x) for x in current.split('.')]
|
||||
|
||||
if l_major <= c_major and \
|
||||
l_minor <= c_minor and \
|
||||
l_patch <= c_patch:
|
||||
return
|
||||
return
|
||||
|
||||
msg = 'You are using dvc version {}, however version {} is available. Consider upgrading.'
|
||||
msg = 'You are using dvc version {}, however version {} is ' \
|
||||
'available. Consider upgrading.'
|
||||
Logger.warn(msg.format(current, latest))
|
||||
|
||||
+64
-61
@@ -2,52 +2,60 @@
|
||||
Helpers for other modules.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import math
|
||||
import json
|
||||
import stat
|
||||
import shutil
|
||||
import hashlib
|
||||
from binaryornot.check import is_binary
|
||||
from multiprocessing.pool import ThreadPool
|
||||
|
||||
from dvc.progress import progress
|
||||
from dvc.istextfile import istextfile
|
||||
from dvc.logger import Logger
|
||||
|
||||
|
||||
LOCAL_CHUNK_SIZE = 1024*1024
|
||||
LARGE_FILE_SIZE = 1024*1024*1024
|
||||
LARGE_DIR_SIZE = 100
|
||||
|
||||
|
||||
def dos2unix(data):
|
||||
return '\n'.join(data.split('\r\n'))
|
||||
return data.replace(b'\r\n', b'\n')
|
||||
|
||||
|
||||
def file_md5(fname):
|
||||
""" get the (md5 hexdigest, md5 digest) of a file """
|
||||
if os.path.exists(fname):
|
||||
hash_md5 = hashlib.md5()
|
||||
binary = is_binary(fname)
|
||||
binary = not istextfile(fname)
|
||||
size = os.path.getsize(fname)
|
||||
bar = False
|
||||
if size >= LARGE_FILE_SIZE:
|
||||
bar = True
|
||||
msg = "Computing md5 for a large file {}. This is only done once."
|
||||
Logger.info(msg.format(os.path.relpath(fname)))
|
||||
name = os.path.relpath(fname)
|
||||
total = 0
|
||||
|
||||
if binary:
|
||||
mode = "rb"
|
||||
else:
|
||||
mode = "r"
|
||||
|
||||
with open(fname, mode) as fobj:
|
||||
with open(fname, 'rb') as fobj:
|
||||
while True:
|
||||
data = fobj.read(LOCAL_CHUNK_SIZE)
|
||||
if not data:
|
||||
break
|
||||
|
||||
if bar:
|
||||
total += len(data)
|
||||
progress.update_target(name, total, size)
|
||||
|
||||
if binary:
|
||||
chunk = data
|
||||
else:
|
||||
if sys.version_info[0] == 2:
|
||||
data = data.decode('utf-8')
|
||||
chunk = dos2unix(data).encode('utf-8')
|
||||
chunk = dos2unix(data)
|
||||
|
||||
hash_md5.update(chunk)
|
||||
|
||||
if bar:
|
||||
progress.finish_target(name)
|
||||
|
||||
return (hash_md5.hexdigest(), hash_md5.digest())
|
||||
else:
|
||||
return (None, None)
|
||||
@@ -64,33 +72,16 @@ def dict_md5(d):
|
||||
return bytes_md5(byts)
|
||||
|
||||
|
||||
def cached_property(func):
|
||||
'''A decorator for caching properties in classes.'''
|
||||
def get(self):
|
||||
'''Try obtaining cache'''
|
||||
try:
|
||||
return self._property_cache[func]
|
||||
except AttributeError:
|
||||
self._property_cache = {}
|
||||
ret = self._property_cache[func] = func(self)
|
||||
return ret
|
||||
except KeyError:
|
||||
ret = self._property_cache[func] = func(self)
|
||||
return ret
|
||||
|
||||
return property(get)
|
||||
|
||||
|
||||
def copyfile(src, dest, no_progress_bar=False):
|
||||
def copyfile(src, dest, no_progress_bar=False, name=None):
|
||||
'''Copy file with progress bar'''
|
||||
copied = 0
|
||||
name = os.path.basename(src)
|
||||
name = name if name else os.path.basename(dest)
|
||||
total = os.stat(src).st_size
|
||||
|
||||
fsrc = open(src, 'rb')
|
||||
|
||||
if os.path.isdir(dest):
|
||||
fdest = open(dest + '/' + name, 'wb+')
|
||||
fdest = open(os.path.join(dest, os.path.basename(src)), 'wb+')
|
||||
else:
|
||||
fdest = open(dest, 'wb+')
|
||||
|
||||
@@ -110,37 +101,16 @@ def copyfile(src, dest, no_progress_bar=False):
|
||||
fdest.close()
|
||||
|
||||
|
||||
def wrap(func, t):
|
||||
try:
|
||||
return func(t)
|
||||
except Exception as exc:
|
||||
Logger.error('Error', exc)
|
||||
raise
|
||||
|
||||
|
||||
def map_progress(func, targets, n_threads):
|
||||
"""
|
||||
Process targets in multi-threaded mode with progress bar
|
||||
"""
|
||||
progress.set_n_total(len(targets))
|
||||
pool = ThreadPool(processes=n_threads)
|
||||
ret = []
|
||||
|
||||
wrapper = lambda t: wrap(func, t)
|
||||
|
||||
try:
|
||||
ret = pool.map(wrapper, targets)
|
||||
except Exception as exc:
|
||||
raise
|
||||
|
||||
return list(zip(targets, ret))
|
||||
|
||||
|
||||
def move(src, dst):
|
||||
dname = os.path.dirname(dst)
|
||||
if not os.path.exists(dname):
|
||||
os.makedirs(dname)
|
||||
|
||||
if os.path.islink(src):
|
||||
shutil.copy(os.readlink(src), dst)
|
||||
os.unlink(src)
|
||||
return
|
||||
|
||||
shutil.move(src, dst)
|
||||
|
||||
|
||||
@@ -153,3 +123,36 @@ def remove(path):
|
||||
os.unlink(path)
|
||||
else:
|
||||
shutil.rmtree(path)
|
||||
|
||||
|
||||
def to_chunks(l, jobs):
|
||||
n = int(math.ceil(len(l) / jobs))
|
||||
|
||||
if len(l) == 1:
|
||||
return [l]
|
||||
|
||||
if n == 0:
|
||||
n = 1
|
||||
|
||||
return [l[x:x+n] for x in range(0, len(l), n)]
|
||||
|
||||
|
||||
# NOTE: Fix env variables modified by PyInstaller
|
||||
# http://pyinstaller.readthedocs.io/en/stable/runtime-information.html
|
||||
def fix_env(env):
|
||||
if env is None:
|
||||
env = os.environ.copy()
|
||||
else:
|
||||
env = env.copy()
|
||||
|
||||
# NOTE: Check if we are in a bundle
|
||||
# https://pythonhosted.org/PyInstaller/runtime-information.html
|
||||
if getattr(sys, 'frozen', False):
|
||||
lp_key = 'LD_LIBRARY_PATH'
|
||||
lp_orig = env.get(lp_key + '_ORIG', None)
|
||||
if lp_orig is not None:
|
||||
env[lp_key] = lp_orig
|
||||
else:
|
||||
env.pop(lp_key, None)
|
||||
|
||||
return env
|
||||
|
||||
+8
-7
@@ -1,24 +1,25 @@
|
||||
azure-storage-blob==1.3.0
|
||||
boto3==1.7.4
|
||||
ply==3.8 # required by google-gax
|
||||
ply>=3.9 # See https://github.com/pyinstaller/pyinstaller/issues/1945
|
||||
configparser>=3.5.0
|
||||
zc.lockfile>=1.2.1
|
||||
future>=0.16.0
|
||||
google-cloud>=0.24.0
|
||||
google-cloud-storage==1.12.0
|
||||
PyInstaller==3.3.1
|
||||
colorama>=0.3.9
|
||||
configobj>=5.0.6
|
||||
networkx>=1.11
|
||||
networkx>=2.1
|
||||
pyyaml>=3.12
|
||||
gitpython>=2.1.8
|
||||
ntfsutils>=0.1.4
|
||||
setuptools>=34.0.0
|
||||
pytest>=3.3.1
|
||||
nanotime>=0.5.2
|
||||
pyasn1>=0.4.1
|
||||
schema>=0.6.7
|
||||
paramiko>=2.4.1
|
||||
binaryornot==0.4.4
|
||||
jsonpath-rw==1.4.0
|
||||
reflink==0.2.0
|
||||
requests>=2.18.4
|
||||
wheel>=0.31.1
|
||||
futures>=3.2.0; python_version == "2.7"
|
||||
grandalf==0.6
|
||||
pydot>=1.2.4
|
||||
asciimatics>=1.10.0
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -x
|
||||
set -e
|
||||
|
||||
#NOTE: ssh keys for ssh test to be able to ssh to the localhost
|
||||
ls -la ~/.ssh/
|
||||
|
||||
ssh-keygen -t rsa -N "" -f mykey
|
||||
mkdir -p ~/.ssh
|
||||
cp mykey ~/.ssh/id_rsa
|
||||
cp mykey.pub ~/.ssh/id_rsa.pub
|
||||
cat mykey.pub >> ~/.ssh/authorized_keys
|
||||
ssh-keyscan localhost >> ~/.ssh/known_hosts
|
||||
ssh-keyscan 127.0.0.1 >> ~/.ssh/known_hosts
|
||||
ssh-keyscan 0.0.0.0 >> ~/.ssh/known_hosts
|
||||
ssh 0.0.0.0 ls &> /dev/null
|
||||
ssh 127.0.0.1 ls &> /dev/null
|
||||
ssh localhost ls &> /dev/null
|
||||
|
||||
scriptdir="$(dirname $0)"
|
||||
|
||||
if [ -n "$TRAVIS_OS_NAME" ] && [ "$TRAVIS_OS_NAME" != "osx" ]; then
|
||||
bash "$scriptdir/install_azurite.sh"
|
||||
source ~/.bashrc
|
||||
fi
|
||||
@@ -0,0 +1,6 @@
|
||||
<configuration>
|
||||
<property>
|
||||
<name>fs.defaultFS</name>
|
||||
<value>hdfs://localhost:8020</value>
|
||||
</property>
|
||||
</configuration>
|
||||
@@ -0,0 +1,6 @@
|
||||
<configuration>
|
||||
<property>
|
||||
<name>dfs.replication</name>
|
||||
<value>1</value>
|
||||
</property>
|
||||
</configuration>
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user