mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 15:28:28 +01:00
+1
-1
@@ -48,7 +48,7 @@ install:
|
||||
- cinst awscli
|
||||
- cinst gsutil
|
||||
- cinst openssl.light
|
||||
- pip install --user --upgrade pip
|
||||
- python -m pip install -U pip setuptools wheel
|
||||
- pip install -r requirements.txt
|
||||
- python setup.py install
|
||||
|
||||
|
||||
+29
-3
@@ -7,6 +7,7 @@ import schema
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.output import Output, Dependency, OutputError
|
||||
from dvc.logger import Logger
|
||||
from dvc.utils import dict_md5
|
||||
|
||||
|
||||
class StageCmdFailedError(DvcException):
|
||||
@@ -36,23 +37,26 @@ class Stage(object):
|
||||
STAGE_FILE = 'Dvcfile'
|
||||
STAGE_FILE_SUFFIX = '.dvc'
|
||||
|
||||
PARAM_MD5 = 'md5'
|
||||
PARAM_CMD = 'cmd'
|
||||
PARAM_DEPS = 'deps'
|
||||
PARAM_OUTS = 'outs'
|
||||
|
||||
SCHEMA = {
|
||||
schema.Optional(PARAM_MD5): schema.Or(str, None),
|
||||
schema.Optional(PARAM_CMD): schema.Or(str, None),
|
||||
schema.Optional(PARAM_DEPS): schema.Or(schema.And(list, schema.Schema([Dependency.SCHEMA])), None),
|
||||
schema.Optional(PARAM_OUTS): schema.Or(schema.And(list, schema.Schema([Output.SCHEMA])), None),
|
||||
}
|
||||
|
||||
def __init__(self, project, path=None, cmd=None, cwd=None, deps=[], outs=[]):
|
||||
def __init__(self, project, path=None, cmd=None, cwd=None, deps=[], outs=[], md5=None):
|
||||
self.project = project
|
||||
self.path = path
|
||||
self.cmd = cmd
|
||||
self.cwd = cwd
|
||||
self.outs = outs
|
||||
self.deps = deps
|
||||
self.md5 = md5
|
||||
|
||||
@property
|
||||
def relpath(self):
|
||||
@@ -72,12 +76,29 @@ class Stage(object):
|
||||
|
||||
return True
|
||||
|
||||
def changed_md5(self):
|
||||
md5 = self.dumpd().get(self.PARAM_MD5, None)
|
||||
|
||||
# backward compatibility
|
||||
if self.md5 == None:
|
||||
return False
|
||||
|
||||
if self.md5 and md5 and self.md5 == md5:
|
||||
return False
|
||||
|
||||
msg = "Dvc file '{}' md5 changed(expected '{}', actual '{}')"
|
||||
self.project.logger.debug(msg.format(self.relpath, self.md5, md5))
|
||||
return True
|
||||
|
||||
def changed(self):
|
||||
ret = False
|
||||
for entry in itertools.chain(self.outs, self.deps):
|
||||
if entry.changed():
|
||||
ret = True
|
||||
|
||||
if self.changed_md5():
|
||||
ret = True
|
||||
|
||||
if ret:
|
||||
self.project.logger.debug(u'Dvc file \'{}\' changed'.format(self.relpath))
|
||||
else:
|
||||
@@ -124,13 +145,16 @@ class Stage(object):
|
||||
cmd = d.get(Stage.PARAM_CMD, None)
|
||||
deps = Dependency.loadd_from(project, d.get(Stage.PARAM_DEPS, []), cwd=cwd)
|
||||
outs = Output.loadd_from(project, d.get(Stage.PARAM_OUTS, []), cwd=cwd)
|
||||
md5 = d.get(Stage.PARAM_MD5, None)
|
||||
|
||||
return Stage(project=project,
|
||||
path=path,
|
||||
cmd=cmd,
|
||||
cwd=cwd,
|
||||
deps=deps,
|
||||
outs=outs)
|
||||
outs=outs,
|
||||
md5=md5)
|
||||
|
||||
@staticmethod
|
||||
def loads(project=None,
|
||||
cmd=None,
|
||||
@@ -171,6 +195,8 @@ class Stage(object):
|
||||
if len(outs):
|
||||
ret[Stage.PARAM_OUTS] = outs
|
||||
|
||||
ret[Stage.PARAM_MD5] = dict_md5(ret)
|
||||
|
||||
return ret
|
||||
|
||||
def dump(self, fname=None):
|
||||
@@ -231,7 +257,7 @@ class Stage(object):
|
||||
ret.update(self._status(self.deps, 'deps'))
|
||||
ret.update(self._status(self.outs, 'outs'))
|
||||
|
||||
if ret:
|
||||
if ret or self.changed_md5():
|
||||
return {self.relpath: ret}
|
||||
|
||||
return {}
|
||||
|
||||
+2
-4
@@ -5,7 +5,7 @@ import threading
|
||||
|
||||
from dvc.system import System
|
||||
from dvc.output import Output
|
||||
from dvc.utils import file_md5, bytes_md5
|
||||
from dvc.utils import file_md5, dict_md5
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
|
||||
@@ -70,9 +70,7 @@ class State(object):
|
||||
def compute_md5(self, path):
|
||||
if os.path.isdir(path):
|
||||
dir_info = self.collect_dir(path)
|
||||
byts = json.dumps(dir_info, sort_keys=True).encode('utf-8')
|
||||
md5 = bytes_md5(byts)
|
||||
return md5 + Output.MD5_DIR_SUFFIX
|
||||
return dict_md5(dir_info) + Output.MD5_DIR_SUFFIX
|
||||
else:
|
||||
return file_md5(path)[0]
|
||||
|
||||
|
||||
+30
-3
@@ -3,10 +3,11 @@ Helpers for other modules.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import json
|
||||
import stat
|
||||
import shutil
|
||||
import hashlib
|
||||
|
||||
from binaryornot.check import is_binary
|
||||
from multiprocessing.pool import ThreadPool
|
||||
|
||||
from dvc.progress import progress
|
||||
@@ -16,13 +17,34 @@ from dvc.logger import Logger
|
||||
LOCAL_CHUNK_SIZE = 1024*1024
|
||||
|
||||
|
||||
def dos2unix(data):
|
||||
return '\n'.join(data.split('\r\n'))
|
||||
|
||||
|
||||
def file_md5(fname):
|
||||
""" get the (md5 hexdigest, md5 digest) of a file """
|
||||
if os.path.exists(fname):
|
||||
hash_md5 = hashlib.md5()
|
||||
with open(fname, "rb") as fobj:
|
||||
for chunk in iter(lambda: fobj.read(1024*1000), b""):
|
||||
binary = is_binary(fname)
|
||||
|
||||
if binary:
|
||||
mode = "rb"
|
||||
else:
|
||||
mode = "r"
|
||||
|
||||
with open(fname, mode) as fobj:
|
||||
while True:
|
||||
data = fobj.read(LOCAL_CHUNK_SIZE)
|
||||
if not data:
|
||||
break
|
||||
|
||||
if binary:
|
||||
chunk = data
|
||||
else:
|
||||
chunk = dos2unix(data).encode('utf-8')
|
||||
|
||||
hash_md5.update(chunk)
|
||||
|
||||
return (hash_md5.hexdigest(), hash_md5.digest())
|
||||
else:
|
||||
return (None, None)
|
||||
@@ -34,6 +56,11 @@ def bytes_md5(byts):
|
||||
return hasher.hexdigest()
|
||||
|
||||
|
||||
def dict_md5(d):
|
||||
byts = json.dumps(d, sort_keys=True).encode('utf-8')
|
||||
return bytes_md5(byts)
|
||||
|
||||
|
||||
def cached_property(func):
|
||||
'''A decorator for caching properties in classes.'''
|
||||
def get(self):
|
||||
|
||||
@@ -16,3 +16,4 @@ nanotime>=0.5.2
|
||||
pyasn1>=0.4.1
|
||||
schema>=0.6.7
|
||||
paramiko>=2.4.1
|
||||
binaryornot==0.4.4
|
||||
|
||||
Reference in New Issue
Block a user