mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 15:28:28 +01:00
dvc: split output and dependency classes
Preparation for the 'external deps' patches. Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
Binary file not shown.
@@ -0,0 +1,126 @@
|
||||
import os
|
||||
import schema
|
||||
import posixpath
|
||||
import ntpath
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.logger import Logger
|
||||
from dvc.utils import remove
|
||||
|
||||
|
||||
class DependencyError(DvcException):
|
||||
def __init__(self, path, msg):
|
||||
super(DependencyError, self).__init__('Dependency \'{}\' error: {}'.format(path, msg))
|
||||
|
||||
|
||||
class DependencyOutsideOfRepoError(DependencyError):
|
||||
def __init__(self, path):
|
||||
super(DependencyOutsideOfRepoError, self).__init__(path, 'outside of repository')
|
||||
|
||||
|
||||
class DependencyDoesNotExistError(DependencyError):
|
||||
def __init__(self, path):
|
||||
super(DependencyDoesNotExistError, self).__init__(path, 'does not exist')
|
||||
|
||||
|
||||
class DependencyIsNotFileOrDirError(DependencyError):
|
||||
def __init__(self, path):
|
||||
super(DependencyIsNotFileOrDirError, self).__init__(path, 'not a file or directory')
|
||||
|
||||
|
||||
class Dependency(object):
|
||||
PARAM_PATH = 'path'
|
||||
PARAM_MD5 = 'md5'
|
||||
MD5_DIR_SUFFIX = '.dir'
|
||||
|
||||
OutsideOfRepoError = DependencyOutsideOfRepoError
|
||||
DoesNotExistError = DependencyDoesNotExistError
|
||||
IsNotFileOrDirError = DependencyIsNotFileOrDirError
|
||||
|
||||
SCHEMA = {
|
||||
PARAM_PATH: str,
|
||||
schema.Optional(PARAM_MD5): schema.Or(str, None),
|
||||
}
|
||||
|
||||
def __init__(self, project, path, md5=None):
|
||||
self.project = project
|
||||
self.path = os.path.abspath(os.path.normpath(path))
|
||||
|
||||
if not self.path.startswith(self.project.root_dir):
|
||||
raise self.OutsideOfRepoError(self.rel_path)
|
||||
|
||||
self.md5 = md5
|
||||
|
||||
@property
|
||||
def rel_path(self):
|
||||
return os.path.relpath(self.path)
|
||||
|
||||
def _changed_md5(self):
|
||||
if not os.path.exists(self.path):
|
||||
return True
|
||||
|
||||
return self.project.state.changed(self.path, self.md5)
|
||||
|
||||
@staticmethod
|
||||
def _changed_msg(changed):
|
||||
if changed:
|
||||
return 'changed'
|
||||
return "didn't change"
|
||||
|
||||
def changed(self):
|
||||
ret = self._changed_md5()
|
||||
|
||||
msg = u'Dependency \'{}\' {}'.format(self.rel_path, self._changed_msg(ret))
|
||||
self.project.logger.debug(msg)
|
||||
|
||||
return ret
|
||||
|
||||
def status(self):
|
||||
if self.changed():
|
||||
#FIXME better msgs
|
||||
return {self.rel_path: 'changed'}
|
||||
return {}
|
||||
|
||||
def save(self):
|
||||
if not os.path.exists(self.path):
|
||||
raise self.DoesNotExistError(self.rel_path)
|
||||
|
||||
if not os.path.isfile(self.path) and not os.path.isdir(self.path):
|
||||
raise self.NotFileOrDirError(self.rel_path)
|
||||
|
||||
if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \
|
||||
(os.path.isdir(self.path) and len(os.listdir(self.path)) == 0):
|
||||
self.project.logger.warn("File/directory '{}' is empty.".format(self.rel_path))
|
||||
|
||||
self.md5 = self.project.state.update(self.path)
|
||||
|
||||
@staticmethod
|
||||
def unixpath(path):
|
||||
assert not ntpath.isabs(path)
|
||||
assert not posixpath.isabs(path)
|
||||
return path.replace('\\', '/')
|
||||
|
||||
def dumpd(self, cwd):
|
||||
return {
|
||||
self.PARAM_PATH: self.unixpath(os.path.relpath(self.path, cwd)),
|
||||
self.PARAM_MD5: self.md5,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def loadd(cls, project, d, cwd=os.curdir):
|
||||
relpath = os.path.normpath(Dependency.unixpath(d[Dependency.PARAM_PATH]))
|
||||
path = os.path.join(cwd, relpath)
|
||||
md5 = d.get(Dependency.PARAM_MD5, None)
|
||||
return cls(project, path, md5=md5)
|
||||
|
||||
@classmethod
|
||||
def loadd_from(cls, project, d_list, cwd=os.curdir):
|
||||
return [cls.loadd(project, x, cwd=cwd) for x in d_list]
|
||||
|
||||
@classmethod
|
||||
def loads(cls, project, s, cwd=os.curdir):
|
||||
return cls(project, os.path.join(cwd, s), md5=None)
|
||||
|
||||
@classmethod
|
||||
def loads_from(cls, project, s_list, cwd=os.curdir):
|
||||
return [cls.loads(project, x, cwd=cwd) for x in s_list]
|
||||
-217
@@ -1,217 +0,0 @@
|
||||
import os
|
||||
import schema
|
||||
import posixpath
|
||||
import ntpath
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.logger import Logger
|
||||
from dvc.utils import remove
|
||||
|
||||
|
||||
class OutputError(DvcException):
|
||||
pass
|
||||
|
||||
|
||||
class CmdOutputError(DvcException):
|
||||
def __init__(self, path, msg):
|
||||
super(CmdOutputError, self).__init__('Output file \'{}\' error: {}'.format(path, msg))
|
||||
|
||||
|
||||
class CmdOutputNoCacheError(CmdOutputError):
|
||||
def __init__(self, path):
|
||||
super(CmdOutputNoCacheError, self).__init__(path, 'no cache')
|
||||
|
||||
|
||||
class CmdOutputOutsideOfRepoError(CmdOutputError):
|
||||
def __init__(self, path):
|
||||
super(CmdOutputOutsideOfRepoError, self).__init__(path, 'outside of repository')
|
||||
|
||||
|
||||
class CmdOutputDoesNotExistError(CmdOutputError):
|
||||
def __init__(self, path):
|
||||
super(CmdOutputDoesNotExistError, self).__init__(path, 'does not exist')
|
||||
|
||||
|
||||
class CmdOutputIsNotFileOrDirError(CmdOutputError):
|
||||
def __init__(self, path):
|
||||
super(CmdOutputIsNotFileOrDirError, self).__init__(path, 'not a file or directory')
|
||||
|
||||
|
||||
class CmdOutputAlreadyTrackedError(CmdOutputError):
|
||||
def __init__(self, path):
|
||||
super(CmdOutputAlreadyTrackedError, self).__init__(path, 'already tracked by scm(e.g. git)')
|
||||
|
||||
|
||||
class Dependency(object):
|
||||
PARAM_PATH = 'path'
|
||||
PARAM_MD5 = 'md5'
|
||||
MD5_DIR_SUFFIX = '.dir'
|
||||
|
||||
SCHEMA = {
|
||||
PARAM_PATH: str,
|
||||
schema.Optional(PARAM_MD5): schema.Or(str, None),
|
||||
}
|
||||
|
||||
def __init__(self, project, path, md5=None):
|
||||
self.project = project
|
||||
self.path = os.path.abspath(os.path.normpath(path))
|
||||
|
||||
if not self.path.startswith(self.project.root_dir):
|
||||
raise CmdOutputOutsideOfRepoError(self.rel_path)
|
||||
|
||||
self.md5 = md5
|
||||
|
||||
@property
|
||||
def rel_path(self):
|
||||
return os.path.relpath(self.path)
|
||||
|
||||
def _changed_md5(self):
|
||||
if not os.path.exists(self.path):
|
||||
return True
|
||||
|
||||
return self.project.state.changed(self.path, self.md5)
|
||||
|
||||
@staticmethod
|
||||
def _changed_msg(changed):
|
||||
if changed:
|
||||
return 'changed'
|
||||
return "didn't change"
|
||||
|
||||
def changed(self):
|
||||
ret = self._changed_md5()
|
||||
|
||||
msg = u'Dependency \'{}\' {}'.format(self.rel_path, self._changed_msg(ret))
|
||||
self.project.logger.debug(msg)
|
||||
|
||||
return ret
|
||||
|
||||
def status(self):
|
||||
if self.changed():
|
||||
#FIXME better msgs
|
||||
return {self.rel_path: 'changed'}
|
||||
return {}
|
||||
|
||||
def save(self):
|
||||
if not os.path.exists(self.path):
|
||||
raise CmdOutputDoesNotExistError(self.rel_path)
|
||||
|
||||
if not os.path.isfile(self.path) and not os.path.isdir(self.path):
|
||||
raise CmdOutputIsNotFileOrDirError(self.rel_path)
|
||||
|
||||
if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \
|
||||
(os.path.isdir(self.path) and len(os.listdir(self.path)) == 0):
|
||||
self.project.logger.warn("File/directory '{}' is empty.".format(self.rel_path))
|
||||
|
||||
self.md5 = self.project.state.update(self.path)
|
||||
|
||||
@staticmethod
|
||||
def unixpath(path):
|
||||
assert not ntpath.isabs(path)
|
||||
assert not posixpath.isabs(path)
|
||||
return path.replace('\\', '/')
|
||||
|
||||
def dumpd(self, cwd):
|
||||
return {
|
||||
Output.PARAM_PATH: self.unixpath(os.path.relpath(self.path, cwd)),
|
||||
Output.PARAM_MD5: self.md5,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def loadd(cls, project, d, cwd=os.curdir):
|
||||
relpath = os.path.normpath(Output.unixpath(d[Output.PARAM_PATH]))
|
||||
path = os.path.join(cwd, relpath)
|
||||
md5 = d.get(Output.PARAM_MD5, None)
|
||||
return cls(project, path, md5=md5)
|
||||
|
||||
@classmethod
|
||||
def loadd_from(cls, project, d_list, cwd=os.curdir):
|
||||
return [cls.loadd(project, x, cwd=cwd) for x in d_list]
|
||||
|
||||
@classmethod
|
||||
def loads(cls, project, s, cwd=os.curdir):
|
||||
return cls(project, os.path.join(cwd, s), md5=None)
|
||||
|
||||
@classmethod
|
||||
def loads_from(cls, project, s_list, cwd=os.curdir):
|
||||
return [cls.loads(project, x, cwd=cwd) for x in s_list]
|
||||
|
||||
|
||||
class Output(Dependency):
|
||||
PARAM_CACHE = 'cache'
|
||||
|
||||
SCHEMA = Dependency.SCHEMA
|
||||
SCHEMA[schema.Optional(PARAM_CACHE)] = bool
|
||||
|
||||
def __init__(self, project, path, md5=None, use_cache=True):
|
||||
super(Output, self).__init__(project, path, md5=md5)
|
||||
self.use_cache = use_cache
|
||||
|
||||
@property
|
||||
def cache(self):
|
||||
if not self.use_cache:
|
||||
return None
|
||||
|
||||
return self.project.cache.get(self.md5)
|
||||
|
||||
def dumpd(self, cwd):
|
||||
ret = super(Output, self).dumpd(cwd)
|
||||
ret[Output.PARAM_CACHE] = self.use_cache
|
||||
return ret
|
||||
|
||||
@classmethod
|
||||
def loadd(cls, project, d, cwd=os.curdir):
|
||||
ret = super(Output, cls).loadd(project, d, cwd=cwd)
|
||||
ret.use_cache = d.get(Output.PARAM_CACHE, True)
|
||||
return ret
|
||||
|
||||
@classmethod
|
||||
def loads(cls, project, s, use_cache=True, cwd=os.curdir):
|
||||
ret = super(Output, cls).loads(project, s, cwd=cwd)
|
||||
ret.use_cache = use_cache
|
||||
return ret
|
||||
|
||||
@classmethod
|
||||
def loads_from(cls, project, s_list, use_cache=False, cwd=os.curdir):
|
||||
return [cls.loads(project, x, use_cache=use_cache, cwd=cwd) for x in s_list]
|
||||
|
||||
def changed(self):
|
||||
if super(Output, self).changed():
|
||||
return True
|
||||
|
||||
if self.use_cache and self.project.cache.changed(self.md5):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def checkout(self):
|
||||
if not self.use_cache:
|
||||
return
|
||||
|
||||
msg = u'Checking out \'{}\' with cache \'{}\''
|
||||
self.project.logger.debug(msg.format(self.rel_path, self.md5))
|
||||
|
||||
if not self.changed():
|
||||
msg = u'Data file \'{}\' with cache \'{}\' didn\'t change, skipping checkout.'
|
||||
self.project.logger.debug(msg.format(self.rel_path, self.md5))
|
||||
return
|
||||
|
||||
self.project.cache.checkout(self.path, self.md5)
|
||||
|
||||
def save(self):
|
||||
super(Output, self).save()
|
||||
|
||||
if not self.use_cache:
|
||||
return
|
||||
|
||||
self.project.logger.debug(u'Saving \'{}\' to \'{}\''.format(self.rel_path, self.md5))
|
||||
|
||||
if self.project.scm.is_tracked(self.path):
|
||||
raise CmdOutputAlreadyTrackedError(self.rel_path)
|
||||
|
||||
if not self.changed():
|
||||
return
|
||||
|
||||
self.project.cache.save(self.path)
|
||||
|
||||
def remove(self):
|
||||
remove(self.path)
|
||||
@@ -0,0 +1,119 @@
|
||||
import os
|
||||
import schema
|
||||
import posixpath
|
||||
import ntpath
|
||||
|
||||
from dvc.dependency import Dependency
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.logger import Logger
|
||||
from dvc.utils import remove
|
||||
|
||||
|
||||
class OutputError(DvcException):
|
||||
def __init__(self, path, msg):
|
||||
super(OutputError, self).__init__('Output \'{}\' error: {}'.format(path, msg))
|
||||
|
||||
|
||||
class OutputOutsideOfRepoError(OutputError):
|
||||
def __init__(self, path):
|
||||
super(OutputOutsideOfRepoError, self).__init__(path, 'outside of repository')
|
||||
|
||||
|
||||
class OutputDoesNotExistError(OutputError):
|
||||
def __init__(self, path):
|
||||
super(OutputDoesNotExistError, self).__init__(path, 'does not exist')
|
||||
|
||||
|
||||
class OutputIsNotFileOrDirError(OutputError):
|
||||
def __init__(self, path):
|
||||
super(OutputIsNotFileOrDirError, self).__init__(path, 'not a file or directory')
|
||||
|
||||
|
||||
class OutputAlreadyTrackedError(OutputError):
|
||||
def __init__(self, path):
|
||||
super(OutputAlreadyTrackedError, self).__init__(path, 'already tracked by scm(e.g. git)')
|
||||
|
||||
|
||||
class Output(Dependency):
|
||||
PARAM_CACHE = 'cache'
|
||||
|
||||
OutsideOfRepoError = OutputOutsideOfRepoError
|
||||
DoesNotExistError = OutputDoesNotExistError
|
||||
IsNotFileOrDirError = OutputIsNotFileOrDirError
|
||||
|
||||
SCHEMA = Dependency.SCHEMA
|
||||
SCHEMA[schema.Optional(PARAM_CACHE)] = bool
|
||||
|
||||
def __init__(self, project, path, md5=None, use_cache=True):
|
||||
super(Output, self).__init__(project, path, md5=md5)
|
||||
self.use_cache = use_cache
|
||||
|
||||
@property
|
||||
def cache(self):
|
||||
if not self.use_cache:
|
||||
return None
|
||||
|
||||
return self.project.cache.get(self.md5)
|
||||
|
||||
def dumpd(self, cwd):
|
||||
ret = super(Output, self).dumpd(cwd)
|
||||
ret[Output.PARAM_CACHE] = self.use_cache
|
||||
return ret
|
||||
|
||||
@classmethod
|
||||
def loadd(cls, project, d, cwd=os.curdir):
|
||||
ret = super(Output, cls).loadd(project, d, cwd=cwd)
|
||||
ret.use_cache = d.get(Output.PARAM_CACHE, True)
|
||||
return ret
|
||||
|
||||
@classmethod
|
||||
def loads(cls, project, s, use_cache=True, cwd=os.curdir):
|
||||
ret = super(Output, cls).loads(project, s, cwd=cwd)
|
||||
ret.use_cache = use_cache
|
||||
return ret
|
||||
|
||||
@classmethod
|
||||
def loads_from(cls, project, s_list, use_cache=False, cwd=os.curdir):
|
||||
return [cls.loads(project, x, use_cache=use_cache, cwd=cwd) for x in s_list]
|
||||
|
||||
def changed(self):
|
||||
if super(Output, self).changed():
|
||||
return True
|
||||
|
||||
if self.use_cache and self.project.cache.changed(self.md5):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def checkout(self):
|
||||
if not self.use_cache:
|
||||
return
|
||||
|
||||
msg = u'Checking out \'{}\' with cache \'{}\''
|
||||
self.project.logger.debug(msg.format(self.rel_path, self.md5))
|
||||
|
||||
if not self.changed():
|
||||
msg = u'Data file \'{}\' with cache \'{}\' didn\'t change, skipping checkout.'
|
||||
self.project.logger.debug(msg.format(self.rel_path, self.md5))
|
||||
return
|
||||
|
||||
self.project.cache.checkout(self.path, self.md5)
|
||||
|
||||
def save(self):
|
||||
super(Output, self).save()
|
||||
|
||||
if not self.use_cache:
|
||||
return
|
||||
|
||||
self.project.logger.debug(u'Saving \'{}\' to \'{}\''.format(self.rel_path, self.md5))
|
||||
|
||||
if self.project.scm.is_tracked(self.path):
|
||||
raise OutputAlreadyTrackedError(self.rel_path)
|
||||
|
||||
if not self.changed():
|
||||
return
|
||||
|
||||
self.project.cache.save(self.path)
|
||||
|
||||
def remove(self):
|
||||
remove(self.path)
|
||||
+3
-2
@@ -5,7 +5,8 @@ import subprocess
|
||||
import schema
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.output import Output, Dependency, OutputError
|
||||
from dvc.output import Output
|
||||
from dvc.dependency import Dependency
|
||||
from dvc.logger import Logger
|
||||
from dvc.utils import dict_md5
|
||||
|
||||
@@ -21,7 +22,7 @@ class StageFileFormatError(DvcException):
|
||||
super(StageFileFormatError, self).__init__('Stage file format error')
|
||||
|
||||
|
||||
class MissingDataSource(OutputError):
|
||||
class MissingDataSource(DvcException):
|
||||
def __init__(self, missing_files):
|
||||
assert len(missing_files) > 0
|
||||
|
||||
|
||||
+5
-6
@@ -6,9 +6,8 @@ import filecmp
|
||||
from dvc.main import main
|
||||
from dvc.utils import file_md5
|
||||
from dvc.stage import Stage
|
||||
from dvc.output import CmdOutputNoCacheError, CmdOutputOutsideOfRepoError
|
||||
from dvc.output import CmdOutputDoesNotExistError, CmdOutputIsNotFileOrDirError
|
||||
from dvc.output import CmdOutputAlreadyTrackedError
|
||||
from dvc.output import OutputOutsideOfRepoError, OutputAlreadyTrackedError
|
||||
from dvc.output import OutputDoesNotExistError, OutputIsNotFileOrDirError
|
||||
from dvc.project import StageNotFoundError
|
||||
from dvc.command.add import CmdAdd
|
||||
|
||||
@@ -31,13 +30,13 @@ class TestAdd(TestDvc):
|
||||
|
||||
class TestAddNonExistentFile(TestDvc):
|
||||
def test(self):
|
||||
with self.assertRaises(CmdOutputDoesNotExistError) as cx:
|
||||
with self.assertRaises(OutputDoesNotExistError) as cx:
|
||||
self.dvc.add('non_existent_file')
|
||||
|
||||
|
||||
class TestAddFileOutsideOfRepo(TestDvc):
|
||||
def test(self):
|
||||
with self.assertRaises(CmdOutputOutsideOfRepoError) as cx:
|
||||
with self.assertRaises(OutputOutsideOfRepoError) as cx:
|
||||
self.dvc.add(os.path.join(os.path.dirname(self.dvc.root_dir), self.FOO))
|
||||
|
||||
|
||||
@@ -63,7 +62,7 @@ class TestAddTrackedFile(TestDvc):
|
||||
self.dvc.scm.add([fname])
|
||||
self.dvc.scm.commit('add {}'.format(fname))
|
||||
|
||||
with self.assertRaises(CmdOutputAlreadyTrackedError) as cx:
|
||||
with self.assertRaises(OutputAlreadyTrackedError) as cx:
|
||||
self.dvc.add(fname)
|
||||
|
||||
|
||||
|
||||
@@ -5,8 +5,6 @@ import filecmp
|
||||
from dvc.main import main
|
||||
from dvc.utils import file_md5
|
||||
from dvc.stage import Stage
|
||||
from dvc.output import CmdOutputNoCacheError, CmdOutputOutsideOfRepoError
|
||||
from dvc.output import CmdOutputDoesNotExistError
|
||||
from dvc.project import StageNotFoundError
|
||||
from dvc.command.gc import CmdGC
|
||||
|
||||
|
||||
@@ -2,7 +2,6 @@ import os
|
||||
|
||||
from dvc.main import main
|
||||
from dvc.stage import Stage
|
||||
from dvc.output import CmdOutputOutsideOfRepoError
|
||||
from dvc.project import StageNotFoundError
|
||||
from dvc.command.remove import CmdRemove
|
||||
|
||||
|
||||
Reference in New Issue
Block a user