dvc: split output and dependency classes

Preparation for the 'external deps' patches.

Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
Ruslan Kuprieiev
2018-05-23 00:01:04 +03:00
parent 4ab9b68fbd
commit 21bf52ee9d
8 changed files with 253 additions and 228 deletions
Binary file not shown.
+126
View File
@@ -0,0 +1,126 @@
import os
import schema
import posixpath
import ntpath
from dvc.exceptions import DvcException
from dvc.logger import Logger
from dvc.utils import remove
class DependencyError(DvcException):
def __init__(self, path, msg):
super(DependencyError, self).__init__('Dependency \'{}\' error: {}'.format(path, msg))
class DependencyOutsideOfRepoError(DependencyError):
def __init__(self, path):
super(DependencyOutsideOfRepoError, self).__init__(path, 'outside of repository')
class DependencyDoesNotExistError(DependencyError):
def __init__(self, path):
super(DependencyDoesNotExistError, self).__init__(path, 'does not exist')
class DependencyIsNotFileOrDirError(DependencyError):
def __init__(self, path):
super(DependencyIsNotFileOrDirError, self).__init__(path, 'not a file or directory')
class Dependency(object):
PARAM_PATH = 'path'
PARAM_MD5 = 'md5'
MD5_DIR_SUFFIX = '.dir'
OutsideOfRepoError = DependencyOutsideOfRepoError
DoesNotExistError = DependencyDoesNotExistError
IsNotFileOrDirError = DependencyIsNotFileOrDirError
SCHEMA = {
PARAM_PATH: str,
schema.Optional(PARAM_MD5): schema.Or(str, None),
}
def __init__(self, project, path, md5=None):
self.project = project
self.path = os.path.abspath(os.path.normpath(path))
if not self.path.startswith(self.project.root_dir):
raise self.OutsideOfRepoError(self.rel_path)
self.md5 = md5
@property
def rel_path(self):
return os.path.relpath(self.path)
def _changed_md5(self):
if not os.path.exists(self.path):
return True
return self.project.state.changed(self.path, self.md5)
@staticmethod
def _changed_msg(changed):
if changed:
return 'changed'
return "didn't change"
def changed(self):
ret = self._changed_md5()
msg = u'Dependency \'{}\' {}'.format(self.rel_path, self._changed_msg(ret))
self.project.logger.debug(msg)
return ret
def status(self):
if self.changed():
#FIXME better msgs
return {self.rel_path: 'changed'}
return {}
def save(self):
if not os.path.exists(self.path):
raise self.DoesNotExistError(self.rel_path)
if not os.path.isfile(self.path) and not os.path.isdir(self.path):
raise self.NotFileOrDirError(self.rel_path)
if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \
(os.path.isdir(self.path) and len(os.listdir(self.path)) == 0):
self.project.logger.warn("File/directory '{}' is empty.".format(self.rel_path))
self.md5 = self.project.state.update(self.path)
@staticmethod
def unixpath(path):
assert not ntpath.isabs(path)
assert not posixpath.isabs(path)
return path.replace('\\', '/')
def dumpd(self, cwd):
return {
self.PARAM_PATH: self.unixpath(os.path.relpath(self.path, cwd)),
self.PARAM_MD5: self.md5,
}
@classmethod
def loadd(cls, project, d, cwd=os.curdir):
relpath = os.path.normpath(Dependency.unixpath(d[Dependency.PARAM_PATH]))
path = os.path.join(cwd, relpath)
md5 = d.get(Dependency.PARAM_MD5, None)
return cls(project, path, md5=md5)
@classmethod
def loadd_from(cls, project, d_list, cwd=os.curdir):
return [cls.loadd(project, x, cwd=cwd) for x in d_list]
@classmethod
def loads(cls, project, s, cwd=os.curdir):
return cls(project, os.path.join(cwd, s), md5=None)
@classmethod
def loads_from(cls, project, s_list, cwd=os.curdir):
return [cls.loads(project, x, cwd=cwd) for x in s_list]
-217
View File
@@ -1,217 +0,0 @@
import os
import schema
import posixpath
import ntpath
from dvc.exceptions import DvcException
from dvc.logger import Logger
from dvc.utils import remove
class OutputError(DvcException):
pass
class CmdOutputError(DvcException):
def __init__(self, path, msg):
super(CmdOutputError, self).__init__('Output file \'{}\' error: {}'.format(path, msg))
class CmdOutputNoCacheError(CmdOutputError):
def __init__(self, path):
super(CmdOutputNoCacheError, self).__init__(path, 'no cache')
class CmdOutputOutsideOfRepoError(CmdOutputError):
def __init__(self, path):
super(CmdOutputOutsideOfRepoError, self).__init__(path, 'outside of repository')
class CmdOutputDoesNotExistError(CmdOutputError):
def __init__(self, path):
super(CmdOutputDoesNotExistError, self).__init__(path, 'does not exist')
class CmdOutputIsNotFileOrDirError(CmdOutputError):
def __init__(self, path):
super(CmdOutputIsNotFileOrDirError, self).__init__(path, 'not a file or directory')
class CmdOutputAlreadyTrackedError(CmdOutputError):
def __init__(self, path):
super(CmdOutputAlreadyTrackedError, self).__init__(path, 'already tracked by scm(e.g. git)')
class Dependency(object):
PARAM_PATH = 'path'
PARAM_MD5 = 'md5'
MD5_DIR_SUFFIX = '.dir'
SCHEMA = {
PARAM_PATH: str,
schema.Optional(PARAM_MD5): schema.Or(str, None),
}
def __init__(self, project, path, md5=None):
self.project = project
self.path = os.path.abspath(os.path.normpath(path))
if not self.path.startswith(self.project.root_dir):
raise CmdOutputOutsideOfRepoError(self.rel_path)
self.md5 = md5
@property
def rel_path(self):
return os.path.relpath(self.path)
def _changed_md5(self):
if not os.path.exists(self.path):
return True
return self.project.state.changed(self.path, self.md5)
@staticmethod
def _changed_msg(changed):
if changed:
return 'changed'
return "didn't change"
def changed(self):
ret = self._changed_md5()
msg = u'Dependency \'{}\' {}'.format(self.rel_path, self._changed_msg(ret))
self.project.logger.debug(msg)
return ret
def status(self):
if self.changed():
#FIXME better msgs
return {self.rel_path: 'changed'}
return {}
def save(self):
if not os.path.exists(self.path):
raise CmdOutputDoesNotExistError(self.rel_path)
if not os.path.isfile(self.path) and not os.path.isdir(self.path):
raise CmdOutputIsNotFileOrDirError(self.rel_path)
if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \
(os.path.isdir(self.path) and len(os.listdir(self.path)) == 0):
self.project.logger.warn("File/directory '{}' is empty.".format(self.rel_path))
self.md5 = self.project.state.update(self.path)
@staticmethod
def unixpath(path):
assert not ntpath.isabs(path)
assert not posixpath.isabs(path)
return path.replace('\\', '/')
def dumpd(self, cwd):
return {
Output.PARAM_PATH: self.unixpath(os.path.relpath(self.path, cwd)),
Output.PARAM_MD5: self.md5,
}
@classmethod
def loadd(cls, project, d, cwd=os.curdir):
relpath = os.path.normpath(Output.unixpath(d[Output.PARAM_PATH]))
path = os.path.join(cwd, relpath)
md5 = d.get(Output.PARAM_MD5, None)
return cls(project, path, md5=md5)
@classmethod
def loadd_from(cls, project, d_list, cwd=os.curdir):
return [cls.loadd(project, x, cwd=cwd) for x in d_list]
@classmethod
def loads(cls, project, s, cwd=os.curdir):
return cls(project, os.path.join(cwd, s), md5=None)
@classmethod
def loads_from(cls, project, s_list, cwd=os.curdir):
return [cls.loads(project, x, cwd=cwd) for x in s_list]
class Output(Dependency):
PARAM_CACHE = 'cache'
SCHEMA = Dependency.SCHEMA
SCHEMA[schema.Optional(PARAM_CACHE)] = bool
def __init__(self, project, path, md5=None, use_cache=True):
super(Output, self).__init__(project, path, md5=md5)
self.use_cache = use_cache
@property
def cache(self):
if not self.use_cache:
return None
return self.project.cache.get(self.md5)
def dumpd(self, cwd):
ret = super(Output, self).dumpd(cwd)
ret[Output.PARAM_CACHE] = self.use_cache
return ret
@classmethod
def loadd(cls, project, d, cwd=os.curdir):
ret = super(Output, cls).loadd(project, d, cwd=cwd)
ret.use_cache = d.get(Output.PARAM_CACHE, True)
return ret
@classmethod
def loads(cls, project, s, use_cache=True, cwd=os.curdir):
ret = super(Output, cls).loads(project, s, cwd=cwd)
ret.use_cache = use_cache
return ret
@classmethod
def loads_from(cls, project, s_list, use_cache=False, cwd=os.curdir):
return [cls.loads(project, x, use_cache=use_cache, cwd=cwd) for x in s_list]
def changed(self):
if super(Output, self).changed():
return True
if self.use_cache and self.project.cache.changed(self.md5):
return True
return False
def checkout(self):
if not self.use_cache:
return
msg = u'Checking out \'{}\' with cache \'{}\''
self.project.logger.debug(msg.format(self.rel_path, self.md5))
if not self.changed():
msg = u'Data file \'{}\' with cache \'{}\' didn\'t change, skipping checkout.'
self.project.logger.debug(msg.format(self.rel_path, self.md5))
return
self.project.cache.checkout(self.path, self.md5)
def save(self):
super(Output, self).save()
if not self.use_cache:
return
self.project.logger.debug(u'Saving \'{}\' to \'{}\''.format(self.rel_path, self.md5))
if self.project.scm.is_tracked(self.path):
raise CmdOutputAlreadyTrackedError(self.rel_path)
if not self.changed():
return
self.project.cache.save(self.path)
def remove(self):
remove(self.path)
+119
View File
@@ -0,0 +1,119 @@
import os
import schema
import posixpath
import ntpath
from dvc.dependency import Dependency
from dvc.exceptions import DvcException
from dvc.logger import Logger
from dvc.utils import remove
class OutputError(DvcException):
def __init__(self, path, msg):
super(OutputError, self).__init__('Output \'{}\' error: {}'.format(path, msg))
class OutputOutsideOfRepoError(OutputError):
def __init__(self, path):
super(OutputOutsideOfRepoError, self).__init__(path, 'outside of repository')
class OutputDoesNotExistError(OutputError):
def __init__(self, path):
super(OutputDoesNotExistError, self).__init__(path, 'does not exist')
class OutputIsNotFileOrDirError(OutputError):
def __init__(self, path):
super(OutputIsNotFileOrDirError, self).__init__(path, 'not a file or directory')
class OutputAlreadyTrackedError(OutputError):
def __init__(self, path):
super(OutputAlreadyTrackedError, self).__init__(path, 'already tracked by scm(e.g. git)')
class Output(Dependency):
PARAM_CACHE = 'cache'
OutsideOfRepoError = OutputOutsideOfRepoError
DoesNotExistError = OutputDoesNotExistError
IsNotFileOrDirError = OutputIsNotFileOrDirError
SCHEMA = Dependency.SCHEMA
SCHEMA[schema.Optional(PARAM_CACHE)] = bool
def __init__(self, project, path, md5=None, use_cache=True):
super(Output, self).__init__(project, path, md5=md5)
self.use_cache = use_cache
@property
def cache(self):
if not self.use_cache:
return None
return self.project.cache.get(self.md5)
def dumpd(self, cwd):
ret = super(Output, self).dumpd(cwd)
ret[Output.PARAM_CACHE] = self.use_cache
return ret
@classmethod
def loadd(cls, project, d, cwd=os.curdir):
ret = super(Output, cls).loadd(project, d, cwd=cwd)
ret.use_cache = d.get(Output.PARAM_CACHE, True)
return ret
@classmethod
def loads(cls, project, s, use_cache=True, cwd=os.curdir):
ret = super(Output, cls).loads(project, s, cwd=cwd)
ret.use_cache = use_cache
return ret
@classmethod
def loads_from(cls, project, s_list, use_cache=False, cwd=os.curdir):
return [cls.loads(project, x, use_cache=use_cache, cwd=cwd) for x in s_list]
def changed(self):
if super(Output, self).changed():
return True
if self.use_cache and self.project.cache.changed(self.md5):
return True
return False
def checkout(self):
if not self.use_cache:
return
msg = u'Checking out \'{}\' with cache \'{}\''
self.project.logger.debug(msg.format(self.rel_path, self.md5))
if not self.changed():
msg = u'Data file \'{}\' with cache \'{}\' didn\'t change, skipping checkout.'
self.project.logger.debug(msg.format(self.rel_path, self.md5))
return
self.project.cache.checkout(self.path, self.md5)
def save(self):
super(Output, self).save()
if not self.use_cache:
return
self.project.logger.debug(u'Saving \'{}\' to \'{}\''.format(self.rel_path, self.md5))
if self.project.scm.is_tracked(self.path):
raise OutputAlreadyTrackedError(self.rel_path)
if not self.changed():
return
self.project.cache.save(self.path)
def remove(self):
remove(self.path)
+3 -2
View File
@@ -5,7 +5,8 @@ import subprocess
import schema
from dvc.exceptions import DvcException
from dvc.output import Output, Dependency, OutputError
from dvc.output import Output
from dvc.dependency import Dependency
from dvc.logger import Logger
from dvc.utils import dict_md5
@@ -21,7 +22,7 @@ class StageFileFormatError(DvcException):
super(StageFileFormatError, self).__init__('Stage file format error')
class MissingDataSource(OutputError):
class MissingDataSource(DvcException):
def __init__(self, missing_files):
assert len(missing_files) > 0
+5 -6
View File
@@ -6,9 +6,8 @@ import filecmp
from dvc.main import main
from dvc.utils import file_md5
from dvc.stage import Stage
from dvc.output import CmdOutputNoCacheError, CmdOutputOutsideOfRepoError
from dvc.output import CmdOutputDoesNotExistError, CmdOutputIsNotFileOrDirError
from dvc.output import CmdOutputAlreadyTrackedError
from dvc.output import OutputOutsideOfRepoError, OutputAlreadyTrackedError
from dvc.output import OutputDoesNotExistError, OutputIsNotFileOrDirError
from dvc.project import StageNotFoundError
from dvc.command.add import CmdAdd
@@ -31,13 +30,13 @@ class TestAdd(TestDvc):
class TestAddNonExistentFile(TestDvc):
def test(self):
with self.assertRaises(CmdOutputDoesNotExistError) as cx:
with self.assertRaises(OutputDoesNotExistError) as cx:
self.dvc.add('non_existent_file')
class TestAddFileOutsideOfRepo(TestDvc):
def test(self):
with self.assertRaises(CmdOutputOutsideOfRepoError) as cx:
with self.assertRaises(OutputOutsideOfRepoError) as cx:
self.dvc.add(os.path.join(os.path.dirname(self.dvc.root_dir), self.FOO))
@@ -63,7 +62,7 @@ class TestAddTrackedFile(TestDvc):
self.dvc.scm.add([fname])
self.dvc.scm.commit('add {}'.format(fname))
with self.assertRaises(CmdOutputAlreadyTrackedError) as cx:
with self.assertRaises(OutputAlreadyTrackedError) as cx:
self.dvc.add(fname)
-2
View File
@@ -5,8 +5,6 @@ import filecmp
from dvc.main import main
from dvc.utils import file_md5
from dvc.stage import Stage
from dvc.output import CmdOutputNoCacheError, CmdOutputOutsideOfRepoError
from dvc.output import CmdOutputDoesNotExistError
from dvc.project import StageNotFoundError
from dvc.command.gc import CmdGC
-1
View File
@@ -2,7 +2,6 @@ import os
from dvc.main import main
from dvc.stage import Stage
from dvc.output import CmdOutputOutsideOfRepoError
from dvc.project import StageNotFoundError
from dvc.command.remove import CmdRemove