diff --git a/dvc/.project.py.swo b/dvc/.project.py.swo new file mode 100644 index 000000000..ea7e9e282 Binary files /dev/null and b/dvc/.project.py.swo differ diff --git a/dvc/dependency/__init__.py b/dvc/dependency/__init__.py new file mode 100644 index 000000000..903f50e76 --- /dev/null +++ b/dvc/dependency/__init__.py @@ -0,0 +1,126 @@ +import os +import schema +import posixpath +import ntpath + +from dvc.exceptions import DvcException +from dvc.logger import Logger +from dvc.utils import remove + + +class DependencyError(DvcException): + def __init__(self, path, msg): + super(DependencyError, self).__init__('Dependency \'{}\' error: {}'.format(path, msg)) + + +class DependencyOutsideOfRepoError(DependencyError): + def __init__(self, path): + super(DependencyOutsideOfRepoError, self).__init__(path, 'outside of repository') + + +class DependencyDoesNotExistError(DependencyError): + def __init__(self, path): + super(DependencyDoesNotExistError, self).__init__(path, 'does not exist') + + +class DependencyIsNotFileOrDirError(DependencyError): + def __init__(self, path): + super(DependencyIsNotFileOrDirError, self).__init__(path, 'not a file or directory') + + +class Dependency(object): + PARAM_PATH = 'path' + PARAM_MD5 = 'md5' + MD5_DIR_SUFFIX = '.dir' + + OutsideOfRepoError = DependencyOutsideOfRepoError + DoesNotExistError = DependencyDoesNotExistError + IsNotFileOrDirError = DependencyIsNotFileOrDirError + + SCHEMA = { + PARAM_PATH: str, + schema.Optional(PARAM_MD5): schema.Or(str, None), + } + + def __init__(self, project, path, md5=None): + self.project = project + self.path = os.path.abspath(os.path.normpath(path)) + + if not self.path.startswith(self.project.root_dir): + raise self.OutsideOfRepoError(self.rel_path) + + self.md5 = md5 + + @property + def rel_path(self): + return os.path.relpath(self.path) + + def _changed_md5(self): + if not os.path.exists(self.path): + return True + + return self.project.state.changed(self.path, self.md5) + + @staticmethod + def _changed_msg(changed): + if changed: + return 'changed' + return "didn't change" + + def changed(self): + ret = self._changed_md5() + + msg = u'Dependency \'{}\' {}'.format(self.rel_path, self._changed_msg(ret)) + self.project.logger.debug(msg) + + return ret + + def status(self): + if self.changed(): + #FIXME better msgs + return {self.rel_path: 'changed'} + return {} + + def save(self): + if not os.path.exists(self.path): + raise self.DoesNotExistError(self.rel_path) + + if not os.path.isfile(self.path) and not os.path.isdir(self.path): + raise self.NotFileOrDirError(self.rel_path) + + if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \ + (os.path.isdir(self.path) and len(os.listdir(self.path)) == 0): + self.project.logger.warn("File/directory '{}' is empty.".format(self.rel_path)) + + self.md5 = self.project.state.update(self.path) + + @staticmethod + def unixpath(path): + assert not ntpath.isabs(path) + assert not posixpath.isabs(path) + return path.replace('\\', '/') + + def dumpd(self, cwd): + return { + self.PARAM_PATH: self.unixpath(os.path.relpath(self.path, cwd)), + self.PARAM_MD5: self.md5, + } + + @classmethod + def loadd(cls, project, d, cwd=os.curdir): + relpath = os.path.normpath(Dependency.unixpath(d[Dependency.PARAM_PATH])) + path = os.path.join(cwd, relpath) + md5 = d.get(Dependency.PARAM_MD5, None) + return cls(project, path, md5=md5) + + @classmethod + def loadd_from(cls, project, d_list, cwd=os.curdir): + return [cls.loadd(project, x, cwd=cwd) for x in d_list] + + @classmethod + def loads(cls, project, s, cwd=os.curdir): + return cls(project, os.path.join(cwd, s), md5=None) + + @classmethod + def loads_from(cls, project, s_list, cwd=os.curdir): + return [cls.loads(project, x, cwd=cwd) for x in s_list] diff --git a/dvc/output.py b/dvc/output.py deleted file mode 100644 index 3448e1fe2..000000000 --- a/dvc/output.py +++ /dev/null @@ -1,217 +0,0 @@ -import os -import schema -import posixpath -import ntpath - -from dvc.exceptions import DvcException -from dvc.logger import Logger -from dvc.utils import remove - - -class OutputError(DvcException): - pass - - -class CmdOutputError(DvcException): - def __init__(self, path, msg): - super(CmdOutputError, self).__init__('Output file \'{}\' error: {}'.format(path, msg)) - - -class CmdOutputNoCacheError(CmdOutputError): - def __init__(self, path): - super(CmdOutputNoCacheError, self).__init__(path, 'no cache') - - -class CmdOutputOutsideOfRepoError(CmdOutputError): - def __init__(self, path): - super(CmdOutputOutsideOfRepoError, self).__init__(path, 'outside of repository') - - -class CmdOutputDoesNotExistError(CmdOutputError): - def __init__(self, path): - super(CmdOutputDoesNotExistError, self).__init__(path, 'does not exist') - - -class CmdOutputIsNotFileOrDirError(CmdOutputError): - def __init__(self, path): - super(CmdOutputIsNotFileOrDirError, self).__init__(path, 'not a file or directory') - - -class CmdOutputAlreadyTrackedError(CmdOutputError): - def __init__(self, path): - super(CmdOutputAlreadyTrackedError, self).__init__(path, 'already tracked by scm(e.g. git)') - - -class Dependency(object): - PARAM_PATH = 'path' - PARAM_MD5 = 'md5' - MD5_DIR_SUFFIX = '.dir' - - SCHEMA = { - PARAM_PATH: str, - schema.Optional(PARAM_MD5): schema.Or(str, None), - } - - def __init__(self, project, path, md5=None): - self.project = project - self.path = os.path.abspath(os.path.normpath(path)) - - if not self.path.startswith(self.project.root_dir): - raise CmdOutputOutsideOfRepoError(self.rel_path) - - self.md5 = md5 - - @property - def rel_path(self): - return os.path.relpath(self.path) - - def _changed_md5(self): - if not os.path.exists(self.path): - return True - - return self.project.state.changed(self.path, self.md5) - - @staticmethod - def _changed_msg(changed): - if changed: - return 'changed' - return "didn't change" - - def changed(self): - ret = self._changed_md5() - - msg = u'Dependency \'{}\' {}'.format(self.rel_path, self._changed_msg(ret)) - self.project.logger.debug(msg) - - return ret - - def status(self): - if self.changed(): - #FIXME better msgs - return {self.rel_path: 'changed'} - return {} - - def save(self): - if not os.path.exists(self.path): - raise CmdOutputDoesNotExistError(self.rel_path) - - if not os.path.isfile(self.path) and not os.path.isdir(self.path): - raise CmdOutputIsNotFileOrDirError(self.rel_path) - - if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \ - (os.path.isdir(self.path) and len(os.listdir(self.path)) == 0): - self.project.logger.warn("File/directory '{}' is empty.".format(self.rel_path)) - - self.md5 = self.project.state.update(self.path) - - @staticmethod - def unixpath(path): - assert not ntpath.isabs(path) - assert not posixpath.isabs(path) - return path.replace('\\', '/') - - def dumpd(self, cwd): - return { - Output.PARAM_PATH: self.unixpath(os.path.relpath(self.path, cwd)), - Output.PARAM_MD5: self.md5, - } - - @classmethod - def loadd(cls, project, d, cwd=os.curdir): - relpath = os.path.normpath(Output.unixpath(d[Output.PARAM_PATH])) - path = os.path.join(cwd, relpath) - md5 = d.get(Output.PARAM_MD5, None) - return cls(project, path, md5=md5) - - @classmethod - def loadd_from(cls, project, d_list, cwd=os.curdir): - return [cls.loadd(project, x, cwd=cwd) for x in d_list] - - @classmethod - def loads(cls, project, s, cwd=os.curdir): - return cls(project, os.path.join(cwd, s), md5=None) - - @classmethod - def loads_from(cls, project, s_list, cwd=os.curdir): - return [cls.loads(project, x, cwd=cwd) for x in s_list] - - -class Output(Dependency): - PARAM_CACHE = 'cache' - - SCHEMA = Dependency.SCHEMA - SCHEMA[schema.Optional(PARAM_CACHE)] = bool - - def __init__(self, project, path, md5=None, use_cache=True): - super(Output, self).__init__(project, path, md5=md5) - self.use_cache = use_cache - - @property - def cache(self): - if not self.use_cache: - return None - - return self.project.cache.get(self.md5) - - def dumpd(self, cwd): - ret = super(Output, self).dumpd(cwd) - ret[Output.PARAM_CACHE] = self.use_cache - return ret - - @classmethod - def loadd(cls, project, d, cwd=os.curdir): - ret = super(Output, cls).loadd(project, d, cwd=cwd) - ret.use_cache = d.get(Output.PARAM_CACHE, True) - return ret - - @classmethod - def loads(cls, project, s, use_cache=True, cwd=os.curdir): - ret = super(Output, cls).loads(project, s, cwd=cwd) - ret.use_cache = use_cache - return ret - - @classmethod - def loads_from(cls, project, s_list, use_cache=False, cwd=os.curdir): - return [cls.loads(project, x, use_cache=use_cache, cwd=cwd) for x in s_list] - - def changed(self): - if super(Output, self).changed(): - return True - - if self.use_cache and self.project.cache.changed(self.md5): - return True - - return False - - def checkout(self): - if not self.use_cache: - return - - msg = u'Checking out \'{}\' with cache \'{}\'' - self.project.logger.debug(msg.format(self.rel_path, self.md5)) - - if not self.changed(): - msg = u'Data file \'{}\' with cache \'{}\' didn\'t change, skipping checkout.' - self.project.logger.debug(msg.format(self.rel_path, self.md5)) - return - - self.project.cache.checkout(self.path, self.md5) - - def save(self): - super(Output, self).save() - - if not self.use_cache: - return - - self.project.logger.debug(u'Saving \'{}\' to \'{}\''.format(self.rel_path, self.md5)) - - if self.project.scm.is_tracked(self.path): - raise CmdOutputAlreadyTrackedError(self.rel_path) - - if not self.changed(): - return - - self.project.cache.save(self.path) - - def remove(self): - remove(self.path) diff --git a/dvc/output/__init__.py b/dvc/output/__init__.py new file mode 100644 index 000000000..ec2b42643 --- /dev/null +++ b/dvc/output/__init__.py @@ -0,0 +1,119 @@ +import os +import schema +import posixpath +import ntpath + +from dvc.dependency import Dependency +from dvc.exceptions import DvcException +from dvc.logger import Logger +from dvc.utils import remove + + +class OutputError(DvcException): + def __init__(self, path, msg): + super(OutputError, self).__init__('Output \'{}\' error: {}'.format(path, msg)) + + +class OutputOutsideOfRepoError(OutputError): + def __init__(self, path): + super(OutputOutsideOfRepoError, self).__init__(path, 'outside of repository') + + +class OutputDoesNotExistError(OutputError): + def __init__(self, path): + super(OutputDoesNotExistError, self).__init__(path, 'does not exist') + + +class OutputIsNotFileOrDirError(OutputError): + def __init__(self, path): + super(OutputIsNotFileOrDirError, self).__init__(path, 'not a file or directory') + + +class OutputAlreadyTrackedError(OutputError): + def __init__(self, path): + super(OutputAlreadyTrackedError, self).__init__(path, 'already tracked by scm(e.g. git)') + + +class Output(Dependency): + PARAM_CACHE = 'cache' + + OutsideOfRepoError = OutputOutsideOfRepoError + DoesNotExistError = OutputDoesNotExistError + IsNotFileOrDirError = OutputIsNotFileOrDirError + + SCHEMA = Dependency.SCHEMA + SCHEMA[schema.Optional(PARAM_CACHE)] = bool + + def __init__(self, project, path, md5=None, use_cache=True): + super(Output, self).__init__(project, path, md5=md5) + self.use_cache = use_cache + + @property + def cache(self): + if not self.use_cache: + return None + + return self.project.cache.get(self.md5) + + def dumpd(self, cwd): + ret = super(Output, self).dumpd(cwd) + ret[Output.PARAM_CACHE] = self.use_cache + return ret + + @classmethod + def loadd(cls, project, d, cwd=os.curdir): + ret = super(Output, cls).loadd(project, d, cwd=cwd) + ret.use_cache = d.get(Output.PARAM_CACHE, True) + return ret + + @classmethod + def loads(cls, project, s, use_cache=True, cwd=os.curdir): + ret = super(Output, cls).loads(project, s, cwd=cwd) + ret.use_cache = use_cache + return ret + + @classmethod + def loads_from(cls, project, s_list, use_cache=False, cwd=os.curdir): + return [cls.loads(project, x, use_cache=use_cache, cwd=cwd) for x in s_list] + + def changed(self): + if super(Output, self).changed(): + return True + + if self.use_cache and self.project.cache.changed(self.md5): + return True + + return False + + def checkout(self): + if not self.use_cache: + return + + msg = u'Checking out \'{}\' with cache \'{}\'' + self.project.logger.debug(msg.format(self.rel_path, self.md5)) + + if not self.changed(): + msg = u'Data file \'{}\' with cache \'{}\' didn\'t change, skipping checkout.' + self.project.logger.debug(msg.format(self.rel_path, self.md5)) + return + + self.project.cache.checkout(self.path, self.md5) + + def save(self): + super(Output, self).save() + + if not self.use_cache: + return + + self.project.logger.debug(u'Saving \'{}\' to \'{}\''.format(self.rel_path, self.md5)) + + if self.project.scm.is_tracked(self.path): + raise OutputAlreadyTrackedError(self.rel_path) + + if not self.changed(): + return + + self.project.cache.save(self.path) + + def remove(self): + remove(self.path) diff --git a/dvc/stage.py b/dvc/stage.py index bef7f9c78..f05447ca9 100644 --- a/dvc/stage.py +++ b/dvc/stage.py @@ -5,7 +5,8 @@ import subprocess import schema from dvc.exceptions import DvcException -from dvc.output import Output, Dependency, OutputError +from dvc.output import Output +from dvc.dependency import Dependency from dvc.logger import Logger from dvc.utils import dict_md5 @@ -21,7 +22,7 @@ class StageFileFormatError(DvcException): super(StageFileFormatError, self).__init__('Stage file format error') -class MissingDataSource(OutputError): +class MissingDataSource(DvcException): def __init__(self, missing_files): assert len(missing_files) > 0 diff --git a/tests/test_add.py b/tests/test_add.py index ccaee916f..3802857c2 100644 --- a/tests/test_add.py +++ b/tests/test_add.py @@ -6,9 +6,8 @@ import filecmp from dvc.main import main from dvc.utils import file_md5 from dvc.stage import Stage -from dvc.output import CmdOutputNoCacheError, CmdOutputOutsideOfRepoError -from dvc.output import CmdOutputDoesNotExistError, CmdOutputIsNotFileOrDirError -from dvc.output import CmdOutputAlreadyTrackedError +from dvc.output import OutputOutsideOfRepoError, OutputAlreadyTrackedError +from dvc.output import OutputDoesNotExistError, OutputIsNotFileOrDirError from dvc.project import StageNotFoundError from dvc.command.add import CmdAdd @@ -31,13 +30,13 @@ class TestAdd(TestDvc): class TestAddNonExistentFile(TestDvc): def test(self): - with self.assertRaises(CmdOutputDoesNotExistError) as cx: + with self.assertRaises(OutputDoesNotExistError) as cx: self.dvc.add('non_existent_file') class TestAddFileOutsideOfRepo(TestDvc): def test(self): - with self.assertRaises(CmdOutputOutsideOfRepoError) as cx: + with self.assertRaises(OutputOutsideOfRepoError) as cx: self.dvc.add(os.path.join(os.path.dirname(self.dvc.root_dir), self.FOO)) @@ -63,7 +62,7 @@ class TestAddTrackedFile(TestDvc): self.dvc.scm.add([fname]) self.dvc.scm.commit('add {}'.format(fname)) - with self.assertRaises(CmdOutputAlreadyTrackedError) as cx: + with self.assertRaises(OutputAlreadyTrackedError) as cx: self.dvc.add(fname) diff --git a/tests/test_gc.py b/tests/test_gc.py index 2bd29ea58..e45e4fa6e 100644 --- a/tests/test_gc.py +++ b/tests/test_gc.py @@ -5,8 +5,6 @@ import filecmp from dvc.main import main from dvc.utils import file_md5 from dvc.stage import Stage -from dvc.output import CmdOutputNoCacheError, CmdOutputOutsideOfRepoError -from dvc.output import CmdOutputDoesNotExistError from dvc.project import StageNotFoundError from dvc.command.gc import CmdGC diff --git a/tests/test_remove.py b/tests/test_remove.py index 9934ec174..b6503feab 100644 --- a/tests/test_remove.py +++ b/tests/test_remove.py @@ -2,7 +2,6 @@ import os from dvc.main import main from dvc.stage import Stage -from dvc.output import CmdOutputOutsideOfRepoError from dvc.project import StageNotFoundError from dvc.command.remove import CmdRemove