From 21bf52ee9df46a20af1080d197024e9861664927 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Tue, 22 May 2018 15:10:01 +0300 Subject: [PATCH] dvc: split output and dependency classes Preparation for the 'external deps' patches. Signed-off-by: Ruslan Kuprieiev --- dvc/.project.py.swo | Bin 0 -> 20480 bytes dvc/dependency/__init__.py | 126 +++++++++++++++++++++ dvc/output.py | 217 ------------------------------------- dvc/output/__init__.py | 119 ++++++++++++++++++++ dvc/stage.py | 5 +- tests/test_add.py | 11 +- tests/test_gc.py | 2 - tests/test_remove.py | 1 - 8 files changed, 253 insertions(+), 228 deletions(-) create mode 100644 dvc/.project.py.swo create mode 100644 dvc/dependency/__init__.py delete mode 100644 dvc/output.py create mode 100644 dvc/output/__init__.py diff --git a/dvc/.project.py.swo b/dvc/.project.py.swo new file mode 100644 index 0000000000000000000000000000000000000000..ea7e9e282aedfd2b1d921fb2e65e0ef8a245d575 GIT binary patch literal 20480 zcmeHNTZ|k>6)l@M4haDS;sX;oxl8MpI5!Wc$ic4y5ho3-(J9V5qTdb?}J?Vafw zx_kDK$O#|;KM2fMh>!@6zlbej8gb#B$IuIk>n^Yrq4{C)m=8ICtH_U$M3p8e+5V9&P>GA25S98mf-j-E*6 zmh{idzPM`6LwO;yuqzIAlj2ak6)6ldc|k;lKiIvht`j;290RX{fnm`(c=*=!d+xsL zPAcX0xp(q6ziItd6zEhP1C9a5fMdWh;23ZWI0hU8j)DI-28zKx_6UsLYYe|&u6xF= zH<;&J&2@83e#IQ0H0j%8@>h(!Z_@udCV!22K47lv#sF)FNp=^k5)fv*9N01pG7 z01gAU0k6ECu@`}7fhT}30uj&#-VMA9xE;6w_zN}>ehB;k_&#t6$biGZUSJQvfZx21 zvCBXo@PUIs6L<^2fGf8!_6OiG;0$m-&;)J-p1+x~Rp1w|Mf-refj{4b`hf2NUk4rs zJ_CFjkidt4dw`pO{lGu>Gxk&9C%{v{mw`*br+{@}8MqC2@kYj80DcC11^6KFHsAoj zfGhjZ7T{^%QD6%g0>^>7fS0hj@^|3(!0&)(fbRmI2Udai058KozXdJ>p94M%kk38_ zoB-YryaRYU@G>@Yo&|mmJP8beqrfe|&A^O3AB4quKTfhV>-O3;fv;s8rG(7W@vrMrr=qsiVi*!P4?dJ6Ky^T3S9+>QJt#rj0NcBUPm` z>rd1biw+Ot*vmz))8yKUBdjbk6U8t~OY1YyQzg);$w|l0q$~ooN}Eo_6*9ozD1pkT zi>2E}tyHC$M}0p@(av7atCVdDWf@UXI#C#%7nUNPbhHU7)mLY-q_SS+REonv92O$; z%{AyGJ#nsVWtA_IX(#FW`V#bH6!ui!=9IjUZu)eZnFEb0gipojJpV~I9d%PI{91N}|3x~irWksVo>VP++X;NM(TZ5!PYE+QNM$AvLZIGsrOF?TR|%1%L7MLa?}e zva-Qusj1p1Xhl`#xw$z;gxKR=v8yOM!cP*1IgICGClZ4qk*UvYHZ81$91<9dxQa1p zBx>1Jn9IlVTWgN+69`~nnNtVD-*xjjCmDx>@g%d|s-&Y?w_XgnI6)Agj%2k_|1?e8 zQW|qhY4TkakgPDeUC}W0G3+VH(Lv)EC8h8CPRrB~eAG-L!{JfnI$U`(~2I_57)jy>{Be|LguP|1(VZuajaVu)u@H?eV7+GDh zDvKpKOyil2nrMnr<76qv6c|gyd71>pPLda8W7J#atTvcePHpz271y%6oSLWkU}jIC zY1~p(mM~dWCzE}e5bWD1Px>^O+i1&5dA^$$qHo1(>Zy`O6m5OH*M@<*wfV|OT+6E7 zn@u$;fK)OXpGJbIzNuabDk;6YTJZ|*D@ji8QdR_p3sK3;XtouqAzewK*{y4KYntU; zp4+19J*+Cj%oL^uJzX$n9*JAc4lROsm~-{SwEBfyrqn%xY}?i`2(w&R?Nk)oGTYoC zy>!Q>l9s)vrrY}{-?FzdXSDvOHT8FBO^x;c0kb&#HP-Y`0}ldcfDZsKVeS4S;1cj9 z;9=k*&;{0j1HiwqZhsDV6nGF=1bpBi@K#_y@F&#ujH&Y}s@ENk0mp!2z%k$$a11yG z90QI4$ADvC3Ildu#eU}*7J0cVK{9+&%K05Fu9j>*J(XhYG3#X7&s5tgDpT%Z2~4VV zD0O^O(4SgptASF~z6os>s605C+O>X}YNa~a4+jI0#@?Fx2-cIK{br?B`qd7C)uJnw zp+%Ao1*<30J+{GVKS7t3uY4pT{bOZSBYC15F^}uJBSB zV{0}RgMjWP3Dj9h=Gjcx3?ZUg^sMeCwQSZJSN5Y}sinSznhm25qJ)`3_J;Z0igYW`+*j)5BR0kH@fcP7;p?Y1{?#90mp!2z%k$$ xa11yG90QJl>tKKuuXMQok-6F9VTuo&^zK!3UH>Di|Hnv0|IymRpD7+={{}s)H}wDj literal 0 HcmV?d00001 diff --git a/dvc/dependency/__init__.py b/dvc/dependency/__init__.py new file mode 100644 index 000000000..903f50e76 --- /dev/null +++ b/dvc/dependency/__init__.py @@ -0,0 +1,126 @@ +import os +import schema +import posixpath +import ntpath + +from dvc.exceptions import DvcException +from dvc.logger import Logger +from dvc.utils import remove + + +class DependencyError(DvcException): + def __init__(self, path, msg): + super(DependencyError, self).__init__('Dependency \'{}\' error: {}'.format(path, msg)) + + +class DependencyOutsideOfRepoError(DependencyError): + def __init__(self, path): + super(DependencyOutsideOfRepoError, self).__init__(path, 'outside of repository') + + +class DependencyDoesNotExistError(DependencyError): + def __init__(self, path): + super(DependencyDoesNotExistError, self).__init__(path, 'does not exist') + + +class DependencyIsNotFileOrDirError(DependencyError): + def __init__(self, path): + super(DependencyIsNotFileOrDirError, self).__init__(path, 'not a file or directory') + + +class Dependency(object): + PARAM_PATH = 'path' + PARAM_MD5 = 'md5' + MD5_DIR_SUFFIX = '.dir' + + OutsideOfRepoError = DependencyOutsideOfRepoError + DoesNotExistError = DependencyDoesNotExistError + IsNotFileOrDirError = DependencyIsNotFileOrDirError + + SCHEMA = { + PARAM_PATH: str, + schema.Optional(PARAM_MD5): schema.Or(str, None), + } + + def __init__(self, project, path, md5=None): + self.project = project + self.path = os.path.abspath(os.path.normpath(path)) + + if not self.path.startswith(self.project.root_dir): + raise self.OutsideOfRepoError(self.rel_path) + + self.md5 = md5 + + @property + def rel_path(self): + return os.path.relpath(self.path) + + def _changed_md5(self): + if not os.path.exists(self.path): + return True + + return self.project.state.changed(self.path, self.md5) + + @staticmethod + def _changed_msg(changed): + if changed: + return 'changed' + return "didn't change" + + def changed(self): + ret = self._changed_md5() + + msg = u'Dependency \'{}\' {}'.format(self.rel_path, self._changed_msg(ret)) + self.project.logger.debug(msg) + + return ret + + def status(self): + if self.changed(): + #FIXME better msgs + return {self.rel_path: 'changed'} + return {} + + def save(self): + if not os.path.exists(self.path): + raise self.DoesNotExistError(self.rel_path) + + if not os.path.isfile(self.path) and not os.path.isdir(self.path): + raise self.NotFileOrDirError(self.rel_path) + + if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \ + (os.path.isdir(self.path) and len(os.listdir(self.path)) == 0): + self.project.logger.warn("File/directory '{}' is empty.".format(self.rel_path)) + + self.md5 = self.project.state.update(self.path) + + @staticmethod + def unixpath(path): + assert not ntpath.isabs(path) + assert not posixpath.isabs(path) + return path.replace('\\', '/') + + def dumpd(self, cwd): + return { + self.PARAM_PATH: self.unixpath(os.path.relpath(self.path, cwd)), + self.PARAM_MD5: self.md5, + } + + @classmethod + def loadd(cls, project, d, cwd=os.curdir): + relpath = os.path.normpath(Dependency.unixpath(d[Dependency.PARAM_PATH])) + path = os.path.join(cwd, relpath) + md5 = d.get(Dependency.PARAM_MD5, None) + return cls(project, path, md5=md5) + + @classmethod + def loadd_from(cls, project, d_list, cwd=os.curdir): + return [cls.loadd(project, x, cwd=cwd) for x in d_list] + + @classmethod + def loads(cls, project, s, cwd=os.curdir): + return cls(project, os.path.join(cwd, s), md5=None) + + @classmethod + def loads_from(cls, project, s_list, cwd=os.curdir): + return [cls.loads(project, x, cwd=cwd) for x in s_list] diff --git a/dvc/output.py b/dvc/output.py deleted file mode 100644 index 3448e1fe2..000000000 --- a/dvc/output.py +++ /dev/null @@ -1,217 +0,0 @@ -import os -import schema -import posixpath -import ntpath - -from dvc.exceptions import DvcException -from dvc.logger import Logger -from dvc.utils import remove - - -class OutputError(DvcException): - pass - - -class CmdOutputError(DvcException): - def __init__(self, path, msg): - super(CmdOutputError, self).__init__('Output file \'{}\' error: {}'.format(path, msg)) - - -class CmdOutputNoCacheError(CmdOutputError): - def __init__(self, path): - super(CmdOutputNoCacheError, self).__init__(path, 'no cache') - - -class CmdOutputOutsideOfRepoError(CmdOutputError): - def __init__(self, path): - super(CmdOutputOutsideOfRepoError, self).__init__(path, 'outside of repository') - - -class CmdOutputDoesNotExistError(CmdOutputError): - def __init__(self, path): - super(CmdOutputDoesNotExistError, self).__init__(path, 'does not exist') - - -class CmdOutputIsNotFileOrDirError(CmdOutputError): - def __init__(self, path): - super(CmdOutputIsNotFileOrDirError, self).__init__(path, 'not a file or directory') - - -class CmdOutputAlreadyTrackedError(CmdOutputError): - def __init__(self, path): - super(CmdOutputAlreadyTrackedError, self).__init__(path, 'already tracked by scm(e.g. git)') - - -class Dependency(object): - PARAM_PATH = 'path' - PARAM_MD5 = 'md5' - MD5_DIR_SUFFIX = '.dir' - - SCHEMA = { - PARAM_PATH: str, - schema.Optional(PARAM_MD5): schema.Or(str, None), - } - - def __init__(self, project, path, md5=None): - self.project = project - self.path = os.path.abspath(os.path.normpath(path)) - - if not self.path.startswith(self.project.root_dir): - raise CmdOutputOutsideOfRepoError(self.rel_path) - - self.md5 = md5 - - @property - def rel_path(self): - return os.path.relpath(self.path) - - def _changed_md5(self): - if not os.path.exists(self.path): - return True - - return self.project.state.changed(self.path, self.md5) - - @staticmethod - def _changed_msg(changed): - if changed: - return 'changed' - return "didn't change" - - def changed(self): - ret = self._changed_md5() - - msg = u'Dependency \'{}\' {}'.format(self.rel_path, self._changed_msg(ret)) - self.project.logger.debug(msg) - - return ret - - def status(self): - if self.changed(): - #FIXME better msgs - return {self.rel_path: 'changed'} - return {} - - def save(self): - if not os.path.exists(self.path): - raise CmdOutputDoesNotExistError(self.rel_path) - - if not os.path.isfile(self.path) and not os.path.isdir(self.path): - raise CmdOutputIsNotFileOrDirError(self.rel_path) - - if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \ - (os.path.isdir(self.path) and len(os.listdir(self.path)) == 0): - self.project.logger.warn("File/directory '{}' is empty.".format(self.rel_path)) - - self.md5 = self.project.state.update(self.path) - - @staticmethod - def unixpath(path): - assert not ntpath.isabs(path) - assert not posixpath.isabs(path) - return path.replace('\\', '/') - - def dumpd(self, cwd): - return { - Output.PARAM_PATH: self.unixpath(os.path.relpath(self.path, cwd)), - Output.PARAM_MD5: self.md5, - } - - @classmethod - def loadd(cls, project, d, cwd=os.curdir): - relpath = os.path.normpath(Output.unixpath(d[Output.PARAM_PATH])) - path = os.path.join(cwd, relpath) - md5 = d.get(Output.PARAM_MD5, None) - return cls(project, path, md5=md5) - - @classmethod - def loadd_from(cls, project, d_list, cwd=os.curdir): - return [cls.loadd(project, x, cwd=cwd) for x in d_list] - - @classmethod - def loads(cls, project, s, cwd=os.curdir): - return cls(project, os.path.join(cwd, s), md5=None) - - @classmethod - def loads_from(cls, project, s_list, cwd=os.curdir): - return [cls.loads(project, x, cwd=cwd) for x in s_list] - - -class Output(Dependency): - PARAM_CACHE = 'cache' - - SCHEMA = Dependency.SCHEMA - SCHEMA[schema.Optional(PARAM_CACHE)] = bool - - def __init__(self, project, path, md5=None, use_cache=True): - super(Output, self).__init__(project, path, md5=md5) - self.use_cache = use_cache - - @property - def cache(self): - if not self.use_cache: - return None - - return self.project.cache.get(self.md5) - - def dumpd(self, cwd): - ret = super(Output, self).dumpd(cwd) - ret[Output.PARAM_CACHE] = self.use_cache - return ret - - @classmethod - def loadd(cls, project, d, cwd=os.curdir): - ret = super(Output, cls).loadd(project, d, cwd=cwd) - ret.use_cache = d.get(Output.PARAM_CACHE, True) - return ret - - @classmethod - def loads(cls, project, s, use_cache=True, cwd=os.curdir): - ret = super(Output, cls).loads(project, s, cwd=cwd) - ret.use_cache = use_cache - return ret - - @classmethod - def loads_from(cls, project, s_list, use_cache=False, cwd=os.curdir): - return [cls.loads(project, x, use_cache=use_cache, cwd=cwd) for x in s_list] - - def changed(self): - if super(Output, self).changed(): - return True - - if self.use_cache and self.project.cache.changed(self.md5): - return True - - return False - - def checkout(self): - if not self.use_cache: - return - - msg = u'Checking out \'{}\' with cache \'{}\'' - self.project.logger.debug(msg.format(self.rel_path, self.md5)) - - if not self.changed(): - msg = u'Data file \'{}\' with cache \'{}\' didn\'t change, skipping checkout.' - self.project.logger.debug(msg.format(self.rel_path, self.md5)) - return - - self.project.cache.checkout(self.path, self.md5) - - def save(self): - super(Output, self).save() - - if not self.use_cache: - return - - self.project.logger.debug(u'Saving \'{}\' to \'{}\''.format(self.rel_path, self.md5)) - - if self.project.scm.is_tracked(self.path): - raise CmdOutputAlreadyTrackedError(self.rel_path) - - if not self.changed(): - return - - self.project.cache.save(self.path) - - def remove(self): - remove(self.path) diff --git a/dvc/output/__init__.py b/dvc/output/__init__.py new file mode 100644 index 000000000..ec2b42643 --- /dev/null +++ b/dvc/output/__init__.py @@ -0,0 +1,119 @@ +import os +import schema +import posixpath +import ntpath + +from dvc.dependency import Dependency +from dvc.exceptions import DvcException +from dvc.logger import Logger +from dvc.utils import remove + + +class OutputError(DvcException): + def __init__(self, path, msg): + super(OutputError, self).__init__('Output \'{}\' error: {}'.format(path, msg)) + + +class OutputOutsideOfRepoError(OutputError): + def __init__(self, path): + super(OutputOutsideOfRepoError, self).__init__(path, 'outside of repository') + + +class OutputDoesNotExistError(OutputError): + def __init__(self, path): + super(OutputDoesNotExistError, self).__init__(path, 'does not exist') + + +class OutputIsNotFileOrDirError(OutputError): + def __init__(self, path): + super(OutputIsNotFileOrDirError, self).__init__(path, 'not a file or directory') + + +class OutputAlreadyTrackedError(OutputError): + def __init__(self, path): + super(OutputAlreadyTrackedError, self).__init__(path, 'already tracked by scm(e.g. git)') + + +class Output(Dependency): + PARAM_CACHE = 'cache' + + OutsideOfRepoError = OutputOutsideOfRepoError + DoesNotExistError = OutputDoesNotExistError + IsNotFileOrDirError = OutputIsNotFileOrDirError + + SCHEMA = Dependency.SCHEMA + SCHEMA[schema.Optional(PARAM_CACHE)] = bool + + def __init__(self, project, path, md5=None, use_cache=True): + super(Output, self).__init__(project, path, md5=md5) + self.use_cache = use_cache + + @property + def cache(self): + if not self.use_cache: + return None + + return self.project.cache.get(self.md5) + + def dumpd(self, cwd): + ret = super(Output, self).dumpd(cwd) + ret[Output.PARAM_CACHE] = self.use_cache + return ret + + @classmethod + def loadd(cls, project, d, cwd=os.curdir): + ret = super(Output, cls).loadd(project, d, cwd=cwd) + ret.use_cache = d.get(Output.PARAM_CACHE, True) + return ret + + @classmethod + def loads(cls, project, s, use_cache=True, cwd=os.curdir): + ret = super(Output, cls).loads(project, s, cwd=cwd) + ret.use_cache = use_cache + return ret + + @classmethod + def loads_from(cls, project, s_list, use_cache=False, cwd=os.curdir): + return [cls.loads(project, x, use_cache=use_cache, cwd=cwd) for x in s_list] + + def changed(self): + if super(Output, self).changed(): + return True + + if self.use_cache and self.project.cache.changed(self.md5): + return True + + return False + + def checkout(self): + if not self.use_cache: + return + + msg = u'Checking out \'{}\' with cache \'{}\'' + self.project.logger.debug(msg.format(self.rel_path, self.md5)) + + if not self.changed(): + msg = u'Data file \'{}\' with cache \'{}\' didn\'t change, skipping checkout.' + self.project.logger.debug(msg.format(self.rel_path, self.md5)) + return + + self.project.cache.checkout(self.path, self.md5) + + def save(self): + super(Output, self).save() + + if not self.use_cache: + return + + self.project.logger.debug(u'Saving \'{}\' to \'{}\''.format(self.rel_path, self.md5)) + + if self.project.scm.is_tracked(self.path): + raise OutputAlreadyTrackedError(self.rel_path) + + if not self.changed(): + return + + self.project.cache.save(self.path) + + def remove(self): + remove(self.path) diff --git a/dvc/stage.py b/dvc/stage.py index bef7f9c78..f05447ca9 100644 --- a/dvc/stage.py +++ b/dvc/stage.py @@ -5,7 +5,8 @@ import subprocess import schema from dvc.exceptions import DvcException -from dvc.output import Output, Dependency, OutputError +from dvc.output import Output +from dvc.dependency import Dependency from dvc.logger import Logger from dvc.utils import dict_md5 @@ -21,7 +22,7 @@ class StageFileFormatError(DvcException): super(StageFileFormatError, self).__init__('Stage file format error') -class MissingDataSource(OutputError): +class MissingDataSource(DvcException): def __init__(self, missing_files): assert len(missing_files) > 0 diff --git a/tests/test_add.py b/tests/test_add.py index ccaee916f..3802857c2 100644 --- a/tests/test_add.py +++ b/tests/test_add.py @@ -6,9 +6,8 @@ import filecmp from dvc.main import main from dvc.utils import file_md5 from dvc.stage import Stage -from dvc.output import CmdOutputNoCacheError, CmdOutputOutsideOfRepoError -from dvc.output import CmdOutputDoesNotExistError, CmdOutputIsNotFileOrDirError -from dvc.output import CmdOutputAlreadyTrackedError +from dvc.output import OutputOutsideOfRepoError, OutputAlreadyTrackedError +from dvc.output import OutputDoesNotExistError, OutputIsNotFileOrDirError from dvc.project import StageNotFoundError from dvc.command.add import CmdAdd @@ -31,13 +30,13 @@ class TestAdd(TestDvc): class TestAddNonExistentFile(TestDvc): def test(self): - with self.assertRaises(CmdOutputDoesNotExistError) as cx: + with self.assertRaises(OutputDoesNotExistError) as cx: self.dvc.add('non_existent_file') class TestAddFileOutsideOfRepo(TestDvc): def test(self): - with self.assertRaises(CmdOutputOutsideOfRepoError) as cx: + with self.assertRaises(OutputOutsideOfRepoError) as cx: self.dvc.add(os.path.join(os.path.dirname(self.dvc.root_dir), self.FOO)) @@ -63,7 +62,7 @@ class TestAddTrackedFile(TestDvc): self.dvc.scm.add([fname]) self.dvc.scm.commit('add {}'.format(fname)) - with self.assertRaises(CmdOutputAlreadyTrackedError) as cx: + with self.assertRaises(OutputAlreadyTrackedError) as cx: self.dvc.add(fname) diff --git a/tests/test_gc.py b/tests/test_gc.py index 2bd29ea58..e45e4fa6e 100644 --- a/tests/test_gc.py +++ b/tests/test_gc.py @@ -5,8 +5,6 @@ import filecmp from dvc.main import main from dvc.utils import file_md5 from dvc.stage import Stage -from dvc.output import CmdOutputNoCacheError, CmdOutputOutsideOfRepoError -from dvc.output import CmdOutputDoesNotExistError from dvc.project import StageNotFoundError from dvc.command.gc import CmdGC diff --git a/tests/test_remove.py b/tests/test_remove.py index 9934ec174..b6503feab 100644 --- a/tests/test_remove.py +++ b/tests/test_remove.py @@ -2,7 +2,6 @@ import os from dvc.main import main from dvc.stage import Stage -from dvc.output import CmdOutputOutsideOfRepoError from dvc.project import StageNotFoundError from dvc.command.remove import CmdRemove