[WIP] dvc: preserve exec bit for tracked files (#5061)

* #4578 added is_user_executable to dvc file

* Addressed PR feedback and introduced test_add_executable

Also fixed all tests in test_add.py

* Added a test to checkout executable file

* Pop isexec in dvc/output/__init__.py:loadd_from

* Implement test that file is executable after checkout

* Address PR feedback

* Address PR feedback

* Changed BaseTree.isexec to use path_info as an argument

* Update tests/func/test_add.py

* Update tests/func/test_add.py

* Update tests/func/test_add.py

* Update tests/func/test_add.py

* fix formatting

* add: preserve exec

* checkout: fix setting exec

* tests: refactor checkout exec test

* tests: fix windows cases

* tests: add: fix exec test on windows

* Update dvc/tree/local.py

* Update dvc/output/base.py

* Update dvc/output/base.py

* Update dvc/cache/base.py

* Update tests/func/test_add.py

* tests: add: skip whole exec test on windows

* save isexec to lockfiles

* fix formatting

Co-authored-by: Ruslan Kuprieiev <[email protected]>
Co-authored-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
Artem Dudarev
2020-12-20 02:31:06 +02:00
committed by GitHub
co-authored by Ruslan Kuprieiev Ruslan Kuprieiev
parent f51bef040a
commit dcea83e75b
16 changed files with 140 additions and 12 deletions
+9
View File
@@ -69,6 +69,7 @@ SCHEMA[BaseOutput.PARAM_CHECKPOINT] = bool
SCHEMA[HashInfo.PARAM_SIZE] = int
SCHEMA[HashInfo.PARAM_NFILES] = int
SCHEMA[BaseOutput.PARAM_DESC] = str
SCHEMA[BaseOutput.PARAM_ISEXEC] = bool
def _get(
@@ -81,6 +82,7 @@ def _get(
persist=False,
checkpoint=False,
desc=None,
isexec=False,
):
parsed = urlparse(p)
@@ -97,6 +99,7 @@ def _get(
persist=persist,
checkpoint=checkpoint,
desc=desc,
isexec=isexec,
)
for o in OUTS:
@@ -112,6 +115,7 @@ def _get(
persist=persist,
checkpoint=checkpoint,
desc=desc,
isexec=isexec,
)
return LocalOutput(
stage,
@@ -124,6 +128,7 @@ def _get(
persist=persist,
checkpoint=checkpoint,
desc=desc,
isexec=isexec,
)
@@ -137,6 +142,7 @@ def loadd_from(stage, d_list):
persist = d.pop(BaseOutput.PARAM_PERSIST, False)
checkpoint = d.pop(BaseOutput.PARAM_CHECKPOINT, False)
desc = d.pop(BaseOutput.PARAM_DESC, False)
isexec = d.pop(BaseOutput.PARAM_ISEXEC, False)
ret.append(
_get(
stage,
@@ -148,6 +154,7 @@ def loadd_from(stage, d_list):
persist=persist,
checkpoint=checkpoint,
desc=desc,
isexec=isexec,
)
)
return ret
@@ -161,6 +168,7 @@ def loads_from(
plot=False,
persist=False,
checkpoint=False,
isexec=False,
):
return [
_get(
@@ -172,6 +180,7 @@ def loads_from(
plot=plot,
persist=persist,
checkpoint=checkpoint,
isexec=isexec,
)
for s in s_list
]
+16 -1
View File
@@ -76,6 +76,7 @@ class BaseOutput:
PARAM_PLOT_HEADER = "header"
PARAM_PERSIST = "persist"
PARAM_DESC = "desc"
PARAM_ISEXEC = "isexec"
METRIC_SCHEMA = Any(
None,
@@ -105,6 +106,7 @@ class BaseOutput:
persist=False,
checkpoint=False,
desc=None,
isexec=False,
):
self._validate_output_path(path, stage)
# This output (and dependency) objects have too many paths/urls
@@ -136,6 +138,8 @@ class BaseOutput:
if self.use_cache and self.cache is None:
raise RemoteCacheRequiredError(self.path_info)
self.isexec = False if self.IS_DEPENDENCY else isexec
def _parse_path(self, tree, path):
if tree:
parsed = urlparse(path)
@@ -287,6 +291,11 @@ class BaseOutput:
return
self.hash_info = self.get_hash()
self.isexec = self.isfile() and self.tree.isexec(self.path_info)
def set_exec(self):
if self.isfile() and self.isexec:
self.tree.set_exec(self.path_info)
def commit(self):
if not self.exists:
@@ -295,6 +304,7 @@ class BaseOutput:
assert self.hash_info
if self.use_cache:
self.cache.save(self.path_info, self.cache.tree, self.hash_info)
self.set_exec()
def dumpd(self):
ret = copy(self.hash_info.to_dict())
@@ -328,6 +338,9 @@ class BaseOutput:
if self.checkpoint:
ret[self.PARAM_CHECKPOINT] = self.checkpoint
if self.isexec:
ret[self.PARAM_ISEXEC] = self.isexec
return ret
def verify_metric(self):
@@ -353,7 +366,7 @@ class BaseOutput:
return None
try:
return self.cache.checkout(
res = self.cache.checkout(
self.path_info,
self.hash_info,
force=force,
@@ -366,6 +379,8 @@ class BaseOutput:
if allow_missing or self.checkpoint:
return None
raise
self.set_exec()
return res
def remove(self, ignore_remove=False):
self.tree.remove(self.path_info)
+1
View File
@@ -25,6 +25,7 @@ DATA_SCHEMA = {
Required("path"): str,
HashInfo.PARAM_SIZE: int,
HashInfo.PARAM_NFILES: int,
BaseOutput.PARAM_ISEXEC: bool,
}
LOCK_FILE_STAGE_SCHEMA = {
Required(StageParams.PARAM_CMD): Any(str, list),
+1
View File
@@ -61,6 +61,7 @@ class StageLoader(Mapping):
info = get_in(checksums, [key, path], {})
info = info.copy()
info.pop("path", None)
item.isexec = info.pop("isexec", None)
item.hash_info = HashInfo.from_dict(info)
@classmethod
+3
View File
@@ -150,6 +150,9 @@ def to_single_stage_lockfile(stage: "Stage") -> dict:
*item.hash_info.to_dict().items(),
]
if item.isexec:
ret.append((item.PARAM_ISEXEC, True))
return OrderedDict(ret)
res = OrderedDict([("cmd", stage.cmd)])
+1
View File
@@ -179,6 +179,7 @@ def compute_md5(stage):
BaseOutput.PARAM_METRIC,
BaseOutput.PARAM_PERSIST,
BaseOutput.PARAM_CHECKPOINT,
BaseOutput.PARAM_ISEXEC,
HashInfo.PARAM_SIZE,
HashInfo.PARAM_NFILES,
],
+9
View File
@@ -195,6 +195,15 @@ class BaseTree:
"""
return True
def set_exec(self, path_info):
raise RemoteActionNotImplemented("set_exec", self.scheme)
def isexec(self, path_info):
"""Optional: Overwrite only if the remote has a way to distinguish
between executable and non-executable file.
"""
return False
def iscopy(self, path_info):
"""Check if this file is an independent copy."""
return False # We can't be sure by default
+1 -1
View File
@@ -234,7 +234,7 @@ class DvcTree(BaseTree): # pylint:disable=abstract-method
recurse = recursive or not strict
return meta.output_exists if recurse else meta.is_output
def isexec(self, path): # pylint: disable=unused-argument
def isexec(self, path_info): # pylint: disable=unused-argument
return False
def get_dir_hash(self, path_info, **kwargs):
+6 -2
View File
@@ -131,8 +131,12 @@ class GitTree(BaseTree): # pylint:disable=abstract-method
)
yield root, dirs, files
def isexec(self, path):
return is_exec(self.stat(path).st_mode)
def isexec(self, path_info):
if not self.exists(path_info):
return False
mode = self.stat(path_info).st_mode
return is_exec(mode)
def stat(self, path):
key = self._get_key(path)
+6 -2
View File
@@ -152,8 +152,12 @@ class LocalTree(BaseTree):
def makedirs(self, path_info):
makedirs(path_info, exist_ok=True, mode=self.dir_mode)
def isexec(self, path):
mode = os.stat(path).st_mode
def set_exec(self, path_info):
mode = self.stat(path_info).st_mode
self.chmod(path_info, mode | stat.S_IEXEC)
def isexec(self, path_info):
mode = self.stat(path_info).st_mode
return is_exec(mode)
def stat(self, path):
+5 -5
View File
@@ -225,11 +225,11 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method
return False
return meta.isfile
def isexec(self, path):
tree, dvc_tree = self._get_tree_pair(path)
if dvc_tree and dvc_tree.exists(path):
return dvc_tree.isexec(path)
return tree.isexec(path)
def isexec(self, path_info):
tree, dvc_tree = self._get_tree_pair(path_info)
if dvc_tree and dvc_tree.exists(path_info):
return dvc_tree.isexec(path_info)
return tree.isexec(path_info)
def stat(self, path):
tree, _ = self._get_tree_pair(path)
+1 -1
View File
@@ -462,7 +462,7 @@ def parse_target(
def is_exec(mode):
return mode & (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
return bool(mode & (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH))
def glob_targets(targets, glob=True, recursive=True):
+21
View File
@@ -2,6 +2,7 @@ import filecmp
import logging
import os
import shutil
import stat
import textwrap
import time
@@ -57,6 +58,26 @@ def test_add(tmp_dir, dvc):
}
@pytest.mark.skipif(os.name == "nt", reason="can't set exec bit on Windows")
def test_add_executable(tmp_dir, dvc):
tmp_dir.gen("foo", "foo")
st = os.stat("foo")
os.chmod("foo", st.st_mode | stat.S_IEXEC)
dvc.add("foo")
assert load_yaml("foo.dvc") == {
"outs": [
{
"md5": "acbd18db4cc2f85cedef654fccc4a4d8",
"path": "foo",
"size": 3,
"isexec": True,
}
],
}
assert os.stat("foo").st_mode & stat.S_IEXEC
def test_add_unicode(tmp_dir, dvc):
with open("\xe1", "wb") as fd:
fd.write(b"something")
+18
View File
@@ -3,6 +3,7 @@ import filecmp
import logging
import os
import shutil
import stat
import pytest
from mock import patch
@@ -862,3 +863,20 @@ def test_checkout_external_modified_file(tmp_dir, dvc, scm, mocker, workspace):
dvc.checkout()
assert (workspace / "foo").read_text() == "foo"
def test_checkout_executable(tmp_dir, dvc):
tmp_dir.dvc_gen("foo", "foo")
contents = load_yaml("foo.dvc")
contents["outs"][0]["isexec"] = True
dump_yaml("foo.dvc", contents)
dvc.checkout("foo")
isexec = os.stat("foo").st_mode & stat.S_IEXEC
if os.name == "nt":
# NOTE: you can't set exec bits on Windows
assert not isexec
else:
assert isexec
@@ -39,6 +39,24 @@ def test_fill_from_lock_deps_outs(dvc, lock_data):
assert stage.outs[0].hash_info == HashInfo("md5", "bar_checksum")
def test_fill_from_lock_outs_isexec(dvc):
stage = create_stage(PipelineStage, dvc, PIPELINE_FILE, outs=["foo"])
assert not stage.outs[0].isexec
StageLoader.fill_from_lock(
stage,
{
"cmd": "command",
"outs": [{"path": "foo", "md5": "foo_checksum", "isexec": True}],
},
)
assert stage.outs[0].def_path == "foo"
assert stage.outs[0].hash_info == HashInfo("md5", "foo_checksum")
assert stage.outs[0].isexec
def test_fill_from_lock_params(dvc, lock_data):
stage = create_stage(
PipelineStage,
@@ -132,6 +132,30 @@ def test_lock_outs(dvc, typ):
)
@pytest.mark.parametrize("typ", ["plots", "metrics", "outs"])
def test_lock_outs_isexec(dvc, typ):
stage = create_stage(PipelineStage, dvc, **{typ: ["input"]}, **kwargs)
stage.outs[0].hash_info = HashInfo("md5", "md-five")
stage.outs[0].isexec = True
assert to_single_stage_lockfile(stage) == OrderedDict(
[
("cmd", "command"),
(
"outs",
[
OrderedDict(
[
("path", "input"),
("md5", "md-five"),
("isexec", True),
]
)
],
),
]
)
@pytest.mark.parametrize("typ", ["plots", "metrics", "outs"])
def test_lock_outs_order(dvc, typ):
stage = create_stage(