mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-05 15:28:27 +01:00
dvc: remove GitWrapper
Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
Ruslan Kuprieiev
@@ -1,448 +0,0 @@
|
||||
import os
|
||||
import re
|
||||
from collections import defaultdict
|
||||
|
||||
from dvc import utils
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.logger import Logger
|
||||
from dvc.config import Config
|
||||
from dvc.executor import Executor, ExecutorError
|
||||
from dvc.system import System
|
||||
from dvc.graph.workflow import Workflow
|
||||
from dvc.graph.commit import Commit
|
||||
|
||||
|
||||
class GitWrapperError(DvcException):
|
||||
def __init__(self, msg):
|
||||
DvcException.__init__(self, msg)
|
||||
|
||||
|
||||
class GitWrapperI(object):
|
||||
COMMIT_LEN = 7
|
||||
|
||||
def __init__(self, git_dir=None, commit=None):
|
||||
self._git_dir = git_dir
|
||||
self._commit = commit
|
||||
|
||||
@property
|
||||
def git_dir(self):
|
||||
return self._git_dir
|
||||
|
||||
@property
|
||||
def lock_file(self):
|
||||
return os.path.join(self.git_dir_abs, Config.CONFIG_DIR, '.' + Config.CONFIG + '.lock')
|
||||
|
||||
@property
|
||||
def git_dir_abs(self):
|
||||
return System.realpath(self.git_dir)
|
||||
|
||||
@property
|
||||
def curr_dir_abs(self):
|
||||
return os.path.abspath(os.curdir)
|
||||
|
||||
@property
|
||||
def curr_commit(self):
|
||||
return self._commit
|
||||
|
||||
def is_ready_to_go(self):
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def git_file_statuses():
|
||||
Logger.debug('[dvc-git] Getting file statuses. Command: git status --porcelain')
|
||||
code, out, err = Executor.exec_cmd(['git', 'status', '--porcelain'])
|
||||
if code != 0:
|
||||
raise ExecutorError('[dvc-git] File status command error - {}'.format(err))
|
||||
Logger.debug('[dvc-git] Getting file statuses. Success.')
|
||||
|
||||
return GitWrapper.parse_porcelain_files(out)
|
||||
|
||||
@staticmethod
|
||||
def git_config_get(name):
|
||||
code, out, err = Executor.exec_cmd(['git', 'config', '--get', name])
|
||||
Logger.debug('[dvc-git] "git config --get {}": code({}), out({}), err({})'.format(name,
|
||||
code,
|
||||
out,
|
||||
err))
|
||||
if code != 0:
|
||||
return None
|
||||
return out
|
||||
|
||||
@staticmethod
|
||||
def git_path_to_system_path(path):
|
||||
if os.name == 'nt':
|
||||
return path.replace('/', '\\')
|
||||
return path
|
||||
|
||||
@staticmethod
|
||||
def parse_porcelain_files(out):
|
||||
result = []
|
||||
if len(out) > 0:
|
||||
lines = out.split('\n')
|
||||
for line in lines:
|
||||
status = line[:2]
|
||||
file = GitWrapperI.git_path_to_system_path(line[3:])
|
||||
result.append((status, file))
|
||||
return result
|
||||
|
||||
def abs_paths_to_dvc(self, files):
|
||||
result = []
|
||||
for file in files:
|
||||
result.append(os.path.relpath(os.path.abspath(file), self.git_dir_abs))
|
||||
|
||||
return result
|
||||
|
||||
def commit_all_changes_and_log_status(self, message):
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def _file_contains(fname, s):
|
||||
if not os.path.isfile(fname):
|
||||
return False
|
||||
|
||||
with open(fname, 'r') as fd:
|
||||
if s in fd.read():
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def modify_gitignore(flist):
|
||||
for fname in flist:
|
||||
# We're modifying .gitignore in the same dir as the file itself, so
|
||||
# that if dir consists only of data files it will be still tracked by
|
||||
# git and we will not have any problems with --porcelain output for
|
||||
# specific changed files.
|
||||
gitignore = os.path.join(os.path.dirname(fname), '.gitignore')
|
||||
entry = os.path.basename(fname)
|
||||
|
||||
if GitWrapperI._file_contains(gitignore, entry + '\n'):
|
||||
return
|
||||
|
||||
with open(gitignore, 'a') as fd:
|
||||
fd.write('\n' + entry)
|
||||
|
||||
|
||||
class GitWrapper(GitWrapperI):
|
||||
FLOATS_FROM_STRING = re.compile(r'[-+]?(?:(?:\d*\.\d+)|(?:\d+\.?))(?:[Ee][+-]?\d+)?')
|
||||
|
||||
def __init__(self):
|
||||
super(GitWrapper, self).__init__()
|
||||
|
||||
def is_inside_work_tree(self):
|
||||
code, out, err = Executor.exec_cmd(['git', 'rev-parse', '--is-inside-work-tree'])
|
||||
if code != 0:
|
||||
Logger.error('[dvc-git] Not inside git tree');
|
||||
return code == 0
|
||||
|
||||
def is_ready_to_go(self):
|
||||
if not self.is_inside_work_tree():
|
||||
return False
|
||||
|
||||
statuses = self.git_file_statuses()
|
||||
if len(statuses) > 0:
|
||||
Logger.error('[dvc-git] Commit all changed files before running '
|
||||
'reproducible command. Changed files:')
|
||||
for status, file in statuses:
|
||||
Logger.error("{} {}".format(status, file))
|
||||
return False
|
||||
|
||||
return self.check_config()
|
||||
|
||||
def check_config(self):
|
||||
# Sanity check to make sure we will be able to commit
|
||||
name = self.git_config_get('user.name')
|
||||
if name == None:
|
||||
Logger.error('[dvc-git] Please setup user.name in git config')
|
||||
return False
|
||||
|
||||
email = self.git_config_get('user.email')
|
||||
if email == None:
|
||||
Logger.error('[dvc-git] Please setup user.email in git config')
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
@property
|
||||
def curr_dir_dvc(self):
|
||||
return os.path.relpath(os.path.abspath(os.curdir), self.git_dir_abs)
|
||||
|
||||
@property
|
||||
def git_dir(self):
|
||||
if self._git_dir:
|
||||
return self._git_dir
|
||||
|
||||
try:
|
||||
Logger.debug('[dvc-git] Getting git directory. Command: git rev-parse --show-toplevel')
|
||||
code, out, err = Executor.exec_cmd(['git', 'rev-parse', '--show-toplevel'])
|
||||
|
||||
if code != 0:
|
||||
raise ExecutorError('[dvc-git] Git directory command error - {}'.format(err))
|
||||
Logger.debug('[dvc-git] Getting git directory. Success.')
|
||||
|
||||
self._git_dir = out
|
||||
return self._git_dir
|
||||
except ExecutorError:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise ExecutorError('Unable to run git command: {}'.format(e))
|
||||
pass
|
||||
|
||||
def curr_commit(self, branches=False):
|
||||
Logger.debug('[dvc-git] Getting current git commit. Command: git rev-parse --short HEAD')
|
||||
|
||||
if branches:
|
||||
cmd = 'git rev-parse --abbrev-ref HEAD'
|
||||
else:
|
||||
cmd = 'git rev-parse HEAD'
|
||||
|
||||
code, out, err = Executor.exec_cmd(cmd.split())
|
||||
if code != 0:
|
||||
raise ExecutorError('[dvc-git] Commit command error - {}'.format(err))
|
||||
Logger.debug('[dvc-git] Getting current git commit. Success.')
|
||||
return out
|
||||
|
||||
@property
|
||||
def curr_branch_or_commit(self):
|
||||
Logger.debug('[dvc-git] Getting current git branch or commit.')
|
||||
|
||||
branch = self.curr_commit(branches=True)
|
||||
if branch == '' or branch == 'HEAD':
|
||||
result = self.curr_commit(branches=False)
|
||||
else:
|
||||
result = branch
|
||||
|
||||
Logger.debug('[dvc-git] Getting current git branch or commit. Result {}. Success.'.format(result))
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def commit_all_changes(message):
|
||||
Logger.debug('[dvc-git] Commit all changes. Commands: {} && {} && {}'.format(
|
||||
'git add --all', 'git status --porcelain', 'git commit -m'))
|
||||
|
||||
Executor.exec_cmd_only_success(['git', 'add', '--all'])
|
||||
out_status = Executor.exec_cmd_only_success(['git', 'status', '--porcelain'])
|
||||
changes = GitWrapper.parse_porcelain_files(out_status)
|
||||
if len(changes) == 0:
|
||||
return []
|
||||
|
||||
Executor.exec_cmd_only_success(['git', 'commit', '-m', '\'{}\''.format(message)])
|
||||
Logger.debug('[dvc-git] Commit all changes. Success.')
|
||||
return changes
|
||||
|
||||
def commit_all_changes_and_log_status(self, message):
|
||||
statuses = self.commit_all_changes(message)
|
||||
Logger.debug('[dvc-git] A new commit {} was made in the current branch. '
|
||||
'Added files:'.format(self.curr_commit()))
|
||||
for status, file in statuses:
|
||||
Logger.debug('[dvc-git]\t{} {}'.format(status, file))
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def abs_paths_to_relative(files):
|
||||
cur_dir = System.realpath(os.curdir)
|
||||
|
||||
result = []
|
||||
for file in files:
|
||||
result.append(os.path.relpath(System.realpath(file), cur_dir))
|
||||
|
||||
return result
|
||||
|
||||
def dvc_paths_to_abs(self, files):
|
||||
results = []
|
||||
|
||||
for file in files:
|
||||
results.append(os.path.join(self.git_dir_abs, file))
|
||||
|
||||
return results
|
||||
|
||||
def were_files_changed(self, code_dependencies, path_factory, changed_files):
|
||||
code_files, code_dirs = self.separate_dependency_files_and_dirs(code_dependencies)
|
||||
code_files_set = set([path_factory.path(x).dvc for x in code_files])
|
||||
for changed_file in changed_files:
|
||||
if changed_file in code_files_set:
|
||||
return True
|
||||
|
||||
for dir in code_dirs:
|
||||
if changed_file.startswith(dir):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def get_changed_files(target_commit):
|
||||
Logger.debug('[dvc-git] Identify changes. Command: '
|
||||
'git diff --name-only HEAD {}'.format(target_commit))
|
||||
|
||||
changed_files_str = Executor.exec_cmd_only_success(['git',
|
||||
'diff',
|
||||
'--name-only',
|
||||
'HEAD',
|
||||
target_commit])
|
||||
changed_files = changed_files_str.strip('"').split('\n')
|
||||
|
||||
Logger.debug('[dvc-git] Identify changes. Success. Changed files: {}'.format(
|
||||
', '.join(changed_files)))
|
||||
return changed_files
|
||||
|
||||
@staticmethod
|
||||
def get_target_commit(fname):
|
||||
Logger.debug('[dvc-git] Get target commit. Command: '
|
||||
'git log -1 --pretty=format:"%h" {}'.format(fname))
|
||||
try:
|
||||
commit = Executor.exec_cmd_only_success(['git',
|
||||
'log',
|
||||
'-1',
|
||||
'--pretty=format:"%h"',
|
||||
fname])
|
||||
return commit.strip('"')
|
||||
except ExecutorError:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def get_last_merge_changed_files():
|
||||
cmd = 'git diff --name-only @{1}'
|
||||
Logger.debug('[dvc-git] Get last merge changed files. Command: {}'.format(cmd))
|
||||
try:
|
||||
fstr = Executor.exec_cmd_only_success(cmd.split())
|
||||
return fstr.split('\n')
|
||||
except ExecutorError:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def checkout_file_before_last_merge(fname):
|
||||
cmd = 'git checkout @{1} -- ' + fname
|
||||
Logger.debug('[dvc-git] Checkout file {} before last merge. Command: {}'.format(fname, cmd))
|
||||
Executor.exec_cmd_only_success(cmd.split())
|
||||
|
||||
@staticmethod
|
||||
def get_file_content_before_last_merge(fname):
|
||||
return GitWrapper.get_file_content(fname, '@{1}')
|
||||
|
||||
@staticmethod
|
||||
def checkout(commit_or_branch, create_new=False):
|
||||
prefix = '-b ' if create_new else ''
|
||||
Logger.debug('[dvc-git] Checkout {}{}'.format(prefix, commit_or_branch))
|
||||
cmd = 'git checkout {}{}'.format(prefix, commit_or_branch)
|
||||
return Executor.exec_cmd(cmd.split())
|
||||
|
||||
@staticmethod
|
||||
def checkout_previous():
|
||||
Logger.debug('[dvc-git] Checkout previous')
|
||||
cmd = 'git checkout -'
|
||||
return Executor.exec_cmd(cmd.split())
|
||||
|
||||
def separate_dependency_files_and_dirs(self, code_dependencies):
|
||||
code_files = []
|
||||
code_dirs = []
|
||||
|
||||
code_dependencies_abs = self.dvc_paths_to_abs(code_dependencies)
|
||||
for code in code_dependencies_abs:
|
||||
if os.path.isdir(code):
|
||||
code_dirs.append(code)
|
||||
else:
|
||||
code_files.append(code)
|
||||
|
||||
return code_files, code_dirs
|
||||
|
||||
LOG_SEPARATOR = '|'
|
||||
LOG_FORMAT = ['%h', '%p', '%an', '%ai', '%s']
|
||||
|
||||
def get_all_commits(self, target, settings):
|
||||
# git log --all --abbrev=7 --pretty=format:"%h|%p|%an|%ai|%s"
|
||||
try:
|
||||
merges_map = GitWrapper.get_merges_map()
|
||||
|
||||
format_str = GitWrapper.LOG_SEPARATOR.join(GitWrapper.LOG_FORMAT)
|
||||
git_cmd = ['git', 'log', '--all', '--abbrev={}'.format(GitWrapper.COMMIT_LEN),
|
||||
'--pretty=format:{}'.format(format_str)]
|
||||
lines = Executor.exec_cmd_only_success(git_cmd).split('\n')
|
||||
|
||||
branches_multimap = GitWrapper.branches_multimap()
|
||||
|
||||
wf = Workflow(target, merges_map, branches_multimap)
|
||||
for line in lines:
|
||||
items = line.split(GitWrapper.LOG_SEPARATOR, len(GitWrapper.LOG_FORMAT))
|
||||
assert len(items) == 5, 'Git wrapper: git log format has {} items, 5 expected'.format(len(items))
|
||||
hash, parent_hash, name, date, comment = items
|
||||
|
||||
commit = Commit(hash, parent_hash, name, date, comment,
|
||||
*self.was_target_changed(hash, target, settings),
|
||||
branch_tips=branches_multimap.get(hash))
|
||||
wf.add_commit(commit)
|
||||
|
||||
return wf
|
||||
except ExecutorError:
|
||||
raise
|
||||
|
||||
def was_target_changed(self, hash, target, settings):
|
||||
git_cmd = ['git', 'show', '--pretty=', '--name-only', hash]
|
||||
changed_files = set(Executor.exec_cmd_only_success(git_cmd).split('\n'))
|
||||
|
||||
if target not in changed_files:
|
||||
return False, None
|
||||
|
||||
metric = utils.parse_target_metric_file(target)
|
||||
if metric is None:
|
||||
return False, None
|
||||
|
||||
return True, metric
|
||||
|
||||
@staticmethod
|
||||
def get_merges_map():
|
||||
# git log --merges --all --abbrev=7
|
||||
# {'a4b56f1': back_to_600_est}
|
||||
git_cmd = ['git', 'log', '--all', '--merges',
|
||||
'--abbrev={}'.format(GitWrapper.COMMIT_LEN)]
|
||||
# lines = map(str.strip, Executor.exec_cmd_only_success(git_cmd).split('\n'))
|
||||
# lines.map
|
||||
return {}
|
||||
|
||||
@staticmethod
|
||||
def branches_multimap():
|
||||
git_cmd = ['git', 'show-ref', '--abbrev={}'.format(GitWrapper.COMMIT_LEN)]
|
||||
lines = Executor.exec_cmd_only_success(git_cmd).split('\n')
|
||||
items_full = map(unicode.split, lines)
|
||||
items = map(lambda it: (it[0], re.sub(r'^refs/heads/', '', it[1])), items_full)
|
||||
|
||||
result = defaultdict(list)
|
||||
for (hash, branch) in items:
|
||||
result[hash].append(branch)
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def git_prev_commit():
|
||||
cmd = 'git check-ref-format --branch @{-1}'
|
||||
code, out, err = Executor.exec_cmd(cmd.split())
|
||||
if code != 0:
|
||||
return ''
|
||||
return out
|
||||
|
||||
@staticmethod
|
||||
def branches(regexp):
|
||||
cmd = 'git branch --format "%(refname)"'
|
||||
lines = Executor.exec_cmd_only_success(cmd.split()).split('\n')
|
||||
lines = map(lambda l: l.strip('"'), lines)
|
||||
|
||||
if not regexp or '/' not in regexp:
|
||||
# remove remotes like 'refs/heads'
|
||||
lines = map(lambda s: s.split('/')[-1], lines)
|
||||
|
||||
if not regexp:
|
||||
return lines
|
||||
|
||||
reg = re.compile(regexp)
|
||||
return list(filter(reg.search, lines))
|
||||
|
||||
@staticmethod
|
||||
def get_file_content(fname, branch):
|
||||
cmd = 'git show {}:{}'.format(branch, fname)
|
||||
return Executor.exec_cmd_only_success(cmd.split())
|
||||
|
||||
@staticmethod
|
||||
def all_untracked_files():
|
||||
cmd = 'git ls-files --others --exclude-standard'
|
||||
res = Executor.exec_cmd_only_success(cmd.split())
|
||||
if not res.strip():
|
||||
return []
|
||||
return res.split('\n')
|
||||
Reference in New Issue
Block a user