import os import re from collections import defaultdict from dvc.exceptions import DvcException from dvc.logger import Logger from dvc.config import Config, ConfigI from dvc.executor import Executor, ExecutorError from dvc.path.data_item import DataItemError from dvc.state_file import StateFile from dvc.system import System from dvc.graph.workflow import Workflow from dvc.graph.commit import Commit class GitWrapperError(DvcException): def __init__(self, msg): DvcException.__init__(self, msg) class GitWrapperI(object): COMMIT_LEN = 7 def __init__(self, git_dir=None, commit=None): self._git_dir = git_dir self._commit = commit @property def git_dir(self): return self._git_dir @property def lock_file(self): return os.path.join(self.git_dir_abs, Config.CONFIG_DIR, '.' + Config.CONFIG + '.lock') @property def git_dir_abs(self): return System.realpath(self.git_dir) @property def curr_dir_abs(self): return os.path.abspath(os.curdir) @property def curr_commit(self): return self._commit def is_ready_to_go(self): return True @staticmethod def git_file_statuses(): Logger.debug('[dvc-git] Getting file statuses. Command: git status --porcelain') code, out, err = Executor.exec_cmd(['git', 'status', '--porcelain']) if code != 0: raise ExecutorError('[dvc-git] File status command error - {}'.format(err)) Logger.debug('[dvc-git] Getting file statuses. Success.') return GitWrapper.parse_porcelain_files(out) @staticmethod def git_config_get(name): code, out, err = Executor.exec_cmd(['git', 'config', '--get', name]) Logger.debug('[dvc-git] "git config --get {}": code({}), out({}), err({})'.format(name, code, out, err)) if code != 0: return None return out @staticmethod def git_path_to_system_path(path): if os.name == 'nt': return path.replace('/', '\\') return path @staticmethod def parse_porcelain_files(out): result = [] if len(out) > 0: lines = out.split('\n') for line in lines: status = line[:2] file = GitWrapperI.git_path_to_system_path(line[3:]) result.append((status, file)) return result def abs_paths_to_dvc(self, files): result = [] for file in files: result.append(os.path.relpath(os.path.abspath(file), self.git_dir_abs)) return result def commit_all_changes_and_log_status(self, message): pass @staticmethod def _file_contains(fname, s): if not os.path.isfile(fname): return False with open(fname, 'r') as fd: if s in fd.read(): return True return False @staticmethod def modify_gitignore(flist): for fname in flist: # We're modifying .gitignore in the same dir as the file itself, so # that if dir consists only of data files it will be still tracked by # git and we will not have any problems with --porcelain output for # specific changed files. gitignore = os.path.join(os.path.dirname(fname), '.gitignore') entry = os.path.basename(fname) if GitWrapperI._file_contains(gitignore, entry + '\n'): return with open(gitignore, 'a') as fd: fd.write('\n' + entry) class GitWrapper(GitWrapperI): FLOATS_FROM_STRING = re.compile(r'[-+]?(?:(?:\d*\.\d+)|(?:\d+\.?))(?:[Ee][+-]?\d+)?') def __init__(self): super(GitWrapper, self).__init__() def is_inside_work_tree(self): code, out, err = Executor.exec_cmd(['git', 'rev-parse', '--is-inside-work-tree']) if code != 0: Logger.error('[dvc-git] Not inside git tree'); return code == 0 def is_ready_to_go(self): if not self.is_inside_work_tree(): return False statuses = self.git_file_statuses() if len(statuses) > 0: Logger.error('[dvc-git] Commit all changed files before running ' 'reproducible command. Changed files:') for status, file in statuses: Logger.error("{} {}".format(status, file)) return False return self.check_config() def check_config(self): # Sanity check to make sure we will be able to commit name = self.git_config_get('user.name') if name == None: Logger.error('[dvc-git] Please setup user.name in git config') return False email = self.git_config_get('user.email') if email == None: Logger.error('[dvc-git] Please setup user.email in git config') return False return True @property def curr_dir_dvc(self): return os.path.relpath(os.path.abspath(os.curdir), self.git_dir_abs) @property def git_dir(self): if self._git_dir: return self._git_dir try: Logger.debug('[dvc-git] Getting git directory. Command: git rev-parse --show-toplevel') code, out, err = Executor.exec_cmd(['git', 'rev-parse', '--show-toplevel']) if code != 0: raise ExecutorError('[dvc-git] Git directory command error - {}'.format(err)) Logger.debug('[dvc-git] Getting git directory. Success.') self._git_dir = out return self._git_dir except ExecutorError: raise except Exception as e: raise ExecutorError('Unable to run git command: {}'.format(e)) pass def curr_commit(self, branches=False): Logger.debug('[dvc-git] Getting current git commit. Command: git rev-parse --short HEAD') if branches: cmd = 'git rev-parse --abbrev-ref HEAD' else: cmd = 'git rev-parse HEAD' code, out, err = Executor.exec_cmd(cmd.split()) if code != 0: raise ExecutorError('[dvc-git] Commit command error - {}'.format(err)) Logger.debug('[dvc-git] Getting current git commit. Success.') return out @property def curr_branch_or_commit(self): Logger.debug('[dvc-git] Getting current git branch or commit.') branch = self.curr_commit(branches=True) if branch == '' or branch == 'HEAD': result = self.curr_commit(branches=False) else: result = branch Logger.debug('[dvc-git] Getting current git branch or commit. Result {}. Success.'.format(result)) return result @staticmethod def commit_all_changes(message): Logger.debug('[dvc-git] Commit all changes. Commands: {} && {} && {}'.format( 'git add --all', 'git status --porcelain', 'git commit -m')) Executor.exec_cmd_only_success(['git', 'add', '--all']) out_status = Executor.exec_cmd_only_success(['git', 'status', '--porcelain']) changes = GitWrapper.parse_porcelain_files(out_status) if len(changes) == 0: return [] Executor.exec_cmd_only_success(['git', 'commit', '-m', '\'{}\''.format(message)]) Logger.debug('[dvc-git] Commit all changes. Success.') return changes def commit_all_changes_and_log_status(self, message): statuses = self.commit_all_changes(message) Logger.debug('[dvc-git] A new commit {} was made in the current branch. ' 'Added files:'.format(self.curr_commit())) for status, file in statuses: Logger.debug('[dvc-git]\t{} {}'.format(status, file)) pass @staticmethod def abs_paths_to_relative(files): cur_dir = System.realpath(os.curdir) result = [] for file in files: result.append(os.path.relpath(System.realpath(file), cur_dir)) return result def dvc_paths_to_abs(self, files): results = [] for file in files: results.append(os.path.join(self.git_dir_abs, file)) return results def were_files_changed(self, code_dependencies, path_factory, changed_files): code_files, code_dirs = self.separate_dependency_files_and_dirs(code_dependencies) code_files_set = set([path_factory.path(x).dvc for x in code_files]) for changed_file in changed_files: if changed_file in code_files_set: return True for dir in code_dirs: if changed_file.startswith(dir): return True return False @staticmethod def get_changed_files(target_commit): Logger.debug('[dvc-git] Identify changes. Command: ' 'git diff --name-only HEAD {}'.format(target_commit)) changed_files_str = Executor.exec_cmd_only_success(['git', 'diff', '--name-only', 'HEAD', target_commit]) changed_files = changed_files_str.strip('"').split('\n') Logger.debug('[dvc-git] Identify changes. Success. Changed files: {}'.format( ', '.join(changed_files))) return changed_files @staticmethod def get_target_commit(fname): Logger.debug('[dvc-git] Get target commit. Command: ' 'git log -1 --pretty=format:"%h" {}'.format(fname)) try: commit = Executor.exec_cmd_only_success(['git', 'log', '-1', '--pretty=format:"%h"', fname]) return commit.strip('"') except ExecutorError: return None @staticmethod def get_last_merge_changed_files(): cmd = 'git diff --name-only @{1}' Logger.debug('[dvc-git] Get last merge changed files. Command: {}'.format(cmd)) try: fstr = Executor.exec_cmd_only_success(cmd.split()) return fstr.split('\n') except ExecutorError: return None @staticmethod def checkout_file_before_last_merge(fname): cmd = 'git checkout @{1} -- ' + fname Logger.debug('[dvc-git] Checkout file {} before last merge. Command: {}'.format(fname, cmd)) Executor.exec_cmd_only_success(cmd.split()) @staticmethod def checkout(commit): Logger.debug('[dvc-git] Checkout {}'.format(commit)) cmd = 'git checkout {}'.format(commit) Executor.exec_cmd(cmd.split()) @staticmethod def checkout_previous(): Logger.debug('[dvc-git] Checkout previous') cmd = 'git checkout -' Executor.exec_cmd(cmd.split()) def separate_dependency_files_and_dirs(self, code_dependencies): code_files = [] code_dirs = [] code_dependencies_abs = self.dvc_paths_to_abs(code_dependencies) for code in code_dependencies_abs: if os.path.isdir(code): code_dirs.append(code) else: code_files.append(code) return code_files, code_dirs LOG_SEPARATOR = '|' LOG_FORMAT = ['%h', '%p', '%an', '%ai', '%s'] def get_all_commits(self, target, settings): # git log --all --abbrev=7 --pretty=format:"%h|%p|%an|%ai|%s" try: merges_map = GitWrapper.get_merges_map() format_str = GitWrapper.LOG_SEPARATOR.join(GitWrapper.LOG_FORMAT) git_cmd = ['git', 'log', '--all', '--abbrev={}'.format(GitWrapper.COMMIT_LEN), '--pretty=format:{}'.format(format_str)] lines = Executor.exec_cmd_only_success(git_cmd).split('\n') branches_multimap = GitWrapper.branches_multimap() wf = Workflow(target, merges_map, branches_multimap) for line in lines: items = line.split(GitWrapper.LOG_SEPARATOR, len(GitWrapper.LOG_FORMAT)) assert len(items) == 5, 'Git wrapper: git log format has {} items, 5 expected'.format(len(items)) hash, parent_hash, name, date, comment = items commit = Commit(hash, parent_hash, name, date, comment, *self.was_target_changed(hash, target, settings), branch_tips=branches_multimap.get(hash)) wf.add_commit(commit) return wf except ExecutorError: raise def was_target_changed(self, hash, target, settings): git_cmd = ['git', 'show', '--pretty=', '--name-only', hash] changed_files = set(Executor.exec_cmd_only_success(git_cmd).split('\n')) if target not in changed_files: return False, None metric = self._read_metric_from_state_file(hash, target, settings) if metric is None: return False, None return True, metric def _read_metric_from_state_file(self, hash, target, settings): try: data_item = settings.path_factory.data_item(target) except DataItemError as ex: Logger.warn('Target file {} is not data item: {}'.format(target, ex)) return None try: cmd_corresponded_state_file = ['git', 'show', '{}:{}'.format(hash, data_item.state.relative)] state_file_content = Executor.exec_cmd_only_success(cmd_corresponded_state_file) except ExecutorError as ex: msg = '[dvc-git] Cannot obtain content of target symbolic file {} with hash {}: {}' Logger.warn(msg.format(target, hash, ex)) return None state_file = StateFile.loads(state_file_content, settings) return state_file.single_target_metric @staticmethod def get_merges_map(): # git log --merges --all --abbrev=7 # {'a4b56f1': back_to_600_est} git_cmd = ['git', 'log', '--all', '--merges', '--abbrev={}'.format(GitWrapper.COMMIT_LEN)] # lines = map(str.strip, Executor.exec_cmd_only_success(git_cmd).split('\n')) # lines.map return {} @staticmethod def branches_multimap(): git_cmd = ['git', 'show-ref', '--abbrev={}'.format(GitWrapper.COMMIT_LEN)] lines = Executor.exec_cmd_only_success(git_cmd).split('\n') items_full = map(unicode.split, lines) items = map(lambda it: (it[0], re.sub(r'^refs/heads/', '', it[1])), items_full) result = defaultdict(list) for (hash, branch) in items: result[hash].append(branch) return result @staticmethod def git_prev_commit(): cmd = 'git check-ref-format --branch @{-1}' code, out, err = Executor.exec_cmd(cmd.split()) if code != 0: return '' return out @staticmethod def state_files_for_previous_commit(): commit = GitWrapper.git_prev_commit() Logger.debug(u'[dvc-git] Previous commit "{}"'.format(commit)) if not commit: Logger.debug(u'[dvc-git] Previous commit does not exist') return [] git_cmd = u'git ls-tree --name-only -r {}'.format(commit) lines = Executor.exec_cmd_only_success(git_cmd.split()).split('\n') return filter(lambda s: s.startswith(ConfigI.STATE_DIR), lines)