Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-07 15:28:28 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #303 from dataversioncontrol/exec_in_branch

Browse Source 
Exec in branch
This commit is contained in:
Dmitry Petrov
2017-11-19 04:54:10 -08:00
committed by GitHub
parent c4d01aa595 d64390e9d0
commit 2d77053dfe
24 changed files with 244 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/cli.py
+16 -3
View File
@@ -25,21 +25,34 @@ from dvc import VERSION
def parse_args(argv=None):
# Common args
parent_parser = argparse.ArgumentParser(add_help=False)
parent_parser.add_argument('-q',
parent_parser.add_argument(
'-q',
'--quiet',
action='store_true',
default=False,
help='Be quiet.')
parent_parser.add_argument('-v',
parent_parser.add_argument(
'-v',
'--verbose',
action='store_true',
default=False,
help='Be verbose.')
parent_parser.add_argument('-G',
parent_parser.add_argument(
'-G',
'--no-git-actions',
action='store_true',
default=False,
help='Skip all git actions including reproducibility check and commits.')
parent_parser.add_argument(
'-b',
'--branch',
metavar='BRANCH',
help='Execute a command in the branch.')
parent_parser.add_argument(
'-n',
'--new-branch',
metavar='BRANCH',
help='Create a new branch and execute a command in the branch.')
# Main parser
desc = 'Data Version Control'
dvc/command/checkout.py
+11 -12
View File
@@ -1,6 +1,6 @@
import os
from dvc.command.base import CmdBase, DvcLock
from dvc.command.common.base import CmdBase
from dvc.logger import Logger
from dvc.system import System
@@ -34,19 +34,18 @@ class CmdCheckout(CmdBase):
Logger.info('Checkout \'{}\''.format(item.data.relative))
def run(self):
with DvcLock(self.is_locker, self.git):
try:
states = self.git.state_files_for_previous_commit()
prev_items = self.settings.path_factory.data_items_from_states(states, existing=False)
except Exception as ex:
Logger.error(u'Unable to get data files from previous commit: {}'.format(ex))
return 1
try:
states = self.git.state_files_for_previous_commit()
prev_items = self.settings.path_factory.data_items_from_states(states, existing=False)
except Exception as ex:
Logger.error(u'Unable to get data files from previous commit: {}'.format(ex))
return 1
curr_items = self.settings.path_factory.all_existing_data_items()
self.checkout(curr_items)
curr_items = self.settings.path_factory.all_existing_data_items()
self.checkout(curr_items)
self.remove_files(list(set(prev_items) - set(curr_items)))
return 0
self.remove_files(list(set(prev_items) - set(curr_items)))
return 0
@staticmethod
def remove_files(removed_items_set):
dvc/command/common/__init__.py
View File
dvc/command/base.py → dvc/command/common/base.py
+14 -41
View File
@@ -1,39 +1,6 @@
import argparse
import fasteners
from multiprocessing import cpu_count
from dvc.config import ConfigError
from dvc.exceptions import DvcException
from dvc.logger import Logger
class CmdBaseError(DvcException):
def __init__(self, msg):
super(CmdBaseError, self).__init__('{}'.format(msg))
class DvcLockerError(CmdBaseError):
def __init__(self, msg):
super(DvcLockerError, self).__init__('DVC locker error: {}'.format(msg))
class DvcLock(object):
def __init__(self, is_locker, git):
self.is_locker = is_locker
self.git = git
self.lock = None
def __enter__(self):
if self.is_locker:
self.lock = fasteners.InterProcessLock(self.git.lock_file)
if not self.lock.acquire(timeout=5):
raise DvcLockerError('Cannot perform the cmd since DVC is busy and locked. Please retry the cmd later.')
return self.lock
def __exit__(self, type, value, traceback):
if self.is_locker:
self.lock.release()
from dvc.command.common.branch_changer import BranchChanger
from dvc.command.common.dvc_lock import DvcLock
class CmdBase(object):
@@ -78,12 +45,6 @@ class CmdBase(object):
def set_git_action(self, value):
self.parsed_args.no_git_actions = not value
@property
def is_locker(self):
if 'no_lock' in self.parsed_args.__dict__:
return not self.parsed_args.no_lock
return True
def set_locker(self, value):
self.parsed_args.no_lock = value
@@ -99,5 +60,17 @@ class CmdBase(object):
def not_committed_changes_warning():
Logger.warn('changes were not committed to git')
def run_cmd(self):
with DvcLock(self.is_locker, self.git):
with BranchChanger(self.parsed_args.branch, self.parsed_args.new_branch, self.git):
return self.run()
# Abstract methods that have to be implemented by any inheritance class
def run(self):
pass
@property
def is_locker(self):
if 'no_lock' in self.parsed_args.__dict__:
return not self.parsed_args.no_lock
return True
dvc/command/common/branch_changer.py
+31
View File
@@ -0,0 +1,31 @@
from dvc.command.common.common_error import CmdCommonError
class BranchChangerError(CmdCommonError):
def __init__(self, branch, msg):
super(CmdCommonError, self).__init__('Error in changing branch \'{}\': {}'.format(branch, msg))
class BranchChanger(object):
def __init__(self, branch, new_branch, git):
if branch and new_branch:
raise BranchChangerError("Commands conflict: --branch and --new-branch cannot be used at the same command")
self.perform_action = branch or new_branch
self.branch = branch if branch else new_branch
self.create_new = new_branch is not None
self.git = git
def __enter__(self):
if self.perform_action:
code, _, err = self.git.checkout(self.branch, self.create_new)
if code != 0:
raise BranchChangerError(self.branch, err)
return self
def __exit__(self, type, value, traceback):
if self.perform_action:
code, _, err = self.git.checkout_previous()
if code != 0:
raise BranchChangerError(self.branch, err)
dvc/command/common/common_error.py
+6
View File
@@ -0,0 +1,6 @@
from dvc.exceptions import DvcException
class CmdCommonError(DvcException):
def __init__(self, msg):
super(CmdCommonError, self).__init__('{}'.format(msg))
dvc/command/common/dvc_lock.py
+28
View File
@@ -0,0 +1,28 @@
import fasteners
from dvc.command.common.common_error import CmdCommonError
class DvcLockerError(CmdCommonError):
def __init__(self, msg):
super(DvcLockerError, self).__init__('DVC locker error: {}'.format(msg))
class DvcLock(object):
TIMEOUT = 5
def __init__(self, is_locker, git):
self.is_locker = is_locker
self.git = git
self.lock = None
def __enter__(self):
if self.is_locker:
self.lock = fasteners.InterProcessLock(self.git.lock_file)
if not self.lock.acquire(timeout=self.TIMEOUT):
raise DvcLockerError('Cannot perform the cmd since DVC is busy and locked. Please retry the cmd later.')
return self.lock
def __exit__(self, type, value, traceback):
if self.is_locker:
self.lock.release()
dvc/command/traverse.py → dvc/command/common/traverse.py
+4 -4
View File
@@ -1,10 +1,11 @@
import os
from dvc.command.base import CmdBase, DvcLock
from dvc.command.common.base import CmdBase
from dvc.exceptions import DvcException
from dvc.logger import Logger
from dvc.path.data_item import DataItemError
class TraverseError(DvcException):
def __init__(self, msg):
DvcException.__init__(self, 'Traverse file tree error: {}'.format(msg))
@@ -17,9 +18,8 @@ class Traverse(CmdBase):
self._do_not_start_from_root = do_not_start_from_root
def run(self):
with DvcLock(self.is_locker, self.git):
if not self._traverse_all():
return 1
if not self._traverse_all():
return 1
return 0
dvc/command/config.py
+2 -6
View File
@@ -1,7 +1,7 @@
import os
import configobj
from dvc.command.base import CmdBase, DvcLock
from dvc.command.common.base import CmdBase
from dvc.logger import Logger
from dvc.config import Config
@@ -75,7 +75,7 @@ class CmdConfig(CmdBase):
return 0
def _run(self):
def run(self):
if self.check_opt() != 0:
return 1
@@ -86,7 +86,3 @@ class CmdConfig(CmdBase):
return self.show()
return self.set()
def run(self):
with DvcLock(self.is_locker, self.git):
return self._run()
dvc/command/data_sync.py
+16 -17
View File
@@ -1,25 +1,27 @@
from dvc.command.base import CmdBase, DvcLock
from dvc.command.common.base import CmdBase
from dvc.logger import Logger
import dvc.data_cloud as cloud
class CmdDataPull(CmdBase):
def __init__(self, settings):
super(CmdDataPull, self).__init__(settings)
class CmdDataBase(CmdBase):
def __init__(self, settings, cmd):
super(CmdDataBase, self).__init__(settings)
self.cmd = cmd
def run(self):
with DvcLock(self.is_locker, self.git):
self.cloud.pull(self.parsed_args.targets, self.parsed_args.jobs)
self.cmd(self.parsed_args.targets, self.parsed_args.jobs)
class CmdDataPush(CmdBase):
class CmdDataPull(CmdDataBase):
def __init__(self, settings):
super(CmdDataPush, self).__init__(settings)
super(CmdDataPull, self).__init__(settings, self.cloud.pull)
class CmdDataPush(CmdDataBase):
def __init__(self, settings):
super(CmdDataPush, self).__init__(settings, self.cloud.push)
def run(self):
with DvcLock(self.is_locker, self.git):
self.cloud.push(self.parsed_args.targets, self.parsed_args.jobs)
class CmdDataStatus(CmdBase):
def __init__(self, settings):
@@ -41,9 +43,6 @@ class CmdDataStatus(CmdBase):
Logger.info('\t{}\t{}'.format(prefix_map[ret], target.data.dvc))
def run(self):
with DvcLock(self.is_locker, self.git):
status = self.cloud.status(self.parsed_args.targets, self.parsed_args.jobs)
self._show(status)
return 0
status = self.cloud.status(self.parsed_args.targets, self.parsed_args.jobs)
self._show(status)
return 0
dvc/command/find.py
+14 -15
View File
@@ -1,5 +1,5 @@
from dvc import utils
from dvc.command.base import CmdBase, DvcLock
from dvc.command.common.base import CmdBase
from dvc.executor import ExecutorError
from dvc.logger import Logger
@@ -9,23 +9,22 @@ class CmdFind(CmdBase):
super(CmdFind, self).__init__(settings)
def run(self):
with DvcLock(self.is_locker, self.git):
fname = self.parsed_args.target
branch_name = self.parsed_args.branch_name
fname = self.parsed_args.target
branch_name = self.parsed_args.branch_name
branches = self.git.branches(branch_name)
metrics = [self.read_metrics(fname, b) for b in branches]
branches = self.git.branches(branch_name)
metrics = [self.read_metrics(fname, b) for b in branches]
zipped = zip(metrics, branches)
res = self.filter(zipped, self.parsed_args.criteria)
res = filter(lambda x: x[0] is not None, res)
zipped = zip(metrics, branches)
res = self.filter(zipped, self.parsed_args.criteria)
res = filter(lambda x: x[0] is not None, res)
for (val, branch) in res:
if self.parsed_args.show_value:
print('{}: {}'.format(branch, val))
else:
print(branch)
return 0
for (val, branch) in res:
if self.parsed_args.show_value:
print('{}: {}'.format(branch, val))
else:
print(branch)
return 0
@staticmethod
def filter(zipped, criteria):
dvc/command/gc.py
+12 -14
View File
@@ -1,7 +1,6 @@
import os
from dvc.command.base import DvcLock
from dvc.command.traverse import Traverse
from dvc.command.common.traverse import Traverse
from dvc.logger import Logger
from dvc.path.data_item import DataItemError
from dvc.config import ConfigI
@@ -13,20 +12,19 @@ class CmdGC(Traverse):
self.clist = []
def run(self):
with DvcLock(self.is_locker, self.git):
if not self._traverse(self.git.git_dir_abs):
Logger.error('Failed to collect used cache')
return 1
if not self._traverse(self.git.git_dir_abs):
Logger.error('Failed to collect used cache')
return 1
for cache in os.listdir(ConfigI.CACHE_DIR):
fname = os.path.join(ConfigI.CACHE_DIR, cache)
if fname in self.clist:
continue
os.remove(fname)
self._remove_cloud_cache(self.settings.path_factory.data_item('.empty', fname))
Logger.info('Cache \'{}\' was removed'.format(fname))
for cache in os.listdir(ConfigI.CACHE_DIR):
fname = os.path.join(ConfigI.CACHE_DIR, cache)
if fname in self.clist:
continue
os.remove(fname)
self._remove_cloud_cache(self.settings.path_factory.data_item('.empty', fname))
Logger.info('Cache \'{}\' was removed'.format(fname))
return 0
return 0
def process_file(self, target):
Logger.debug(u'[Cmd-GC] GC file {}.'.format(target))
dvc/command/import_file.py
+22 -26
View File
@@ -1,5 +1,4 @@
import os
import re
try:
from urlparse import urlparse
@@ -8,12 +7,11 @@ except ImportError:
# pylint: disable=no-name-in-module, import-error
from urllib.parse import urlparse
from dvc.command.base import CmdBase, DvcLock
from dvc.data_cloud import sizeof_fmt, file_md5
from dvc.command.common.base import CmdBase
from dvc.logger import Logger
from dvc.exceptions import DvcException
from dvc.state_file import StateFile
from dvc.system import System
class ImportFileError(DvcException):
def __init__(self, msg):
@@ -26,32 +24,30 @@ class CmdImportFile(CmdBase):
def run(self):
targets = []
with DvcLock(self.is_locker, self.git):
output = self.parsed_args.output
self.verify_output(output, self.parsed_args.input)
output = self.parsed_args.output
self.verify_output(output, self.parsed_args.input)
for input in self.parsed_args.input:
if not os.path.isdir(input):
targets.append((input, output))
else:
input_dir = os.path.basename(input)
for root, dirs, files in os.walk(input):
for file in files:
filename = os.path.join(root, file)
for input in self.parsed_args.input:
if not os.path.isdir(input):
targets.append((input, output))
else:
input_dir = os.path.basename(input)
for root, dirs, files in os.walk(input):
for file in files:
filename = os.path.join(root, file)
rel = os.path.relpath(filename, input)
out = os.path.join(output, input_dir, rel)
rel = os.path.relpath(filename, input)
out = os.path.join(output, input_dir, rel)
out_dir = os.path.dirname(out)
if not os.path.exists(out_dir):
os.mkdir(out_dir)
out_dir = os.path.dirname(out)
if not os.path.exists(out_dir):
os.mkdir(out_dir)
targets.append((filename, out))
pass
self.import_files(targets, self.parsed_args.lock, self.parsed_args.jobs)
message = 'DVC import files: {} -> {}'.format(str(self.parsed_args.input), output)
self.commit_if_needed(message)
pass
targets.append((filename, out))
pass
self.import_files(targets, self.parsed_args.lock, self.parsed_args.jobs)
message = 'DVC import files: {} -> {}'.format(str(self.parsed_args.input), output)
self.commit_if_needed(message)
@staticmethod
def verify_output(output, input):
dvc/command/init.py
+5 -1
View File
@@ -1,7 +1,7 @@
import os
from pathlib import Path
from dvc.command.base import CmdBase
from dvc.command.common.base import CmdBase
from dvc.logger import Logger
from dvc.config import Config
from dvc.exceptions import DvcException
@@ -62,6 +62,10 @@ ProjectName =
def __init__(self, settings):
super(CmdInit, self).__init__(settings)
@property
def is_locker(self):
return False
def get_not_existing_path(self, *args):
path = Path(os.path.join(self.git.git_dir, *args))
if path.exists():
dvc/command/instance_create.py
+14 -15
View File
@@ -1,5 +1,5 @@
from dvc.cloud.instance_manager import InstanceManager
from dvc.command.base import CmdBase, DvcLock
from dvc.command.common.base import CmdBase
from dvc.exceptions import DvcException
from dvc.logger import Logger
@@ -9,21 +9,20 @@ class CmdInstanceCreate(CmdBase):
super(CmdInstanceCreate, self).__init__(settings)
def run(self):
with DvcLock(self.is_locker, self.git):
name = self.parsed_args.name
cloud = self.parsed_args.cloud or self.settings.config.cloud
name = self.parsed_args.name
cloud = self.parsed_args.cloud or self.settings.config.cloud
# print('NAME: {}'.format(name))
# print('CLOUD: {}'.format(cloud))
# print('NAME: {}'.format(name))
# print('CLOUD: {}'.format(cloud))
if not name:
Logger.error('Instance name is not defined')
return 1
if not name:
Logger.error('Instance name is not defined')
return 1
try:
InstanceManager().create(name, cloud, self.parsed_args, self.settings.config)
except DvcException as ex:
Logger.error('Instance creation error: {}'.format(ex))
return 1
try:
InstanceManager().create(name, cloud, self.parsed_args, self.settings.config)
except DvcException as ex:
Logger.error('Instance creation error: {}'.format(ex))
return 1
return 0
return 0
dvc/command/lock.py
+2 -3
View File
@@ -1,4 +1,4 @@
from dvc.command.base import CmdBase, DvcLock
from dvc.command.common.base import CmdBase
from dvc.logger import Logger
from dvc.state_file import StateFile
@@ -8,8 +8,7 @@ class CmdLock(CmdBase):
super(CmdLock, self).__init__(settings)
def run(self):
with DvcLock(self.is_locker, self.git):
return self.lock_files(self.parsed_args.files, not self.parsed_args.unlock)
return self.lock_files(self.parsed_args.files, not self.parsed_args.unlock)
def lock_files(self, files, target):
cmd = 'lock' if target else 'unlock'
dvc/command/merge.py
+8 -9
View File
@@ -1,13 +1,13 @@
import os
import json
from dvc.command.base import CmdBase, DvcLock
from dvc.command.common.base import CmdBase
from dvc.config import ConfigI
from dvc.logger import Logger
from dvc.state_file import StateFile
from dvc.path.data_item import DataItem
from dvc.system import System
class CmdMerge(CmdBase):
def __init__(self, settings):
super(CmdMerge, self).__init__(settings)
@@ -63,12 +63,11 @@ class CmdMerge(CmdBase):
self.commit_if_needed(msg)
def run(self):
with DvcLock(self.is_locker, self.git):
targets = self.collect_targets()
if not targets:
return 1
targets = self.collect_targets()
if not targets:
return 1
self.checkout_targets(targets)
self.print_info(targets)
self.checkout_targets(targets)
self.print_info(targets)
return 0
return 0
dvc/command/remove.py
+8 -1
View File
@@ -1,6 +1,6 @@
import os
from dvc.command.traverse import Traverse
from dvc.command.common.traverse import Traverse
from dvc.logger import Logger
@@ -8,6 +8,13 @@ class CmdRemove(Traverse):
def __init__(self, settings):
super(CmdRemove, self).__init__(settings, "remove")
def run(self):
if not self.parsed_args.target:
Logger.error('Nothing specified, nothing removed.')
return 1
return super(CmdRemove, self).run()
def process_file(self, target):
Logger.debug(u'[Cmd-Remove] Remove file {}.'.format(target))
dvc/command/repro.py
+13 -16
View File
@@ -1,7 +1,6 @@
import os
import copy
from dvc.command.base import DvcLock
from dvc.command.import_file import CmdImportFile
from dvc.command.run import CmdRun
from dvc.logger import Logger
@@ -40,23 +39,21 @@ class CmdRepro(CmdRun):
return []
def run(self):
with DvcLock(self.is_locker, self.git):
recursive = not self.parsed_args.single_item
targets = []
recursive = not self.parsed_args.single_item
targets = []
if self.parsed_args.target:
targets += self.parsed_args.target
else:
target = self.settings.config._config['Global'].get('Target', None)
if not target or len(target) == 0:
Logger.error('Reproduction target is not defined. ' +
'Specify data file or set target by ' +
'`dvc config global.target target` command.')
return 1
targets += [target]
if self.parsed_args.target:
targets += self.parsed_args.target
else:
target = self.settings.config._config['Global'].get('Target', None)
if not target or len(target) == 0:
Logger.error('Reproduction target is not defined. ' +
'Specify data file or set target by ' +
'`dvc config global.target target` command.')
return 1
targets += [target]
return self.repro_target(targets, recursive, self.parsed_args.force)
pass
return self.repro_target(targets, recursive, self.parsed_args.force)
def repro_target(self, target, recursive, force):
if not self.no_git_actions and not self.git.is_ready_to_go():
dvc/command/run.py
+9 -13
View File
@@ -1,11 +1,9 @@
import os
import sys
from dvc.command.base import CmdBase, DvcLock
from dvc.command.common.base import CmdBase
from dvc.exceptions import DvcException
from dvc.git_wrapper import GitWrapper
from dvc.logger import Logger
from dvc.path.data_item import DataDirError, NotInGitDirError
from dvc.repository_change import RepositoryChange
from dvc.state_file import StateFile
from dvc.utils import cached_property
@@ -37,16 +35,14 @@ class CmdRun(CmdBase):
return self._data_items_from_params(self.parsed_args.output, 'Output')
def run(self):
with DvcLock(self.is_locker, self.git):
cmd = [self.parsed_args.command] + self.parsed_args.args
data_items_from_args, not_data_items_from_args = self.argv_files_by_type(cmd)
return self.run_and_commit_if_needed(cmd,
data_items_from_args,
not_data_items_from_args,
self.parsed_args.stdout,
self.parsed_args.stderr,
self.parsed_args.shell)
pass
cmd = [self.parsed_args.command] + self.parsed_args.args
data_items_from_args, not_data_items_from_args = self.argv_files_by_type(cmd)
return self.run_and_commit_if_needed(cmd,
data_items_from_args,
not_data_items_from_args,
self.parsed_args.stdout,
self.parsed_args.stderr,
self.parsed_args.shell)
def run_and_commit_if_needed(self, command_args, data_items_from_args, not_data_items_from_args,
stdout, stderr, shell,
dvc/command/show_pipeline.py
+1 -2
View File
@@ -1,8 +1,7 @@
import os
import networkx as nx
from dvc.command.traverse import Traverse
from dvc.command.init import CmdInit
from dvc.command.common.traverse import Traverse
from dvc.logger import Logger
from dvc.state_file import StateFile
dvc/command/show_workflow.py
+1 -1
View File
@@ -1,6 +1,6 @@
import os
from dvc.command.base import CmdBase
from dvc.command.common.base import CmdBase
from dvc.logger import Logger
dvc/git_wrapper.py
+6 -5
View File
@@ -317,16 +317,17 @@ class GitWrapper(GitWrapperI):
Executor.exec_cmd_only_success(cmd.split())
@staticmethod
def checkout(commit):
Logger.debug('[dvc-git] Checkout {}'.format(commit))
cmd = 'git checkout {}'.format(commit)
Executor.exec_cmd(cmd.split())
def checkout(commit_or_branch, create_new=False):
prefix = '-b ' if create_new else ''
Logger.debug('[dvc-git] Checkout {}{}'.format(prefix, commit_or_branch))
cmd = 'git checkout {}{}'.format(prefix, commit_or_branch)
return Executor.exec_cmd(cmd.split())
@staticmethod
def checkout_previous():
Logger.debug('[dvc-git] Checkout previous')
cmd = 'git checkout -'
Executor.exec_cmd(cmd.split())
return Executor.exec_cmd(cmd.split())
def separate_dependency_files_and_dirs(self, code_dependencies):
code_files = []
dvc/main.py
+1 -1
View File
@@ -18,7 +18,7 @@ def main():
return 255
try:
ret = instance.run()
ret = instance.run_cmd()
except Exception as e:
exc_info = settings.parsed_args.verbose
Logger.error("{} error: {}".format(instance.__class__.__name__, e), exc_info=exc_info)