Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-08 15:28:26 +01:00
Code Issues Packages Projects Releases Wiki Activity

Use DvcLock from CmdBase class only

Browse Source
This commit is contained in:
Dmitry
2017-11-19 04:14:12 -08:00
parent 47e72559f2
commit db625e8213
16 changed files with 133 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/command/checkout.py
+11 -12
View File
@@ -1,6 +1,6 @@
import os
from dvc.command.common.base import CmdBase, DvcLock
from dvc.command.common.base import CmdBase
from dvc.logger import Logger
from dvc.system import System
@@ -34,19 +34,18 @@ class CmdCheckout(CmdBase):
Logger.info('Checkout \'{}\''.format(item.data.relative))
def run(self):
with DvcLock(self.is_locker, self.git):
try:
states = self.git.state_files_for_previous_commit()
prev_items = self.settings.path_factory.data_items_from_states(states, existing=False)
except Exception as ex:
Logger.error(u'Unable to get data files from previous commit: {}'.format(ex))
return 1
try:
states = self.git.state_files_for_previous_commit()
prev_items = self.settings.path_factory.data_items_from_states(states, existing=False)
except Exception as ex:
Logger.error(u'Unable to get data files from previous commit: {}'.format(ex))
return 1
curr_items = self.settings.path_factory.all_existing_data_items()
self.checkout(curr_items)
curr_items = self.settings.path_factory.all_existing_data_items()
self.checkout(curr_items)
self.remove_files(list(set(prev_items) - set(curr_items)))
return 0
self.remove_files(list(set(prev_items) - set(curr_items)))
return 0
@staticmethod
def remove_files(removed_items_set):
dvc/command/common/base.py
+10 -9
View File
@@ -45,12 +45,6 @@ class CmdBase(object):
def set_git_action(self, value):
self.parsed_args.no_git_actions = not value
@property
def is_locker(self):
if 'no_lock' in self.parsed_args.__dict__:
return not self.parsed_args.no_lock
return True
def set_locker(self, value):
self.parsed_args.no_lock = value
@@ -67,9 +61,16 @@ class CmdBase(object):
Logger.warn('changes were not committed to git')
def run_cmd(self):
with BranchChanger(self.parsed_args.branch, self.parsed_args.new_branch, self.git):
return self.run()
with DvcLock(self.is_locker, self.git):
with BranchChanger(self.parsed_args.branch, self.parsed_args.new_branch, self.git):
return self.run()
# Abstract method that has to be implemented by any inheritance class
# Abstract methods that have to be implemented by any inheritance class
def run(self):
pass
@property
def is_locker(self):
if 'no_lock' in self.parsed_args.__dict__:
return not self.parsed_args.no_lock
return True
dvc/command/common/branch_changer.py
-1
View File
@@ -19,7 +19,6 @@ class BranchChanger(object):
def __enter__(self):
if self.perform_action:
print('========PERFORM')
self.git.checkout(self.branch, self.create_new)
return self
dvc/command/common/dvc_lock.py
+3 -1
View File
@@ -9,6 +9,8 @@ class DvcLockerError(CmdCommonError):
class DvcLock(object):
TIMEOUT = 5
def __init__(self, is_locker, git):
self.is_locker = is_locker
self.git = git
@@ -17,7 +19,7 @@ class DvcLock(object):
def __enter__(self):
if self.is_locker:
self.lock = fasteners.InterProcessLock(self.git.lock_file)
if not self.lock.acquire(timeout=5):
if not self.lock.acquire(timeout=self.TIMEOUT):
raise DvcLockerError('Cannot perform the cmd since DVC is busy and locked. Please retry the cmd later.')
return self.lock
dvc/command/config.py
+2 -6
View File
@@ -1,7 +1,7 @@
import os
import configobj
from dvc.command.common.base import CmdBase, DvcLock
from dvc.command.common.base import CmdBase
from dvc.logger import Logger
from dvc.config import Config
@@ -75,7 +75,7 @@ class CmdConfig(CmdBase):
return 0
def _run(self):
def run(self):
if self.check_opt() != 0:
return 1
@@ -86,7 +86,3 @@ class CmdConfig(CmdBase):
return self.show()
return self.set()
def run(self):
with DvcLock(self.is_locker, self.git):
return self._run()
dvc/command/data_sync.py
+7 -11
View File
@@ -1,4 +1,4 @@
from dvc.command.common.base import CmdBase, DvcLock
from dvc.command.common.base import CmdBase
from dvc.logger import Logger
import dvc.data_cloud as cloud
@@ -9,8 +9,7 @@ class CmdDataPull(CmdBase):
super(CmdDataPull, self).__init__(settings)
def run(self):
with DvcLock(self.is_locker, self.git):
self.cloud.pull(self.parsed_args.targets, self.parsed_args.jobs)
self.cloud.pull(self.parsed_args.targets, self.parsed_args.jobs)
class CmdDataPush(CmdBase):
@@ -18,8 +17,8 @@ class CmdDataPush(CmdBase):
super(CmdDataPush, self).__init__(settings)
def run(self):
with DvcLock(self.is_locker, self.git):
self.cloud.push(self.parsed_args.targets, self.parsed_args.jobs)
self.cloud.push(self.parsed_args.targets, self.parsed_args.jobs)
class CmdDataStatus(CmdBase):
def __init__(self, settings):
@@ -41,9 +40,6 @@ class CmdDataStatus(CmdBase):
Logger.info('\t{}\t{}'.format(prefix_map[ret], target.data.dvc))
def run(self):
with DvcLock(self.is_locker, self.git):
status = self.cloud.status(self.parsed_args.targets, self.parsed_args.jobs)
self._show(status)
return 0
status = self.cloud.status(self.parsed_args.targets, self.parsed_args.jobs)
self._show(status)
return 0
dvc/command/find.py
+14 -15
View File
@@ -1,5 +1,5 @@
from dvc import utils
from dvc.command.common.base import CmdBase, DvcLock
from dvc.command.common.base import CmdBase
from dvc.executor import ExecutorError
from dvc.logger import Logger
@@ -9,23 +9,22 @@ class CmdFind(CmdBase):
super(CmdFind, self).__init__(settings)
def run(self):
with DvcLock(self.is_locker, self.git):
fname = self.parsed_args.target
branch_name = self.parsed_args.branch_name
fname = self.parsed_args.target
branch_name = self.parsed_args.branch_name
branches = self.git.branches(branch_name)
metrics = [self.read_metrics(fname, b) for b in branches]
branches = self.git.branches(branch_name)
metrics = [self.read_metrics(fname, b) for b in branches]
zipped = zip(metrics, branches)
res = self.filter(zipped, self.parsed_args.criteria)
res = filter(lambda x: x[0] is not None, res)
zipped = zip(metrics, branches)
res = self.filter(zipped, self.parsed_args.criteria)
res = filter(lambda x: x[0] is not None, res)
for (val, branch) in res:
if self.parsed_args.show_value:
print('{}: {}'.format(branch, val))
else:
print(branch)
return 0
for (val, branch) in res:
if self.parsed_args.show_value:
print('{}: {}'.format(branch, val))
else:
print(branch)
return 0
@staticmethod
def filter(zipped, criteria):
dvc/command/gc.py
+11 -13
View File
@@ -1,6 +1,5 @@
import os
from dvc.command.common.base import DvcLock
from dvc.command.traverse import Traverse
from dvc.logger import Logger
from dvc.path.data_item import DataItemError
@@ -13,20 +12,19 @@ class CmdGC(Traverse):
self.clist = []
def run(self):
with DvcLock(self.is_locker, self.git):
if not self._traverse(self.git.git_dir_abs):
Logger.error('Failed to collect used cache')
return 1
if not self._traverse(self.git.git_dir_abs):
Logger.error('Failed to collect used cache')
return 1
for cache in os.listdir(ConfigI.CACHE_DIR):
fname = os.path.join(ConfigI.CACHE_DIR, cache)
if fname in self.clist:
continue
os.remove(fname)
self._remove_cloud_cache(self.settings.path_factory.data_item('.empty', fname))
Logger.info('Cache \'{}\' was removed'.format(fname))
for cache in os.listdir(ConfigI.CACHE_DIR):
fname = os.path.join(ConfigI.CACHE_DIR, cache)
if fname in self.clist:
continue
os.remove(fname)
self._remove_cloud_cache(self.settings.path_factory.data_item('.empty', fname))
Logger.info('Cache \'{}\' was removed'.format(fname))
return 0
return 0
def process_file(self, target):
Logger.debug(u'[Cmd-GC] GC file {}.'.format(target))
dvc/command/import_file.py
+22 -26
View File
@@ -1,5 +1,4 @@
import os
import re
try:
from urlparse import urlparse
@@ -8,12 +7,11 @@ except ImportError:
# pylint: disable=no-name-in-module, import-error
from urllib.parse import urlparse
from dvc.command.common.base import CmdBase, DvcLock
from dvc.data_cloud import sizeof_fmt, file_md5
from dvc.command.common.base import CmdBase
from dvc.logger import Logger
from dvc.exceptions import DvcException
from dvc.state_file import StateFile
from dvc.system import System
class ImportFileError(DvcException):
def __init__(self, msg):
@@ -26,32 +24,30 @@ class CmdImportFile(CmdBase):
def run(self):
targets = []
with DvcLock(self.is_locker, self.git):
output = self.parsed_args.output
self.verify_output(output, self.parsed_args.input)
output = self.parsed_args.output
self.verify_output(output, self.parsed_args.input)
for input in self.parsed_args.input:
if not os.path.isdir(input):
targets.append((input, output))
else:
input_dir = os.path.basename(input)
for root, dirs, files in os.walk(input):
for file in files:
filename = os.path.join(root, file)
for input in self.parsed_args.input:
if not os.path.isdir(input):
targets.append((input, output))
else:
input_dir = os.path.basename(input)
for root, dirs, files in os.walk(input):
for file in files:
filename = os.path.join(root, file)
rel = os.path.relpath(filename, input)
out = os.path.join(output, input_dir, rel)
rel = os.path.relpath(filename, input)
out = os.path.join(output, input_dir, rel)
out_dir = os.path.dirname(out)
if not os.path.exists(out_dir):
os.mkdir(out_dir)
out_dir = os.path.dirname(out)
if not os.path.exists(out_dir):
os.mkdir(out_dir)
targets.append((filename, out))
pass
self.import_files(targets, self.parsed_args.lock, self.parsed_args.jobs)
message = 'DVC import files: {} -> {}'.format(str(self.parsed_args.input), output)
self.commit_if_needed(message)
pass
targets.append((filename, out))
pass
self.import_files(targets, self.parsed_args.lock, self.parsed_args.jobs)
message = 'DVC import files: {} -> {}'.format(str(self.parsed_args.input), output)
self.commit_if_needed(message)
@staticmethod
def verify_output(output, input):
dvc/command/init.py
+4
View File
@@ -62,6 +62,10 @@ ProjectName =
def __init__(self, settings):
super(CmdInit, self).__init__(settings)
@property
def is_locker(self):
return False
def get_not_existing_path(self, *args):
path = Path(os.path.join(self.git.git_dir, *args))
if path.exists():
dvc/command/instance_create.py
+14 -15
View File
@@ -1,5 +1,5 @@
from dvc.cloud.instance_manager import InstanceManager
from dvc.command.common.base import CmdBase, DvcLock
from dvc.command.common.base import CmdBase
from dvc.exceptions import DvcException
from dvc.logger import Logger
@@ -9,21 +9,20 @@ class CmdInstanceCreate(CmdBase):
super(CmdInstanceCreate, self).__init__(settings)
def run(self):
with DvcLock(self.is_locker, self.git):
name = self.parsed_args.name
cloud = self.parsed_args.cloud or self.settings.config.cloud
name = self.parsed_args.name
cloud = self.parsed_args.cloud or self.settings.config.cloud
# print('NAME: {}'.format(name))
# print('CLOUD: {}'.format(cloud))
# print('NAME: {}'.format(name))
# print('CLOUD: {}'.format(cloud))
if not name:
Logger.error('Instance name is not defined')
return 1
if not name:
Logger.error('Instance name is not defined')
return 1
try:
InstanceManager().create(name, cloud, self.parsed_args, self.settings.config)
except DvcException as ex:
Logger.error('Instance creation error: {}'.format(ex))
return 1
try:
InstanceManager().create(name, cloud, self.parsed_args, self.settings.config)
except DvcException as ex:
Logger.error('Instance creation error: {}'.format(ex))
return 1
return 0
return 0
dvc/command/lock.py
+2 -3
View File
@@ -1,4 +1,4 @@
from dvc.command.common.base import CmdBase, DvcLock
from dvc.command.common.base import CmdBase
from dvc.logger import Logger
from dvc.state_file import StateFile
@@ -8,8 +8,7 @@ class CmdLock(CmdBase):
super(CmdLock, self).__init__(settings)
def run(self):
with DvcLock(self.is_locker, self.git):
return self.lock_files(self.parsed_args.files, not self.parsed_args.unlock)
return self.lock_files(self.parsed_args.files, not self.parsed_args.unlock)
def lock_files(self, files, target):
cmd = 'lock' if target else 'unlock'
dvc/command/merge.py
+7 -8
View File
@@ -1,6 +1,6 @@
import os
from dvc.command.common.base import CmdBase, DvcLock
from dvc.command.common.base import CmdBase
from dvc.config import ConfigI
from dvc.logger import Logger
from dvc.state_file import StateFile
@@ -63,12 +63,11 @@ class CmdMerge(CmdBase):
self.commit_if_needed(msg)
def run(self):
with DvcLock(self.is_locker, self.git):
targets = self.collect_targets()
if not targets:
return 1
targets = self.collect_targets()
if not targets:
return 1
self.checkout_targets(targets)
self.print_info(targets)
self.checkout_targets(targets)
self.print_info(targets)
return 0
return 0
dvc/command/repro.py
+13 -16
View File
@@ -1,7 +1,6 @@
import os
import copy
from dvc.command.common.base import DvcLock
from dvc.command.import_file import CmdImportFile
from dvc.command.run import CmdRun
from dvc.logger import Logger
@@ -40,23 +39,21 @@ class CmdRepro(CmdRun):
return []
def run(self):
with DvcLock(self.is_locker, self.git):
recursive = not self.parsed_args.single_item
targets = []
recursive = not self.parsed_args.single_item
targets = []
if self.parsed_args.target:
targets += self.parsed_args.target
else:
target = self.settings.config._config['Global'].get('Target', None)
if not target or len(target) == 0:
Logger.error('Reproduction target is not defined. ' +
'Specify data file or set target by ' +
'`dvc config global.target target` command.')
return 1
targets += [target]
if self.parsed_args.target:
targets += self.parsed_args.target
else:
target = self.settings.config._config['Global'].get('Target', None)
if not target or len(target) == 0:
Logger.error('Reproduction target is not defined. ' +
'Specify data file or set target by ' +
'`dvc config global.target target` command.')
return 1
targets += [target]
return self.repro_target(targets, recursive, self.parsed_args.force)
pass
return self.repro_target(targets, recursive, self.parsed_args.force)
def repro_target(self, target, recursive, force):
if not self.no_git_actions and not self.git.is_ready_to_go():
dvc/command/run.py
+9 -11
View File
@@ -1,6 +1,6 @@
import os
from dvc.command.common.base import CmdBase, DvcLock
from dvc.command.common.base import CmdBase
from dvc.exceptions import DvcException
from dvc.git_wrapper import GitWrapper
from dvc.logger import Logger
@@ -35,16 +35,14 @@ class CmdRun(CmdBase):
return self._data_items_from_params(self.parsed_args.output, 'Output')
def run(self):
with DvcLock(self.is_locker, self.git):
cmd = [self.parsed_args.command] + self.parsed_args.args
data_items_from_args, not_data_items_from_args = self.argv_files_by_type(cmd)
return self.run_and_commit_if_needed(cmd,
data_items_from_args,
not_data_items_from_args,
self.parsed_args.stdout,
self.parsed_args.stderr,
self.parsed_args.shell)
pass
cmd = [self.parsed_args.command] + self.parsed_args.args
data_items_from_args, not_data_items_from_args = self.argv_files_by_type(cmd)
return self.run_and_commit_if_needed(cmd,
data_items_from_args,
not_data_items_from_args,
self.parsed_args.stdout,
self.parsed_args.stderr,
self.parsed_args.shell)
def run_and_commit_if_needed(self, command_args, data_items_from_args, not_data_items_from_args,
stdout, stderr, shell,
dvc/command/traverse.py
+4 -4
View File
@@ -1,10 +1,11 @@
import os
from dvc.command.common.base import CmdBase, DvcLock
from dvc.command.common.base import CmdBase
from dvc.exceptions import DvcException
from dvc.logger import Logger
from dvc.path.data_item import DataItemError
class TraverseError(DvcException):
def __init__(self, msg):
DvcException.__init__(self, 'Traverse file tree error: {}'.format(msg))
@@ -17,9 +18,8 @@ class Traverse(CmdBase):
self._do_not_start_from_root = do_not_start_from_root
def run(self):
with DvcLock(self.is_locker, self.git):
if not self._traverse_all():
return 1
if not self._traverse_all():
return 1
return 0