mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 23:11:44 +01:00
README. Small fixes: data remove, locker
This commit is contained in:
@@ -0,0 +1,26 @@
|
||||
Company name: Capital Chi
|
||||
Capital 𝛘
|
||||
Logo1: C𝛘
|
||||
Logo2: 𝛘 inside C
|
||||
|
||||
Short form: cpx
|
||||
|
||||
|
||||
Product line
|
||||
|
||||
1) Free tool: Capital Chi (cpx)
|
||||
2) Service: Capital Chi Service
|
||||
3) Enterprise: Capital Chi enterprise (cpxe)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Value proposition:
|
||||
Instead, I think you have to be able to articulate your value prop using a sentence like this: "Developers want to do X. But they don't want to do Y, so we do it. All they have to do is Z." The more specific and evidence-based this madlib is, the better.
|
||||
|
||||
instead of trying to make their lives easier, you should try and give them super powers or help them do something that's almost impossible.
|
||||
|
||||
I think a killer feature for any dev product is its trustworthiness.
|
||||
|
||||
FROM http://www.heavybit.com/library/video/developer-product-design/
|
||||
@@ -3,6 +3,7 @@ import os
|
||||
|
||||
from neatlynx.git_wrapper import GitWrapper
|
||||
from neatlynx.config import Config, ConfigError
|
||||
from neatlynx.logger import Logger
|
||||
|
||||
|
||||
class CmdBase(object):
|
||||
@@ -48,6 +49,18 @@ class CmdBase(object):
|
||||
def define_args(self):
|
||||
pass
|
||||
|
||||
def set_skip_git_actions(self, parser):
|
||||
parser.add_argument('--skip-git-actions', '-s', action='store_true',
|
||||
help='skip all git actions including reproducibility check and commits')
|
||||
|
||||
@property
|
||||
def skip_git_actions(self):
|
||||
return self.args.skip_git_actions
|
||||
|
||||
@staticmethod
|
||||
def not_committed_changes_warning():
|
||||
Logger.warn('Warning: changes were not committed to git')
|
||||
|
||||
def add_string_arg(self, parser, name, message, default = None,
|
||||
conf_section=None, conf_name=None):
|
||||
if conf_section and conf_name:
|
||||
|
||||
@@ -24,6 +24,8 @@ class CmdDataImport(CmdBase):
|
||||
pass
|
||||
|
||||
def define_args(self, parser):
|
||||
self.set_skip_git_actions(parser)
|
||||
|
||||
parser.add_argument('input',
|
||||
metavar='',
|
||||
help='Input file',
|
||||
@@ -40,13 +42,17 @@ class CmdDataImport(CmdBase):
|
||||
return 1
|
||||
|
||||
try:
|
||||
if not self.git.is_ready_to_go():
|
||||
if not self.skip_git_actions and not self.git.is_ready_to_go():
|
||||
return 1
|
||||
|
||||
output = self.args.output
|
||||
for file in self.args.input:
|
||||
self.import_file(file, output)
|
||||
|
||||
if self.skip_git_actions:
|
||||
self.not_committed_changes_warning()
|
||||
return 0
|
||||
|
||||
message = 'NLX data import: {} {}'.format(' '.join(self.args.input), self.args.output)
|
||||
self.git.commit_all_changes_and_log_status(message)
|
||||
finally:
|
||||
|
||||
+28
-15
@@ -19,7 +19,9 @@ class CmdDataRemove(CmdBase):
|
||||
pass
|
||||
|
||||
def define_args(self, parser):
|
||||
self.add_string_arg(parser, 'target', 'Target to remove - file or directory')
|
||||
self.set_skip_git_actions(parser)
|
||||
|
||||
parser.add_argument('target', metavar='', help='Target to remove - file or directory', nargs='*')
|
||||
parser.add_argument('-r', '--recursive', action='store_true', help='Remove directory recursively')
|
||||
parser.add_argument('-c', '--remove-from-cloud', action='store_true', help='Keep file in cloud')
|
||||
pass
|
||||
@@ -32,26 +34,37 @@ class CmdDataRemove(CmdBase):
|
||||
return 1
|
||||
|
||||
try:
|
||||
target = self.args.target
|
||||
if not self.skip_git_actions and not self.git.is_ready_to_go():
|
||||
return 1
|
||||
|
||||
if os.path.isdir(target):
|
||||
if not self.args.recursive:
|
||||
raise DataRemoveError('Directory cannot be removed. Use --recurcive flag.')
|
||||
for target in self.args.target:
|
||||
self.remove_target(target)
|
||||
|
||||
if os.path.realpath(target) == \
|
||||
os.path.realpath(os.path.join(self.git.git_dir_abs, self.config.data_dir)):
|
||||
raise DataRemoveError('data directory cannot be removed')
|
||||
if self.skip_git_actions:
|
||||
self.not_committed_changes_warning()
|
||||
return 0
|
||||
|
||||
return self.remove_dir(target)
|
||||
|
||||
dobj = DataFileObjExisting(target, self.git, self.config)
|
||||
if os.path.islink(dobj.data_file_relative):
|
||||
return self.remove_symlink(dobj.data_file_relative)
|
||||
|
||||
raise DataRemoveError('Cannot remove a regular file "{}"'.format(target))
|
||||
message = 'NLX data remove: {}'.format(' '.join(self.args.target))
|
||||
self.git.commit_all_changes_and_log_status(message)
|
||||
finally:
|
||||
lock.release()
|
||||
|
||||
def remove_target(self, target):
|
||||
if os.path.isdir(target):
|
||||
if not self.args.recursive:
|
||||
raise DataRemoveError('Directory cannot be removed. Use --recurcive flag.')
|
||||
|
||||
if os.path.realpath(target) == \
|
||||
os.path.realpath(os.path.join(self.git.git_dir_abs, self.config.data_dir)):
|
||||
raise DataRemoveError('data directory cannot be removed')
|
||||
|
||||
return self.remove_dir(target)
|
||||
dobj = DataFileObjExisting(target, self.git, self.config)
|
||||
if os.path.islink(dobj.data_file_relative):
|
||||
return self.remove_symlink(dobj.data_file_relative)
|
||||
|
||||
raise DataRemoveError('Cannot remove a regular file "{}"'.format(target))
|
||||
|
||||
def remove_symlink(self, file):
|
||||
dobj = DataFileObjExisting(file, self.git, self.config)
|
||||
|
||||
|
||||
+58
-24
@@ -22,26 +22,53 @@ class CmdRun(CmdBase):
|
||||
pass
|
||||
|
||||
def define_args(self, parser):
|
||||
parser.add_argument('--ignore-git-status', help='ignore git status', action='store_true')
|
||||
self.set_skip_git_actions(parser)
|
||||
|
||||
parser.add_argument('--random', help='not reproducible, output is random', action='store_true')
|
||||
parser.add_argument('--stdout', help='output std output to a file')
|
||||
parser.add_argument('--stderr', help='output std error to a file')
|
||||
pass
|
||||
|
||||
def run(self):
|
||||
lock = fasteners.InterProcessLock(self.lock.lock_file)
|
||||
lock = fasteners.InterProcessLock(self.git.lock_file)
|
||||
gotten = lock.acquire(timeout=5)
|
||||
if not gotten:
|
||||
Logger.info('Cannot perform the command since NLX is busy and locked. Please retry the command later.')
|
||||
return 1
|
||||
|
||||
try:
|
||||
if not self.args.ignore_git_status and not self.git.is_ready_to_go():
|
||||
if not self.skip_git_actions and not self.git.is_ready_to_go():
|
||||
return 1
|
||||
|
||||
old_files = set(self.get_changed_files())
|
||||
|
||||
GitWrapper.exec_cmd(self._args_unkn, self.args.stdout, self.args.stderr)
|
||||
|
||||
dobjs = self.get_new_file_objects()
|
||||
new_files = set(self.get_changed_files())
|
||||
|
||||
removed_files = old_files - new_files
|
||||
if removed_files != set():
|
||||
Logger.error('Error: existing files were removed in run command'.format(' '.join(removed_files)))
|
||||
|
||||
created_files = new_files - old_files
|
||||
|
||||
if created_files == set():
|
||||
Logger.error('Error: no files were changes in run command')
|
||||
return 1
|
||||
|
||||
dobjs, errors = self.files_to_dobjs(created_files)
|
||||
|
||||
if errors:
|
||||
for file in GitWrapper.abs_paths_to_relative(errors):
|
||||
Logger.error('Error: file "{}" was created outside of the data directory'.format(file))
|
||||
|
||||
for file in created_files:
|
||||
rel_path = GitWrapper.abs_paths_to_relative([file])[0]
|
||||
Logger.error('Removing created file: {}'.format(rel_path))
|
||||
os.remove(file)
|
||||
|
||||
Logger.error('Errors occurred. Please fix the errors and re-run the command.')
|
||||
return 1
|
||||
|
||||
for dobj in dobjs:
|
||||
os.makedirs(os.path.dirname(dobj.cache_file_relative), exist_ok=True)
|
||||
@@ -53,6 +80,10 @@ class CmdRun(CmdBase):
|
||||
state_file.save()
|
||||
pass
|
||||
|
||||
if self.skip_git_actions:
|
||||
self.not_committed_changes_warning()
|
||||
return 0
|
||||
|
||||
message = 'NLX run: {}'.format(' '.join(sys.argv))
|
||||
self.git.commit_all_changes_and_log_status(message)
|
||||
finally:
|
||||
@@ -60,30 +91,33 @@ class CmdRun(CmdBase):
|
||||
|
||||
return 0
|
||||
|
||||
def get_new_file_objects(self):
|
||||
def files_to_dobjs(self, files):
|
||||
result = []
|
||||
errors = []
|
||||
|
||||
for file in files:
|
||||
try:
|
||||
result.append(DataFileObj(file, self.git, self.config))
|
||||
except NotInDataDirError:
|
||||
errors.append(file)
|
||||
|
||||
return result, errors
|
||||
|
||||
def get_changed_files(self):
|
||||
statuses = GitWrapper.status_files()
|
||||
|
||||
error = False
|
||||
dobjs = []
|
||||
result = []
|
||||
for status, file in statuses:
|
||||
try:
|
||||
file_path = os.path.join(self.git.git_dir_abs, file)
|
||||
if os.path.isfile(file_path):
|
||||
dobjs.append(DataFileObj(file_path, self.git, self.config))
|
||||
else:
|
||||
files = []
|
||||
self.get_all_files_from_dir(file_path, files)
|
||||
for f in files:
|
||||
dobjs.append(DataFileObj(f, self.git, self.config))
|
||||
except NotInDataDirError:
|
||||
Logger.error('Error: file "{}" was created outside of the data directory'.format(file_path))
|
||||
error = True
|
||||
file_path = os.path.join(self.git.git_dir_abs, file)
|
||||
if os.path.isfile(file_path):
|
||||
result.append(file_path)
|
||||
else:
|
||||
files = []
|
||||
self.get_all_files_from_dir(file_path, files)
|
||||
for f in files:
|
||||
result.append(f)
|
||||
|
||||
if error:
|
||||
Logger.error('Please fix the errors and re-run the command')
|
||||
return None
|
||||
|
||||
return dobjs
|
||||
return result
|
||||
|
||||
def get_all_files_from_dir(self, dir, result):
|
||||
if not os.path.isdir(dir):
|
||||
|
||||
+26
-6
@@ -1,4 +1,5 @@
|
||||
import os
|
||||
import sys
|
||||
import subprocess
|
||||
|
||||
from neatlynx.exceptions import NeatLynxException
|
||||
@@ -37,15 +38,21 @@ class GitWrapper(GitWrapperI):
|
||||
def exec_cmd(cmd, stdout_file=None, stderr_file=None, cwd=None):
|
||||
stdout_fd = None
|
||||
if stdout_file is not None:
|
||||
stdout_fd = open(stdout_file, 'w')
|
||||
stdout = stdout_fd
|
||||
if stdout_file == '-':
|
||||
stdout = sys.stdout
|
||||
else:
|
||||
stdout_fd = open(stdout_file, 'w')
|
||||
stdout = stdout_fd
|
||||
else:
|
||||
stdout = subprocess.PIPE
|
||||
|
||||
stderr_fd = None
|
||||
if stderr_file is not None:
|
||||
stderr_fd = open(stderr_file, 'w')
|
||||
stderr = stderr_fd
|
||||
if stderr_file == '-':
|
||||
stderr = sys.stderr
|
||||
else:
|
||||
stderr_fd = open(stderr_file, 'w')
|
||||
stderr = stderr_fd
|
||||
else:
|
||||
stderr = subprocess.PIPE
|
||||
|
||||
@@ -65,13 +72,16 @@ class GitWrapper(GitWrapperI):
|
||||
def exec_cmd_only_success(cmd, stdout_file=None, stderr_file=None, cwd=None):
|
||||
code, out, err = GitWrapper.exec_cmd(cmd, stdout_file=None, stderr_file=None, cwd=None)
|
||||
if code != 0:
|
||||
raise GitCmdError('Git command error ({}): {}'.format(' '.join(cmd), err))
|
||||
if err:
|
||||
sys.stderr.write(err + '\n')
|
||||
raise GitCmdError('Git command error ({}):\n{}'.format(' '.join(cmd), out))
|
||||
return out
|
||||
|
||||
def is_ready_to_go(self):
|
||||
statuses = self.status_files()
|
||||
if len(statuses) > 0:
|
||||
Logger.error('Commit changed files before reproducible command (nlx-repro):')
|
||||
Logger.error('Commit all changed files before running reproducible command.')
|
||||
Logger.error('Changed files:')
|
||||
for status, file in statuses:
|
||||
Logger.error("{} {}".format(status, file))
|
||||
return False
|
||||
@@ -141,3 +151,13 @@ class GitWrapper(GitWrapperI):
|
||||
@property
|
||||
def lock_file(self):
|
||||
return os.path.join(self.git_dir_abs, '.' + Config.CONFIG + '.lock')
|
||||
|
||||
@staticmethod
|
||||
def abs_paths_to_relative(files):
|
||||
cur_dir = os.path.realpath(os.curdir)
|
||||
|
||||
result = []
|
||||
for file in files:
|
||||
result.append(os.path.relpath(file, cur_dir))
|
||||
|
||||
return result
|
||||
Reference in New Issue
Block a user