README. Small fixes: data remove, locker

This commit is contained in:
Dmitry Petrov
2017-03-13 23:26:16 -07:00
parent 63c8aa6050
commit 4f6f2bfc89
6 changed files with 158 additions and 46 deletions
+26
View File
@@ -0,0 +1,26 @@
Company name: Capital Chi
Capital 𝛘
Logo1: C𝛘
Logo2: 𝛘 inside C
Short form: cpx
Product line
1) Free tool: Capital Chi (cpx)
2) Service: Capital Chi Service
3) Enterprise: Capital Chi enterprise (cpxe)
Value proposition:
Instead, I think you have to be able to articulate your value prop using a sentence like this: "Developers want to do X. But they don't want to do Y, so we do it. All they have to do is Z." The more specific and evidence-based this madlib is, the better.
instead of trying to make their lives easier, you should try and give them super powers or help them do something that's almost impossible.
I think a killer feature for any dev product is its trustworthiness.
FROM http://www.heavybit.com/library/video/developer-product-design/
+13
View File
@@ -3,6 +3,7 @@ import os
from neatlynx.git_wrapper import GitWrapper
from neatlynx.config import Config, ConfigError
from neatlynx.logger import Logger
class CmdBase(object):
@@ -48,6 +49,18 @@ class CmdBase(object):
def define_args(self):
pass
def set_skip_git_actions(self, parser):
parser.add_argument('--skip-git-actions', '-s', action='store_true',
help='skip all git actions including reproducibility check and commits')
@property
def skip_git_actions(self):
return self.args.skip_git_actions
@staticmethod
def not_committed_changes_warning():
Logger.warn('Warning: changes were not committed to git')
def add_string_arg(self, parser, name, message, default = None,
conf_section=None, conf_name=None):
if conf_section and conf_name:
+7 -1
View File
@@ -24,6 +24,8 @@ class CmdDataImport(CmdBase):
pass
def define_args(self, parser):
self.set_skip_git_actions(parser)
parser.add_argument('input',
metavar='',
help='Input file',
@@ -40,13 +42,17 @@ class CmdDataImport(CmdBase):
return 1
try:
if not self.git.is_ready_to_go():
if not self.skip_git_actions and not self.git.is_ready_to_go():
return 1
output = self.args.output
for file in self.args.input:
self.import_file(file, output)
if self.skip_git_actions:
self.not_committed_changes_warning()
return 0
message = 'NLX data import: {} {}'.format(' '.join(self.args.input), self.args.output)
self.git.commit_all_changes_and_log_status(message)
finally:
+28 -15
View File
@@ -19,7 +19,9 @@ class CmdDataRemove(CmdBase):
pass
def define_args(self, parser):
self.add_string_arg(parser, 'target', 'Target to remove - file or directory')
self.set_skip_git_actions(parser)
parser.add_argument('target', metavar='', help='Target to remove - file or directory', nargs='*')
parser.add_argument('-r', '--recursive', action='store_true', help='Remove directory recursively')
parser.add_argument('-c', '--remove-from-cloud', action='store_true', help='Keep file in cloud')
pass
@@ -32,26 +34,37 @@ class CmdDataRemove(CmdBase):
return 1
try:
target = self.args.target
if not self.skip_git_actions and not self.git.is_ready_to_go():
return 1
if os.path.isdir(target):
if not self.args.recursive:
raise DataRemoveError('Directory cannot be removed. Use --recurcive flag.')
for target in self.args.target:
self.remove_target(target)
if os.path.realpath(target) == \
os.path.realpath(os.path.join(self.git.git_dir_abs, self.config.data_dir)):
raise DataRemoveError('data directory cannot be removed')
if self.skip_git_actions:
self.not_committed_changes_warning()
return 0
return self.remove_dir(target)
dobj = DataFileObjExisting(target, self.git, self.config)
if os.path.islink(dobj.data_file_relative):
return self.remove_symlink(dobj.data_file_relative)
raise DataRemoveError('Cannot remove a regular file "{}"'.format(target))
message = 'NLX data remove: {}'.format(' '.join(self.args.target))
self.git.commit_all_changes_and_log_status(message)
finally:
lock.release()
def remove_target(self, target):
if os.path.isdir(target):
if not self.args.recursive:
raise DataRemoveError('Directory cannot be removed. Use --recurcive flag.')
if os.path.realpath(target) == \
os.path.realpath(os.path.join(self.git.git_dir_abs, self.config.data_dir)):
raise DataRemoveError('data directory cannot be removed')
return self.remove_dir(target)
dobj = DataFileObjExisting(target, self.git, self.config)
if os.path.islink(dobj.data_file_relative):
return self.remove_symlink(dobj.data_file_relative)
raise DataRemoveError('Cannot remove a regular file "{}"'.format(target))
def remove_symlink(self, file):
dobj = DataFileObjExisting(file, self.git, self.config)
+58 -24
View File
@@ -22,26 +22,53 @@ class CmdRun(CmdBase):
pass
def define_args(self, parser):
parser.add_argument('--ignore-git-status', help='ignore git status', action='store_true')
self.set_skip_git_actions(parser)
parser.add_argument('--random', help='not reproducible, output is random', action='store_true')
parser.add_argument('--stdout', help='output std output to a file')
parser.add_argument('--stderr', help='output std error to a file')
pass
def run(self):
lock = fasteners.InterProcessLock(self.lock.lock_file)
lock = fasteners.InterProcessLock(self.git.lock_file)
gotten = lock.acquire(timeout=5)
if not gotten:
Logger.info('Cannot perform the command since NLX is busy and locked. Please retry the command later.')
return 1
try:
if not self.args.ignore_git_status and not self.git.is_ready_to_go():
if not self.skip_git_actions and not self.git.is_ready_to_go():
return 1
old_files = set(self.get_changed_files())
GitWrapper.exec_cmd(self._args_unkn, self.args.stdout, self.args.stderr)
dobjs = self.get_new_file_objects()
new_files = set(self.get_changed_files())
removed_files = old_files - new_files
if removed_files != set():
Logger.error('Error: existing files were removed in run command'.format(' '.join(removed_files)))
created_files = new_files - old_files
if created_files == set():
Logger.error('Error: no files were changes in run command')
return 1
dobjs, errors = self.files_to_dobjs(created_files)
if errors:
for file in GitWrapper.abs_paths_to_relative(errors):
Logger.error('Error: file "{}" was created outside of the data directory'.format(file))
for file in created_files:
rel_path = GitWrapper.abs_paths_to_relative([file])[0]
Logger.error('Removing created file: {}'.format(rel_path))
os.remove(file)
Logger.error('Errors occurred. Please fix the errors and re-run the command.')
return 1
for dobj in dobjs:
os.makedirs(os.path.dirname(dobj.cache_file_relative), exist_ok=True)
@@ -53,6 +80,10 @@ class CmdRun(CmdBase):
state_file.save()
pass
if self.skip_git_actions:
self.not_committed_changes_warning()
return 0
message = 'NLX run: {}'.format(' '.join(sys.argv))
self.git.commit_all_changes_and_log_status(message)
finally:
@@ -60,30 +91,33 @@ class CmdRun(CmdBase):
return 0
def get_new_file_objects(self):
def files_to_dobjs(self, files):
result = []
errors = []
for file in files:
try:
result.append(DataFileObj(file, self.git, self.config))
except NotInDataDirError:
errors.append(file)
return result, errors
def get_changed_files(self):
statuses = GitWrapper.status_files()
error = False
dobjs = []
result = []
for status, file in statuses:
try:
file_path = os.path.join(self.git.git_dir_abs, file)
if os.path.isfile(file_path):
dobjs.append(DataFileObj(file_path, self.git, self.config))
else:
files = []
self.get_all_files_from_dir(file_path, files)
for f in files:
dobjs.append(DataFileObj(f, self.git, self.config))
except NotInDataDirError:
Logger.error('Error: file "{}" was created outside of the data directory'.format(file_path))
error = True
file_path = os.path.join(self.git.git_dir_abs, file)
if os.path.isfile(file_path):
result.append(file_path)
else:
files = []
self.get_all_files_from_dir(file_path, files)
for f in files:
result.append(f)
if error:
Logger.error('Please fix the errors and re-run the command')
return None
return dobjs
return result
def get_all_files_from_dir(self, dir, result):
if not os.path.isdir(dir):
+26 -6
View File
@@ -1,4 +1,5 @@
import os
import sys
import subprocess
from neatlynx.exceptions import NeatLynxException
@@ -37,15 +38,21 @@ class GitWrapper(GitWrapperI):
def exec_cmd(cmd, stdout_file=None, stderr_file=None, cwd=None):
stdout_fd = None
if stdout_file is not None:
stdout_fd = open(stdout_file, 'w')
stdout = stdout_fd
if stdout_file == '-':
stdout = sys.stdout
else:
stdout_fd = open(stdout_file, 'w')
stdout = stdout_fd
else:
stdout = subprocess.PIPE
stderr_fd = None
if stderr_file is not None:
stderr_fd = open(stderr_file, 'w')
stderr = stderr_fd
if stderr_file == '-':
stderr = sys.stderr
else:
stderr_fd = open(stderr_file, 'w')
stderr = stderr_fd
else:
stderr = subprocess.PIPE
@@ -65,13 +72,16 @@ class GitWrapper(GitWrapperI):
def exec_cmd_only_success(cmd, stdout_file=None, stderr_file=None, cwd=None):
code, out, err = GitWrapper.exec_cmd(cmd, stdout_file=None, stderr_file=None, cwd=None)
if code != 0:
raise GitCmdError('Git command error ({}): {}'.format(' '.join(cmd), err))
if err:
sys.stderr.write(err + '\n')
raise GitCmdError('Git command error ({}):\n{}'.format(' '.join(cmd), out))
return out
def is_ready_to_go(self):
statuses = self.status_files()
if len(statuses) > 0:
Logger.error('Commit changed files before reproducible command (nlx-repro):')
Logger.error('Commit all changed files before running reproducible command.')
Logger.error('Changed files:')
for status, file in statuses:
Logger.error("{} {}".format(status, file))
return False
@@ -141,3 +151,13 @@ class GitWrapper(GitWrapperI):
@property
def lock_file(self):
return os.path.join(self.git_dir_abs, '.' + Config.CONFIG + '.lock')
@staticmethod
def abs_paths_to_relative(files):
cur_dir = os.path.realpath(os.curdir)
result = []
for file in files:
result.append(os.path.relpath(file, cur_dir))
return result