diff --git a/README b/README new file mode 100644 index 000000000..af78cb8ad --- /dev/null +++ b/README @@ -0,0 +1,26 @@ +Company name: Capital Chi +Capital 𝛘 +Logo1: C𝛘 +Logo2: 𝛘 inside C + +Short form: cpx + + + Product line + +1) Free tool: Capital Chi (cpx) +2) Service: Capital Chi Service +3) Enterprise: Capital Chi enterprise (cpxe) + + + + + +Value proposition: +Instead, I think you have to be able to articulate your value prop using a sentence like this: "Developers want to do X. But they don't want to do Y, so we do it. All they have to do is Z." The more specific and evidence-based this madlib is, the better. + + instead of trying to make their lives easier, you should try and give them super powers or help them do something that's almost impossible. + + I think a killer feature for any dev product is its trustworthiness. + +FROM http://www.heavybit.com/library/video/developer-product-design/ diff --git a/neatlynx/cmd_base.py b/neatlynx/cmd_base.py index 5836c507f..4469d21b2 100644 --- a/neatlynx/cmd_base.py +++ b/neatlynx/cmd_base.py @@ -3,6 +3,7 @@ import os from neatlynx.git_wrapper import GitWrapper from neatlynx.config import Config, ConfigError +from neatlynx.logger import Logger class CmdBase(object): @@ -48,6 +49,18 @@ class CmdBase(object): def define_args(self): pass + def set_skip_git_actions(self, parser): + parser.add_argument('--skip-git-actions', '-s', action='store_true', + help='skip all git actions including reproducibility check and commits') + + @property + def skip_git_actions(self): + return self.args.skip_git_actions + + @staticmethod + def not_committed_changes_warning(): + Logger.warn('Warning: changes were not committed to git') + def add_string_arg(self, parser, name, message, default = None, conf_section=None, conf_name=None): if conf_section and conf_name: diff --git a/neatlynx/cmd_data_import.py b/neatlynx/cmd_data_import.py index 01623b17e..9cdc3e8cb 100644 --- a/neatlynx/cmd_data_import.py +++ b/neatlynx/cmd_data_import.py @@ -24,6 +24,8 @@ class CmdDataImport(CmdBase): pass def define_args(self, parser): + self.set_skip_git_actions(parser) + parser.add_argument('input', metavar='', help='Input file', @@ -40,13 +42,17 @@ class CmdDataImport(CmdBase): return 1 try: - if not self.git.is_ready_to_go(): + if not self.skip_git_actions and not self.git.is_ready_to_go(): return 1 output = self.args.output for file in self.args.input: self.import_file(file, output) + if self.skip_git_actions: + self.not_committed_changes_warning() + return 0 + message = 'NLX data import: {} {}'.format(' '.join(self.args.input), self.args.output) self.git.commit_all_changes_and_log_status(message) finally: diff --git a/neatlynx/cmd_data_remove.py b/neatlynx/cmd_data_remove.py index fb61ad47b..15859841b 100644 --- a/neatlynx/cmd_data_remove.py +++ b/neatlynx/cmd_data_remove.py @@ -19,7 +19,9 @@ class CmdDataRemove(CmdBase): pass def define_args(self, parser): - self.add_string_arg(parser, 'target', 'Target to remove - file or directory') + self.set_skip_git_actions(parser) + + parser.add_argument('target', metavar='', help='Target to remove - file or directory', nargs='*') parser.add_argument('-r', '--recursive', action='store_true', help='Remove directory recursively') parser.add_argument('-c', '--remove-from-cloud', action='store_true', help='Keep file in cloud') pass @@ -32,26 +34,37 @@ class CmdDataRemove(CmdBase): return 1 try: - target = self.args.target + if not self.skip_git_actions and not self.git.is_ready_to_go(): + return 1 - if os.path.isdir(target): - if not self.args.recursive: - raise DataRemoveError('Directory cannot be removed. Use --recurcive flag.') + for target in self.args.target: + self.remove_target(target) - if os.path.realpath(target) == \ - os.path.realpath(os.path.join(self.git.git_dir_abs, self.config.data_dir)): - raise DataRemoveError('data directory cannot be removed') + if self.skip_git_actions: + self.not_committed_changes_warning() + return 0 - return self.remove_dir(target) - - dobj = DataFileObjExisting(target, self.git, self.config) - if os.path.islink(dobj.data_file_relative): - return self.remove_symlink(dobj.data_file_relative) - - raise DataRemoveError('Cannot remove a regular file "{}"'.format(target)) + message = 'NLX data remove: {}'.format(' '.join(self.args.target)) + self.git.commit_all_changes_and_log_status(message) finally: lock.release() + def remove_target(self, target): + if os.path.isdir(target): + if not self.args.recursive: + raise DataRemoveError('Directory cannot be removed. Use --recurcive flag.') + + if os.path.realpath(target) == \ + os.path.realpath(os.path.join(self.git.git_dir_abs, self.config.data_dir)): + raise DataRemoveError('data directory cannot be removed') + + return self.remove_dir(target) + dobj = DataFileObjExisting(target, self.git, self.config) + if os.path.islink(dobj.data_file_relative): + return self.remove_symlink(dobj.data_file_relative) + + raise DataRemoveError('Cannot remove a regular file "{}"'.format(target)) + def remove_symlink(self, file): dobj = DataFileObjExisting(file, self.git, self.config) diff --git a/neatlynx/cmd_run.py b/neatlynx/cmd_run.py index 384c0f020..2301402a5 100644 --- a/neatlynx/cmd_run.py +++ b/neatlynx/cmd_run.py @@ -22,26 +22,53 @@ class CmdRun(CmdBase): pass def define_args(self, parser): - parser.add_argument('--ignore-git-status', help='ignore git status', action='store_true') + self.set_skip_git_actions(parser) + parser.add_argument('--random', help='not reproducible, output is random', action='store_true') parser.add_argument('--stdout', help='output std output to a file') parser.add_argument('--stderr', help='output std error to a file') pass def run(self): - lock = fasteners.InterProcessLock(self.lock.lock_file) + lock = fasteners.InterProcessLock(self.git.lock_file) gotten = lock.acquire(timeout=5) if not gotten: Logger.info('Cannot perform the command since NLX is busy and locked. Please retry the command later.') return 1 try: - if not self.args.ignore_git_status and not self.git.is_ready_to_go(): + if not self.skip_git_actions and not self.git.is_ready_to_go(): return 1 + old_files = set(self.get_changed_files()) + GitWrapper.exec_cmd(self._args_unkn, self.args.stdout, self.args.stderr) - dobjs = self.get_new_file_objects() + new_files = set(self.get_changed_files()) + + removed_files = old_files - new_files + if removed_files != set(): + Logger.error('Error: existing files were removed in run command'.format(' '.join(removed_files))) + + created_files = new_files - old_files + + if created_files == set(): + Logger.error('Error: no files were changes in run command') + return 1 + + dobjs, errors = self.files_to_dobjs(created_files) + + if errors: + for file in GitWrapper.abs_paths_to_relative(errors): + Logger.error('Error: file "{}" was created outside of the data directory'.format(file)) + + for file in created_files: + rel_path = GitWrapper.abs_paths_to_relative([file])[0] + Logger.error('Removing created file: {}'.format(rel_path)) + os.remove(file) + + Logger.error('Errors occurred. Please fix the errors and re-run the command.') + return 1 for dobj in dobjs: os.makedirs(os.path.dirname(dobj.cache_file_relative), exist_ok=True) @@ -53,6 +80,10 @@ class CmdRun(CmdBase): state_file.save() pass + if self.skip_git_actions: + self.not_committed_changes_warning() + return 0 + message = 'NLX run: {}'.format(' '.join(sys.argv)) self.git.commit_all_changes_and_log_status(message) finally: @@ -60,30 +91,33 @@ class CmdRun(CmdBase): return 0 - def get_new_file_objects(self): + def files_to_dobjs(self, files): + result = [] + errors = [] + + for file in files: + try: + result.append(DataFileObj(file, self.git, self.config)) + except NotInDataDirError: + errors.append(file) + + return result, errors + + def get_changed_files(self): statuses = GitWrapper.status_files() - error = False - dobjs = [] + result = [] for status, file in statuses: - try: - file_path = os.path.join(self.git.git_dir_abs, file) - if os.path.isfile(file_path): - dobjs.append(DataFileObj(file_path, self.git, self.config)) - else: - files = [] - self.get_all_files_from_dir(file_path, files) - for f in files: - dobjs.append(DataFileObj(f, self.git, self.config)) - except NotInDataDirError: - Logger.error('Error: file "{}" was created outside of the data directory'.format(file_path)) - error = True + file_path = os.path.join(self.git.git_dir_abs, file) + if os.path.isfile(file_path): + result.append(file_path) + else: + files = [] + self.get_all_files_from_dir(file_path, files) + for f in files: + result.append(f) - if error: - Logger.error('Please fix the errors and re-run the command') - return None - - return dobjs + return result def get_all_files_from_dir(self, dir, result): if not os.path.isdir(dir): diff --git a/neatlynx/git_wrapper.py b/neatlynx/git_wrapper.py index cfc377d09..d23f26e47 100644 --- a/neatlynx/git_wrapper.py +++ b/neatlynx/git_wrapper.py @@ -1,4 +1,5 @@ import os +import sys import subprocess from neatlynx.exceptions import NeatLynxException @@ -37,15 +38,21 @@ class GitWrapper(GitWrapperI): def exec_cmd(cmd, stdout_file=None, stderr_file=None, cwd=None): stdout_fd = None if stdout_file is not None: - stdout_fd = open(stdout_file, 'w') - stdout = stdout_fd + if stdout_file == '-': + stdout = sys.stdout + else: + stdout_fd = open(stdout_file, 'w') + stdout = stdout_fd else: stdout = subprocess.PIPE stderr_fd = None if stderr_file is not None: - stderr_fd = open(stderr_file, 'w') - stderr = stderr_fd + if stderr_file == '-': + stderr = sys.stderr + else: + stderr_fd = open(stderr_file, 'w') + stderr = stderr_fd else: stderr = subprocess.PIPE @@ -65,13 +72,16 @@ class GitWrapper(GitWrapperI): def exec_cmd_only_success(cmd, stdout_file=None, stderr_file=None, cwd=None): code, out, err = GitWrapper.exec_cmd(cmd, stdout_file=None, stderr_file=None, cwd=None) if code != 0: - raise GitCmdError('Git command error ({}): {}'.format(' '.join(cmd), err)) + if err: + sys.stderr.write(err + '\n') + raise GitCmdError('Git command error ({}):\n{}'.format(' '.join(cmd), out)) return out def is_ready_to_go(self): statuses = self.status_files() if len(statuses) > 0: - Logger.error('Commit changed files before reproducible command (nlx-repro):') + Logger.error('Commit all changed files before running reproducible command.') + Logger.error('Changed files:') for status, file in statuses: Logger.error("{} {}".format(status, file)) return False @@ -141,3 +151,13 @@ class GitWrapper(GitWrapperI): @property def lock_file(self): return os.path.join(self.git_dir_abs, '.' + Config.CONFIG + '.lock') + + @staticmethod + def abs_paths_to_relative(files): + cur_dir = os.path.realpath(os.curdir) + + result = [] + for file in files: + result.append(os.path.relpath(file, cur_dir)) + + return result \ No newline at end of file