diff --git a/dvc/cli.py b/dvc/cli.py index 4dcc4ce80..6aab2f0e5 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -14,6 +14,7 @@ from dvc.command.import_file import CmdImportFile from dvc.command.target import CmdTarget from dvc.command.instance_create import CmdInstanceCreate from dvc.command.config import CmdConfig +from dvc.command.visual import CmdVisual from dvc.command.test import CmdTest from dvc.config import Config from dvc import VERSION @@ -380,6 +381,11 @@ def parse_args(argv=None): 'config', parents=[parent_parser], help='Get or set repository options') + config_parser.add_argument('-u', + '--unset', + default=False, + action='store_true', + help='Unset option') config_parser.add_argument('name', help='Option name') config_parser.add_argument('value', @@ -388,6 +394,17 @@ def parse_args(argv=None): help='Option value') config_parser.set_defaults(func=CmdConfig) + # Visual + vz_parser = subparsers.add_parser( + 'visual', + parents=[parent_parser], + help='Create a dependency graph for data') + vz_parser.add_argument( + 'target', + nargs='*', + help='Target data') + vz_parser.set_defaults(func=CmdVisual) + if isinstance(argv, str): argv = argv.split() diff --git a/dvc/command/config.py b/dvc/command/config.py index 340c40fbf..78391624d 100644 --- a/dvc/command/config.py +++ b/dvc/command/config.py @@ -1,7 +1,7 @@ import os import configobj -from dvc.command.base import CmdBase +from dvc.command.base import CmdBase, DvcLock from dvc.logger import Logger from dvc.config import Config @@ -9,43 +9,80 @@ class CmdConfig(CmdBase): def __init__(self, settings): super(CmdConfig, self).__init__(settings) + # Using configobj because it doesn't + # drop comments like configparser does. + self.configobj = configobj.ConfigObj(self._config_path) + @property def _config_path(self): return os.path.join(self.git.git_dir, Config.CONFIG) - def _find_key(self, l, name): - for k in l: + def _get_key(self, d, name, add=False): + for k in d.keys(): if k.lower() == name.lower(): return k + if add: + d[name] = {} + return name + return None - def run(self): - _section, _opt = self.parsed_args.name.strip().split('.') - - # Using configobj because it doesn't - # drop comments like configparser does. - config = configobj.ConfigObj(self._config_path) - - section = self._find_key(config.keys(), _section) - if not section: - return 1 - - opt = self._find_key(config[section].keys(), _opt) - if not opt: - return 1 - - if self.parsed_args.value == None: - Logger.info(config[section][opt]) - return 0 - + def unset(self): try: - config[section][opt] = self.parsed_args.value - config.write() + del self.configobj[self.section][self.opt] + self.configobj.write() + except Exception as exc: + Logger.error('Failed to unset \'{}\': {}'.format(self.parsed_args.name, exc)) + return 1 + + return 0 + + def show(self): + Logger.info(self.configobj[self.section][self.opt]) + return 0 + + def set(self): + try: + self.configobj[self.section][self.opt] = self.parsed_args.value + self.configobj.write() except Exception as exc: Logger.error('Failed to set \'{}\' to \'{}\': {}'.format(self.parsed_args.name, - self.parsed_args.value, + value, exc)) return 1 return 0 + + def check_opt(self): + _section, _opt = self.parsed_args.name.strip().split('.') + add = (self.parsed_args.value != None and self.parsed_args.unset == False) + + section = self._get_key(self.configobj, _section, add) + if not section: + return 1 + + opt = self._get_key(self.configobj[section], _opt, add) + if not opt: + return 1 + + self.section = section + self.opt = opt + + return 0 + + def _run(self): + if self.check_opt() != 0: + return 1 + + if self.parsed_args.unset: + return self.unset() + + if self.parsed_args.value == None: + return self.show() + + return self.set() + + def run(self): + with DvcLock(self.is_locker, self.git): + return self._run() diff --git a/dvc/command/init.py b/dvc/command/init.py index 4979de0ce..b765dc635 100644 --- a/dvc/command/init.py +++ b/dvc/command/init.py @@ -63,7 +63,7 @@ StoragePath = ProjectName = ''' - EMPTY_FILE_NAME = 'empty' + EMPTY_FILE_NAME = '.empty' EMPTY_FILE_CHECKSUM = '0000000' def __init__(self, settings): diff --git a/dvc/command/visual.py b/dvc/command/visual.py new file mode 100644 index 000000000..41abcc3d2 --- /dev/null +++ b/dvc/command/visual.py @@ -0,0 +1,90 @@ +import os +import networkx as nx + +from dvc.command.traverse import Traverse +from dvc.command.init import CmdInit + +from dvc.logger import Logger +from dvc.state_file import StateFile + +class CmdVisual(Traverse): + def __init__(self, settings): + super(CmdVisual, self).__init__(settings, "collect", do_not_start_from_root=False) + self.g = nx.DiGraph() + self.subs = [] + + def draw(self, g, target, fname): + try: + A = nx.nx_agraph.to_agraph(g) + A.write(fname + '.dot') + A.draw(fname + '.svg', prog='dot') + except Exception as exc: + Logger.error('Failed to draw dependency graph for {}: {}'.format(target, exc)) + return 1 + + return 0 + + def find_sub(self, fname): + for s in self.subs: + if fname in s.nodes(): + return s + + return self.g + + def draw_targets(self): + if self.parsed_args.target == None: + target = self.settings.config.data_dir + return self.draw(self.g, target, target) + + for t in self.parsed_args.target: + fname = os.path.basename(os.path.normpath(t)) + s = self.find_sub(t) + + ret = self.draw(s, t, fname) + if ret != 0: + return ret + + return 0 + + def run(self): + saved_targets = self.settings.parsed_args.target + self.settings.parsed_args.target = [self.settings.config.data_dir] + + ret = super(CmdVisual, self).run() + + self.settings.parsed_args.target = saved_targets + + if ret != 0: + Logger.error('Failed to build dependency graph for the project') + return 1 + + # Try to find independent clusters which might occure + # when a bunch of data items were used independently. + self.subs = nx.weakly_connected_component_subgraphs(self.g) + + return self.draw_targets() + + def process_file(self, target): + data_item = self._get_data_item(target) + name = data_item.data.relative + state = StateFile.load(data_item.state.relative, self.git) + + self.g.add_node(name) + + for i in state.input_files: + self.g.add_node(i) + self.g.add_edge(i, name) + + for o in state.output_files: + if o == name: + continue + self.g.add_node(o) + self.g.add_edge(name, o) + + @property + def no_git_actions(self): + return True + + @staticmethod + def not_committed_changes_warning(): + pass diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index 52f9d52fb..766b15337 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -342,43 +342,6 @@ class DataCloudAWS(DataCloudBase): return 's3.amazonaws.com' return 's3.%s.amazonaws.com' % region - def get_aws_credentials(self): - """ gets aws credentials, looking in various places - - Params: - - Searches: - 1 any override in dvc.conf [AWS] CredentialPath; - 2 ~/.aws/credentials - - - Returns: - if successfully found, (access_key_id, secret) - None otherwise - """ - - # FIX: It won't work in Windows. - default = os.path.expanduser('~/.aws/credentials') - - paths = self.credential_paths(default) - for path in paths: - try: - cc = configparser.SafeConfigParser() - - # use readfp(open( ... to aid mocking. - cc.readfp(open(path, 'r')) - - if 'default' in cc.keys(): - access_key = cc['default'].get('aws_access_key_id', None) - secret = cc['default'].get('aws_secret_access_key', None) - - if access_key is not None and secret is not None: - return (access_key, secret) - except Exception as e: - pass - - return None - def credential_paths(self, default): paths = [] credpath = self._cloud_settings.cloud_config.get('CredentialPath', None) @@ -393,12 +356,6 @@ class DataCloudAWS(DataCloudBase): paths.append(default) return paths - def sanity_check(self): - creds = self.get_aws_credentials() - if creds is None: - Logger.info("can't find aws credetials, assuming envirment variables or iam role") - self._aws_creds = creds - def _get_bucket_aws(self, bucket_name): """ get a bucket object, aws """ if all([self._aws_creds.access_key_id, self._aws_creds.secret_access_key, self.aws_region_host]): diff --git a/functest/common.sh b/functest/common.sh index 094296428..53f018231 100644 --- a/functest/common.sh +++ b/functest/common.sh @@ -16,6 +16,9 @@ BASE_DIR_S3=https://s3-$REGION.amazonaws.com/dataversioncontrol/functests DATA_S3=$BASE_DIR_S3/stackoverflow_raw_small CODE_S3=$BASE_DIR_S3/code +TEST_REPO_S3='dvc-test/myrepo' +TEST_REPO_REGION='us-east-2' + function dvc_print() { echo -e "$1" } @@ -103,6 +106,11 @@ function dvc_create_repo() { dvc import $DATA_CACHE/* data/xml } +function dvc_clean_cloud_aws() { + dvc_info "Cleaning $TEST_REPO_S3" + aws s3 rm --recursive "s3://$TEST_REPO_S3/" +} + function dvc_check_dirs() { for d in $1; do if [ ! -d "$d" ]; then diff --git a/functest/run.sh b/functest/run.sh index e6d3ef6db..cc36fa836 100755 --- a/functest/run.sh +++ b/functest/run.sh @@ -6,6 +6,7 @@ source common.sh TESTS+=" test-init.sh" TESTS+=" test-import.sh" +TESTS+=" test-sync-aws.sh" TESTS+=" test-repro-code.sh" for t in $TESTS; do diff --git a/functest/test-sync-aws.sh b/functest/test-sync-aws.sh new file mode 100755 index 000000000..1490ee553 --- /dev/null +++ b/functest/test-sync-aws.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +set -e + +source common.sh + +if [ -z $TRAVIS ]; then + dvc_create_repo + + dvc_info "Setting up AWS cloud" + dvc_clean_cloud_aws + dvc config aws.storagepath $TEST_REPO_S3 + dvc config aws.region $TEST_REPO_REGION + + dvc_info "Checking status" + dvc status data/xml + + dvc_info "Pushing to AWS" + dvc push data/xml + + dvc_info "Checking status" + dvc status data/xml + + dvc_info "Modifying data" + echo "123456" >> data/xml/Tags.xml + + dvc_info "Checking status" + dvc status data/xml + + dvc_info "Pulling data" + dvc pull data/xml + + dvc_info "Checking status" + dvc status data/xml + + dvc_clean_cloud_aws +else + dvc_info "AWS test is disabled for travis" +fi + +dvc_pass diff --git a/requirements.txt b/requirements.txt index 8e3d4ad16..8754dcc29 100644 --- a/requirements.txt +++ b/requirements.txt @@ -87,3 +87,4 @@ wcwidth==0.1.7 colorama==0.3.9 google-compute-engine==2.4.1 configobj==5.0.6 +networkx==1.11 diff --git a/setup.py b/setup.py index d47341ba2..28868597e 100644 --- a/setup.py +++ b/setup.py @@ -93,6 +93,7 @@ install_requires = [ 'colorama==0.3.9', 'google-compute-engine==2.4.1', 'configobj==5.0.6', + 'networkx==1.11', ] if platform.system() == 'Darwin':