Merge pull request #183 from efiop/master

misc patches
This commit is contained in:
Ruslan Kuprieiev
2017-08-26 11:34:38 +03:00
committed by GitHub
10 changed files with 222 additions and 69 deletions
+17
View File
@@ -14,6 +14,7 @@ from dvc.command.import_file import CmdImportFile
from dvc.command.target import CmdTarget
from dvc.command.instance_create import CmdInstanceCreate
from dvc.command.config import CmdConfig
from dvc.command.visual import CmdVisual
from dvc.command.test import CmdTest
from dvc.config import Config
from dvc import VERSION
@@ -380,6 +381,11 @@ def parse_args(argv=None):
'config',
parents=[parent_parser],
help='Get or set repository options')
config_parser.add_argument('-u',
'--unset',
default=False,
action='store_true',
help='Unset option')
config_parser.add_argument('name',
help='Option name')
config_parser.add_argument('value',
@@ -388,6 +394,17 @@ def parse_args(argv=None):
help='Option value')
config_parser.set_defaults(func=CmdConfig)
# Visual
vz_parser = subparsers.add_parser(
'visual',
parents=[parent_parser],
help='Create a dependency graph for data')
vz_parser.add_argument(
'target',
nargs='*',
help='Target data')
vz_parser.set_defaults(func=CmdVisual)
if isinstance(argv, str):
argv = argv.split()
+62 -25
View File
@@ -1,7 +1,7 @@
import os
import configobj
from dvc.command.base import CmdBase
from dvc.command.base import CmdBase, DvcLock
from dvc.logger import Logger
from dvc.config import Config
@@ -9,43 +9,80 @@ class CmdConfig(CmdBase):
def __init__(self, settings):
super(CmdConfig, self).__init__(settings)
# Using configobj because it doesn't
# drop comments like configparser does.
self.configobj = configobj.ConfigObj(self._config_path)
@property
def _config_path(self):
return os.path.join(self.git.git_dir, Config.CONFIG)
def _find_key(self, l, name):
for k in l:
def _get_key(self, d, name, add=False):
for k in d.keys():
if k.lower() == name.lower():
return k
if add:
d[name] = {}
return name
return None
def run(self):
_section, _opt = self.parsed_args.name.strip().split('.')
# Using configobj because it doesn't
# drop comments like configparser does.
config = configobj.ConfigObj(self._config_path)
section = self._find_key(config.keys(), _section)
if not section:
return 1
opt = self._find_key(config[section].keys(), _opt)
if not opt:
return 1
if self.parsed_args.value == None:
Logger.info(config[section][opt])
return 0
def unset(self):
try:
config[section][opt] = self.parsed_args.value
config.write()
del self.configobj[self.section][self.opt]
self.configobj.write()
except Exception as exc:
Logger.error('Failed to unset \'{}\': {}'.format(self.parsed_args.name, exc))
return 1
return 0
def show(self):
Logger.info(self.configobj[self.section][self.opt])
return 0
def set(self):
try:
self.configobj[self.section][self.opt] = self.parsed_args.value
self.configobj.write()
except Exception as exc:
Logger.error('Failed to set \'{}\' to \'{}\': {}'.format(self.parsed_args.name,
self.parsed_args.value,
value,
exc))
return 1
return 0
def check_opt(self):
_section, _opt = self.parsed_args.name.strip().split('.')
add = (self.parsed_args.value != None and self.parsed_args.unset == False)
section = self._get_key(self.configobj, _section, add)
if not section:
return 1
opt = self._get_key(self.configobj[section], _opt, add)
if not opt:
return 1
self.section = section
self.opt = opt
return 0
def _run(self):
if self.check_opt() != 0:
return 1
if self.parsed_args.unset:
return self.unset()
if self.parsed_args.value == None:
return self.show()
return self.set()
def run(self):
with DvcLock(self.is_locker, self.git):
return self._run()
+1 -1
View File
@@ -63,7 +63,7 @@ StoragePath =
ProjectName =
'''
EMPTY_FILE_NAME = 'empty'
EMPTY_FILE_NAME = '.empty'
EMPTY_FILE_CHECKSUM = '0000000'
def __init__(self, settings):
+90
View File
@@ -0,0 +1,90 @@
import os
import networkx as nx
from dvc.command.traverse import Traverse
from dvc.command.init import CmdInit
from dvc.logger import Logger
from dvc.state_file import StateFile
class CmdVisual(Traverse):
def __init__(self, settings):
super(CmdVisual, self).__init__(settings, "collect", do_not_start_from_root=False)
self.g = nx.DiGraph()
self.subs = []
def draw(self, g, target, fname):
try:
A = nx.nx_agraph.to_agraph(g)
A.write(fname + '.dot')
A.draw(fname + '.svg', prog='dot')
except Exception as exc:
Logger.error('Failed to draw dependency graph for {}: {}'.format(target, exc))
return 1
return 0
def find_sub(self, fname):
for s in self.subs:
if fname in s.nodes():
return s
return self.g
def draw_targets(self):
if self.parsed_args.target == None:
target = self.settings.config.data_dir
return self.draw(self.g, target, target)
for t in self.parsed_args.target:
fname = os.path.basename(os.path.normpath(t))
s = self.find_sub(t)
ret = self.draw(s, t, fname)
if ret != 0:
return ret
return 0
def run(self):
saved_targets = self.settings.parsed_args.target
self.settings.parsed_args.target = [self.settings.config.data_dir]
ret = super(CmdVisual, self).run()
self.settings.parsed_args.target = saved_targets
if ret != 0:
Logger.error('Failed to build dependency graph for the project')
return 1
# Try to find independent clusters which might occure
# when a bunch of data items were used independently.
self.subs = nx.weakly_connected_component_subgraphs(self.g)
return self.draw_targets()
def process_file(self, target):
data_item = self._get_data_item(target)
name = data_item.data.relative
state = StateFile.load(data_item.state.relative, self.git)
self.g.add_node(name)
for i in state.input_files:
self.g.add_node(i)
self.g.add_edge(i, name)
for o in state.output_files:
if o == name:
continue
self.g.add_node(o)
self.g.add_edge(name, o)
@property
def no_git_actions(self):
return True
@staticmethod
def not_committed_changes_warning():
pass
-43
View File
@@ -342,43 +342,6 @@ class DataCloudAWS(DataCloudBase):
return 's3.amazonaws.com'
return 's3.%s.amazonaws.com' % region
def get_aws_credentials(self):
""" gets aws credentials, looking in various places
Params:
Searches:
1 any override in dvc.conf [AWS] CredentialPath;
2 ~/.aws/credentials
Returns:
if successfully found, (access_key_id, secret)
None otherwise
"""
# FIX: It won't work in Windows.
default = os.path.expanduser('~/.aws/credentials')
paths = self.credential_paths(default)
for path in paths:
try:
cc = configparser.SafeConfigParser()
# use readfp(open( ... to aid mocking.
cc.readfp(open(path, 'r'))
if 'default' in cc.keys():
access_key = cc['default'].get('aws_access_key_id', None)
secret = cc['default'].get('aws_secret_access_key', None)
if access_key is not None and secret is not None:
return (access_key, secret)
except Exception as e:
pass
return None
def credential_paths(self, default):
paths = []
credpath = self._cloud_settings.cloud_config.get('CredentialPath', None)
@@ -393,12 +356,6 @@ class DataCloudAWS(DataCloudBase):
paths.append(default)
return paths
def sanity_check(self):
creds = self.get_aws_credentials()
if creds is None:
Logger.info("can't find aws credetials, assuming envirment variables or iam role")
self._aws_creds = creds
def _get_bucket_aws(self, bucket_name):
""" get a bucket object, aws """
if all([self._aws_creds.access_key_id, self._aws_creds.secret_access_key, self.aws_region_host]):
+8
View File
@@ -16,6 +16,9 @@ BASE_DIR_S3=https://s3-$REGION.amazonaws.com/dataversioncontrol/functests
DATA_S3=$BASE_DIR_S3/stackoverflow_raw_small
CODE_S3=$BASE_DIR_S3/code
TEST_REPO_S3='dvc-test/myrepo'
TEST_REPO_REGION='us-east-2'
function dvc_print() {
echo -e "$1"
}
@@ -103,6 +106,11 @@ function dvc_create_repo() {
dvc import $DATA_CACHE/* data/xml
}
function dvc_clean_cloud_aws() {
dvc_info "Cleaning $TEST_REPO_S3"
aws s3 rm --recursive "s3://$TEST_REPO_S3/"
}
function dvc_check_dirs() {
for d in $1; do
if [ ! -d "$d" ]; then
+1
View File
@@ -6,6 +6,7 @@ source common.sh
TESTS+=" test-init.sh"
TESTS+=" test-import.sh"
TESTS+=" test-sync-aws.sh"
TESTS+=" test-repro-code.sh"
for t in $TESTS; do
+41
View File
@@ -0,0 +1,41 @@
#!/bin/bash
set -e
source common.sh
if [ -z $TRAVIS ]; then
dvc_create_repo
dvc_info "Setting up AWS cloud"
dvc_clean_cloud_aws
dvc config aws.storagepath $TEST_REPO_S3
dvc config aws.region $TEST_REPO_REGION
dvc_info "Checking status"
dvc status data/xml
dvc_info "Pushing to AWS"
dvc push data/xml
dvc_info "Checking status"
dvc status data/xml
dvc_info "Modifying data"
echo "123456" >> data/xml/Tags.xml
dvc_info "Checking status"
dvc status data/xml
dvc_info "Pulling data"
dvc pull data/xml
dvc_info "Checking status"
dvc status data/xml
dvc_clean_cloud_aws
else
dvc_info "AWS test is disabled for travis"
fi
dvc_pass
+1
View File
@@ -87,3 +87,4 @@ wcwidth==0.1.7
colorama==0.3.9
google-compute-engine==2.4.1
configobj==5.0.6
networkx==1.11
+1
View File
@@ -93,6 +93,7 @@ install_requires = [
'colorama==0.3.9',
'google-compute-engine==2.4.1',
'configobj==5.0.6',
'networkx==1.11',
]
if platform.system() == 'Darwin':