mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 15:28:28 +01:00
+17
@@ -14,6 +14,7 @@ from dvc.command.import_file import CmdImportFile
|
||||
from dvc.command.target import CmdTarget
|
||||
from dvc.command.instance_create import CmdInstanceCreate
|
||||
from dvc.command.config import CmdConfig
|
||||
from dvc.command.visual import CmdVisual
|
||||
from dvc.command.test import CmdTest
|
||||
from dvc.config import Config
|
||||
from dvc import VERSION
|
||||
@@ -380,6 +381,11 @@ def parse_args(argv=None):
|
||||
'config',
|
||||
parents=[parent_parser],
|
||||
help='Get or set repository options')
|
||||
config_parser.add_argument('-u',
|
||||
'--unset',
|
||||
default=False,
|
||||
action='store_true',
|
||||
help='Unset option')
|
||||
config_parser.add_argument('name',
|
||||
help='Option name')
|
||||
config_parser.add_argument('value',
|
||||
@@ -388,6 +394,17 @@ def parse_args(argv=None):
|
||||
help='Option value')
|
||||
config_parser.set_defaults(func=CmdConfig)
|
||||
|
||||
# Visual
|
||||
vz_parser = subparsers.add_parser(
|
||||
'visual',
|
||||
parents=[parent_parser],
|
||||
help='Create a dependency graph for data')
|
||||
vz_parser.add_argument(
|
||||
'target',
|
||||
nargs='*',
|
||||
help='Target data')
|
||||
vz_parser.set_defaults(func=CmdVisual)
|
||||
|
||||
if isinstance(argv, str):
|
||||
argv = argv.split()
|
||||
|
||||
|
||||
+62
-25
@@ -1,7 +1,7 @@
|
||||
import os
|
||||
import configobj
|
||||
|
||||
from dvc.command.base import CmdBase
|
||||
from dvc.command.base import CmdBase, DvcLock
|
||||
from dvc.logger import Logger
|
||||
from dvc.config import Config
|
||||
|
||||
@@ -9,43 +9,80 @@ class CmdConfig(CmdBase):
|
||||
def __init__(self, settings):
|
||||
super(CmdConfig, self).__init__(settings)
|
||||
|
||||
# Using configobj because it doesn't
|
||||
# drop comments like configparser does.
|
||||
self.configobj = configobj.ConfigObj(self._config_path)
|
||||
|
||||
@property
|
||||
def _config_path(self):
|
||||
return os.path.join(self.git.git_dir, Config.CONFIG)
|
||||
|
||||
def _find_key(self, l, name):
|
||||
for k in l:
|
||||
def _get_key(self, d, name, add=False):
|
||||
for k in d.keys():
|
||||
if k.lower() == name.lower():
|
||||
return k
|
||||
|
||||
if add:
|
||||
d[name] = {}
|
||||
return name
|
||||
|
||||
return None
|
||||
|
||||
def run(self):
|
||||
_section, _opt = self.parsed_args.name.strip().split('.')
|
||||
|
||||
# Using configobj because it doesn't
|
||||
# drop comments like configparser does.
|
||||
config = configobj.ConfigObj(self._config_path)
|
||||
|
||||
section = self._find_key(config.keys(), _section)
|
||||
if not section:
|
||||
return 1
|
||||
|
||||
opt = self._find_key(config[section].keys(), _opt)
|
||||
if not opt:
|
||||
return 1
|
||||
|
||||
if self.parsed_args.value == None:
|
||||
Logger.info(config[section][opt])
|
||||
return 0
|
||||
|
||||
def unset(self):
|
||||
try:
|
||||
config[section][opt] = self.parsed_args.value
|
||||
config.write()
|
||||
del self.configobj[self.section][self.opt]
|
||||
self.configobj.write()
|
||||
except Exception as exc:
|
||||
Logger.error('Failed to unset \'{}\': {}'.format(self.parsed_args.name, exc))
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
def show(self):
|
||||
Logger.info(self.configobj[self.section][self.opt])
|
||||
return 0
|
||||
|
||||
def set(self):
|
||||
try:
|
||||
self.configobj[self.section][self.opt] = self.parsed_args.value
|
||||
self.configobj.write()
|
||||
except Exception as exc:
|
||||
Logger.error('Failed to set \'{}\' to \'{}\': {}'.format(self.parsed_args.name,
|
||||
self.parsed_args.value,
|
||||
value,
|
||||
exc))
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
def check_opt(self):
|
||||
_section, _opt = self.parsed_args.name.strip().split('.')
|
||||
add = (self.parsed_args.value != None and self.parsed_args.unset == False)
|
||||
|
||||
section = self._get_key(self.configobj, _section, add)
|
||||
if not section:
|
||||
return 1
|
||||
|
||||
opt = self._get_key(self.configobj[section], _opt, add)
|
||||
if not opt:
|
||||
return 1
|
||||
|
||||
self.section = section
|
||||
self.opt = opt
|
||||
|
||||
return 0
|
||||
|
||||
def _run(self):
|
||||
if self.check_opt() != 0:
|
||||
return 1
|
||||
|
||||
if self.parsed_args.unset:
|
||||
return self.unset()
|
||||
|
||||
if self.parsed_args.value == None:
|
||||
return self.show()
|
||||
|
||||
return self.set()
|
||||
|
||||
def run(self):
|
||||
with DvcLock(self.is_locker, self.git):
|
||||
return self._run()
|
||||
|
||||
+1
-1
@@ -63,7 +63,7 @@ StoragePath =
|
||||
ProjectName =
|
||||
'''
|
||||
|
||||
EMPTY_FILE_NAME = 'empty'
|
||||
EMPTY_FILE_NAME = '.empty'
|
||||
EMPTY_FILE_CHECKSUM = '0000000'
|
||||
|
||||
def __init__(self, settings):
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
import os
|
||||
import networkx as nx
|
||||
|
||||
from dvc.command.traverse import Traverse
|
||||
from dvc.command.init import CmdInit
|
||||
|
||||
from dvc.logger import Logger
|
||||
from dvc.state_file import StateFile
|
||||
|
||||
class CmdVisual(Traverse):
|
||||
def __init__(self, settings):
|
||||
super(CmdVisual, self).__init__(settings, "collect", do_not_start_from_root=False)
|
||||
self.g = nx.DiGraph()
|
||||
self.subs = []
|
||||
|
||||
def draw(self, g, target, fname):
|
||||
try:
|
||||
A = nx.nx_agraph.to_agraph(g)
|
||||
A.write(fname + '.dot')
|
||||
A.draw(fname + '.svg', prog='dot')
|
||||
except Exception as exc:
|
||||
Logger.error('Failed to draw dependency graph for {}: {}'.format(target, exc))
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
def find_sub(self, fname):
|
||||
for s in self.subs:
|
||||
if fname in s.nodes():
|
||||
return s
|
||||
|
||||
return self.g
|
||||
|
||||
def draw_targets(self):
|
||||
if self.parsed_args.target == None:
|
||||
target = self.settings.config.data_dir
|
||||
return self.draw(self.g, target, target)
|
||||
|
||||
for t in self.parsed_args.target:
|
||||
fname = os.path.basename(os.path.normpath(t))
|
||||
s = self.find_sub(t)
|
||||
|
||||
ret = self.draw(s, t, fname)
|
||||
if ret != 0:
|
||||
return ret
|
||||
|
||||
return 0
|
||||
|
||||
def run(self):
|
||||
saved_targets = self.settings.parsed_args.target
|
||||
self.settings.parsed_args.target = [self.settings.config.data_dir]
|
||||
|
||||
ret = super(CmdVisual, self).run()
|
||||
|
||||
self.settings.parsed_args.target = saved_targets
|
||||
|
||||
if ret != 0:
|
||||
Logger.error('Failed to build dependency graph for the project')
|
||||
return 1
|
||||
|
||||
# Try to find independent clusters which might occure
|
||||
# when a bunch of data items were used independently.
|
||||
self.subs = nx.weakly_connected_component_subgraphs(self.g)
|
||||
|
||||
return self.draw_targets()
|
||||
|
||||
def process_file(self, target):
|
||||
data_item = self._get_data_item(target)
|
||||
name = data_item.data.relative
|
||||
state = StateFile.load(data_item.state.relative, self.git)
|
||||
|
||||
self.g.add_node(name)
|
||||
|
||||
for i in state.input_files:
|
||||
self.g.add_node(i)
|
||||
self.g.add_edge(i, name)
|
||||
|
||||
for o in state.output_files:
|
||||
if o == name:
|
||||
continue
|
||||
self.g.add_node(o)
|
||||
self.g.add_edge(name, o)
|
||||
|
||||
@property
|
||||
def no_git_actions(self):
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def not_committed_changes_warning():
|
||||
pass
|
||||
@@ -342,43 +342,6 @@ class DataCloudAWS(DataCloudBase):
|
||||
return 's3.amazonaws.com'
|
||||
return 's3.%s.amazonaws.com' % region
|
||||
|
||||
def get_aws_credentials(self):
|
||||
""" gets aws credentials, looking in various places
|
||||
|
||||
Params:
|
||||
|
||||
Searches:
|
||||
1 any override in dvc.conf [AWS] CredentialPath;
|
||||
2 ~/.aws/credentials
|
||||
|
||||
|
||||
Returns:
|
||||
if successfully found, (access_key_id, secret)
|
||||
None otherwise
|
||||
"""
|
||||
|
||||
# FIX: It won't work in Windows.
|
||||
default = os.path.expanduser('~/.aws/credentials')
|
||||
|
||||
paths = self.credential_paths(default)
|
||||
for path in paths:
|
||||
try:
|
||||
cc = configparser.SafeConfigParser()
|
||||
|
||||
# use readfp(open( ... to aid mocking.
|
||||
cc.readfp(open(path, 'r'))
|
||||
|
||||
if 'default' in cc.keys():
|
||||
access_key = cc['default'].get('aws_access_key_id', None)
|
||||
secret = cc['default'].get('aws_secret_access_key', None)
|
||||
|
||||
if access_key is not None and secret is not None:
|
||||
return (access_key, secret)
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
def credential_paths(self, default):
|
||||
paths = []
|
||||
credpath = self._cloud_settings.cloud_config.get('CredentialPath', None)
|
||||
@@ -393,12 +356,6 @@ class DataCloudAWS(DataCloudBase):
|
||||
paths.append(default)
|
||||
return paths
|
||||
|
||||
def sanity_check(self):
|
||||
creds = self.get_aws_credentials()
|
||||
if creds is None:
|
||||
Logger.info("can't find aws credetials, assuming envirment variables or iam role")
|
||||
self._aws_creds = creds
|
||||
|
||||
def _get_bucket_aws(self, bucket_name):
|
||||
""" get a bucket object, aws """
|
||||
if all([self._aws_creds.access_key_id, self._aws_creds.secret_access_key, self.aws_region_host]):
|
||||
|
||||
@@ -16,6 +16,9 @@ BASE_DIR_S3=https://s3-$REGION.amazonaws.com/dataversioncontrol/functests
|
||||
DATA_S3=$BASE_DIR_S3/stackoverflow_raw_small
|
||||
CODE_S3=$BASE_DIR_S3/code
|
||||
|
||||
TEST_REPO_S3='dvc-test/myrepo'
|
||||
TEST_REPO_REGION='us-east-2'
|
||||
|
||||
function dvc_print() {
|
||||
echo -e "$1"
|
||||
}
|
||||
@@ -103,6 +106,11 @@ function dvc_create_repo() {
|
||||
dvc import $DATA_CACHE/* data/xml
|
||||
}
|
||||
|
||||
function dvc_clean_cloud_aws() {
|
||||
dvc_info "Cleaning $TEST_REPO_S3"
|
||||
aws s3 rm --recursive "s3://$TEST_REPO_S3/"
|
||||
}
|
||||
|
||||
function dvc_check_dirs() {
|
||||
for d in $1; do
|
||||
if [ ! -d "$d" ]; then
|
||||
|
||||
@@ -6,6 +6,7 @@ source common.sh
|
||||
|
||||
TESTS+=" test-init.sh"
|
||||
TESTS+=" test-import.sh"
|
||||
TESTS+=" test-sync-aws.sh"
|
||||
TESTS+=" test-repro-code.sh"
|
||||
|
||||
for t in $TESTS; do
|
||||
|
||||
Executable
+41
@@ -0,0 +1,41 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
source common.sh
|
||||
|
||||
if [ -z $TRAVIS ]; then
|
||||
dvc_create_repo
|
||||
|
||||
dvc_info "Setting up AWS cloud"
|
||||
dvc_clean_cloud_aws
|
||||
dvc config aws.storagepath $TEST_REPO_S3
|
||||
dvc config aws.region $TEST_REPO_REGION
|
||||
|
||||
dvc_info "Checking status"
|
||||
dvc status data/xml
|
||||
|
||||
dvc_info "Pushing to AWS"
|
||||
dvc push data/xml
|
||||
|
||||
dvc_info "Checking status"
|
||||
dvc status data/xml
|
||||
|
||||
dvc_info "Modifying data"
|
||||
echo "123456" >> data/xml/Tags.xml
|
||||
|
||||
dvc_info "Checking status"
|
||||
dvc status data/xml
|
||||
|
||||
dvc_info "Pulling data"
|
||||
dvc pull data/xml
|
||||
|
||||
dvc_info "Checking status"
|
||||
dvc status data/xml
|
||||
|
||||
dvc_clean_cloud_aws
|
||||
else
|
||||
dvc_info "AWS test is disabled for travis"
|
||||
fi
|
||||
|
||||
dvc_pass
|
||||
@@ -87,3 +87,4 @@ wcwidth==0.1.7
|
||||
colorama==0.3.9
|
||||
google-compute-engine==2.4.1
|
||||
configobj==5.0.6
|
||||
networkx==1.11
|
||||
|
||||
Reference in New Issue
Block a user