mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-04 15:28:27 +01:00
Compare commits
26
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4bca30e4d6 | ||
|
|
7a84c104fd | ||
|
|
1aa7e5dbec | ||
|
|
fe0d6ec3e0 | ||
|
|
a3891995e6 | ||
|
|
05f3eb67ce | ||
|
|
29db8a1a95 | ||
|
|
4082757aa1 | ||
|
|
8a11595a93 | ||
|
|
60546dcd98 | ||
|
|
1b4326aefd | ||
|
|
1138afe495 | ||
|
|
7916ce4bfb | ||
|
|
c5db57900d | ||
|
|
c9ef5d3ccb | ||
|
|
cd72149b5f | ||
|
|
80c1f10f08 | ||
|
|
573546d824 | ||
|
|
41b20cd675 | ||
|
|
cb90bd0c8d | ||
|
|
82896be988 | ||
|
|
84ae8e02f7 | ||
|
|
a97d5fef72 | ||
|
|
ed02ff7c30 | ||
|
|
363ad46326 | ||
|
|
776ed1d2dd |
@@ -1,8 +1,6 @@
|
|||||||
name: Build packages
|
name: Build packages
|
||||||
on:
|
on:
|
||||||
push:
|
push: {}
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
release:
|
release:
|
||||||
types: [released]
|
types: [released]
|
||||||
jobs:
|
jobs:
|
||||||
@@ -121,11 +119,11 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
store_login: ${{ secrets.STORE_LOGIN }}
|
store_login: ${{ secrets.STORE_LOGIN }}
|
||||||
snap: ${{ steps.snapcraft.outputs.snap }}
|
snap: ${{ steps.snapcraft.outputs.snap }}
|
||||||
release: edge
|
release: edge,v1/edge
|
||||||
- name: Publish to the stable channel
|
- name: Publish to the stable channel
|
||||||
uses: snapcore/action-publish@v1
|
uses: snapcore/action-publish@v1
|
||||||
if: github.event_name == 'release'
|
if: github.event_name == 'release'
|
||||||
with:
|
with:
|
||||||
store_login: ${{ secrets.STORE_LOGIN }}
|
store_login: ${{ secrets.STORE_LOGIN }}
|
||||||
snap: ${{ steps.snapcraft.outputs.snap }}
|
snap: ${{ steps.snapcraft.outputs.snap }}
|
||||||
release: stable
|
release: stable,v1/stable
|
||||||
|
|||||||
@@ -1,12 +1,8 @@
|
|||||||
name: Tests
|
name: Tests
|
||||||
|
|
||||||
on:
|
on:
|
||||||
pull_request:
|
- pull_request
|
||||||
branches:
|
- push
|
||||||
- master
|
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
|
|
||||||
env:
|
env:
|
||||||
DVC_TEST: "true"
|
DVC_TEST: "true"
|
||||||
|
|||||||
@@ -45,14 +45,6 @@ class CmdBase(ABC):
|
|||||||
updater = Updater(self.repo.tmp_dir, hardlink_lock=hardlink_lock)
|
updater = Updater(self.repo.tmp_dir, hardlink_lock=hardlink_lock)
|
||||||
updater.check()
|
updater.check()
|
||||||
|
|
||||||
@property
|
|
||||||
def default_targets(self):
|
|
||||||
"""Default targets for `dvc repro`."""
|
|
||||||
from dvc.dvcfile import PIPELINE_FILE
|
|
||||||
|
|
||||||
logger.debug(f"assuming default target '{PIPELINE_FILE}'.")
|
|
||||||
return [PIPELINE_FILE]
|
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def run(self):
|
def run(self):
|
||||||
pass
|
pass
|
||||||
|
|||||||
+38
-10
@@ -6,6 +6,25 @@ from dvc.config import Config, ConfigError
|
|||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
NAME_REGEX = r"^(?P<remote>remote\.)?(?P<section>[^\.]*)\.(?P<option>[^\.]*)$"
|
||||||
|
|
||||||
|
|
||||||
|
def _name_type(value):
|
||||||
|
import re
|
||||||
|
|
||||||
|
match = re.match(NAME_REGEX, value)
|
||||||
|
if not match:
|
||||||
|
raise argparse.ArgumentTypeError(
|
||||||
|
"name argument should look like "
|
||||||
|
"remote.name.option or "
|
||||||
|
"section.option"
|
||||||
|
)
|
||||||
|
return (
|
||||||
|
bool(match.group("remote")),
|
||||||
|
match.group("section").lower(),
|
||||||
|
match.group("option").lower(),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class CmdConfig(CmdBaseNoRepo):
|
class CmdConfig(CmdBaseNoRepo):
|
||||||
def __init__(self, args):
|
def __init__(self, args):
|
||||||
@@ -14,20 +33,24 @@ class CmdConfig(CmdBaseNoRepo):
|
|||||||
self.config = Config(validate=False)
|
self.config = Config(validate=False)
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
section, opt = self.args.name.lower().strip().split(".", 1)
|
remote, section, opt = self.args.name
|
||||||
|
|
||||||
if self.args.value is None and not self.args.unset:
|
if self.args.value is None and not self.args.unset:
|
||||||
conf = self.config.load_one(self.args.level)
|
conf = self.config.load_one(self.args.level)
|
||||||
self._check(conf, section, opt)
|
if remote:
|
||||||
|
conf = conf["remote"]
|
||||||
|
self._check(conf, remote, section, opt)
|
||||||
logger.info(conf[section][opt])
|
logger.info(conf[section][opt])
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
with self.config.edit(self.args.level) as conf:
|
with self.config.edit(self.args.level) as conf:
|
||||||
|
if remote:
|
||||||
|
conf = conf["remote"]
|
||||||
if self.args.unset:
|
if self.args.unset:
|
||||||
self._check(conf, section, opt)
|
self._check(conf, remote, section, opt)
|
||||||
del conf[section][opt]
|
del conf[section][opt]
|
||||||
else:
|
else:
|
||||||
self._check(conf, section)
|
self._check(conf, remote, section)
|
||||||
conf[section][opt] = self.args.value
|
conf[section][opt] = self.args.value
|
||||||
|
|
||||||
if self.args.name == "cache.type":
|
if self.args.name == "cache.type":
|
||||||
@@ -39,14 +62,15 @@ class CmdConfig(CmdBaseNoRepo):
|
|||||||
|
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
def _check(self, conf, section, opt=None):
|
def _check(self, conf, remote, section, opt=None):
|
||||||
|
name = "remote" if remote else "section"
|
||||||
if section not in conf:
|
if section not in conf:
|
||||||
msg = "section {} doesn't exist"
|
raise ConfigError(f"{name} '{section}' doesn't exist")
|
||||||
raise ConfigError(msg.format(self.args.name))
|
|
||||||
|
|
||||||
if opt and opt not in conf[section]:
|
if opt and opt not in conf[section]:
|
||||||
msg = "option {} doesn't exist"
|
raise ConfigError(
|
||||||
raise ConfigError(msg.format(self.args.name))
|
f"option '{opt}' doesn't exist in {name} '{section}'"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
parent_config_parser = argparse.ArgumentParser(add_help=False)
|
parent_config_parser = argparse.ArgumentParser(add_help=False)
|
||||||
@@ -92,6 +116,10 @@ def add_parser(subparsers, parent_parser):
|
|||||||
action="store_true",
|
action="store_true",
|
||||||
help="Unset option.",
|
help="Unset option.",
|
||||||
)
|
)
|
||||||
config_parser.add_argument("name", help="Option name.")
|
config_parser.add_argument(
|
||||||
|
"name",
|
||||||
|
type=_name_type,
|
||||||
|
help="Option name (remote.name.option or section.option).",
|
||||||
|
)
|
||||||
config_parser.add_argument("value", nargs="?", help="Option value.")
|
config_parser.add_argument("value", nargs="?", help="Option value.")
|
||||||
config_parser.set_defaults(func=CmdConfig)
|
config_parser.set_defaults(func=CmdConfig)
|
||||||
|
|||||||
+10
-24
@@ -451,32 +451,18 @@ class CmdExperimentsRun(CmdRepro):
|
|||||||
saved_dir = os.path.realpath(os.curdir)
|
saved_dir = os.path.realpath(os.curdir)
|
||||||
os.chdir(self.args.cwd)
|
os.chdir(self.args.cwd)
|
||||||
|
|
||||||
# Dirty hack so the for loop below can at least enter once
|
self.repo.experiments.run(
|
||||||
if self.args.all_pipelines:
|
name=self.args.name,
|
||||||
self.args.targets = [None]
|
queue=self.args.queue,
|
||||||
elif not self.args.targets:
|
run_all=self.args.run_all,
|
||||||
self.args.targets = self.default_targets
|
jobs=self.args.jobs,
|
||||||
|
params=self.args.params,
|
||||||
ret = 0
|
checkpoint_resume=self.args.checkpoint_resume,
|
||||||
for target in self.args.targets:
|
**self._repro_kwargs,
|
||||||
try:
|
)
|
||||||
self.repo.experiments.run(
|
|
||||||
target,
|
|
||||||
name=self.args.name,
|
|
||||||
queue=self.args.queue,
|
|
||||||
run_all=self.args.run_all,
|
|
||||||
jobs=self.args.jobs,
|
|
||||||
params=self.args.params,
|
|
||||||
checkpoint_resume=self.args.checkpoint_resume,
|
|
||||||
**self._repro_kwargs,
|
|
||||||
)
|
|
||||||
except DvcException:
|
|
||||||
logger.exception("")
|
|
||||||
ret = 1
|
|
||||||
break
|
|
||||||
|
|
||||||
os.chdir(saved_dir)
|
os.chdir(saved_dir)
|
||||||
return ret
|
return 0
|
||||||
|
|
||||||
|
|
||||||
class CmdExperimentsGC(CmdRepro):
|
class CmdExperimentsGC(CmdRepro):
|
||||||
|
|||||||
+12
-28
@@ -7,7 +7,6 @@ from dvc.command.base import CmdBase, append_doc_link
|
|||||||
from dvc.command.metrics import _show_metrics
|
from dvc.command.metrics import _show_metrics
|
||||||
from dvc.command.status import CmdDataStatus
|
from dvc.command.status import CmdDataStatus
|
||||||
from dvc.dvcfile import PIPELINE_FILE
|
from dvc.dvcfile import PIPELINE_FILE
|
||||||
from dvc.exceptions import DvcException
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -17,40 +16,25 @@ class CmdRepro(CmdBase):
|
|||||||
saved_dir = os.path.realpath(os.curdir)
|
saved_dir = os.path.realpath(os.curdir)
|
||||||
os.chdir(self.args.cwd)
|
os.chdir(self.args.cwd)
|
||||||
|
|
||||||
# Dirty hack so the for loop below can at least enter once
|
stages = self.repo.reproduce(**self._repro_kwargs)
|
||||||
if self.args.all_pipelines:
|
if len(stages) == 0:
|
||||||
self.args.targets = [None]
|
logger.info(CmdDataStatus.UP_TO_DATE_MSG)
|
||||||
elif not self.args.targets:
|
else:
|
||||||
self.args.targets = self.default_targets
|
logger.info(
|
||||||
|
"Use `dvc push` to send your updates to " "remote storage."
|
||||||
|
)
|
||||||
|
|
||||||
ret = 0
|
if self.args.metrics:
|
||||||
for target in self.args.targets:
|
metrics = self.repo.metrics.show()
|
||||||
try:
|
logger.info(_show_metrics(metrics))
|
||||||
stages = self.repo.reproduce(target, **self._repro_kwargs)
|
|
||||||
|
|
||||||
if len(stages) == 0:
|
|
||||||
logger.info(CmdDataStatus.UP_TO_DATE_MSG)
|
|
||||||
else:
|
|
||||||
logger.info(
|
|
||||||
"Use `dvc push` to send your updates to "
|
|
||||||
"remote storage."
|
|
||||||
)
|
|
||||||
|
|
||||||
if self.args.metrics:
|
|
||||||
metrics = self.repo.metrics.show()
|
|
||||||
logger.info(_show_metrics(metrics))
|
|
||||||
|
|
||||||
except DvcException:
|
|
||||||
logger.exception("")
|
|
||||||
ret = 1
|
|
||||||
break
|
|
||||||
|
|
||||||
os.chdir(saved_dir)
|
os.chdir(saved_dir)
|
||||||
return ret
|
return 0
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def _repro_kwargs(self):
|
def _repro_kwargs(self):
|
||||||
return {
|
return {
|
||||||
|
"targets": self.args.targets,
|
||||||
"single_item": self.args.single_item,
|
"single_item": self.args.single_item,
|
||||||
"force": self.args.force,
|
"force": self.args.force,
|
||||||
"dry": self.args.dry,
|
"dry": self.args.dry,
|
||||||
|
|||||||
+1
-2
@@ -148,9 +148,8 @@ class HardlinkLock(flufl.lock.Lock, LockBase):
|
|||||||
): # pylint: disable=super-init-not-called
|
): # pylint: disable=super-init-not-called
|
||||||
import socket
|
import socket
|
||||||
|
|
||||||
super().__init__(lockfile)
|
|
||||||
|
|
||||||
self._tmp_dir = tmp_dir
|
self._tmp_dir = tmp_dir
|
||||||
|
super().__init__(lockfile)
|
||||||
|
|
||||||
# NOTE: this is basically Lock.__init__ copy-paste, except that
|
# NOTE: this is basically Lock.__init__ copy-paste, except that
|
||||||
# instead of using `socket.getfqdn()` we use `socket.gethostname()`
|
# instead of using `socket.getfqdn()` we use `socket.gethostname()`
|
||||||
|
|||||||
@@ -37,13 +37,13 @@ def _parse_params(path_params: Iterable):
|
|||||||
@locked
|
@locked
|
||||||
def run(
|
def run(
|
||||||
repo,
|
repo,
|
||||||
target: Optional[str] = None,
|
targets: Optional[Iterable] = None,
|
||||||
params: Optional[Iterable] = None,
|
params: Optional[Iterable] = None,
|
||||||
run_all: Optional[bool] = False,
|
run_all: Optional[bool] = False,
|
||||||
jobs: Optional[int] = 1,
|
jobs: Optional[int] = 1,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> dict:
|
) -> dict:
|
||||||
"""Reproduce the specified target as an experiment.
|
"""Reproduce the specified targets as an experiment.
|
||||||
|
|
||||||
Accepts the same additional kwargs as Repo.reproduce.
|
Accepts the same additional kwargs as Repo.reproduce.
|
||||||
|
|
||||||
@@ -59,10 +59,10 @@ def run(
|
|||||||
params = []
|
params = []
|
||||||
try:
|
try:
|
||||||
return repo.experiments.reproduce_one(
|
return repo.experiments.reproduce_one(
|
||||||
target=target, params=params, **kwargs
|
targets=targets, params=params, **kwargs
|
||||||
)
|
)
|
||||||
except UnchangedExperimentError:
|
except UnchangedExperimentError:
|
||||||
# If experiment contains no changes, just run regular repro
|
# If experiment contains no changes, just run regular repro
|
||||||
kwargs.pop("queue", None)
|
kwargs.pop("queue", None)
|
||||||
kwargs.pop("checkpoint_resume", None)
|
kwargs.pop("checkpoint_resume", None)
|
||||||
return {None: repo.reproduce(target=target, **kwargs)}
|
return {None: repo.reproduce(targets=targets, **kwargs)}
|
||||||
|
|||||||
@@ -3,12 +3,10 @@ from dvc.utils.diff import diff as _diff
|
|||||||
from dvc.utils.diff import format_dict
|
from dvc.utils.diff import format_dict
|
||||||
|
|
||||||
|
|
||||||
def _get_metrics(repo, *args, rev=None, **kwargs):
|
def _get_metrics(repo, *args, revs=None, **kwargs):
|
||||||
try:
|
try:
|
||||||
metrics = repo.metrics.show(
|
metrics = repo.metrics.show(*args, **kwargs, revs=revs)
|
||||||
*args, **kwargs, revs=[rev] if rev else None
|
return metrics
|
||||||
)
|
|
||||||
return metrics.get(rev or "", {})
|
|
||||||
except NoMetricsError:
|
except NoMetricsError:
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
@@ -19,8 +17,12 @@ def diff(repo, *args, a_rev=None, b_rev=None, **kwargs):
|
|||||||
|
|
||||||
with_unchanged = kwargs.pop("all", False)
|
with_unchanged = kwargs.pop("all", False)
|
||||||
|
|
||||||
old = _get_metrics(repo, *args, **kwargs, rev=(a_rev or "HEAD"))
|
a_rev = a_rev or "HEAD"
|
||||||
new = _get_metrics(repo, *args, **kwargs, rev=b_rev)
|
b_rev = b_rev or "workspace"
|
||||||
|
|
||||||
|
metrics = _get_metrics(repo, *args, **kwargs, revs=[a_rev, b_rev])
|
||||||
|
old = metrics.get(a_rev, {})
|
||||||
|
new = metrics.get(b_rev, {})
|
||||||
|
|
||||||
return _diff(
|
return _diff(
|
||||||
format_dict(old), format_dict(new), with_unchanged=with_unchanged
|
format_dict(old), format_dict(new), with_unchanged=with_unchanged
|
||||||
|
|||||||
@@ -4,10 +4,10 @@ from dvc.utils.diff import format_dict
|
|||||||
from .show import NoParamsError
|
from .show import NoParamsError
|
||||||
|
|
||||||
|
|
||||||
def _get_params(repo, *args, rev=None, **kwargs):
|
def _get_params(repo, *args, revs=None, **kwargs):
|
||||||
try:
|
try:
|
||||||
params = repo.params.show(*args, **kwargs, revs=[rev] if rev else None)
|
params = repo.params.show(*args, **kwargs, revs=revs)
|
||||||
return params.get(rev or "", {})
|
return params
|
||||||
except NoParamsError:
|
except NoParamsError:
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
@@ -18,8 +18,12 @@ def diff(repo, *args, a_rev=None, b_rev=None, **kwargs):
|
|||||||
|
|
||||||
with_unchanged = kwargs.pop("all", False)
|
with_unchanged = kwargs.pop("all", False)
|
||||||
|
|
||||||
old = _get_params(repo, *args, **kwargs, rev=(a_rev or "HEAD"))
|
a_rev = a_rev or "HEAD"
|
||||||
new = _get_params(repo, *args, **kwargs, rev=b_rev)
|
b_rev = b_rev or "workspace"
|
||||||
|
|
||||||
|
params = _get_params(repo, *args, **kwargs, revs=[a_rev, b_rev])
|
||||||
|
old = params.get(a_rev, {})
|
||||||
|
new = params.get(b_rev, {})
|
||||||
|
|
||||||
return _diff(
|
return _diff(
|
||||||
format_dict(old), format_dict(new), with_unchanged=with_unchanged
|
format_dict(old), format_dict(new), with_unchanged=with_unchanged
|
||||||
|
|||||||
+11
-5
@@ -1,4 +1,5 @@
|
|||||||
import logging
|
import logging
|
||||||
|
from collections import defaultdict
|
||||||
from typing import TYPE_CHECKING, List
|
from typing import TYPE_CHECKING, List
|
||||||
|
|
||||||
from dvc.dependency.param import ParamsDependency
|
from dvc.dependency.param import ParamsDependency
|
||||||
@@ -50,17 +51,18 @@ def _read_params(repo, configs, rev):
|
|||||||
return res
|
return res
|
||||||
|
|
||||||
|
|
||||||
def _add_vars(repo, configs, params):
|
def _collect_vars(repo, params):
|
||||||
|
vars_params = defaultdict(dict)
|
||||||
for stage in repo.stages:
|
for stage in repo.stages:
|
||||||
if isinstance(stage, PipelineStage) and stage.tracked_vars:
|
if isinstance(stage, PipelineStage) and stage.tracked_vars:
|
||||||
for file, vars_ in stage.tracked_vars.items():
|
for file, vars_ in stage.tracked_vars.items():
|
||||||
# `params` file are shown regardless of `tracked` or not
|
# `params` file are shown regardless of `tracked` or not
|
||||||
# to reduce noise and duplication, they are skipped
|
# to reduce noise and duplication, they are skipped
|
||||||
if file in configs:
|
if file in params:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
params[file] = params.get(file, {})
|
vars_params[file].update(vars_)
|
||||||
params[file].update(vars_)
|
return vars_params
|
||||||
|
|
||||||
|
|
||||||
@locked
|
@locked
|
||||||
@@ -70,7 +72,11 @@ def show(repo, revs=None):
|
|||||||
for branch in repo.brancher(revs=revs):
|
for branch in repo.brancher(revs=revs):
|
||||||
configs = _collect_configs(repo, branch)
|
configs = _collect_configs(repo, branch)
|
||||||
params = _read_params(repo, configs, branch)
|
params = _read_params(repo, configs, branch)
|
||||||
_add_vars(repo, configs, params)
|
vars_params = _collect_vars(repo, params)
|
||||||
|
|
||||||
|
# NOTE: only those that are not added as a ParamDependency are included
|
||||||
|
# so we don't need to recursively merge them yet.
|
||||||
|
params.update(vars_params)
|
||||||
|
|
||||||
if params:
|
if params:
|
||||||
res[branch] = params
|
res[branch] = params
|
||||||
|
|||||||
+65
-75
@@ -2,7 +2,7 @@ import logging
|
|||||||
import typing
|
import typing
|
||||||
from functools import partial
|
from functools import partial
|
||||||
|
|
||||||
from dvc.exceptions import InvalidArgumentError, ReproductionError
|
from dvc.exceptions import ReproductionError
|
||||||
from dvc.repo.scm_context import scm_context
|
from dvc.repo.scm_context import scm_context
|
||||||
from dvc.stage.run import CheckpointKilledError
|
from dvc.stage.run import CheckpointKilledError
|
||||||
|
|
||||||
@@ -54,33 +54,11 @@ def _dump_stage(stage):
|
|||||||
dvcfile.dump(stage, update_pipeline=False)
|
dvcfile.dump(stage, update_pipeline=False)
|
||||||
|
|
||||||
|
|
||||||
def _get_active_graph(G):
|
|
||||||
import networkx as nx
|
|
||||||
|
|
||||||
active = G.copy()
|
|
||||||
for stage in G:
|
|
||||||
if not stage.frozen:
|
|
||||||
continue
|
|
||||||
active.remove_edges_from(G.out_edges(stage))
|
|
||||||
for edge in G.out_edges(stage):
|
|
||||||
_, to_stage = edge
|
|
||||||
for node in nx.dfs_preorder_nodes(G, to_stage):
|
|
||||||
# NOTE: `in_degree` will return InDegreeView({}) if stage
|
|
||||||
# no longer exists in the `active` DAG.
|
|
||||||
if not active.in_degree(node):
|
|
||||||
# NOTE: if some edge no longer exists `remove_edges_from`
|
|
||||||
# will ignore it without error.
|
|
||||||
active.remove_edges_from(G.out_edges(node))
|
|
||||||
active.remove_node(node)
|
|
||||||
|
|
||||||
return active
|
|
||||||
|
|
||||||
|
|
||||||
@locked
|
@locked
|
||||||
@scm_context
|
@scm_context
|
||||||
def reproduce(
|
def reproduce(
|
||||||
self: "Repo",
|
self: "Repo",
|
||||||
target=None,
|
targets=None,
|
||||||
recursive=False,
|
recursive=False,
|
||||||
pipeline=False,
|
pipeline=False,
|
||||||
all_pipelines=False,
|
all_pipelines=False,
|
||||||
@@ -88,41 +66,46 @@ def reproduce(
|
|||||||
):
|
):
|
||||||
glob = kwargs.pop("glob", False)
|
glob = kwargs.pop("glob", False)
|
||||||
accept_group = not glob
|
accept_group = not glob
|
||||||
assert target is None or isinstance(target, str)
|
|
||||||
if not target and not all_pipelines:
|
if isinstance(targets, str):
|
||||||
raise InvalidArgumentError(
|
targets = [targets]
|
||||||
"Neither `target` nor `--all-pipelines` are specified."
|
|
||||||
)
|
if not all_pipelines and not targets:
|
||||||
|
from dvc.dvcfile import PIPELINE_FILE
|
||||||
|
|
||||||
|
targets = [PIPELINE_FILE]
|
||||||
|
|
||||||
interactive = kwargs.get("interactive", False)
|
interactive = kwargs.get("interactive", False)
|
||||||
if not interactive:
|
if not interactive:
|
||||||
kwargs["interactive"] = self.config["core"].get("interactive", False)
|
kwargs["interactive"] = self.config["core"].get("interactive", False)
|
||||||
|
|
||||||
active_graph = _get_active_graph(self.graph)
|
stages = set()
|
||||||
active_pipelines = get_pipelines(active_graph)
|
|
||||||
|
|
||||||
if pipeline or all_pipelines:
|
if pipeline or all_pipelines:
|
||||||
|
pipelines = get_pipelines(self.graph)
|
||||||
if all_pipelines:
|
if all_pipelines:
|
||||||
pipelines = active_pipelines
|
used_pipelines = pipelines
|
||||||
else:
|
else:
|
||||||
stage = self.stage.get_target(target)
|
used_pipelines = []
|
||||||
pipelines = [get_pipeline(active_pipelines, stage)]
|
for target in targets:
|
||||||
|
stage = self.stage.get_target(target)
|
||||||
|
used_pipelines.append(get_pipeline(pipelines, stage))
|
||||||
|
|
||||||
targets = []
|
for pline in used_pipelines:
|
||||||
for pipeline in pipelines:
|
for stage in pline:
|
||||||
for stage in pipeline:
|
if pline.in_degree(stage) == 0:
|
||||||
if pipeline.in_degree(stage) == 0:
|
stages.add(stage)
|
||||||
targets.append(stage)
|
|
||||||
else:
|
else:
|
||||||
targets = self.stage.collect(
|
for target in targets:
|
||||||
target,
|
stages.update(
|
||||||
recursive=recursive,
|
self.stage.collect(
|
||||||
graph=active_graph,
|
target,
|
||||||
accept_group=accept_group,
|
recursive=recursive,
|
||||||
glob=glob,
|
accept_group=accept_group,
|
||||||
)
|
glob=glob,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
return _reproduce_stages(active_graph, targets, **kwargs)
|
return _reproduce_stages(self.graph, list(stages), **kwargs)
|
||||||
|
|
||||||
|
|
||||||
def _reproduce_stages(
|
def _reproduce_stages(
|
||||||
@@ -163,7 +146,7 @@ def _reproduce_stages(
|
|||||||
|
|
||||||
The derived evaluation of _downstream_ B would be: [B, D, E]
|
The derived evaluation of _downstream_ B would be: [B, D, E]
|
||||||
"""
|
"""
|
||||||
pipeline = _get_pipeline(G, stages, downstream, single_item)
|
steps = _get_steps(G, stages, downstream, single_item)
|
||||||
|
|
||||||
force_downstream = kwargs.pop("force_downstream", False)
|
force_downstream = kwargs.pop("force_downstream", False)
|
||||||
result = []
|
result = []
|
||||||
@@ -171,7 +154,7 @@ def _reproduce_stages(
|
|||||||
# `ret` is used to add a cosmetic newline.
|
# `ret` is used to add a cosmetic newline.
|
||||||
ret = []
|
ret = []
|
||||||
checkpoint_func = kwargs.pop("checkpoint_func", None)
|
checkpoint_func = kwargs.pop("checkpoint_func", None)
|
||||||
for stage in pipeline:
|
for stage in steps:
|
||||||
if ret:
|
if ret:
|
||||||
logger.info("")
|
logger.info("")
|
||||||
|
|
||||||
@@ -205,36 +188,43 @@ def _reproduce_stages(
|
|||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
def _get_pipeline(G, stages, downstream, single_item):
|
def _get_steps(G, stages, downstream, single_item):
|
||||||
import networkx as nx
|
import networkx as nx
|
||||||
|
|
||||||
if single_item:
|
active = G.copy()
|
||||||
all_pipelines = stages
|
if not single_item:
|
||||||
else:
|
# NOTE: frozen stages don't matter for single_item
|
||||||
all_pipelines = []
|
for stage in G:
|
||||||
for stage in stages:
|
if stage.frozen:
|
||||||
if downstream:
|
# NOTE: disconnect frozen stage from its dependencies
|
||||||
# NOTE (py3 only):
|
active.remove_edges_from(G.out_edges(stage))
|
||||||
# Python's `deepcopy` defaults to pickle/unpickle the object.
|
|
||||||
# Stages are complex objects (with references to `repo`,
|
|
||||||
# `outs`, and `deps`) that cause struggles when you try
|
|
||||||
# to serialize them. We need to create a copy of the graph
|
|
||||||
# itself, and then reverse it, instead of using
|
|
||||||
# graph.reverse() directly because it calls `deepcopy`
|
|
||||||
# underneath -- unless copy=False is specified.
|
|
||||||
nodes = nx.dfs_postorder_nodes(
|
|
||||||
G.copy().reverse(copy=False), stage
|
|
||||||
)
|
|
||||||
all_pipelines += reversed(list(nodes))
|
|
||||||
else:
|
|
||||||
all_pipelines += nx.dfs_postorder_nodes(G, stage)
|
|
||||||
|
|
||||||
pipeline = []
|
all_pipelines = []
|
||||||
|
for stage in stages:
|
||||||
|
if downstream:
|
||||||
|
# NOTE (py3 only):
|
||||||
|
# Python's `deepcopy` defaults to pickle/unpickle the object.
|
||||||
|
# Stages are complex objects (with references to `repo`,
|
||||||
|
# `outs`, and `deps`) that cause struggles when you try
|
||||||
|
# to serialize them. We need to create a copy of the graph
|
||||||
|
# itself, and then reverse it, instead of using
|
||||||
|
# graph.reverse() directly because it calls `deepcopy`
|
||||||
|
# underneath -- unless copy=False is specified.
|
||||||
|
nodes = nx.dfs_postorder_nodes(active.reverse(copy=False), stage)
|
||||||
|
all_pipelines += reversed(list(nodes))
|
||||||
|
else:
|
||||||
|
all_pipelines += nx.dfs_postorder_nodes(active, stage)
|
||||||
|
|
||||||
|
steps = []
|
||||||
for stage in all_pipelines:
|
for stage in all_pipelines:
|
||||||
if stage not in pipeline:
|
if stage not in steps:
|
||||||
pipeline.append(stage)
|
# NOTE: order of steps still matters for single_item
|
||||||
|
if single_item and stage not in stages:
|
||||||
|
continue
|
||||||
|
|
||||||
return pipeline
|
steps.append(stage)
|
||||||
|
|
||||||
|
return steps
|
||||||
|
|
||||||
|
|
||||||
def _repro_callback(experiments_callback, unchanged, stages):
|
def _repro_callback(experiments_callback, unchanged, stages):
|
||||||
|
|||||||
+2
-2
@@ -71,8 +71,8 @@ class System:
|
|||||||
clonefile.restype = ctypes.c_int
|
clonefile.restype = ctypes.c_int
|
||||||
|
|
||||||
return clonefile(
|
return clonefile(
|
||||||
ctypes.c_char_p(src.encode("utf-8")),
|
ctypes.c_char_p(os.fsencode(src)),
|
||||||
ctypes.c_char_p(dst.encode("utf-8")),
|
ctypes.c_char_p(os.fsencode(dst)),
|
||||||
ctypes.c_int(0),
|
ctypes.c_int(0),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -11,7 +11,7 @@ def get_connection(conn_func, *args, **kwargs):
|
|||||||
conn = pool.get_connection()
|
conn = pool.get_connection()
|
||||||
try:
|
try:
|
||||||
yield conn
|
yield conn
|
||||||
except BaseException:
|
except Exception:
|
||||||
conn.close()
|
conn.close()
|
||||||
raise
|
raise
|
||||||
else:
|
else:
|
||||||
|
|||||||
+1
-1
@@ -6,7 +6,7 @@
|
|||||||
import os
|
import os
|
||||||
import subprocess
|
import subprocess
|
||||||
|
|
||||||
_BASE_VERSION = "1.11.0"
|
_BASE_VERSION = "1.11.9"
|
||||||
|
|
||||||
|
|
||||||
def _generate_version(base_version):
|
def _generate_version(base_version):
|
||||||
|
|||||||
@@ -1,2 +1,3 @@
|
|||||||
PyInstaller==3.5
|
PyInstaller==3.5
|
||||||
|
numpy==1.19.3
|
||||||
psutil
|
psutil
|
||||||
|
|||||||
@@ -71,7 +71,7 @@ install_requires = [
|
|||||||
"flufl.lock>=3.2,<4",
|
"flufl.lock>=3.2,<4",
|
||||||
"win-unicode-console>=0.5; sys_platform == 'win32'",
|
"win-unicode-console>=0.5; sys_platform == 'win32'",
|
||||||
"pywin32>=225; sys_platform == 'win32'",
|
"pywin32>=225; sys_platform == 'win32'",
|
||||||
"networkx>=2.1,<2.5",
|
"networkx>=2.1",
|
||||||
"pydot>=1.2.4",
|
"pydot>=1.2.4",
|
||||||
"speedcopy>=2.0.1; python_version < '3.8' and sys_platform == 'win32'",
|
"speedcopy>=2.0.1; python_version < '3.8' and sys_platform == 'win32'",
|
||||||
"dataclasses==0.7; python_version < '3.7'",
|
"dataclasses==0.7; python_version < '3.7'",
|
||||||
|
|||||||
@@ -141,14 +141,14 @@ class TestGCMultipleDvcRepos(TestDvc):
|
|||||||
|
|
||||||
class TestConfig(TestDvc):
|
class TestConfig(TestDvc):
|
||||||
def test(self):
|
def test(self):
|
||||||
name = "param"
|
name = "section.option"
|
||||||
value = "1"
|
value = "1"
|
||||||
|
|
||||||
args = parse_args(["config", "-u", "--unset", name, value])
|
args = parse_args(["config", "-u", "--unset", name, value])
|
||||||
|
|
||||||
self.assertIsInstance(args.func(args), CmdConfig)
|
self.assertIsInstance(args.func(args), CmdConfig)
|
||||||
self.assertEqual(args.unset, True)
|
self.assertEqual(args.unset, True)
|
||||||
self.assertEqual(args.name, name)
|
self.assertEqual(args.name, (False, "section", "option"))
|
||||||
self.assertEqual(args.value, value)
|
self.assertEqual(args.value, value)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -145,3 +145,19 @@ def test_load_relative_paths(dvc, field, remote_url):
|
|||||||
assert cfg["remote"]["test"][field] == os.path.join(
|
assert cfg["remote"]["test"][field] == os.path.join(
|
||||||
dvc_dir, "..", "file.txt"
|
dvc_dir, "..", "file.txt"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_config_remote(tmp_dir, dvc, caplog):
|
||||||
|
(tmp_dir / ".dvc" / "config").write_text(
|
||||||
|
"['remote \"myremote\"']\n"
|
||||||
|
" url = s3://bucket/path\n"
|
||||||
|
" region = myregion\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
caplog.clear()
|
||||||
|
assert main(["config", "remote.myremote.url"]) == 0
|
||||||
|
assert "s3://bucket/path" in caplog.text
|
||||||
|
|
||||||
|
caplog.clear()
|
||||||
|
assert main(["config", "remote.myremote.region"]) == 0
|
||||||
|
assert "myregion" in caplog.text
|
||||||
|
|||||||
@@ -170,6 +170,19 @@ def test_non_existing_stage_name(tmp_dir, dvc, run_copy):
|
|||||||
assert main(["freeze", ":copy-file1-file3"]) != 0
|
assert main(["freeze", ":copy-file1-file3"]) != 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_repro_frozen(tmp_dir, dvc, run_copy):
|
||||||
|
(data_stage,) = tmp_dir.dvc_gen("data", "foo")
|
||||||
|
stage0 = run_copy("data", "stage0", name="copy-data-stage0")
|
||||||
|
run_copy("stage0", "stage1", name="copy-data-stage1")
|
||||||
|
run_copy("stage1", "stage2", name="copy-data-stage2")
|
||||||
|
|
||||||
|
dvc.freeze("copy-data-stage1")
|
||||||
|
|
||||||
|
tmp_dir.gen("data", "bar")
|
||||||
|
stages = dvc.reproduce()
|
||||||
|
assert stages == [data_stage, stage0]
|
||||||
|
|
||||||
|
|
||||||
def test_downstream(tmp_dir, dvc):
|
def test_downstream(tmp_dir, dvc):
|
||||||
# The dependency graph should look like this:
|
# The dependency graph should look like this:
|
||||||
#
|
#
|
||||||
|
|||||||
@@ -0,0 +1,11 @@
|
|||||||
|
import pytest
|
||||||
|
|
||||||
|
from dvc.cli import DvcParserError, parse_args
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"name", ["way.too.long", "no_option", "remote.way.too.long"],
|
||||||
|
)
|
||||||
|
def test_config_bad_name(name):
|
||||||
|
with pytest.raises(DvcParserError):
|
||||||
|
parse_args(["config", name])
|
||||||
@@ -9,7 +9,6 @@ from dvc.command.experiments import (
|
|||||||
CmdExperimentsRun,
|
CmdExperimentsRun,
|
||||||
CmdExperimentsShow,
|
CmdExperimentsShow,
|
||||||
)
|
)
|
||||||
from dvc.dvcfile import PIPELINE_FILE
|
|
||||||
from dvc.exceptions import InvalidArgumentError
|
from dvc.exceptions import InvalidArgumentError
|
||||||
|
|
||||||
from .test_repro import default_arguments as repro_arguments
|
from .test_repro import default_arguments as repro_arguments
|
||||||
@@ -100,9 +99,7 @@ def test_experiments_run(dvc, mocker, args, resume):
|
|||||||
mocker.patch.object(cmd.repo, "reproduce")
|
mocker.patch.object(cmd.repo, "reproduce")
|
||||||
mocker.patch.object(cmd.repo.experiments, "run")
|
mocker.patch.object(cmd.repo.experiments, "run")
|
||||||
cmd.run()
|
cmd.run()
|
||||||
cmd.repo.experiments.run.assert_called_with(
|
cmd.repo.experiments.run.assert_called_with(**default_arguments)
|
||||||
PIPELINE_FILE, **default_arguments
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_experiments_gc(dvc, mocker):
|
def test_experiments_gc(dvc, mocker):
|
||||||
|
|||||||
@@ -1,6 +1,5 @@
|
|||||||
from dvc.cli import parse_args
|
from dvc.cli import parse_args
|
||||||
from dvc.command.repro import CmdRepro
|
from dvc.command.repro import CmdRepro
|
||||||
from dvc.dvcfile import PIPELINE_FILE
|
|
||||||
|
|
||||||
default_arguments = {
|
default_arguments = {
|
||||||
"all_pipelines": False,
|
"all_pipelines": False,
|
||||||
@@ -16,6 +15,7 @@ default_arguments = {
|
|||||||
"force_downstream": False,
|
"force_downstream": False,
|
||||||
"pull": False,
|
"pull": False,
|
||||||
"glob": False,
|
"glob": False,
|
||||||
|
"targets": [],
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -23,7 +23,7 @@ def test_default_arguments(dvc, mocker):
|
|||||||
cmd = CmdRepro(parse_args(["repro"]))
|
cmd = CmdRepro(parse_args(["repro"]))
|
||||||
mocker.patch.object(cmd.repo, "reproduce")
|
mocker.patch.object(cmd.repo, "reproduce")
|
||||||
cmd.run()
|
cmd.run()
|
||||||
cmd.repo.reproduce.assert_called_with(PIPELINE_FILE, **default_arguments)
|
cmd.repo.reproduce.assert_called_with(**default_arguments)
|
||||||
|
|
||||||
|
|
||||||
def test_downstream(dvc, mocker):
|
def test_downstream(dvc, mocker):
|
||||||
@@ -32,4 +32,4 @@ def test_downstream(dvc, mocker):
|
|||||||
cmd.run()
|
cmd.run()
|
||||||
arguments = default_arguments.copy()
|
arguments = default_arguments.copy()
|
||||||
arguments.update({"downstream": True})
|
arguments.update({"downstream": True})
|
||||||
cmd.repo.reproduce.assert_called_with(PIPELINE_FILE, **arguments)
|
cmd.repo.reproduce.assert_called_with(**arguments)
|
||||||
|
|||||||
@@ -1,37 +1,5 @@
|
|||||||
import mock
|
import mock
|
||||||
|
|
||||||
from dvc.repo.reproduce import _get_active_graph
|
|
||||||
|
|
||||||
|
|
||||||
def test_get_active_graph(tmp_dir, dvc):
|
|
||||||
(pre_foo_stage,) = tmp_dir.dvc_gen({"pre-foo": "pre-foo"})
|
|
||||||
foo_stage = dvc.run(
|
|
||||||
single_stage=True, deps=["pre-foo"], outs=["foo"], cmd="echo foo > foo"
|
|
||||||
)
|
|
||||||
bar_stage = dvc.run(
|
|
||||||
single_stage=True, deps=["foo"], outs=["bar"], cmd="echo bar > bar"
|
|
||||||
)
|
|
||||||
baz_stage = dvc.run(
|
|
||||||
single_stage=True, deps=["foo"], outs=["baz"], cmd="echo baz > baz"
|
|
||||||
)
|
|
||||||
|
|
||||||
dvc.freeze("bar.dvc")
|
|
||||||
|
|
||||||
graph = dvc.graph
|
|
||||||
active_graph = _get_active_graph(graph)
|
|
||||||
assert active_graph.nodes == graph.nodes
|
|
||||||
assert set(active_graph.edges) == {
|
|
||||||
(foo_stage, pre_foo_stage),
|
|
||||||
(baz_stage, foo_stage),
|
|
||||||
}
|
|
||||||
|
|
||||||
dvc.freeze("baz.dvc")
|
|
||||||
|
|
||||||
graph = dvc.graph
|
|
||||||
active_graph = _get_active_graph(graph)
|
|
||||||
assert set(active_graph.nodes) == {bar_stage, baz_stage}
|
|
||||||
assert not active_graph.edges
|
|
||||||
|
|
||||||
|
|
||||||
@mock.patch("dvc.repo.reproduce._reproduce_stage", returns=[])
|
@mock.patch("dvc.repo.reproduce._reproduce_stage", returns=[])
|
||||||
def test_number_reproduces(reproduce_stage_mock, tmp_dir, dvc):
|
def test_number_reproduces(reproduce_stage_mock, tmp_dir, dvc):
|
||||||
|
|||||||
Reference in New Issue
Block a user