database: fix ui issues (#10129)

This commit is contained in:
skshetry
2023-12-04 13:27:05 +05:45
committed by GitHub
parent 8768b50aed
commit e187508f40
4 changed files with 67 additions and 18 deletions
+3 -2
View File
@@ -38,7 +38,7 @@ class CmdTestDb(CmdBase):
dbt_config = merge(_get_dbt_config(self.repo.config), cli_dbt_config)
project_dir = self.repo.root_dir
if not conn_config or not dbt_config:
if not (conn_config or dbt_config):
if is_dbt_project(project_dir):
ui.write("Using", DBT_PROJECT_FILE, "for testing", styled=True)
else:
@@ -151,10 +151,11 @@ def add_parser(subparsers, parent_parser):
import_parser.set_defaults(func=CmdImportDb)
TEST_DB_HELP = "Test the database connection"
test_db_parser = subparsers.add_parser(
"test-db",
parents=[parent_parser],
description=append_doc_link(IMPORT_HELP, "test-db"),
description=append_doc_link(TEST_DB_HELP, "test-db"),
add_help=False,
)
test_db_parser.add_argument("--conn", dest="connection")
+12 -6
View File
@@ -6,21 +6,26 @@ if TYPE_CHECKING:
from agate import Table
def noop(_):
pass
@dataclass
class PandasSQLSerializer:
sql: "Union[sa.TextClause, str]"
con: "sa.Connection"
chunksize: int = 10_000
def to_csv(self, file: str) -> None:
def to_csv(self, file: str, progress=noop) -> None:
import pandas as pd
with open(file, mode="wb") as f:
idfs = pd.read_sql_query(self.sql, self.con, chunksize=self.chunksize)
for i, df in enumerate(idfs):
df.to_csv(f, header=i == 0, index=False)
progress(len(df))
def to_json(self, file: str) -> None:
def to_json(self, file: str, progress=noop) -> None: # noqa: ARG002
import pandas as pd
df = pd.read_sql_query(self.sql, self.con)
@@ -31,10 +36,10 @@ class PandasSQLSerializer:
class AgateSerializer:
table: "Table"
def to_csv(self, file: str) -> None:
def to_csv(self, file: str, progress=noop) -> None: # noqa: ARG002
return self.table.to_csv(file)
def to_json(self, file: str) -> None:
def to_json(self, file: str, progress=noop) -> None: # noqa: ARG002
return self.table.to_json(file)
@@ -42,7 +47,8 @@ def export(
serializer: Union[PandasSQLSerializer, AgateSerializer],
file: str,
format: str = "csv", # noqa: A002
progress=noop,
) -> None:
if format == "csv":
return serializer.to_csv(file)
return serializer.to_json(file)
return serializer.to_csv(file, progress=progress)
return serializer.to_json(file, progress=progress)
+30 -10
View File
@@ -1,8 +1,8 @@
import os
from contextlib import contextmanager, nullcontext
from typing import TYPE_CHECKING, Any, Dict, Iterator, Optional, Union
from typing import TYPE_CHECKING, Any, Callable, Dict, Iterator, Optional, Union
from funcy import compact
from funcy import compact, log_durations
from dvc.exceptions import DvcException
from dvc.log import logger
@@ -53,6 +53,19 @@ def chdir(path):
os.chdir(wdir)
@contextmanager
def download_progress(to: "Output") -> Iterator[Callable[[int], Any]]:
from dvc.ui import ui
from dvc.ui._rich_progress import DbDownloadProgress
with log_durations(logger.debug, f"Saving to {to}"), DbDownloadProgress(
console=ui.error_console,
) as progress:
task = progress.add_task("Saving", total=None, output=to)
yield lambda n: progress.advance(task, advance=n)
progress.update(task, description="Saved", total=0)
class AbstractDependency(Dependency):
"""Dependency without workspace/fs/fs_path"""
@@ -102,7 +115,13 @@ class DbDependency(AbstractDependency):
def update(self, rev=None):
"""nothing to update."""
def download(self, to, jobs=None, file_format=None, **kwargs): # noqa: ARG002
def download(
self,
to: "Output",
jobs: Optional[int] = None, # noqa: ARG002
file_format: Optional[str] = None,
**kwargs: Any,
) -> None:
from dvc.database import export, get_adapter
db_info = self.info.get(PARAM_DB, {})
@@ -129,10 +148,14 @@ class DbDependency(AbstractDependency):
db.test_connection(onerror=status.stop)
file_format = file_format or db_info.get(PARAM_FILE_FORMAT, "csv")
assert file_format
with log_status("Executing query") as status, db.query(query) as serializer:
status.stop()
logger.debug("using serializer: %s", serializer)
with log_status(f"Saving to {to}", status=status):
return export(serializer, to.fs_path, format=file_format)
with download_progress(to) as progress:
return export(
serializer, to.fs_path, format=file_format, progress=progress
)
class DbtDependency(AbstractDependency):
@@ -246,8 +269,6 @@ class DbtDependency(AbstractDependency):
jobs: Optional[int] = None, # noqa: ARG002
file_format: Optional[str] = None,
) -> None:
from dvc.ui import ui
from .repo import RepoDependency
project_dir = self.info.get(PARAM_DB, {}).get(self.PARAM_PROJECT_DIR, "")
@@ -263,7 +284,6 @@ class DbtDependency(AbstractDependency):
project_path = os.path.join(wdir, project_dir) if project_dir else root
with ctx, chdir(project_path):
self._download_db(to, file_format=file_format)
ui.write(f"Saved file to {to}", styled=True)
def _download_db(
self,
@@ -290,8 +310,8 @@ class DbtDependency(AbstractDependency):
model, version=version, profile=profile, target=target
)
# NOTE: we keep everything in memory, and then export it out later.
with log_status(f"Saving to {to}"):
export(serializer, to.fs_path, format=file_format)
with download_progress(to) as progress:
export(serializer, to.fs_path, format=file_format, progress=progress)
DB_SCHEMA = {
+22
View File
@@ -4,6 +4,7 @@ from rich.progress import (
DownloadColumn,
MofNCompleteColumn,
Progress,
SpinnerColumn,
TextColumn,
TimeElapsedColumn,
TimeRemainingColumn,
@@ -52,3 +53,24 @@ class RichTransferProgress(RichProgress):
yield self.make_tasks_table(summary_tasks)
self.columns = self.TRANSFER_COLS
yield self.make_tasks_table(other_tasks)
class DbDownloadProgress(RichProgress):
PROGRESS_COLS = (
SpinnerColumn(),
TextColumn("[progress.description]{task.description}"),
TextColumn("[progress.download]{task.completed:,} rows"),
TextColumn("to [repr.filename]{task.fields[output]}"),
)
STATUS_COLS = (
SpinnerColumn(),
TextColumn("[progress.description]{task.description}"),
TextColumn("to [repr.filename]{task.fields[output]}"),
)
def get_renderables(self):
if self.tasks:
(task, *_) = self.tasks
cols = self.PROGRESS_COLS if task.completed else self.STATUS_COLS
self.columns = cols[1:] if task.finished else cols
yield self.make_tasks_table(self.tasks)