mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-07 07:18:27 +01:00
129 lines
3.6 KiB
Python
129 lines
3.6 KiB
Python
"""E2E tests involving dvc/dbt and a ~~real~~ sqlite db.
|
|
|
|
Any other tests should likely be mocked tests in `test_import_db` rather than here.
|
|
"""
|
|
|
|
import io
|
|
import os
|
|
import sqlite3
|
|
from contextlib import closing
|
|
from typing import TYPE_CHECKING
|
|
|
|
import pytest
|
|
|
|
if TYPE_CHECKING:
|
|
from agate import Table
|
|
|
|
|
|
dbt_connections = pytest.importorskip(
|
|
"dbt.adapters.sql.connections", reason="dbt-core not installed"
|
|
)
|
|
SQLConnectionManager = dbt_connections.SQLConnectionManager
|
|
pytest.importorskip("dbt.adapters.sqlite", reason="dbt-sqlite not installed")
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def disable_tracking(monkeypatch):
|
|
monkeypatch.setenv("DO_NOT_TRACK", "1")
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def mock_sqlite_adapter(mocker):
|
|
# https://github.com/codeforkjeff/dbt-sqlite/issues/47
|
|
mocker.patch(
|
|
"dbt.adapters.sqlite.connections.SQLiteCredentials.unique_field", "sqlite_host"
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def db_path(tmp_dir):
|
|
d = tmp_dir / "db"
|
|
d.mkdir()
|
|
return d / "main.db"
|
|
|
|
|
|
@pytest.fixture
|
|
def seed_db(db_path):
|
|
conn = sqlite3.connect(db_path)
|
|
conn.execute("CREATE TABLE model (id INTEGER PRIMARY KEY, value INTEGER)")
|
|
|
|
def inner(values):
|
|
conn.executemany("INSERT INTO model(value) VALUES(?)", [(i,) for i in values])
|
|
conn.commit()
|
|
with closing(conn.execute("SELECT * FROM model")) as cursor:
|
|
return SQLConnectionManager.get_result_from_cursor(cursor, None)
|
|
|
|
with closing(conn):
|
|
yield inner
|
|
|
|
|
|
@pytest.fixture
|
|
def dbt_profile(tmp_dir, db_path):
|
|
(tmp_dir / "profiles.yml").dump(
|
|
{
|
|
"sqlite_profile": {
|
|
"target": "dev",
|
|
"outputs": {
|
|
"dev": {
|
|
"type": "sqlite",
|
|
"threads": 1,
|
|
"database": "database",
|
|
"schema": "main",
|
|
"schemas_and_paths": {"main": os.fspath(db_path)},
|
|
"schema_directory": os.fspath(db_path.parent),
|
|
}
|
|
},
|
|
}
|
|
}
|
|
)
|
|
return "sqlite_profile"
|
|
|
|
|
|
@pytest.fixture
|
|
def dbt_project(tmp_dir, dbt_profile):
|
|
(tmp_dir / "dbt_project.yml").dump({"name": "project", "profile": dbt_profile})
|
|
return tmp_dir
|
|
|
|
|
|
@pytest.fixture
|
|
def dbt_model(dbt_project):
|
|
dbt_project.gen({"models": {"model.sql": "select * from models"}})
|
|
return "model"
|
|
|
|
|
|
@pytest.fixture(params=("sql", "model"))
|
|
def import_db_parameters(request):
|
|
if request.param == "sql":
|
|
profile = request.getfixturevalue("dbt_profile")
|
|
return {"sql": "select * from model", "profile": profile}
|
|
return {"model": request.getfixturevalue("dbt_model")}
|
|
|
|
|
|
@pytest.fixture
|
|
def file_name(import_db_parameters):
|
|
if model := import_db_parameters.get("model"):
|
|
return model
|
|
return "results"
|
|
|
|
|
|
def format_output(table: "Table", output_format: str):
|
|
output = io.StringIO()
|
|
getattr(table, f"to_{output_format}")(output)
|
|
return output.getvalue()
|
|
|
|
|
|
@pytest.mark.filterwarnings("ignore::ResourceWarning") # dbt leaks fileobj from logger
|
|
@pytest.mark.parametrize("output_format", ("csv", "json"))
|
|
def test_e2e(tmp_dir, dvc, seed_db, import_db_parameters, file_name, output_format):
|
|
results: "Table" = seed_db(values=range(5))
|
|
|
|
stage = dvc.imp_db(**import_db_parameters, output_format=output_format)
|
|
|
|
output_file = tmp_dir / f"{file_name}.{output_format}"
|
|
assert output_file.read_text() == format_output(results, output_format)
|
|
|
|
results: "Table" = seed_db(values=range(6, 10))
|
|
|
|
dvc.update(stage.addressing)
|
|
assert output_file.read_text() == format_output(results, output_format)
|