mirror of
https://github.com/sandronator/finetuning_aufgabe4.git
synced 2026-09-04 00:26:06 +02:00
Label output with Aufgabe + strategy + index config, fix lock hangs
Two related improvements to make the test runs both usable and
diagnosable:
1. Avoid lock waits / "deadlock" hangs during CLUSTER
The Manager connection ran EXPLAIN ANALYZE without committing, so
psycopg2's default autocommit=False left an open transaction
holding AccessShareLock on publ and auth. The next setup_db then
blocked on DROP INDEX / CLUSTER (both need AccessExclusiveLock),
visible as a hang -- typically at the CLUSTER step. Setting
conn_postsql.autocommit = True releases the locks immediately after
each SELECT, which is safe here because all Manager queries are
read-only.
2. Make the run output identify which test is running
The "Running: ..." line previously printed only the query key
(query_1/query_2), shadowing 'strategy' with the dict key and never
surfacing which join strategy or index config was active. Output
now reads e.g. "[Aufgabe 3b | SortMergeStrategy | idx=nc-both]" by:
- using self.__class__.__name__ so subclasses surface correctly
- passing the current index_config from Manager into the strategy
- threading an optional 'aufgabe' label through Manager.execute()
and into each strategy's run() (including the Nested/SortMerge/
Hash wrappers that override run()).
main.py now tags each execute() call with its assignment number.
Co-Authored-By: Claude Opus 4.7 <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 4.7
parent
d8a906e2d4
commit
da2453da32
+17
-7
@@ -12,25 +12,35 @@ class BaseStrategy:
|
||||
self.cursor = conn.cursor()
|
||||
self.db_name = db_name
|
||||
self.queries = queries
|
||||
self.index_config = None # vom Manager via setup_db gesetzt
|
||||
|
||||
def set_queries(self, queries):
|
||||
self.queries = queries
|
||||
|
||||
def run(self):
|
||||
for strategy, query in self.queries.items():
|
||||
print("Running: " + strategy + "\n")
|
||||
def run(self, aufgabe: str | None = None):
|
||||
strategy_name = self.__class__.__name__
|
||||
prefix_parts = []
|
||||
if aufgabe:
|
||||
prefix_parts.append(aufgabe)
|
||||
prefix_parts.append(strategy_name)
|
||||
if self.index_config:
|
||||
prefix_parts.append(f"idx={self.index_config}")
|
||||
prefix = " | ".join(prefix_parts)
|
||||
|
||||
for query_name, query in self.queries.items():
|
||||
print(f"Running: [{prefix}] / {query_name}\n")
|
||||
start = time.time()
|
||||
self.cursor.execute(query)
|
||||
end = time.time()
|
||||
self._print_result(strategy, end, start)
|
||||
self._print_result(query_name, end, start, prefix)
|
||||
|
||||
|
||||
def _print_result(self, strategy, end, start):
|
||||
def _print_result(self, query_name, end, start, prefix):
|
||||
duration = end - start
|
||||
minutes= int(duration // 60)
|
||||
minutes = int(duration // 60)
|
||||
seconds = duration % 60
|
||||
|
||||
print(f"---- RESULTS FROM {self.db_name} Strategy: {strategy} ----\n")
|
||||
print(f"---- RESULTS FROM {self.db_name} | {prefix} | Query: {query_name} ----\n")
|
||||
print(f"duration: {minutes}m {seconds:.2f}s")
|
||||
_print_cursor(self.cursor)
|
||||
|
||||
|
||||
+2
-2
@@ -5,14 +5,14 @@ class HashJoinStrategy(BaseStrategy):
|
||||
def __init__(self, conn, db_name, all_queries):
|
||||
super().__init__(conn, db_name, all_queries)
|
||||
|
||||
def run(self):
|
||||
def run(self, aufgabe: str | None = None):
|
||||
if self.db_name not in resolve:
|
||||
raise Exception("Hash Join not supported for " + self.db_name)
|
||||
|
||||
for disable in resolve[self.db_name]["disable"]:
|
||||
self.cursor.execute(disable)
|
||||
|
||||
super().run()
|
||||
super().run(aufgabe=aufgabe)
|
||||
|
||||
for reset in resolve[self.db_name]["reset"]:
|
||||
self.cursor.execute(reset)
|
||||
@@ -8,6 +8,9 @@ from repositories.all_queries import queries_postgres
|
||||
|
||||
if __name__ == '__main__':
|
||||
db_post, conn_postsql = get_connection(maria=False)
|
||||
# Autocommit verhindert, dass EXPLAIN ANALYZE Locks (AccessShareLock auf publ/auth)
|
||||
# bis zum naechsten Commit haelt -- sonst blockiert das spaeter DROP INDEX/CLUSTER.
|
||||
conn_postsql.autocommit = True
|
||||
|
||||
# Daten einmalig laden; danach werden zwischen den Tests nur die Indexe getauscht.
|
||||
load_data_postgres()
|
||||
@@ -15,33 +18,33 @@ if __name__ == '__main__':
|
||||
# Aufgabe 1
|
||||
join_manager = Manager(BaseStrategy(conn_postsql, db_post, queries_postgres["with_index"]))
|
||||
# join_manager.setup_db("no-index")
|
||||
# join_manager.execute() # 3 Mrd. Tupel via Kreuzprodukt -> > 10 min
|
||||
# join_manager.execute(aufgabe="Aufgabe 1a") # 3 Mrd. Tupel via Kreuzprodukt -> > 10 min
|
||||
join_manager.setup_db("unique-publ")
|
||||
join_manager.execute()
|
||||
join_manager.execute(aufgabe="Aufgabe 1b")
|
||||
join_manager.setup_db("cl-both")
|
||||
join_manager.execute()
|
||||
join_manager.execute(aufgabe="Aufgabe 1c")
|
||||
|
||||
# Aufgabe 2 — vorheriger Lauf war 'cl-both' (CLUSTER hat Tabelle physisch sortiert);
|
||||
# reload_data=True stellt die ursprüngliche Ladereihenfolge wieder her.
|
||||
join_manager.setStrategy(NestedInnerLoopStrategy(conn_postsql, db_post, queries_postgres["with_index"]))
|
||||
join_manager.setup_db("nc-publ", reload_data=True)
|
||||
join_manager.execute()
|
||||
join_manager.execute(aufgabe="Aufgabe 2a")
|
||||
join_manager.setup_db("nc-auth")
|
||||
join_manager.execute()
|
||||
join_manager.execute(aufgabe="Aufgabe 2b")
|
||||
join_manager.setup_db("nc-both")
|
||||
join_manager.execute()
|
||||
join_manager.execute(aufgabe="Aufgabe 2c")
|
||||
|
||||
# Aufgabe 3 — Tabelle ist noch im Original-Layout, kein Reload nötig.
|
||||
join_manager.setStrategy(SortMergeStrategy(conn_postsql, db_post, queries_postgres["no_index"]))
|
||||
join_manager.setup_db("no-index")
|
||||
join_manager.execute()
|
||||
join_manager.execute(aufgabe="Aufgabe 3a")
|
||||
join_manager.setQueries(queries_postgres["with_index"])
|
||||
join_manager.setup_db("nc-both")
|
||||
join_manager.execute()
|
||||
join_manager.execute(aufgabe="Aufgabe 3b")
|
||||
join_manager.setup_db("cl-both")
|
||||
join_manager.execute()
|
||||
join_manager.execute(aufgabe="Aufgabe 3c")
|
||||
|
||||
# Aufgabe 4 — wieder vom CLUSTER-Zustand wegkommen.
|
||||
join_manager.setStrategy(HashJoinStrategy(conn_postsql, db_post, queries_postgres["no_index"]))
|
||||
join_manager.setup_db("no-index", reload_data=True)
|
||||
join_manager.execute()
|
||||
join_manager.execute(aufgabe="Aufgabe 4")
|
||||
|
||||
+7
-2
@@ -4,17 +4,22 @@ import setup
|
||||
class Manager:
|
||||
def __init__(self, strategy: BaseStrategy, ):
|
||||
self.strategy = strategy
|
||||
self.current_index_config = None
|
||||
|
||||
def setStrategy(self, strategy: BaseStrategy):
|
||||
self.strategy = strategy
|
||||
# Aktuellen Index-Config an die neue Strategy weiterreichen
|
||||
self.strategy.index_config = self.current_index_config
|
||||
|
||||
def setQueries(self, queries: dict[str, str]):
|
||||
self.strategy.set_queries(queries)
|
||||
|
||||
# Only Postgresql and MariaDb available at the moment
|
||||
def setup_db(self, index_config, reload_data=False):
|
||||
self.current_index_config = index_config
|
||||
self.strategy.index_config = index_config
|
||||
setup.reset_postgres(index_config, reload_data=reload_data)
|
||||
|
||||
def execute(self):
|
||||
self.strategy.run()
|
||||
def execute(self, aufgabe: str | None = None):
|
||||
self.strategy.run(aufgabe=aufgabe)
|
||||
|
||||
@@ -5,14 +5,14 @@ class NestedInnerLoopStrategy(BaseStrategy):
|
||||
def __init__(self, conn, db_name, all_queries):
|
||||
super().__init__(conn, db_name, all_queries)
|
||||
|
||||
def run(self):
|
||||
def run(self, aufgabe: str | None = None):
|
||||
if self.db_name not in resolve:
|
||||
raise Exception("No Inner Loop Resolver Found")
|
||||
|
||||
for disable in resolve[self.db_name]["disable"]:
|
||||
self.cursor.execute(disable)
|
||||
|
||||
super().run()
|
||||
super().run(aufgabe=aufgabe)
|
||||
|
||||
for reset in resolve[self.db_name]["reset"]:
|
||||
self.cursor.execute(reset)
|
||||
@@ -5,14 +5,14 @@ class SortMergeStrategy(BaseStrategy):
|
||||
def __init__(self, conn, db_name, all_queries):
|
||||
super().__init__(conn, db_name, all_queries)
|
||||
|
||||
def run(self):
|
||||
def run(self, aufgabe: str | None = None):
|
||||
if self.db_name not in resolve:
|
||||
raise Exception("Sort-Merge Join not supported for " + self.db_name)
|
||||
|
||||
for disable in resolve[self.db_name]["disable"]:
|
||||
self.cursor.execute(disable)
|
||||
|
||||
super().run()
|
||||
super().run(aufgabe=aufgabe)
|
||||
|
||||
for reset in resolve[self.db_name]["reset"]:
|
||||
self.cursor.execute(reset)
|
||||
Reference in New Issue
Block a user