Another attempt to resolve contradictory between oid-based and relname-based

approaches to organize ML storage base. In this we store list of oids of persistent tables for each record in the aqo_data table to have a possibility of cleaning records which depends on removed tables. On the other hand, we use relnames (tupDesc hash for TEMP tables) to form a kind of signature of a table. This signature is used for a feature subspace generation.

Author: danolivo

Makefile

@@ -1,7 +1,7 @@
 # contrib/aqo/Makefile
 
 EXTENSION = aqo
-EXTVERSION = 1.4
+EXTVERSION = 1.5
 PGFILEDESC = "AQO - Adaptive Query Optimization"
 MODULE_big = aqo
 OBJS = aqo.o auto_tuning.o cardinality_estimation.o cardinality_hooks.o \
@@ -24,6 +24,7 @@ REGRESS =	aqo_disabled \
 clean_aqo_data \
 plancache	\
 statement_timeout \
+temp_tables \
 top_queries
 
 fdw_srcdir = $(top_srcdir)/contrib/postgres_fdw
@@ -33,7 +34,7 @@ EXTRA_REGRESS_OPTS=--temp-config=$(top_srcdir)/$(subdir)/conf.add
 EXTRA_INSTALL = contrib/postgres_fdw contrib/pg_stat_statements
 
 DATA = aqo--1.0.sql aqo--1.0--1.1.sql aqo--1.1--1.2.sql aqo--1.2.sql \
-aqo--1.2--1.3.sql aqo--1.3--1.4.sql
+aqo--1.2--1.3.sql aqo--1.3--1.4.sql aqo--1.4--1.5.sql
 
 ifdef USE_PGXS
 PG_CONFIG ?= pg_config

aqo--1.4--1.5.sql

@@ -0,0 +1,75 @@
+/* contrib/aqo/aqo--1.4--1.5.sql */
+
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION aqo UPDATE TO '1.5'" to load this file. \quit
+
+--
+-- Re-create the aqo_data table. Do so to keep the columns order.
+--
+DROP TABLE public.aqo_data CASCADE;
+CREATE TABLE public.aqo_data (
+fspace_hash bigint NOT NULL REFERENCES public.aqo_queries ON DELETE CASCADE,
+fsspace_hash	int NOT NULL,
+nfeatures int NOT NULL,
+features double precision[][],
+targets double precision[],
+oids oid [] DEFAULT NULL,
+reliability double precision []
+);
+CREATE UNIQUE INDEX aqo_fss_access_idx ON public.aqo_data (fspace_hash, fsspace_hash);
+
+
+--
+-- Remove rows from the AQO ML knowledge base, related to previously dropped
+-- tables of the database.
+--
+CREATE OR REPLACE FUNCTION public.clean_aqo_data() RETURNS void AS $$
+DECLARE
+aqo_data_row aqo_data%ROWTYPE;
+aqo_queries_row aqo_queries%ROWTYPE;
+aqo_query_texts_row aqo_query_texts%ROWTYPE;
+aqo_query_stat_row aqo_query_stat%ROWTYPE;
+oid_var oid;
+fspace_hash_var bigint;
+delete_row boolean DEFAULT false;
+BEGIN
+FOR aqo_data_row IN (SELECT * FROM aqo_data)
+LOOP
+delete_row = false;
+SELECT aqo_data_row.fspace_hash INTO fspace_hash_var FROM aqo_data;
+
+IF (aqo_data_row.oids IS NOT NULL) THEN
+FOREACH oid_var IN ARRAY aqo_data_row.oids
+LOOP
+IF NOT EXISTS (SELECT relname FROM pg_class WHERE oid = oid_var) THEN
+delete_row = true;
+END IF;
+END LOOP;
+END IF;
+
+FOR aqo_queries_row IN (SELECT * FROM public.aqo_queries)
+LOOP
+IF (delete_row = true AND fspace_hash_var <> 0 AND
+fspace_hash_var = aqo_queries_row.fspace_hash AND
+aqo_queries_row.fspace_hash = aqo_queries_row.query_hash) THEN
+DELETE FROM aqo_data WHERE aqo_data = aqo_data_row;
+DELETE FROM aqo_queries WHERE aqo_queries = aqo_queries_row;
+
+FOR aqo_query_texts_row IN (SELECT * FROM aqo_query_texts)
+LOOP
+DELETE FROM aqo_query_texts
+WHERE aqo_query_texts_row.query_hash = fspace_hash_var AND
+aqo_query_texts = aqo_query_texts_row;
+END LOOP;
+
+FOR aqo_query_stat_row IN (SELECT * FROM aqo_query_stat)
+LOOP
+DELETE FROM aqo_query_stat
+WHERE aqo_query_stat_row.query_hash = fspace_hash_var AND
+aqo_query_stat = aqo_query_stat_row;
+END LOOP;
+END IF;
+END LOOP;
+END LOOP;
+END;
+$$ LANGUAGE plpgsql;

aqo.control

@@ -1,5 +1,5 @@
 # AQO extension
 comment = 'machine learning for cardinality estimation in optimizer'
-default_version = '1.4'
+default_version = '1.5'
 module_pathname = '$libdir/aqo'
 relocatable = false

aqo.h

@@ -280,12 +280,12 @@ extern bool find_query(uint64 qhash, QueryContextData *ctx);
 extern bool update_query(uint64 qhash, uint64 fhash,
 bool learn_aqo, bool use_aqo, bool auto_tuning);
 extern bool add_query_text(uint64 query_hash, const char *query_string);
-extern bool load_fss_ext(uint64 fs, int fss, OkNNrdata *data, List **relnames,
+extern bool load_fss_ext(uint64 fs, int fss, OkNNrdata *data, List **reloids,
 bool isSafe);
-extern bool load_fss(uint64 fs, int fss, OkNNrdata *data, List **relnames);
+extern bool load_fss(uint64 fs, int fss, OkNNrdata *data, List **reloids);
 extern bool update_fss_ext(uint64 fs, int fss, OkNNrdata *data,
-List *relnames, bool isTimedOut);
-extern bool update_fss(uint64 fs, int fss, OkNNrdata *data, List *relnames);
+List *reloids, bool isTimedOut);
+extern bool update_fss(uint64 fs, int fss, OkNNrdata *data, List *reloids);
 QueryStat *get_aqo_stat(uint64 query_hash);
 void update_aqo_stat(uint64 query_hash, QueryStat * stat);
 extern bool my_index_insert(Relation indexRelation,	Datum *values, bool *isnull,
@@ -305,8 +305,8 @@ extern void print_into_explain(PlannedStmt *plannedstmt, IntoClause *into,
 extern void print_node_explain(ExplainState *es, PlanState *ps, Plan *plan);
 
 /* Cardinality estimation */
-double predict_for_relation(List *restrict_clauses, List *selectivities,
-List *relnames, int *fss);
+extern double predict_for_relation(List *restrict_clauses, List *selectivities,
+List *relsigns, int *fss);
 
 /* Query execution statistics collecting hooks */
 void aqo_ExecutorStart(QueryDesc *queryDesc, int eflags);

cardinality_estimation.c

@@ -26,7 +26,7 @@
 #ifdef AQO_DEBUG_PRINT
 static void
 predict_debug_output(List *clauses, List *selectivities,
-List *relnames, int fss, double result)
+List *reloids, int fss, double result)
 {
 StringInfoData debug_str;
 ListCell *lc;
@@ -42,8 +42,8 @@ predict_debug_output(List *clauses, List *selectivities,
 appendStringInfo(&debug_str, "%lf ", *s);
 }
 
-appendStringInfoString(&debug_str, "}, relnames: { ");
-foreach(lc, relnames)
+appendStringInfoString(&debug_str, "}, reloids: { ");
+foreach(lc, reloids)
 {
 Value *relname = lfirst_node(String, lc);
 appendStringInfo(&debug_str, "%s ", valStr(relname));
@@ -59,22 +59,22 @@ predict_debug_output(List *clauses, List *selectivities,
 * General method for prediction the cardinality of given relation.
 */
 double
-predict_for_relation(List *clauses, List *selectivities,
-List *relnames, int *fss)
+predict_for_relation(List *clauses, List *selectivities, List *relsigns,
+int *fss)
 {
 double *features;
 double result;
 int i;
 OkNNrdata	data;
 
-if (relnames == NIL)
+if (relsigns == NIL)
 /*
 * Don't make prediction for query plans without any underlying plane
 * tables. Use return value -4 for debug purposes.
 */
 return -4.;
 
-*fss = get_fss_for_object(relnames, clauses, selectivities,
+*fss = get_fss_for_object(relsigns, clauses, selectivities,
 &data.cols, &features);
 
 if (data.cols > 0)
@@ -94,7 +94,7 @@ predict_for_relation(List *clauses, List *selectivities,
 result = -1;
 }
 #ifdef AQO_DEBUG_PRINT
-predict_debug_output(clauses, selectivities, relnames, *fss, result);
+predict_debug_output(clauses, selectivities, relsigns, *fss, result);
 #endif
 pfree(features);
 if (data.cols > 0)

cardinality_hooks.c

@@ -138,12 +138,12 @@ default_estimate_num_groups(PlannerInfo *root, List *groupExprs,
 void
 aqo_set_baserel_rows_estimate(PlannerInfo *root, RelOptInfo *rel)
 {
-double	predicted;
-RangeTblEntry *rte;
-List *relnames = NIL;
-List *selectivities = NULL;
-List *clauses;
-int fss = 0;
+double	predicted;
+RangeTblEntry *rte;
+RelSortOut rels = {NIL, NIL};
+List *selectivities = NULL;
+List *clauses;
+int fss = 0;
 
 if (IsQueryDisabled())
 /* Fast path. */
@@ -166,16 +166,18 @@ aqo_set_baserel_rows_estimate(PlannerInfo *root, RelOptInfo *rel)
 {
 /* Predict for a plane table. */
 Assert(rte->eref && rte->eref->aliasname);
-relnames = list_make1(makeString(pstrdup(rte->eref->aliasname)));
+get_list_of_relids(root, rel->relids, &rels);
 }
 
 clauses = aqo_get_clauses(root, rel->baserestrictinfo);
-predicted = predict_for_relation(clauses, selectivities, relnames, &fss);
+predicted = predict_for_relation(clauses, selectivities, rels.signatures,
+&fss);
 rel->fss_hash = fss;
 
+list_free(rels.hrels);
+list_free(rels.signatures);
 list_free_deep(selectivities);
 list_free(clauses);
-list_free(relnames);
 
 if (predicted >= 0)
 {
@@ -212,7 +214,7 @@ aqo_get_parameterized_baserel_size(PlannerInfo *root,
 {
 double predicted;
 RangeTblEntry *rte = NULL;
-List *relnames = NIL;
+RelSortOut	rels = {NIL, NIL};
 List *allclauses = NULL;
 List *selectivities = NULL;
 ListCell *l;
@@ -268,10 +270,12 @@ aqo_get_parameterized_baserel_size(PlannerInfo *root,
 {
 /* Predict for a plane table. */
 Assert(rte->eref && rte->eref->aliasname);
-relnames = list_make1(makeString(pstrdup(rte->eref->aliasname)));
+get_list_of_relids(root, rel->relids, &rels);
 }
 
-predicted = predict_for_relation(allclauses, selectivities, relnames, &fss);
+predicted = predict_for_relation(allclauses, selectivities, rels.signatures, &fss);
+list_free(rels.hrels);
+list_free(rels.signatures);
 
 predicted_ppi_rows = predicted;
 fss_ppi_hash = fss;
@@ -296,7 +300,7 @@ aqo_set_joinrel_size_estimates(PlannerInfo *root, RelOptInfo *rel,
 List *restrictlist)
 {
 double predicted;
-List *relnames;
+RelSortOut rels = {NIL, NIL};
 List *outer_clauses;
 List *inner_clauses;
 List *allclauses;
@@ -322,7 +326,7 @@ aqo_set_joinrel_size_estimates(PlannerInfo *root, RelOptInfo *rel,
 goto default_estimator;
 }
 
-relnames = get_relnames(root, rel->relids);
+get_list_of_relids(root, rel->relids, &rels);
 outer_clauses = get_path_clauses(outer_rel->cheapest_total_path, root,
 &outer_selectivities);
 inner_clauses = get_path_clauses(inner_rel->cheapest_total_path, root,
@@ -333,7 +337,11 @@ aqo_set_joinrel_size_estimates(PlannerInfo *root, RelOptInfo *rel,
 list_concat(outer_selectivities,
 inner_selectivities));
 
-predicted = predict_for_relation(allclauses, selectivities, relnames, &fss);
+predicted = predict_for_relation(allclauses, selectivities, rels.signatures,
+&fss);
+list_free(rels.hrels);
+list_free(rels.signatures);
+
 rel->fss_hash = fss;
 
 if (predicted >= 0)
@@ -364,7 +372,7 @@ aqo_get_parameterized_joinrel_size(PlannerInfo *root,
 List *clauses)
 {
 double predicted;
-List *relnames;
+RelSortOut	rels = {NIL, NIL};
 List *outer_clauses;
 List *inner_clauses;
 List *allclauses;
@@ -390,7 +398,7 @@ aqo_get_parameterized_joinrel_size(PlannerInfo *root,
 goto default_estimator;
 }
 
-relnames = get_relnames(root, rel->relids);
+get_list_of_relids(root, rel->relids, &rels);
 outer_clauses = get_path_clauses(outer_path, root, &outer_selectivities);
 inner_clauses = get_path_clauses(inner_path, root, &inner_selectivities);
 allclauses = list_concat(aqo_get_clauses(root, clauses),
@@ -399,7 +407,10 @@ aqo_get_parameterized_joinrel_size(PlannerInfo *root,
 list_concat(outer_selectivities,
 inner_selectivities));
 
-predicted = predict_for_relation(allclauses, selectivities, relnames, &fss);
+predicted = predict_for_relation(allclauses, selectivities, rels.signatures,
+&fss);
+list_free(rels.hrels);
+list_free(rels.signatures);
 
 predicted_ppi_rows = predicted;
 fss_ppi_hash = fss;
@@ -426,13 +437,16 @@ predict_num_groups(PlannerInfo *root, Path *subpath, List *group_exprs,
 child_fss = subpath->parent->fss_hash;
 else
 {
-List *relnames;
-List *clauses;
-List *selectivities = NIL;
+RelSortOut rels = {NIL, NIL};
+List *clauses;
+List *selectivities = NIL;
 
-relnames = get_relnames(root, subpath->parent->relids);
+get_list_of_relids(root, subpath->parent->relids, &rels);
 clauses = get_path_clauses(subpath, root, &selectivities);
-(void) predict_for_relation(clauses, selectivities, relnames, &child_fss);
+(void) predict_for_relation(clauses, selectivities, rels.signatures,
+&child_fss);
+list_free(rels.hrels);
+list_free(rels.signatures);
 }
 
 *fss = get_grouped_exprs_hash(child_fss, group_exprs);