Move aqo_query_stat table into file storage. Use shmem hash table and file.

Replace UI according to this change.

Author: danolivo

aqo--1.4--1.5.sql

@@ -33,17 +33,38 @@ CREATE TABLE aqo_query_texts (
 query_text text NOT NULL
 );
 
-CREATE TABLE aqo_query_stat (
-query_hash bigint CONSTRAINT aqo_query_stat_idx PRIMARY KEY REFERENCES aqo_queries ON DELETE CASCADE,
-execution_time_with_aqo double precision[],
-execution_time_without_aqo double precision[],
-planning_time_with_aqo double precision[],
-planning_time_without_aqo double precision[],
-cardinality_error_with_aqo double precision[],
-cardinality_error_without_aqo double precision[],
-executions_with_aqo bigint,
-executions_without_aqo bigint
-);
+/* Now redefine */
+CREATE FUNCTION aqo_query_stat(
+OUT queryid bigint,
+OUT execution_time_with_aqo double precision[],
+OUT execution_time_without_aqo	double precision[],
+OUT planning_time_with_aqo double precision[],
+OUT planning_time_without_aqo double precision[],
+OUT cardinality_error_with_aqo	double precision[],
+OUT cardinality_error_without_aqo	double precision[],
+OUT executions_with_aqo bigint,
+OUT executions_without_aqo bigint
+)
+RETURNS SETOF record
+AS 'MODULE_PATHNAME', 'aqo_query_stat'
+LANGUAGE C STRICT VOLATILE PARALLEL SAFE;
+
+CREATE VIEW aqo_query_stat AS SELECT * FROM aqo_query_stat();
+
+--
+-- Remove all records in the AQO statistics.
+-- Return number of rows removed.
+--
+CREATE FUNCTION aqo_stat_reset() RETURNS bigint
+AS 'MODULE_PATHNAME'
+LANGUAGE C PARALLEL SAFE;
+
+COMMENT ON FUNCTION aqo_stat_reset() IS
+'Reset query statistics gathered by AQO';
+
+CREATE FUNCTION aqo_stat_remove(fs bigint) RETURNS bool
+AS 'MODULE_PATHNAME'
+LANGUAGE C STRICT PARALLEL SAFE;
 
 --
 -- Re-create the aqo_data table.
@@ -97,7 +118,7 @@ IF (controlled) THEN
 execution_time_with_aqo[array_length(execution_time_with_aqo, 1)] AS exectime,
 executions_with_aqo AS execs
 FROM aqo_queries aq JOIN aqo_query_stat aqs
-ON aq.query_hash = aqs.query_hash
+ON aq.query_hash = aqs.queryid
 WHERE TRUE = ANY (SELECT unnest(execution_time_with_aqo) IS NOT NULL)
 ) AS q1
 ORDER BY nn ASC;
@@ -116,7 +137,7 @@ ELSE
 (SELECT AVG(t) FROM unnest(execution_time_without_aqo) t) AS exectime,
 executions_without_aqo AS execs
 FROM aqo_queries aq JOIN aqo_query_stat aqs
-ON aq.query_hash = aqs.query_hash
+ON aq.query_hash = aqs.queryid
 WHERE TRUE = ANY (SELECT unnest(execution_time_without_aqo) IS NOT NULL)
 ) AS q1
 ORDER BY (nn) ASC;
@@ -195,6 +216,7 @@ BEGIN
 
 -- Remove ALL feature space if one of oids isn't exists
 DELETE FROM aqo_queries WHERE fspace_hash = fs;
+PERFORM * FROM aqo_stat_remove(fs);
 END LOOP;
 
 -- Calculate difference with previous state of knowledge base
@@ -235,7 +257,7 @@ IF (controlled) THEN
 cardinality_error_with_aqo[array_length(cardinality_error_with_aqo, 1)] AS cerror,
 executions_with_aqo AS execs
 FROM aqo_queries aq JOIN aqo_query_stat aqs
-ON aq.query_hash = aqs.query_hash
+ON aq.query_hash = aqs.queryid
 WHERE TRUE = ANY (SELECT unnest(cardinality_error_with_aqo) IS NOT NULL)
 ) AS q1
 ORDER BY nn ASC;
@@ -251,7 +273,7 @@ ELSE
 (SELECT AVG(t) FROM unnest(cardinality_error_without_aqo) t) AS cerror,
 executions_without_aqo AS execs
 FROM aqo_queries aq JOIN aqo_query_stat aqs
-ON aq.query_hash = aqs.query_hash
+ON aq.query_hash = aqs.queryid
 WHERE TRUE = ANY (SELECT unnest(cardinality_error_without_aqo) IS NOT NULL)
 ) AS q1
 ORDER BY (nn) ASC;

aqo.c

@@ -63,7 +63,7 @@ static const struct config_enum_entry format_options[] = {
 };
 
 /* Parameters of autotuning */
-int aqo_stat_size = 20;
+int aqo_stat_size = STAT_SAMPLE_SIZE;
 int auto_tuning_window_size = 5;
 double auto_tuning_exploration = 0.1;
 int auto_tuning_max_iterations = 50;
@@ -205,7 +205,7 @@ _PG_init(void)
 "Used for smooth transition from table storage",
 NULL,
 &aqo_use_file_storage,
-false,
+true,
 PGC_USERSET,
 0,
 NULL,

aqo.h

@@ -135,7 +135,6 @@
 #include "optimizer/cost.h"
 #include "parser/analyze.h"
 #include "parser/parsetree.h"
-#include "utils/array.h"
 #include "utils/builtins.h"
 #include "utils/guc.h"
 #include "utils/hsearch.h"
@@ -145,6 +144,7 @@
 #include "utils/snapmgr.h"
 
 #include "machine_learning.h"
+#include "storage.h"
 
 /* Check PostgreSQL version (9.6.0 contains important changes in planner) */
 #if PG_VERSION_NUM < 90600
@@ -175,32 +175,6 @@ extern bool aqo_show_hash;
 extern bool aqo_show_details;
 extern int aqo_join_threshold;
 
-/*
-* It is mostly needed for auto tuning of query. with auto tuning mode aqo
-* checks stability of last executions of the query, bad influence of strong
-* cardinality estimation on query execution (planner bug?) and so on.
-* It can induce aqo to suppress machine learning for this query.
-*/
-typedef struct
-{
-double *execution_time_with_aqo;
-double *execution_time_without_aqo;
-double *planning_time_with_aqo;
-double *planning_time_without_aqo;
-double *cardinality_error_with_aqo;
-double *cardinality_error_without_aqo;
-
-int execution_time_with_aqo_size;
-int execution_time_without_aqo_size;
-int planning_time_with_aqo_size;
-int planning_time_without_aqo_size;
-int cardinality_error_with_aqo_size;
-int cardinality_error_without_aqo_size;
-
-int64 executions_with_aqo;
-int64 executions_without_aqo;
-}	QueryStat;
-
 /* Parameters for current query */
 typedef struct QueryContextData
 {
@@ -288,8 +262,6 @@ extern bool load_fss(uint64 fs, int fss, OkNNrdata *data, List **reloids,
 extern bool update_fss_ext(uint64 fs, int fss, OkNNrdata *data,
 List *reloids, bool isTimedOut);
 extern bool update_fss(uint64 fs, int fss, OkNNrdata *data, List *reloids);
-QueryStat *get_aqo_stat(uint64 query_hash);
-void update_aqo_stat(uint64 query_hash, QueryStat * stat);
 extern bool my_index_insert(Relation indexRelation,	Datum *values, bool *isnull,
 ItemPointer heap_t_ctid, Relation heapRelation,
 IndexUniqueCheck checkUnique);
@@ -317,7 +289,7 @@ void aqo_ExecutorRun(QueryDesc *queryDesc, ScanDirection direction,
 void aqo_ExecutorEnd(QueryDesc *queryDesc);
 
 /* Automatic query tuning */
-extern void automatical_query_tuning(uint64 query_hash, QueryStat * stat);
+extern void automatical_query_tuning(uint64 query_hash, StatEntry *stat);
 
 /* Utilities */
 extern int int64_compare(const void *a, const void *b);
@@ -326,8 +298,6 @@ extern int double_cmp(const void *a, const void *b);
 extern int *argsort(void *a, int n, size_t es,
 int (*cmp) (const void *, const void *));
 extern int *inverse_permutation(int *a, int n);
-extern QueryStat *palloc_query_stat(void);
-extern void pfree_query_stat(QueryStat *stat);
 
 /* Selectivity cache for parametrized baserels */
 extern void cache_selectivity(int clause_hash, int relid, int global_relid,

aqo_shared.c

@@ -4,9 +4,12 @@
 
 #include "postgres.h"
 
+#include "lib/dshash.h"
+#include "miscadmin.h"
 #include "storage/shmem.h"
 
 #include "aqo_shared.h"
+#include "storage.h"
 
 
 typedef struct
@@ -23,11 +26,13 @@ shmem_startup_hook_type prev_shmem_startup_hook = NULL;
 AQOSharedState *aqo_state = NULL;
 HTAB *fss_htab = NULL;
 static int aqo_htab_max_items = 1000;
+static int fs_max_items = 1000; /* Max number of different feature spaces in ML model */
 static uint32 temp_storage_size = 1024 * 1024 * 10; /* Storage size, in bytes */
 static dsm_segment *seg = NULL;
 
 
 static void aqo_detach_shmem(int code, Datum arg);
+static void on_shmem_shutdown(int code, Datum arg);
 
 
 void *
@@ -169,16 +174,23 @@ aqo_init_shmem(void)
 bool found;
 HASHCTL info;
 
+if (prev_shmem_startup_hook)
+prev_shmem_startup_hook();
+
 aqo_state = NULL;
 fss_htab = NULL;
+stat_htab = NULL;
 
 LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE);
 aqo_state = ShmemInitStruct("aqo", sizeof(AQOSharedState), &found);
 if (!found)
 {
 /* First time through ... */
+
 LWLockInitialize(&aqo_state->lock, LWLockNewTrancheId());
 aqo_state->dsm_handler = DSM_HANDLE_INVALID;
+
+LWLockInitialize(&aqo_state->stat_lock, LWLockNewTrancheId());
 }
 
 info.keysize = sizeof(htab_key);
@@ -188,8 +200,31 @@ aqo_init_shmem(void)
 &info,
 HASH_ELEM | HASH_BLOBS);
 
+info.keysize = sizeof(((StatEntry *) 0)->queryid);
+info.entrysize = sizeof(StatEntry);
+stat_htab = ShmemInitHash("aqo stat hash",
+fs_max_items, fs_max_items,
+&info,
+HASH_ELEM | HASH_BLOBS);
+
 LWLockRelease(AddinShmemInitLock);
 LWLockRegisterTranche(aqo_state->lock.tranche, "aqo");
+LWLockRegisterTranche(aqo_state->stat_lock.tranche, "aqo stat storage");
+
+if (!IsUnderPostmaster)
+{
+on_shmem_exit(on_shmem_shutdown, (Datum) 0);
+aqo_stat_load();
+}
+}
+
+/*
+* Main idea here is to store all ML data in temp files on postmaster shutdown.
+*/
+static void
+on_shmem_shutdown(int code, Datum arg)
+{
+aqo_stat_flush();
 }
 
 Size
@@ -199,6 +234,7 @@ aqo_memsize(void)
 
 size = MAXALIGN(sizeof(AQOSharedState));
 size = add_size(size, hash_estimate_size(aqo_htab_max_items, sizeof(htab_entry)));
+size = add_size(size, hash_estimate_size(fs_max_items, sizeof(AQOSharedState)));
 
 return size;
 }

aqo_shared.h

@@ -5,6 +5,8 @@
 #include "storage/dsm.h"
 #include "storage/ipc.h"
 #include "storage/lwlock.h"
+#include "utils/dsa.h"
+#include "lib/dshash.h"
 
 #define AQO_SHARED_MAGIC	0x053163
 
@@ -25,6 +27,9 @@ typedef struct AQOSharedState
 {
 LWLock lock; /* mutual exclusion */
 dsm_handle	dsm_handler;
+
+/* Storage fields */
+LWLock stat_lock; /* lock for access to stat storage */
 } AQOSharedState;