Add search by fss hash in aqo_data table and hash table.

If we didn't any neibours with fs and fss hash indexes in aqo_data, we write new object in aqo_data with target value as average value of neirest neibours by fss_hash. I don't consider fs_hash in find neirest neibour for calculating average value of target for new object because I think fs_hash contain a description of a completely different query with a different table that matches or almost matches the indicative descriptions of the current object, but they are not an entity.

Author: Alena Rybakina

Makefile

@@ -26,7 +26,8 @@ REGRESS =	aqo_disabled \
 statement_timeout \
 temp_tables \
 top_queries \
-relocatable
+relocatable\
+look_a_like
 
 fdw_srcdir = $(top_srcdir)/contrib/postgres_fdw
 stat_srcdir = $(top_srcdir)/contrib/pg_stat_statements

aqo--1.0.sql

@@ -50,3 +50,5 @@ CREATE FUNCTION invalidate_deactivated_queries_cache() RETURNS trigger
 CREATE TRIGGER aqo_queries_invalidate AFTER UPDATE OR DELETE OR TRUNCATE
 ON public.aqo_queries FOR EACH STATEMENT
 EXECUTE PROCEDURE invalidate_deactivated_queries_cache();
+CREATE INDEX aqo_fss_idx
+on public.aqo_data (fsspace_hash);

aqo.h

@@ -283,7 +283,7 @@ extern bool update_query(uint64 qhash, uint64 fhash,
 extern bool add_query_text(uint64 query_hash, const char *query_string);
 extern bool load_fss_ext(uint64 fs, int fss, OkNNrdata *data, List **reloids,
 bool isSafe);
-extern bool load_fss(uint64 fs, int fss, OkNNrdata *data, List **reloids);
+extern bool load_fss(uint64 fs, int fss, OkNNrdata *data, List **reloids, bool use_idx_fss);
 extern bool update_fss_ext(uint64 fs, int fss, OkNNrdata *data,
 List *reloids, bool isTimedOut);
 extern bool update_fss(uint64 fs, int fss, OkNNrdata *data, List *reloids);

cardinality_estimation.c

@@ -91,7 +91,10 @@ predict_for_relation(List *clauses, List *selectivities, List *relsigns,
 * small part of paths was used for AQO learning and fetch into the AQO
 * knowledge base.
 */
-result = -1;
+if (!load_fss(query_context.fspace_hash, *fss, &data, NULL, false))
+result = -1;
+else
+result = OkNNr_predict(&data, features);
 }
 #ifdef AQO_DEBUG_PRINT
 predict_debug_output(clauses, selectivities, relsigns, *fss, result);

expected/aqo_fdw.out

@@ -88,32 +88,22 @@ SELECT str FROM expln('
 EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF)
 SELECT * FROM frgn AS a, frgn AS b WHERE a.x=b.x;
 ') AS str WHERE str NOT LIKE '%Sort Method%';
-str
-------------------------------------------------------------
-Merge Join (actual rows=1 loops=1)
+str
+-------------------------------------------
+Foreign Scan (actual rows=1 loops=1)
 AQO not used
-Merge Cond: (a.x = b.x)
--> Sort (actual rows=1 loops=1)
-AQO not used
-Sort Key: a.x
--> Foreign Scan on frgn a (actual rows=1 loops=1)
-AQO not used
--> Sort (actual rows=1 loops=1)
-AQO not used
-Sort Key: b.x
--> Foreign Scan on frgn b (actual rows=1 loops=1)
-AQO not used
+Relations: (frgn a) INNER JOIN (frgn b)
 Using aqo: true
 AQO mode: LEARN
 JOINS: 0
-(16 rows)
+(6 rows)
 
 EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF, VERBOSE)
 SELECT * FROM frgn AS a, frgn AS b WHERE a.x=b.x;
 QUERY PLAN
 --------------------------------------------------------------------------------------------------------
 Foreign Scan (actual rows=1 loops=1)
-AQO: rows=1, error=0%
+AQO not used
 Output: a.x, b.x
 Relations: (public.frgn a) INNER JOIN (public.frgn b)
 Remote SQL: SELECT r1.x, r2.x FROM (public.local r1 INNER JOIN public.local r2 ON (((r1.x = r2.x))))

expected/aqo_learn.out

@@ -265,12 +265,9 @@ ORDER BY (md5(query_text))
 | | WHERE t1.a < 1 AND t2.b < 1 AND t2.c < 1 AND t2.d < 1 AND t1.a = t2.a;
 {1,1,1} | 4 | EXPLAIN SELECT * FROM aqo_test0 +
 | | WHERE a < 3 AND b < 3 AND c < 3 AND d < 3;
-{1} | 0 | SELECT count(*) FROM tmp1;
-{1} | 0 | SELECT count(*) FROM tmp1;
-{1} | 6 | EXPlAIN SELECT t1.a, t2.b, t3.c +
-| | FROM aqo_test1 AS t1, aqo_test0 AS t2, aqo_test0 AS t3 +
-| | WHERE t1.a < 1 AND t3.b < 1 AND t2.c < 1 AND t3.d < 0 AND t1.a = t2.a AND t1.b = t3.b;
-{1} | 1 | EXPlAIN SELECT t1.a, t2.b, t3.c +
+{1,1,1,1,1} | 0 | SELECT count(*) FROM tmp1;
+{1,1,1,1,1} | 0 | SELECT count(*) FROM tmp1;
+{1} | 2 | EXPlAIN SELECT t1.a, t2.b, t3.c +
 | | FROM aqo_test1 AS t1, aqo_test0 AS t2, aqo_test0 AS t3 +
 | | WHERE t1.a < 1 AND t3.b < 1 AND t2.c < 1 AND t3.d < 0 AND t1.a = t2.a AND t1.b = t3.b;
 {1} | 2 | EXPlAIN SELECT t1.a, t2.b, t3.c +
@@ -279,34 +276,37 @@ ORDER BY (md5(query_text))
 {1} | 3 | EXPlAIN SELECT t1.a, t2.b, t3.c +
 | | FROM aqo_test1 AS t1, aqo_test0 AS t2, aqo_test0 AS t3 +
 | | WHERE t1.a < 1 AND t3.b < 1 AND t2.c < 1 AND t3.d < 0 AND t1.a = t2.a AND t1.b = t3.b;
-{1} | 2 | EXPlAIN SELECT t1.a, t2.b, t3.c +
+{1} | 6 | EXPlAIN SELECT t1.a, t2.b, t3.c +
 | | FROM aqo_test1 AS t1, aqo_test0 AS t2, aqo_test0 AS t3 +
 | | WHERE t1.a < 1 AND t3.b < 1 AND t2.c < 1 AND t3.d < 0 AND t1.a = t2.a AND t1.b = t3.b;
-{1} | 2 | EXPLAIN SELECT t1.a, t2.b FROM aqo_test0 AS t1, aqo_test0 AS t2 +
+{1,1} | 1 | EXPlAIN SELECT t1.a, t2.b, t3.c +
+| | FROM aqo_test1 AS t1, aqo_test0 AS t2, aqo_test0 AS t3 +
+| | WHERE t1.a < 1 AND t3.b < 1 AND t2.c < 1 AND t3.d < 0 AND t1.a = t2.a AND t1.b = t3.b;
+{1} | 4 | EXPLAIN SELECT t1.a, t2.b FROM aqo_test0 AS t1, aqo_test0 AS t2 +
 | | WHERE t1.a < 1 AND t1.b < 1 AND t2.c < 1 AND t2.d < 1;
 {1} | 2 | EXPLAIN SELECT t1.a, t2.b FROM aqo_test0 AS t1, aqo_test0 AS t2 +
 | | WHERE t1.a < 1 AND t1.b < 1 AND t2.c < 1 AND t2.d < 1;
-{1} | 4 | EXPLAIN SELECT t1.a, t2.b FROM aqo_test0 AS t1, aqo_test0 AS t2 +
+{1} | 2 | EXPLAIN SELECT t1.a, t2.b FROM aqo_test0 AS t1, aqo_test0 AS t2 +
 | | WHERE t1.a < 1 AND t1.b < 1 AND t2.c < 1 AND t2.d < 1;
-{1} | 2 | EXPLAIN SELECT t1.a AS a, t2.a AS b, t3.a AS c +
+{1} | 1 | EXPLAIN SELECT t1.a AS a, t2.a AS b, t3.a AS c +
 | | FROM aqo_test1 AS t1, aqo_test1 AS t2, aqo_test1 AS t3 +
 | | WHERE t1.a = t2.b AND t2.a = t3.b;
-{1} | 1 | EXPLAIN SELECT t1.a AS a, t2.a AS b, t3.a AS c +
+{1} | 2 | EXPLAIN SELECT t1.a AS a, t2.a AS b, t3.a AS c +
 | | FROM aqo_test1 AS t1, aqo_test1 AS t2, aqo_test1 AS t3 +
 | | WHERE t1.a = t2.b AND t2.a = t3.b;
 {1} | 0 | EXPLAIN SELECT t1.a AS a, t2.a AS b, t3.a AS c +
 | | FROM aqo_test1 AS t1, aqo_test1 AS t2, aqo_test1 AS t3 +
 | | WHERE t1.a = t2.b AND t2.a = t3.b;
-{1} | 3 | EXPLAIN SELECT t1.a AS a, t2.a AS b, t3.a AS c, t4.a AS d +
+{1} | 2 | EXPLAIN SELECT t1.a AS a, t2.a AS b, t3.a AS c, t4.a AS d +
 | | FROM aqo_test1 AS t1, aqo_test1 AS t2, aqo_test1 AS t3, aqo_test1 AS t4 +
 | | WHERE t1.a = t2.b AND t2.a = t3.b AND t3.a = t4.b;
-{1} | 0 | EXPLAIN SELECT t1.a AS a, t2.a AS b, t3.a AS c, t4.a AS d +
+{1} | 3 | EXPLAIN SELECT t1.a AS a, t2.a AS b, t3.a AS c, t4.a AS d +
 | | FROM aqo_test1 AS t1, aqo_test1 AS t2, aqo_test1 AS t3, aqo_test1 AS t4 +
 | | WHERE t1.a = t2.b AND t2.a = t3.b AND t3.a = t4.b;
 {1} | 1 | EXPLAIN SELECT t1.a AS a, t2.a AS b, t3.a AS c, t4.a AS d +
 | | FROM aqo_test1 AS t1, aqo_test1 AS t2, aqo_test1 AS t3, aqo_test1 AS t4 +
 | | WHERE t1.a = t2.b AND t2.a = t3.b AND t3.a = t4.b;
-{1} | 2 | EXPLAIN SELECT t1.a AS a, t2.a AS b, t3.a AS c, t4.a AS d +
+{1} | 0 | EXPLAIN SELECT t1.a AS a, t2.a AS b, t3.a AS c, t4.a AS d +
 | | FROM aqo_test1 AS t1, aqo_test1 AS t2, aqo_test1 AS t3, aqo_test1 AS t4 +
 | | WHERE t1.a = t2.b AND t2.a = t3.b AND t3.a = t4.b;
 (21 rows)
@@ -587,7 +587,7 @@ SELECT * FROM check_estimated_rows('SELECT * FROM aqo_test1;');
 SELECT * FROM check_estimated_rows('SELECT * FROM aqo_test1 AS t1, aqo_test1 AS t2 WHERE t1.a = t2.b');
 estimated | actual
 -----------+--------
-20 | 19
+19 | 19
 (1 row)
 
 SELECT count(*) FROM

expected/look_a_like.out

@@ -0,0 +1,240 @@
+CREATE IF NOT EXISTS EXTENSION aqo;
+ERROR: syntax error at or near "IF"
+LINE 1: CREATE IF NOT EXISTS EXTENSION aqo;
+^
+SET aqo.join_threshold = 0;
+SET aqo.mode = 'learn';
+SET aqo.show_details = 'on';
+DROP TABLE IF EXISTS a,b CASCADE;
+NOTICE: table "a" does not exist, skipping
+NOTICE: table "b" does not exist, skipping
+CREATE TABLE a (x int);
+INSERT INTO a (x) SELECT mod(ival,10) FROM generate_series(1,1000) As ival;
+CREATE TABLE b (y int);
+INSERT INTO b (y) SELECT mod(ival + 1,10) FROM generate_series(1,1000) As ival;
+--
+-- Returns string-by-string explain of a query. Made for removing some strings
+-- from the explain output.
+--
+CREATE OR REPLACE FUNCTION expln(query_string text) RETURNS SETOF text AS $$
+BEGIN
+RETURN QUERY
+EXECUTE format('EXPLAIN (ANALYZE, VERBOSE, COSTS OFF, TIMING OFF, SUMMARY OFF) %s', query_string);
+RETURN;
+END;
+$$ LANGUAGE PLPGSQL;
+-- no one predicted rows. we use knowledge cardinalities of the query
+-- in the next queries with the same fss_hash
+SELECT str AS result
+FROM expln('
+SELECT x FROM A where x = 5;') AS str
+WHERE str NOT LIKE 'Query Identifier%';
+result
+------------------------------------------------
+Seq Scan on public.a (actual rows=100 loops=1)
+AQO not used
+Output: x
+Filter: (a.x = 5)
+Rows Removed by Filter: 900
+Using aqo: true
+AQO mode: LEARN
+JOINS: 0
+(8 rows)
+
+-- cardinality 100 in the first Seq Scan on a
+SELECT str AS result
+FROM expln('
+SELECT x FROM A,B WHERE x = 5 AND A.x = B.y;') AS str
+WHERE str NOT LIKE 'Query Identifier%';
+result
+------------------------------------------------------------
+Nested Loop (actual rows=10000 loops=1)
+AQO not used
+Output: a.x
+-> Seq Scan on public.a (actual rows=100 loops=1)
+AQO: rows=100, error=0%
+Output: a.x
+Filter: (a.x = 5)
+Rows Removed by Filter: 900
+-> Materialize (actual rows=100 loops=100)
+AQO not used
+Output: b.y
+-> Seq Scan on public.b (actual rows=100 loops=1)
+AQO not used
+Output: b.y
+Filter: (b.y = 5)
+Rows Removed by Filter: 900
+Using aqo: true
+AQO mode: LEARN
+JOINS: 0
+(19 rows)
+
+-- cardinality 100 in Nesteed Loop in the first Seq Scan on a
+SELECT str AS result
+FROM expln('
+SELECT x, sum(x) FROM A,B WHERE y = 5 AND A.x = B.y group by(x);') AS str
+WHERE str NOT LIKE 'Query Identifier%';
+result
+------------------------------------------------------------------
+GroupAggregate (actual rows=1 loops=1)
+AQO not used
+Output: a.x, sum(a.x)
+Group Key: a.x
+-> Nested Loop (actual rows=10000 loops=1)
+AQO not used
+Output: a.x
+-> Seq Scan on public.a (actual rows=100 loops=1)
+AQO: rows=100, error=0%
+Output: a.x
+Filter: (a.x = 5)
+Rows Removed by Filter: 900
+-> Materialize (actual rows=100 loops=100)
+AQO: rows=100, error=0%
+Output: b.y
+-> Seq Scan on public.b (actual rows=100 loops=1)
+AQO: rows=100, error=0%
+Output: b.y
+Filter: (b.y = 5)
+Rows Removed by Filter: 900
+Using aqo: true
+AQO mode: LEARN
+JOINS: 1
+(23 rows)
+
+-- cardinality 100 in the first Seq Scan on a
+SELECT str AS result
+FROM expln('
+SELECT x, sum(x) FROM A WHERE x = 5 group by(x);') AS str
+WHERE str NOT LIKE 'Query Identifier%';
+result
+------------------------------------------------------
+GroupAggregate (actual rows=1 loops=1)
+AQO not used
+Output: x, sum(x)
+Group Key: a.x
+-> Seq Scan on public.a (actual rows=100 loops=1)
+AQO: rows=100, error=0%
+Output: x
+Filter: (a.x = 5)
+Rows Removed by Filter: 900
+Using aqo: true
+AQO mode: LEARN
+JOINS: 0
+(12 rows)
+
+-- no one predicted rows. we use knowledge cardinalities of the query
+-- in the next queries with the same fss_hash
+SELECT str AS result
+FROM expln('
+SELECT x FROM A where x < 10 group by(x);') AS str
+WHERE str NOT LIKE 'Query Identifier%';
+result
+-------------------------------------------------------
+HashAggregate (actual rows=10 loops=1)
+AQO not used
+Output: x
+Group Key: a.x
+Batches: 1 Memory Usage: 40kB
+-> Seq Scan on public.a (actual rows=1000 loops=1)
+AQO not used
+Output: x
+Filter: (a.x < 10)
+Using aqo: true
+AQO mode: LEARN
+JOINS: 0
+(12 rows)
+
+-- cardinality 1000 in Seq Scan on a
+SELECT str AS result
+FROM expln('
+SELECT x,y FROM A,B WHERE x < 10 AND A.x = B.y;') AS str
+WHERE str NOT LIKE 'Query Identifier%';
+result
+-------------------------------------------------------------
+Merge Join (actual rows=100000 loops=1)
+AQO not used
+Output: a.x, b.y
+Merge Cond: (a.x = b.y)
+-> Sort (actual rows=1000 loops=1)
+AQO not used
+Output: a.x
+Sort Key: a.x
+Sort Method: quicksort Memory: 79kB
+-> Seq Scan on public.a (actual rows=1000 loops=1)
+AQO: rows=1000, error=0%
+Output: a.x
+Filter: (a.x < 10)
+-> Sort (actual rows=99901 loops=1)
+AQO not used
+Output: b.y
+Sort Key: b.y
+Sort Method: quicksort Memory: 79kB
+-> Seq Scan on public.b (actual rows=1000 loops=1)
+AQO not used
+Output: b.y
+Using aqo: true
+AQO mode: LEARN
+JOINS: 0
+(24 rows)
+
+-- cardinality 100 in Seq Scan on a and Seq Scan on b
+SELECT str AS result
+FROM expln('
+SELECT x FROM A,B where x < 10 and y > 10 group by(x);') AS str
+WHERE str NOT LIKE 'Query Identifier%';
+result
+----------------------------------------------------------------
+HashAggregate (actual rows=0 loops=1)
+AQO not used
+Output: a.x
+Group Key: a.x
+Batches: 1 Memory Usage: 40kB
+-> Nested Loop (actual rows=0 loops=1)
+AQO not used
+Output: a.x
+-> Seq Scan on public.a (actual rows=1000 loops=1)
+AQO: rows=1000, error=0%
+Output: a.x
+Filter: (a.x < 10)
+-> Materialize (actual rows=0 loops=1000)
+AQO not used
+-> Seq Scan on public.b (actual rows=0 loops=1)
+AQO not used
+Filter: (b.y > 10)
+Rows Removed by Filter: 1000
+Using aqo: true
+AQO mode: LEARN
+JOINS: 1
+(21 rows)
+
+-- cardinality 1000 Hash Cond: (a.x = b.y) and 1 Seq Scan on b
+-- this cardinality is wrong because we take it from bad neibours (previous query).
+-- clause y > 10 give count of rows with the same clauses.
+SELECT str AS result
+FROM expln('
+SELECT x,y FROM A,B WHERE x < 10 and y > 10 AND A.x = B.y;') AS str
+WHERE str NOT LIKE 'Query Identifier%';
+result
+----------------------------------------------------------
+Hash Join (actual rows=0 loops=1)
+AQO not used
+Output: a.x, b.y
+Hash Cond: (a.x = b.y)
+-> Seq Scan on public.a (actual rows=1 loops=1)
+AQO: rows=1000, error=100%
+Output: a.x
+Filter: (a.x < 10)
+-> Hash (actual rows=0 loops=1)
+AQO not used
+Output: b.y
+Buckets: 1024 Batches: 1 Memory Usage: 8kB
+-> Seq Scan on public.b (actual rows=0 loops=1)
+AQO: rows=1, error=100%
+Output: b.y
+Filter: (b.y > 10)
+Rows Removed by Filter: 1000
+Using aqo: true
+AQO mode: LEARN
+JOINS: 0
+(20 rows)
+

expected/unsupported.out

@@ -381,7 +381,7 @@ EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF)
 -> Aggregate (actual rows=1 loops=1000)
 AQO not used
 -> Seq Scan on t t0 (actual rows=50 loops=1000)
-AQO not used
+AQO: rows=50, error=0%
 Filter: (x = t.x)
 Rows Removed by Filter: 950
 SubPlan 2